From 18f71fc9e02eff91b6c7f7d6b83701cab98f800b Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 21:26:34 +0000 Subject: [PATCH 01/17] feat(skills): add docs-handoff skill for post-delivery documentation handoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new standalone simple skill at skills/docs-handoff/ that handles post-delivery documentation handoff for a single [DOCS] Jira issue. The skill gathers implementation evidence from contributing DEV/UI stories, validates delivery status, and produces a bounded enrichment plan with PR Remote Links and an owning-source narrative proposal. Package contents: - SKILL.md entry point (v0.1.0) - references/evidence-schema.md — input validation and evidence contracts - references/jira-enrichment.md — approved metadata/Remote Link protocol - references/handoff-protocol.md — preview/apply two-phase workflow - scripts/evidence.py — deterministic Python CLI (validate, build-plan, check-enrichment) with URL normalization, contributor extraction, change plan building, idempotency checks, and handoff rendering - scripts/test_evidence.py — 67 unit tests covering all contracts Key design decisions: - Contributors extracted from Contributing stories line, not Dependencies - Source/narrative apply disabled; proposals route to design /revise - Forbidden field policy enforced in check-enrichment (no summary, description, status, assignee, sprint, parent, labels, comment writes) - All Python operations are read-only with input non-mutation assertions - Consumer readiness dimensions are recorded but not auto-evaluated - Synthetic .example/.invalid fixtures throughout Integration: - Registered in CI test.yaml unit-tests job - Added to AGENTS.md (simple skills list, file organization, Jira writes) - Added to README.md (skills section, structure, usage examples) - Installer discovers via skills/*/SKILL.md; standalone --packages works Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test.yaml | 1 + AGENTS.md | 7 +- README.md | 9 + skills/docs-handoff/SKILL.md | 24 + .../references/evidence-schema.md | 97 ++ .../references/handoff-protocol.md | 141 ++ .../references/jira-enrichment.md | 106 ++ skills/docs-handoff/scripts/evidence.py | 1190 +++++++++++++++++ skills/docs-handoff/scripts/test_evidence.py | 935 +++++++++++++ 9 files changed, 2509 insertions(+), 1 deletion(-) create mode 100644 skills/docs-handoff/SKILL.md create mode 100644 skills/docs-handoff/references/evidence-schema.md create mode 100644 skills/docs-handoff/references/handoff-protocol.md create mode 100644 skills/docs-handoff/references/jira-enrichment.md create mode 100644 skills/docs-handoff/scripts/evidence.py create mode 100644 skills/docs-handoff/scripts/test_evidence.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5d301ca1..7bfa3cba 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -66,4 +66,5 @@ jobs: - run: python3 -m unittest discover -s sizing/scripts -p 'test_*.py' -v - run: python3 -m unittest discover -s triage/scripts -p 'test_*.py' -v - run: python3 -m unittest discover -s skills/report-bug/scripts -p 'test_*.py' -v + - run: python3 -m unittest discover -s skills/docs-handoff/scripts -p 'test_*.py' -v - run: python3 -m unittest discover -s design/scripts -p 'test_*.py' -v diff --git a/AGENTS.md b/AGENTS.md index 0f21d0fb..3cf3a7a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ This file provides guidance to AI coding assistants when working with this repos This repository contains reusable AI coding workflows and focused skills that can be installed globally or per-project in any environment (Cursor, Claude Code, Gemini, Codex). Each package is a self-contained directory with structured markdown files that AI agents can read and execute. **Current simple skills:** +- **docs-handoff** — Post-delivery documentation handoff for a single [DOCS] Jira issue with evidence gathering and enrichment - **gh-stack** — Manages stacked PRs with gh-stack (creation, viewing, editing, push, submit, sync, rebase, merge, checkout) - **report-bug** — Configurable, evidence-based Jira Bug reporting with explicit confirmation @@ -272,6 +273,10 @@ ai-workflows/ ├── triage/ ├── ui-design/ ├── skills/ # Focused skills (auto-discovered via SKILL.md) +│ ├── docs-handoff/ +│ │ ├── SKILL.md +│ │ ├── references/ +│ │ └── scripts/ │ ├── gh-stack/ │ │ ├── SKILL.md │ │ └── references/ @@ -295,5 +300,5 @@ ai-workflows/ When a workflow or simple skill invokes commands that could affect shared systems: - **Git operations**: Always verify with `git status` before destructive operations - **PR/MR creation**: Confirm branch and base before pushing -- **Jira writes**: cve-fix `/close`, design `/sync`, ui-design `/sync`, sizing `/apply`, and `report-bug` may write to Jira; all require explicit approval. `report-bug` may create only the fully previewed issue and approved follow-up links/attachments +- **Jira writes**: cve-fix `/close`, design `/sync`, ui-design `/sync`, sizing `/apply`, `report-bug`, and `docs-handoff` may write to Jira; all require explicit approval. `report-bug` may create only the fully previewed issue and approved follow-up links/attachments. `docs-handoff` may add only approved PR Remote Links and eligible metadata to the target DOCS issue - **Documentation changes**: Run Vale validation before applying changes to repository files diff --git a/README.md b/README.md index 9bcd31ed..25b0dd77 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ Reusable AI coding workflows and focused skills a team member can install global ### Skills +- **Docs Handoff** -- Post-delivery documentation handoff for a single `[DOCS]` + Jira issue. Gathers implementation evidence from contributing stories, + validates delivery, and produces a bounded enrichment plan with PR Remote + Links and an owning-source narrative proposal. + See [skills/docs-handoff/SKILL.md](skills/docs-handoff/SKILL.md). + - **gh-stack** -- Manages stacked PRs and splits multi-part work into reviewable branches with `gh stack`. Covers creation, viewing, editing, push, submit, sync, rebase, merge, and checkout. @@ -88,6 +94,8 @@ supporting references or scripts they need. The installer discovers both forms. docs-writer/ SKILL.md, skills/, commands/ skills/ + docs-handoff/ + SKILL.md, references/, scripts/ gh-stack/ SKILL.md, references/ report-bug/ @@ -212,6 +220,7 @@ Each workflow or skill is intended for a specific project or use case: - **triage** -- teams that want bulk Jira triage, categorization, and HTML reports from this repo or a clone - **sizing** -- teams sizing Features for cycle planning using T-shirt sizes with per-team effort breakdowns - **skill-reviewer** -- reviewing or standardizing Cursor/agent skills and skill packs (structure, clarity, completeness) +- **docs-handoff** -- post-delivery documentation handoff for [DOCS] Jira issues with evidence gathering and PR Remote Link enrichment - **gh-stack** -- managing stacked PRs with `gh stack` in any project using stacked branches - **report-bug** -- filing complete Jira bugs with consumer-configurable targets and templates while leaving Priority for triage diff --git a/skills/docs-handoff/SKILL.md b/skills/docs-handoff/SKILL.md new file mode 100644 index 00000000..b8b2185b --- /dev/null +++ b/skills/docs-handoff/SKILL.md @@ -0,0 +1,24 @@ +--- +name: docs-handoff +version: 0.1.0 +description: >- + Post-delivery documentation handoff for a single [DOCS] Jira issue. Gathers + implementation evidence from contributing stories, validates delivery, and + produces a bounded enrichment plan. Use after implementation PRs merge. +--- + +# Docs Handoff + +Gather delivery evidence for one `[DOCS]` Jira issue and produce an +actionable handoff for the documentation writer. + +1. Read [references/evidence-schema.md](references/evidence-schema.md) + for input validation, contributor extraction, and evidence contracts. +2. Read [references/jira-enrichment.md](references/jira-enrichment.md) + for the approved metadata and PR Remote Link write protocol. +3. Read [references/handoff-protocol.md](references/handoff-protocol.md) + for the preview/apply two-phase workflow and source-change proposals. + +Inputs: one DOCS issue key, a selected release/target, and an optional +caller-supplied consumer profile. Outputs: delivery-evidence findings, +enrichment plan, and an owning-source narrative proposal when supported. diff --git a/skills/docs-handoff/references/evidence-schema.md b/skills/docs-handoff/references/evidence-schema.md new file mode 100644 index 00000000..933a19a3 --- /dev/null +++ b/skills/docs-handoff/references/evidence-schema.md @@ -0,0 +1,97 @@ +# Evidence Schema and Validation + +Contract for DOCS issue validation, contributor extraction, and +delivery-evidence normalization. + +## DOCS Issue Snapshot + +The skill operates on a single `[DOCS]` Jira issue at a time. The +caller provides a JSON snapshot with at minimum: + +| Field | Type | Required | Source | +|---|---|---|---| +| `key` | string | Yes | Jira issue key (e.g. `TEST-100`) | +| `summary` | string | Yes | Must contain `[DOCS]` prefix | +| `description` | string | Yes | Rendered Markdown from Jira | +| `status` | string | No | Current issue status | +| `parent` | object | No | Parent issue key and type | +| `remote_links` | array | No | Existing Remote Links | +| `issue_links` | array | No | Existing issue links | + +The script `../scripts/evidence.py validate` validates the snapshot and +extracts structured data. + +## Required Description Sections + +A valid `[DOCS]` issue description must contain these `##` sections: + +- **Documentation Inputs** — planned user-facing artifacts grouped by + contributing story +- **Documentation Scope** — what the reader needs to understand +- **Acceptance Criteria** — reader-verifiable documentation outcomes + +Missing sections produce validation errors. + +## Contributor Extraction + +Contributors are extracted from the `Contributing stories:` line in the +Design Reference section. In a rendered (synced) issue, these are Jira +keys (e.g. `TEST-201`). Unresolved `Story N.NN` placeholders are +extracted as qualified IDs and flagged as warnings. + +**Contributors vs Dependencies**: the `Dependencies` section lists +prerequisite stories that must complete before documentation work begins. +Dependencies are NOT automatically direct contributors — they do not +produce user-facing artifacts that belong in Documentation Inputs. The +script extracts these separately and the plan does not generate +missing-evidence findings for dependency-only references. + +## Documentation Inputs Parsing + +The Documentation Inputs section is parsed into structured items: + +``` +**JIRA-KEY -- title:** +- Input item text +- Another item [Assumption] +- Open item [Open: §9.2] +``` + +Each item records: +- `contributor`: the Jira key or qualified story ID from the header +- `text`: the input description +- `marker`: `planned` (default), `assumption`, or `open` +- `open_ref`: the referenced open question (when marker is `open`) + +## PR Evidence Items + +Each gathered PR evidence item must have: + +| Field | Type | Required | +|---|---|---| +| `pr_url` | string | Yes — HTTP(S) PR/MR URL | +| `contributor_key` | string | Yes — Jira key of contributing story | +| `target_branch` | string | No — PR target branch | +| `state` | string | No — PR state (open, merged, closed) | +| `title` | string | No — PR title | +| `merged` | boolean | No — whether PR is merged | +| `merge_commit` | string | No — merge commit SHA | +| `observations` | array | No — AI-generated semantic observations | + +The script `normalize-evidence` (internal to `build-plan`) validates +URLs, rejects embedded credentials, deduplicates by normalized URL, +and preserves distinct repository identities. + +## Input Status Values + +After comparing planned inputs against gathered evidence, each input +receives a status: + +- `delivered` — contributor has at least one merged PR +- `in_progress` — contributor has PRs but none merged +- `missing` — no PR evidence found for this contributor + +These are *delivery* statuses, not semantic confirmation. A `delivered` +status means a merged PR exists — it does not prove the actual +implementation matches the planned input. Semantic planned-vs-actual +assessment is an AI responsibility recorded in `observations`. diff --git a/skills/docs-handoff/references/handoff-protocol.md b/skills/docs-handoff/references/handoff-protocol.md new file mode 100644 index 00000000..f092fefe --- /dev/null +++ b/skills/docs-handoff/references/handoff-protocol.md @@ -0,0 +1,141 @@ +# Handoff Protocol + +Preview/apply two-phase workflow for the docs-handoff skill. This +protocol governs the end-to-end flow from issue intake to enrichment +and source-change proposals. + +## Phase 1: Preview (No Side Effects) + +All operations in the preview phase are read-only. No Jira writes, +no source file modifications, no issue state changes. + +### Step 1 — Validate Issue + +Provide the DOCS issue snapshot to `evidence.py validate`. The script +validates structure, extracts contributors, dependencies, and +Documentation Inputs items. If validation fails (missing sections, +invalid key, no contributors), stop and report the errors. + +### Step 2 — Gather Evidence (AI Responsibility) + +For each extracted contributor (NOT dependencies): + +1. Fetch the contributor's Jira issue to discover linked PRs +2. For each PR, read metadata: URL, title, target branch, merge status, + merge commit +3. Read the PR diff and assess what user-facing surfaces changed +4. Record semantic observations comparing planned inputs against actual + implementation + +**Evidence quality requirements:** + +- A merged PR status alone does not prove the feature is delivered to + the target release. Record the target branch and note ambiguity when + it differs from the expected release branch. +- String equality between a planned input and a PR title is not semantic + confirmation. Actual code/diff review is required. +- Partial evidence (some PRs found, some contributors without PRs) is a + valid intermediate state. Report it accurately — do not claim + completion or fabricate evidence. +- Unknown or contradictory results (e.g., PR merged but reverted) must + be reported as findings, not silently resolved. + +### Step 3 — Build Plan + +Provide the validation result, normalized evidence, target, and optional +profile to `evidence.py build-plan`. The script produces: + +- Per-input delivery status (delivered/in\_progress/missing) +- PR Remote Links to add +- Metadata proposals +- Findings (missing evidence, new surfaces, unresolved refs, target + ambiguity) +- Source-change proposal (disabled; advisory) +- Consumer readiness dimensions (when profile provided) + +### Step 4 — Check Enrichment + +Provide the plan and current issue state (with existing Remote Links) to +`evidence.py check-enrichment`. The script identifies: + +- Eligible links (new, not already present) +- Existing links (already on the issue) +- Skipped links (invalid URLs) +- Forbidden fields (blocked by policy) +- Baseline fingerprint (for drift detection) + +### Step 5 — Present Preview + +Show the user the complete bounded preview: + +- Target issue key and current baseline +- Delivery summary (counts per status) +- Per-input status with PR references and target notes +- Findings requiring attention +- Eligible PR Remote Links (the set that would be added) +- Existing links (already present, no action needed) +- Metadata proposals with evidence +- Source-change proposal (disabled, with intended changes and owning + workflow action) +- Consumer readiness state (dimensions recorded, results pending) +- Operation eligibility summary + +## Phase 2: Apply (Explicit Approval Required) + +The user reviews the preview and explicitly approves or rejects the +enrichment plan as a single bounded operation. + +### On Approval + +1. **Re-read** the issue's current Remote Links from Jira (fresh, not + cached from preview). +2. **Detect drift**: if the baseline fingerprint changed since the + preview, the plan is stale. Report the conflict and regenerate. +3. **Apply** eligible PR Remote Links via the runtime's available Jira + write capability. +4. **Re-read** the issue after writes. +5. **Produce receipt**: per-link confirmed/uncertain status. +6. **Report**: complete results including any uncertain outcomes. + +### On Rejection + +No writes. The preview artifacts remain available for review. + +## Source-Change Proposals + +The skill produces structured proposals for updating the Documentation +Inputs section in the owning design workflow's source artifacts. These +proposals include: + +- Specific intended changes (add, update) with evidence references +- The owning workflow (`design`) and action (`/revise`) +- Factual observations from PR review + +**Source/narrative apply is disabled.** The docs-handoff skill does NOT +directly modify design source artifacts or Jira issue descriptions. The +proposal is advisory — the human or owning workflow applies changes +through the established `/revise` phase. + +This is an explicit design choice, not a limitation to work around: + +- The design workflow owns the Documentation Inputs source +- Direct edits could conflict with the sync manifest and content hashes +- Writer-managed content in the description must not be overwritten +- The `/revise` phase has the correct reconciliation and consistency + checks + +If a supported explicit owning-source handoff can be established in a +future version, it must be tested for producer/renderer/sync preservation +and writer/concurrency handling BEFORE enabling apply. + +## Late Contributor Changes + +If evidence gathering reveals that a contributing story's actual scope +differs from the planned Documentation Inputs (e.g., a new API field was +added that is not in the plan), this is reported as a finding with type +`new_surface`. It is a PROPOSAL for the user to evaluate, not a silent +scope expansion. The user decides whether to: + +1. Accept the new surface and request a `/revise` to update the source +2. Defer it to a future documentation cycle +3. Determine it is out of scope for this DOCS story diff --git a/skills/docs-handoff/references/jira-enrichment.md b/skills/docs-handoff/references/jira-enrichment.md new file mode 100644 index 00000000..040ba91d --- /dev/null +++ b/skills/docs-handoff/references/jira-enrichment.md @@ -0,0 +1,106 @@ +# Jira Enrichment Protocol + +Approved metadata and PR Remote Link write protocol for the docs-handoff +skill. All writes require explicit user approval of the complete bounded +plan — not per-operation prompts. + +## Approved Operations + +| Operation | Scope | Confirmation | +|---|---|---| +| Add PR Remote Links | Per-PR URL on the target DOCS issue | One plan approval | +| Propose metadata fields | Target release, when value is from approved catalog | One plan approval | + +## Forbidden Operations + +The skill must NEVER perform these operations, regardless of profile or +caller input: + +- Modify `summary` or `description` +- Change issue `status` or `resolution` +- Set or change `assignee` +- Modify `sprint`, `parent`, or `labels` +- Add `comments` +- Set `priority` or `issuetype` +- Create new issues +- Trigger documentation authoring or publication +- Execute workflow hooks or batch operations + +A consumer profile cannot override these restrictions. The +`check-enrichment` subcommand rejects any plan that includes forbidden +fields. + +## PR Remote Links + +Remote Links connect the DOCS issue to implementation PRs. Each link +has: + +- **URL**: normalized PR/MR URL (validated, no credentials) +- **Title**: PR title for display +- **Relationship**: `implements` (the PR implements a contributing story) + +### Idempotency + +Before proposing links, the skill checks existing Remote Links on the +issue. Links whose normalized URL already exists are reported as +`existing` and skipped. Only genuinely new links are `eligible`. + +### Approval Flow + +1. The preview shows ALL proposed links, existing links, and the complete + enrichment plan as a single bounded operation. +2. The user approves or rejects the entire plan. +3. On approval, eligible links are added via the runtime's available Jira + write capability. +4. After writes, the skill re-reads the issue's Remote Links and produces + a per-link receipt with `confirmed` or `uncertain` status. + +The approval is a genuine user decision — not a trusted `approved: true` +JSON toggle. The skill waits for the runtime's interactive confirmation. + +## Metadata Fields + +Metadata proposals require actual project field catalog and valid values. +Unknown or uncatalogued values remain gaps — the skill never guesses +release versions, priority values, or component names. + +The `proposed_target_release` field is included when the caller provides +a target release. Applying it requires verifying the value exists in the +project's Fix Version or Target Version catalog. + +## Baseline Drift Detection + +The `check-enrichment` subcommand computes a baseline fingerprint from +the issue's current Remote Links. If the baseline changes between plan +creation and apply (concurrent modification), the plan is stale and must +be regenerated. + +## Readback and Receipts + +After applying approved writes, the skill: + +1. Re-reads the issue's Remote Links from Jira +2. Compares expected links against the readback +3. Produces a receipt for each operation: + - `confirmed`: link found in readback + - `uncertain`: link not found — may indicate write failure or + propagation delay + +On uncertain results, the skill reports the discrepancy. It does NOT +automatically retry or claim rollback. + +## Consumer Profile + +An optional caller-supplied consumer profile may specify: + +- `readiness_dimensions`: list of assessment dimensions the consumer + cares about (e.g. `["technical_accuracy", "completeness"]`). These + are recorded but NOT automatically evaluated — the caller must populate + actual assessments. Absent dimensions are a visible gap, not universal + readiness. + +The profile cannot: +- Override forbidden field restrictions +- Bypass the approval flow +- Enable automatic writes +- Provide internal service endpoints or credentials diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py new file mode 100644 index 00000000..77bbc409 --- /dev/null +++ b/skills/docs-handoff/scripts/evidence.py @@ -0,0 +1,1190 @@ +#!/usr/bin/env python3 +"""Deterministic delivery-evidence operations for the docs-handoff skill. + +Provides subcommands for validating DOCS issue structure, extracting +contributor references, normalizing PR evidence, building bounded +change plans, and checking enrichment idempotency. + +Subcommands: + validate Validate a DOCS issue snapshot and extract contributors + build-plan Build a delivery-evidence change plan from gathered evidence + check-enrichment Check proposed enrichment against existing issue state + +All inputs come from files or stdin (JSON). This script never contacts +Jira, GitHub, or any external service. ``urllib.parse`` is used for +URL validation only -- no network I/O. + +Exit codes: + 0 -- success (result JSON on stdout) + 1 -- input validation error or data inconsistency (diagnostic on stderr) + 2 -- invalid arguments (from argparse) +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import sys +from pathlib import Path +from typing import Any, NoReturn +from urllib.parse import urlsplit + +# --------------------------------------------------------------------------- +# Exit codes +# --------------------------------------------------------------------------- + +EXIT_SUCCESS = 0 +EXIT_ERROR = 1 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def fail(msg: str, code: int = EXIT_ERROR) -> NoReturn: + """Print an error to stderr and exit.""" + print(f"error: {msg}", file=sys.stderr) + sys.exit(code) + + +def _load_json(path: Path) -> Any: + """Load and return parsed JSON from a file.""" + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError) as exc: + fail(f"cannot read {path}: {exc}") + + +def _write_json(data: Any) -> None: + """Write JSON to stdout.""" + json.dump(data, sys.stdout, indent=2, ensure_ascii=False) + sys.stdout.write("\n") + + +# --------------------------------------------------------------------------- +# URL validation +# --------------------------------------------------------------------------- + +# Allowed PR URL host patterns (public forges) +_PR_URL_RE = re.compile( + r"^https://(?:github\.com|gitlab\.com|gitlab\.[a-z0-9.-]+)" + r"/[^/]+/[^/]+/(?:pull|merge_requests)/\d+/?$" +) + + +def validate_url(url: str, label: str = "url") -> str: + """Validate an HTTP(S) URL without embedded credentials. + + Returns the normalized URL (trailing slash stripped) or raises + ValueError with a descriptive message. + """ + if not isinstance(url, str) or not url.strip(): + raise ValueError(f"{label}: must be a non-empty string") + url = url.strip() + + # Reject control characters and whitespace + if any(ord(c) < 32 or c.isspace() for c in url): + raise ValueError(f"{label}: contains whitespace or control characters") + + parsed = urlsplit(url) + if parsed.scheme.lower() not in ("http", "https"): + raise ValueError(f"{label}: must use http or https scheme") + if not parsed.hostname: + raise ValueError(f"{label}: missing hostname") + if parsed.username is not None or parsed.password is not None: + raise ValueError(f"{label}: must not contain embedded credentials") + + # Strip trailing slash for normalization + normalized = url.rstrip("/") + return normalized + + +def validate_pr_url(url: str, label: str = "pr_url") -> str: + """Validate and normalize a pull/merge request URL. + + Returns the normalized URL or raises ValueError. + """ + normalized = validate_url(url, label) + # Strip fragment and query for identity comparison + parsed = urlsplit(normalized) + clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + clean = clean.rstrip("/") + return clean + + +def extract_pr_identity(url: str) -> dict[str, str]: + """Extract owner, repo, and PR number from a normalized PR URL. + + Returns a dict with ``owner``, ``repo``, ``number``, ``host``. + Raises ValueError if the URL is not a recognized PR URL. + """ + parsed = urlsplit(url) + parts = parsed.path.strip("/").split("/") + # GitHub: owner/repo/pull/N GitLab: owner/repo/-/merge_requests/N + if len(parts) >= 4 and parts[2] == "pull": + return { + "host": parsed.hostname or "", + "owner": parts[0], + "repo": parts[1], + "number": parts[3], + } + # GitLab with /-/ prefix + if len(parts) >= 5 and parts[2] == "-" and parts[3] == "merge_requests": + return { + "host": parsed.hostname or "", + "owner": parts[0], + "repo": parts[1], + "number": parts[4], + } + # GitLab without /-/ prefix + if len(parts) >= 4 and parts[2] == "merge_requests": + return { + "host": parsed.hostname or "", + "owner": parts[0], + "repo": parts[1], + "number": parts[3], + } + raise ValueError(f"unrecognized PR/MR URL structure: {url}") + + +# --------------------------------------------------------------------------- +# Jira key and reference parsing +# --------------------------------------------------------------------------- + +_JIRA_KEY_RE = re.compile(r"\b([A-Z][A-Z0-9]+-\d+)\b") +_STORY_REF_RE = re.compile( + r"(?:Stories|Story)\s+(\d+\.\d+(?:\s*,\s*\d+\.\d+)*)" +) + + +def parse_jira_key(text: str) -> str | None: + """Extract a single Jira key from text, or None.""" + m = _JIRA_KEY_RE.search(text) + return m.group(1) if m else None + + +def extract_all_jira_keys(text: str) -> list[str]: + """Extract all Jira keys from text, preserving order, deduplicated.""" + seen: set[str] = set() + result: list[str] = [] + for m in _JIRA_KEY_RE.finditer(text): + key = m.group(1) + if key not in seen: + seen.add(key) + result.append(key) + return result + + +def extract_story_refs(text: str) -> list[str]: + """Extract Story N.NN qualified references from text. + + Returns deduplicated list of qualified IDs like ``story-1.01``. + """ + seen: set[str] = set() + result: list[str] = [] + for m in _STORY_REF_RE.finditer(text): + for part in m.group(1).split(","): + part = part.strip() + if part: + qid = f"story-{part}" + if qid not in seen: + seen.add(qid) + result.append(qid) + return result + + +# --------------------------------------------------------------------------- +# DOCS issue validation and contributor extraction +# --------------------------------------------------------------------------- + +# Sections expected in a rendered DOCS issue description +_DOCS_SECTIONS = { + "Documentation Inputs", + "Documentation Scope", + "Acceptance Criteria", +} + + +def _extract_sections(text: str) -> dict[str, str]: + """Extract ## sections from rendered Jira markdown description. + + Returns heading -> body text mapping. + """ + sections: dict[str, str] = {} + current: str | None = None + lines: list[str] = [] + + for raw in text.splitlines(): + stripped = raw.strip() + if stripped.startswith("## "): + if current is not None: + sections[current] = "\n".join(lines).strip() + current = stripped[3:].strip() + lines = [] + elif current is not None: + lines.append(raw) + + if current is not None: + sections[current] = "\n".join(lines).strip() + + return sections + + +def _extract_contributing_stories(description: str) -> list[str]: + """Extract Contributing stories references from a rendered description. + + Looks for a ``Contributing stories:`` line in the Design Reference + section or in the full description. Returns a list of Jira keys + found on that line. Local ``Story N.NN`` placeholders without Jira + keys are returned as qualified IDs (``story-N.NN``) and flagged as + unresolved. + """ + result: list[str] = [] + seen: set[str] = set() + for line in description.splitlines(): + stripped = line.strip() + # Normalize list-item prefixes + for pfx in ("- ", "* ", "+ "): + if stripped.startswith(pfx): + stripped = stripped[len(pfx):].strip() + break + if not stripped.lower().startswith("contributing stories:"): + continue + value = stripped.split(":", 1)[1].strip() + # Extract Jira keys (rendered form) + for key in extract_all_jira_keys(value): + if key not in seen: + seen.add(key) + result.append(key) + # Extract unresolved Story N.NN placeholders + for qid in extract_story_refs(value): + if qid not in seen: + seen.add(qid) + result.append(qid) + return result + + +def _extract_dependencies(description: str) -> list[str]: + """Extract Dependencies references from a rendered description. + + Returns Jira keys and unresolved story refs found in the Dependencies + section only. + """ + sections = _extract_sections(description) + deps_text = sections.get("Dependencies", "") + if not deps_text: + # Try inline Dependencies: line + for line in description.splitlines(): + stripped = line.strip() + for pfx in ("- ", "* ", "+ "): + if stripped.startswith(pfx): + stripped = stripped[len(pfx):].strip() + break + if stripped.lower().startswith("dependencies:"): + deps_text = stripped.split(":", 1)[1].strip() + break + + result: list[str] = [] + seen: set[str] = set() + for key in extract_all_jira_keys(deps_text): + if key not in seen: + seen.add(key) + result.append(key) + for qid in extract_story_refs(deps_text): + if qid not in seen: + seen.add(qid) + result.append(qid) + return result + + +def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: + """Parse Documentation Inputs section into structured items. + + Recognizes bold contributor headers (``**Story N.NN -- title:**`` or + ``**JIRA-KEY -- title:**``) followed by bullet-pointed input items. + Each input may have ``[Assumption]`` or ``[Open: ...]`` markers. + """ + items: list[dict[str, Any]] = [] + current_contributor: str | None = None + contributor_key: str | None = None + + for line in text.splitlines(): + stripped = line.strip() + # Bold contributor header: **KEY -- title:** or **Story N.NN -- title:** + header_match = re.match( + r"\*\*(.+?)(?:\s*(?:--|—)\s*.+?)?\s*:?\*\*:?\s*$", + stripped, + ) + if header_match: + header_text = header_match.group(1).strip() + # Extract key from header + key = parse_jira_key(header_text) + if key: + contributor_key = key + current_contributor = header_text + else: + # May be Story N.NN format + refs = extract_story_refs(header_text) + if refs: + contributor_key = refs[0] + current_contributor = header_text + else: + contributor_key = header_text + current_contributor = header_text + continue + + # Bullet item under a contributor + bullet_match = re.match(r"[-*+]\s+(.*)", stripped) + if bullet_match and current_contributor is not None: + item_text = bullet_match.group(1).strip() + marker = "planned" + open_ref = None + if "[Assumption]" in item_text: + marker = "assumption" + item_text = item_text.replace("[Assumption]", "").strip() + open_match = re.search(r"\[Open:\s*([^\]]+)\]", item_text) + if open_match: + marker = "open" + open_ref = open_match.group(1).strip() + item_text = re.sub(r"\[Open:\s*[^\]]+\]", "", item_text).strip() + + items.append({ + "contributor": contributor_key, + "contributor_header": current_contributor, + "text": item_text, + "marker": marker, + "open_ref": open_ref, + }) + + return items + + +def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: + """Validate a DOCS issue snapshot and extract structured data. + + Input: a JSON object with at minimum ``key``, ``summary``, + ``description``. May include ``status``, ``parent``, + ``remote_links``, ``issue_links``. + + Returns a result object with ``valid``, ``errors``, ``warnings``, + and extracted ``contributors``, ``dependencies``, + ``documentation_inputs``, ``sections``. + """ + errors: list[str] = [] + warnings: list[str] = [] + + key = snapshot.get("key") + if not key or not isinstance(key, str): + errors.append("missing or invalid 'key' field") + elif not _JIRA_KEY_RE.fullmatch(key): + errors.append(f"invalid Jira key format: {key}") + + summary = snapshot.get("summary", "") + if not isinstance(summary, str) or not summary.strip(): + errors.append("missing or empty 'summary' field") + elif "[DOCS]" not in summary: + errors.append( + f"summary does not contain [DOCS] prefix: {summary!r}" + ) + + description = snapshot.get("description", "") + if not isinstance(description, str) or not description.strip(): + errors.append("missing or empty 'description' field") + return { + "valid": False, + "errors": errors, + "warnings": warnings, + "key": key, + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + + sections = _extract_sections(description) + + # Check required sections + missing_sections = _DOCS_SECTIONS - set(sections.keys()) + for ms in sorted(missing_sections): + errors.append(f"missing required section: {ms}") + + # Extract contributors + contributors = _extract_contributing_stories(description) + unresolved = [c for c in contributors if c.startswith("story-")] + if unresolved: + for u in unresolved: + warnings.append(f"unresolved local reference: {u}") + + if not contributors: + errors.append("no Contributing stories found in description") + + # Extract dependencies (distinct from contributors) + dependencies = _extract_dependencies(description) + + # Parse documentation inputs + doc_inputs_text = sections.get("Documentation Inputs", "") + documentation_inputs = _parse_documentation_inputs(doc_inputs_text) + if not documentation_inputs and "Documentation Inputs" in sections: + warnings.append( + "Documentation Inputs section exists but no structured items parsed" + ) + + return { + "valid": len(errors) == 0, + "errors": errors, + "warnings": warnings, + "key": key, + "contributors": contributors, + "dependencies": dependencies, + "documentation_inputs": documentation_inputs, + "sections": {k: v for k, v in sections.items()}, + } + + +# --------------------------------------------------------------------------- +# Evidence normalization +# --------------------------------------------------------------------------- + + +def normalize_evidence( + evidence_items: list[dict[str, Any]], +) -> dict[str, Any]: + """Normalize and deduplicate gathered PR evidence. + + Each evidence item must have ``pr_url``, ``contributor_key``, and + optionally ``target_branch``, ``state``, ``title``, ``merged``, + ``merge_commit``, ``observations`` (list of strings from AI review). + + Returns ``{"items": [...], "errors": [...], "warnings": [...]}``. + Deduplication is by normalized PR URL. + """ + errors: list[str] = [] + warnings: list[str] = [] + seen_urls: dict[str, int] = {} + normalized: list[dict[str, Any]] = [] + + if not isinstance(evidence_items, list): + return {"items": [], "errors": ["evidence must be an array"], "warnings": []} + + for i, item in enumerate(evidence_items): + if not isinstance(item, dict): + errors.append(f"evidence[{i}]: must be an object") + continue + + pr_url_raw = item.get("pr_url", "") + contributor = item.get("contributor_key", "") + + if not contributor or not isinstance(contributor, str): + errors.append(f"evidence[{i}]: missing contributor_key") + continue + + try: + pr_url = validate_pr_url(pr_url_raw, f"evidence[{i}].pr_url") + except ValueError as exc: + errors.append(str(exc)) + continue + + # Deduplicate by normalized URL + if pr_url in seen_urls: + existing_idx = seen_urls[pr_url] + # Merge contributor if different + existing = normalized[existing_idx] + if contributor not in existing.get("contributor_keys", []): + existing.setdefault("contributor_keys", [contributor]) + if contributor not in existing["contributor_keys"]: + existing["contributor_keys"].append(contributor) + warnings.append( + f"evidence[{i}]: duplicate PR URL {pr_url} " + f"(merged with evidence[{existing_idx}])" + ) + continue + + # Extract PR identity + try: + pr_id = extract_pr_identity(pr_url) + except ValueError: + pr_id = {} + + entry: dict[str, Any] = { + "pr_url": pr_url, + "contributor_keys": [contributor], + "pr_identity": pr_id, + "target_branch": item.get("target_branch", ""), + "state": item.get("state", ""), + "title": item.get("title", ""), + "merged": item.get("merged"), + "merge_commit": item.get("merge_commit", ""), + "observations": item.get("observations", []), + } + + seen_urls[pr_url] = len(normalized) + normalized.append(entry) + + return {"items": normalized, "errors": errors, "warnings": warnings} + + +# --------------------------------------------------------------------------- +# Change plan +# --------------------------------------------------------------------------- + + +def _fingerprint(data: Any) -> str: + """Compute a stable SHA-256 fingerprint of JSON-serializable data.""" + raw = json.dumps(data, sort_keys=True, ensure_ascii=True) + return hashlib.sha256(raw.encode()).hexdigest() + + +def build_change_plan( + validation: dict[str, Any], + evidence: dict[str, Any], + target: dict[str, Any], + profile: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build a delivery-evidence change plan. + + Args: + validation: output of ``validate_issue`` + evidence: output of ``normalize_evidence`` + target: ``{"release": "...", "target_branch": "..."}`` + profile: optional caller-supplied consumer profile + + Returns a plan object with ``status``, ``inputs``, ``pr_links``, + ``metadata``, ``findings``, ``source_proposal``, and ``fingerprint``. + """ + errors: list[str] = [] + findings: list[dict[str, str]] = [] + + if not validation.get("valid"): + return { + "status": "invalid_issue", + "errors": validation.get("errors", []), + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": findings, + "source_proposal": None, + "readiness": None, + "fingerprint": "", + } + + issue_key = validation["key"] + contributors = set(validation.get("contributors", [])) + dependencies = set(validation.get("dependencies", [])) + doc_inputs = validation.get("documentation_inputs", []) + evidence_items = evidence.get("items", []) + target_release = target.get("release", "") + target_branch = target.get("target_branch", "") + + # --- Per-input delivery status --- + input_statuses: list[dict[str, Any]] = [] + contributor_evidence: dict[str, list[dict[str, Any]]] = {} + + for ev in evidence_items: + for ck in ev.get("contributor_keys", []): + contributor_evidence.setdefault(ck, []).append(ev) + + # Track which contributors had evidence + contributors_with_evidence: set[str] = set() + + for doc_input in doc_inputs: + contrib = doc_input.get("contributor") + evs = contributor_evidence.get(contrib, []) + + status = "missing" + pr_refs: list[str] = [] + target_notes: list[str] = [] + + if evs: + contributors_with_evidence.add(contrib) + has_merged = any(e.get("merged") is True for e in evs) + status = "delivered" if has_merged else "in_progress" + pr_refs = [e["pr_url"] for e in evs] + + for ev in evs: + ev_target = ev.get("target_branch", "") + if target_branch and ev_target and ev_target != target_branch: + target_notes.append( + f"PR {ev['pr_url']} targets {ev_target}, " + f"expected {target_branch}" + ) + + input_statuses.append({ + "text": doc_input.get("text", ""), + "contributor": contrib, + "marker": doc_input.get("marker", "planned"), + "status": status, + "pr_refs": pr_refs, + "target_notes": target_notes, + }) + + # --- Contributors without evidence --- + for contrib in sorted(contributors): + if contrib not in contributors_with_evidence: + if contrib.startswith("story-"): + findings.append({ + "type": "unresolved_contributor", + "ref": contrib, + "detail": ( + f"Contributor {contrib} is an unresolved local " + f"reference with no Jira key mapping" + ), + }) + elif contrib not in dependencies: + findings.append({ + "type": "missing_evidence", + "ref": contrib, + "detail": ( + f"Contributing story {contrib} has no " + f"associated PR evidence" + ), + }) + + # --- Evidence for non-contributors (new surfaces) --- + all_contributor_keys: set[str] = set() + for di in doc_inputs: + c = di.get("contributor") + if c: + all_contributor_keys.add(c) + all_contributor_keys |= contributors + + for ev in evidence_items: + for ck in ev.get("contributor_keys", []): + if ck not in all_contributor_keys: + findings.append({ + "type": "new_surface", + "ref": ck, + "detail": ( + f"PR {ev['pr_url']} is attributed to {ck} " + f"which is not a listed contributor" + ), + }) + + # --- PR Remote Links to add --- + pr_links: list[dict[str, Any]] = [] + for ev in evidence_items: + pr_links.append({ + "url": ev["pr_url"], + "title": ev.get("title", ""), + "relationship": "implements", + "contributor_keys": ev.get("contributor_keys", []), + }) + + # --- Metadata proposals --- + metadata: dict[str, Any] = {} + if target_release: + metadata["proposed_target_release"] = target_release + + # Profile-based readiness (independent of delivery) + readiness: dict[str, Any] | None = None + if profile: + dims = profile.get("readiness_dimensions", []) + if dims: + readiness = { + "dimensions": dims, + "results": {}, + "complete": False, + "note": ( + "Consumer readiness assessment requires evaluating " + "each dimension against actual delivery evidence. " + "Dimensions are recorded; results must be populated " + "by the caller with actual assessments." + ), + } + else: + readiness = { + "dimensions": [], + "results": {}, + "complete": False, + "note": "No readiness dimensions in consumer profile", + } + + # --- Source-change proposal --- + source_proposal = _build_source_proposal( + doc_inputs, input_statuses, findings, evidence_items + ) + + plan = { + "status": "complete", + "issue_key": issue_key, + "target": target, + "inputs": input_statuses, + "pr_links": pr_links, + "metadata": metadata, + "findings": findings, + "source_proposal": source_proposal, + "readiness": readiness, + "fingerprint": "", + } + plan["fingerprint"] = _fingerprint({ + "issue_key": issue_key, + "inputs": input_statuses, + "pr_links": pr_links, + "metadata": metadata, + }) + + return plan + + +def _build_source_proposal( + doc_inputs: list[dict[str, Any]], + input_statuses: list[dict[str, Any]], + findings: list[dict[str, str]], + evidence_items: list[dict[str, Any]], +) -> dict[str, Any]: + """Build an owning-source narrative-change proposal. + + This produces a structured proposal that the owning design workflow + could apply via its ``/revise`` phase. The proposal includes + specific intended changes to the Documentation Inputs section with + supporting evidence references. + + The docs-handoff skill does NOT directly apply these changes. + Source/narrative apply is disabled; the proposal is advisory. + """ + proposed_changes: list[dict[str, Any]] = [] + + for inp_status in input_statuses: + status = inp_status["status"] + contrib = inp_status["contributor"] + text = inp_status["text"] + + if status == "delivered": + # Check if observations suggest the actual implementation + # differs from the planned input + relevant_evs = [ + e for e in evidence_items + if contrib in e.get("contributor_keys", []) + ] + has_observations = any( + e.get("observations") for e in relevant_evs + ) + if has_observations: + all_obs = [] + for e in relevant_evs: + all_obs.extend(e.get("observations", [])) + proposed_changes.append({ + "action": "update", + "contributor": contrib, + "input_text": text, + "reason": "Implementation observations suggest differences", + "observations": all_obs, + "evidence_urls": [e["pr_url"] for e in relevant_evs], + }) + + # New surfaces from non-contributors + for finding in findings: + if finding["type"] == "new_surface": + proposed_changes.append({ + "action": "add", + "contributor": finding["ref"], + "input_text": "", + "reason": finding["detail"], + "observations": [], + "evidence_urls": [], + }) + + return { + "apply_enabled": False, + "reason": ( + "Source/narrative apply is disabled. No supported " + "source-preserving handoff path is established for this " + "invocation. Proposed changes should be reviewed and applied " + "through the owning design workflow's /revise phase." + ), + "owning_workflow": "design", + "owning_action": "/revise", + "proposed_changes": proposed_changes, + } + + +# --------------------------------------------------------------------------- +# Enrichment check +# --------------------------------------------------------------------------- + + +def check_enrichment( + plan: dict[str, Any], + current_state: dict[str, Any], +) -> dict[str, Any]: + """Check proposed enrichment against current issue state. + + Args: + plan: output of ``build_change_plan`` + current_state: current issue snapshot with ``remote_links`` + (list of ``{"url": ..., "title": ...}``) and + optionally ``fields`` (dict of field names to values) + + Returns an enrichment result with ``eligible_links``, + ``existing_links``, ``skipped_links``, ``eligible_metadata``, + ``forbidden_fields``, ``plan_fingerprint``, ``baseline_fingerprint``, + ``stale``. + """ + # Forbidden fields that this skill must never modify + FORBIDDEN = { + "summary", "description", "status", "assignee", + "sprint", "parent", "labels", "comment", + "resolution", "priority", "issuetype", + } + + existing_links = current_state.get("remote_links", []) + existing_urls: set[str] = set() + for link in existing_links: + url = link.get("url", "") + try: + existing_urls.add(validate_pr_url(url, "existing_link")) + except ValueError: + existing_urls.add(url) + + eligible: list[dict[str, Any]] = [] + existing: list[dict[str, Any]] = [] + skipped: list[dict[str, Any]] = [] + + for proposed in plan.get("pr_links", []): + url = proposed.get("url", "") + try: + norm_url = validate_pr_url(url, "proposed_link") + except ValueError: + skipped.append({**proposed, "reason": f"invalid URL: {url}"}) + continue + + if norm_url in existing_urls: + existing.append({**proposed, "url": norm_url}) + else: + eligible.append({**proposed, "url": norm_url}) + + # Metadata eligibility + eligible_metadata: dict[str, Any] = {} + forbidden_found: list[str] = [] + for field, value in plan.get("metadata", {}).items(): + base_field = field.replace("proposed_", "") + if base_field in FORBIDDEN: + forbidden_found.append(field) + else: + eligible_metadata[field] = value + + # Check profile-based field overrides + profile_fields = current_state.get("profile_fields", {}) + for pf in profile_fields: + if pf in FORBIDDEN: + forbidden_found.append(f"profile:{pf}") + + # Baseline fingerprint for drift detection + baseline_fp = _fingerprint({ + "remote_links": [ + {"url": l.get("url", "")} for l in existing_links + ], + }) + + return { + "eligible_links": eligible, + "existing_links": existing, + "skipped_links": skipped, + "eligible_metadata": eligible_metadata, + "forbidden_fields": forbidden_found, + "plan_fingerprint": plan.get("fingerprint", ""), + "baseline_fingerprint": baseline_fp, + "stale": False, + } + + +def verify_enrichment( + expected: dict[str, Any], + readback: dict[str, Any], +) -> dict[str, Any]: + """Verify enrichment results against a readback snapshot. + + Args: + expected: the enrichment result from ``check_enrichment`` + readback: fresh issue snapshot after writes + + Returns a receipt with per-operation status. + """ + readback_links = readback.get("remote_links", []) + readback_urls: set[str] = set() + for link in readback_links: + url = link.get("url", "") + try: + readback_urls.add(validate_pr_url(url, "readback")) + except ValueError: + readback_urls.add(url) + + receipts: list[dict[str, Any]] = [] + for link in expected.get("eligible_links", []): + url = link.get("url", "") + try: + norm = validate_pr_url(url, "receipt") + except ValueError: + norm = url + found = norm in readback_urls + receipts.append({ + "url": norm, + "applied": found, + "status": "confirmed" if found else "uncertain", + }) + + # Baseline drift check + new_baseline = _fingerprint({ + "remote_links": [ + {"url": l.get("url", "")} for l in readback_links + ], + }) + + return { + "receipts": receipts, + "baseline_fingerprint": new_baseline, + "all_confirmed": all(r["applied"] for r in receipts), + } + + +# --------------------------------------------------------------------------- +# Handoff rendering +# --------------------------------------------------------------------------- + + +def render_handoff(plan: dict[str, Any]) -> str: + """Render a human-readable handoff document from a change plan. + + Produces deterministic Markdown suitable for writer consumption. + """ + parts: list[str] = [] + issue_key = plan.get("issue_key", "unknown") + target = plan.get("target", {}) + + parts.append(f"# Documentation Handoff — {issue_key}") + parts.append("") + parts.append(f"**Target release:** {target.get('release', 'not specified')}") + parts.append(f"**Target branch:** {target.get('target_branch', 'not specified')}") + parts.append("") + + # Delivery summary + inputs = plan.get("inputs", []) + delivered = sum(1 for i in inputs if i["status"] == "delivered") + in_progress = sum(1 for i in inputs if i["status"] == "in_progress") + missing = sum(1 for i in inputs if i["status"] == "missing") + + parts.append("## Delivery Summary") + parts.append("") + parts.append(f"- Delivered: {delivered}") + parts.append(f"- In progress: {in_progress}") + parts.append(f"- Missing evidence: {missing}") + parts.append(f"- Total inputs: {len(inputs)}") + parts.append("") + + # Per-input details + if inputs: + parts.append("## Input Status") + parts.append("") + parts.append("| Input | Contributor | Status | PRs |") + parts.append("|---|---|---|---|") + for inp in inputs: + text = inp.get("text", "")[:60] + contrib = inp.get("contributor", "") + status = inp.get("status", "unknown") + prs = ", ".join(inp.get("pr_refs", [])) + if not prs: + prs = "none" + parts.append(f"| {text} | {contrib} | {status} | {prs} |") + parts.append("") + + # Target notes + target_notes = [ + n for inp in inputs for n in inp.get("target_notes", []) + ] + if target_notes: + parts.append("## Target Branch Notes") + parts.append("") + for note in target_notes: + parts.append(f"- {note}") + parts.append("") + + # Findings + findings = plan.get("findings", []) + if findings: + parts.append("## Findings") + parts.append("") + for f in findings: + parts.append(f"- **{f['type']}** ({f.get('ref', '')}): {f.get('detail', '')}") + parts.append("") + + # PR links to add + pr_links = plan.get("pr_links", []) + if pr_links: + parts.append("## PR Remote Links") + parts.append("") + parts.append("| URL | Relationship | Contributors |") + parts.append("|---|---|---|") + for link in pr_links: + url = link.get("url", "") + rel = link.get("relationship", "") + contribs = ", ".join(link.get("contributor_keys", [])) + parts.append(f"| {url} | {rel} | {contribs} |") + parts.append("") + + # Source proposal + source = plan.get("source_proposal") + if source: + parts.append("## Source Change Proposal") + parts.append("") + parts.append(f"**Apply enabled:** {source.get('apply_enabled', False)}") + parts.append(f"**Reason:** {source.get('reason', '')}") + if source.get("proposed_changes"): + parts.append("") + parts.append(f"**Owning workflow:** {source.get('owning_workflow', '')}") + parts.append(f"**Owning action:** {source.get('owning_action', '')}") + parts.append("") + for change in source["proposed_changes"]: + action = change.get("action", "") + contrib = change.get("contributor", "") + reason = change.get("reason", "") + parts.append(f"- **{action}** {contrib}: {reason}") + parts.append("") + + # Readiness + readiness = plan.get("readiness") + if readiness: + parts.append("## Consumer Readiness") + parts.append("") + dims = readiness.get("dimensions", []) + if dims: + parts.append(f"**Dimensions:** {', '.join(dims)}") + else: + parts.append("**Dimensions:** none specified") + parts.append(f"**Complete:** {readiness.get('complete', False)}") + parts.append(f"**Note:** {readiness.get('note', '')}") + parts.append("") + + return "\n".join(parts) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def cmd_validate(args: argparse.Namespace) -> int: + """Run the validate subcommand.""" + snapshot = _load_json(args.input) + if not isinstance(snapshot, dict): + fail("input must be a JSON object") + + # Verify input is not mutated + original = copy.deepcopy(snapshot) + + result = validate_issue(snapshot) + + # Assert read-only behavior + if snapshot != original: + fail("BUG: validate_issue mutated its input") + + _write_json(result) + return EXIT_SUCCESS if result["valid"] else EXIT_ERROR + + +def cmd_build_plan(args: argparse.Namespace) -> int: + """Run the build-plan subcommand.""" + data = _load_json(args.input) + if not isinstance(data, dict): + fail("input must be a JSON object") + + validation = data.get("validation") + evidence_raw = data.get("evidence") + target = data.get("target", {}) + profile = data.get("profile") + + if not isinstance(validation, dict): + fail("input.validation must be an object") + if not isinstance(evidence_raw, dict): + fail("input.evidence must be an object") + if not isinstance(target, dict): + fail("input.target must be an object") + + # Verify input is not mutated + original = copy.deepcopy(data) + + plan = build_change_plan(validation, evidence_raw, target, profile) + + if data != original: + fail("BUG: build_change_plan mutated its input") + + _write_json(plan) + return EXIT_SUCCESS + + +def cmd_check_enrichment(args: argparse.Namespace) -> int: + """Run the check-enrichment subcommand.""" + data = _load_json(args.input) + if not isinstance(data, dict): + fail("input must be a JSON object") + + plan = data.get("plan") + current = data.get("current_state") + + if not isinstance(plan, dict): + fail("input.plan must be an object") + if not isinstance(current, dict): + fail("input.current_state must be an object") + + if args.readback: + readback = _load_json(Path(args.readback)) + if not isinstance(readback, dict): + fail("readback must be a JSON object") + enrichment = check_enrichment(plan, current) + receipt = verify_enrichment(enrichment, readback) + _write_json({"enrichment": enrichment, "receipt": receipt}) + else: + # Verify input is not mutated + original = copy.deepcopy(data) + result = check_enrichment(plan, current) + if data != original: + fail("BUG: check_enrichment mutated its input") + _write_json(result) + + return EXIT_SUCCESS + + +def main(argv: list[str] | None = None) -> int: + """Entry point for the evidence CLI.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command") + sub.required = True + + p_validate = sub.add_parser( + "validate", + help="Validate a DOCS issue snapshot and extract contributors", + ) + p_validate.add_argument("--input", required=True, type=Path) + + p_plan = sub.add_parser( + "build-plan", + help="Build a delivery-evidence change plan", + ) + p_plan.add_argument("--input", required=True, type=Path) + + p_enrich = sub.add_parser( + "check-enrichment", + help="Check proposed enrichment against current state", + ) + p_enrich.add_argument("--input", required=True, type=Path) + p_enrich.add_argument("--readback", type=Path, default=None) + + args = parser.parse_args(argv) + + if args.command == "validate": + return cmd_validate(args) + elif args.command == "build-plan": + return cmd_build_plan(args) + elif args.command == "check-enrichment": + return cmd_check_enrichment(args) + else: + fail(f"unknown command: {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py new file mode 100644 index 00000000..4efa5d75 --- /dev/null +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -0,0 +1,935 @@ +#!/usr/bin/env python3 +"""Tests for the docs-handoff evidence helper. + +All fixtures use synthetic .example/.invalid data — no real Jira +instances, private product keys, or personal identifiers. +""" + +from __future__ import annotations + +import copy +import json +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path +from unittest.mock import patch + +# Resolve the script under test relative to this file so the tests +# work regardless of whether the skill is installed via symlink or +# run from the repo checkout. +_SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPT_DIR)) + +import evidence # noqa: E402 + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +def _make_docs_snapshot( + *, + key: str = "TEST-100", + summary: str = "[DOCS] Widget configuration reference", + description: str | None = None, + extra: dict | None = None, +) -> dict: + """Build a minimal valid DOCS issue snapshot.""" + if description is None: + description = textwrap.dedent("""\ + ## User Story + + **As a** fleet administrator, + **I want to** understand widget configuration options, + **So that** I can configure widgets correctly. + + ## Acceptance Criteria + + - [ ] Reader can identify all configuration fields + - [ ] Reader can follow a worked example + + ## Documentation Scope + + This feature introduces new widget configuration CLI commands. + + ## Documentation Inputs + + **TEST-201 -- Add widget config command:** + - New `widget config set` CLI command with `--name` and `--value` flags + - Config validation error messages and recovery steps + + **TEST-202 -- Widget status endpoint:** + - New `/api/v1/widgets/status` GET endpoint response fields + - Status values: `active`, `degraded`, `offline` [Assumption] + + ## Dependencies + + TEST-300 + + ## Design Reference + + Design document: https://github.example.invalid/org/docs/pull/42 + Epic: TEST-50 + PRD Requirements: FR-1, FR-2 + Design section: Section 4.3 API Changes + Contributing stories: TEST-201, TEST-202 + """) + snapshot = {"key": key, "summary": summary, "description": description} + if extra: + snapshot.update(extra) + return snapshot + + +def _make_evidence_items() -> list[dict]: + """Build sample PR evidence items.""" + return [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/101", + "contributor_key": "TEST-201", + "target_branch": "main", + "state": "merged", + "title": "feat: add widget config command", + "merged": True, + "merge_commit": "abc123def", + "observations": ["Adds --name and --value flags as planned"], + }, + { + "pr_url": "https://github.example.invalid/org/repo/pull/102", + "contributor_key": "TEST-202", + "target_branch": "release-4.19", + "state": "merged", + "title": "feat: widget status endpoint", + "merged": True, + "merge_commit": "def456abc", + "observations": [ + "Adds /api/v1/widgets/status endpoint", + "Status values include 'maintenance' not in plan", + ], + }, + ] + + +# --------------------------------------------------------------------------- +# URL validation +# --------------------------------------------------------------------------- + +class TestUrlValidation(unittest.TestCase): + """Test URL validation and normalization.""" + + def test_valid_https_url(self) -> None: + result = evidence.validate_url("https://github.example.invalid/org/repo") + self.assertEqual(result, "https://github.example.invalid/org/repo") + + def test_strips_trailing_slash(self) -> None: + result = evidence.validate_url("https://github.example.invalid/org/repo/") + self.assertEqual(result, "https://github.example.invalid/org/repo") + + def test_rejects_embedded_credentials(self) -> None: + with self.assertRaises(ValueError) as ctx: + evidence.validate_url("https://user:pass@github.example.invalid/org/repo") + self.assertIn("credentials", str(ctx.exception)) + + def test_rejects_empty(self) -> None: + with self.assertRaises(ValueError): + evidence.validate_url("") + + def test_rejects_ftp(self) -> None: + with self.assertRaises(ValueError): + evidence.validate_url("ftp://example.invalid/file") + + def test_rejects_control_chars(self) -> None: + with self.assertRaises(ValueError): + evidence.validate_url("https://example.invalid/\x00bad") + + def test_pr_url_strips_fragment(self) -> None: + result = evidence.validate_pr_url( + "https://github.example.invalid/org/repo/pull/42#discussion" + ) + self.assertEqual( + result, "https://github.example.invalid/org/repo/pull/42" + ) + + def test_pr_url_strips_query(self) -> None: + result = evidence.validate_pr_url( + "https://github.example.invalid/org/repo/pull/42?tab=files" + ) + self.assertEqual( + result, "https://github.example.invalid/org/repo/pull/42" + ) + + +class TestPrIdentity(unittest.TestCase): + """Test PR identity extraction from URLs.""" + + def test_github_pr(self) -> None: + result = evidence.extract_pr_identity( + "https://github.example.invalid/org/repo/pull/42" + ) + self.assertEqual(result["owner"], "org") + self.assertEqual(result["repo"], "repo") + self.assertEqual(result["number"], "42") + + def test_gitlab_mr(self) -> None: + result = evidence.extract_pr_identity( + "https://gitlab.example.invalid/org/repo/-/merge_requests/7" + ) + self.assertEqual(result["owner"], "org") + self.assertEqual(result["repo"], "repo") + self.assertEqual(result["number"], "7") + + def test_gitlab_mr_no_dash(self) -> None: + result = evidence.extract_pr_identity( + "https://gitlab.example.invalid/org/repo/merge_requests/7" + ) + self.assertEqual(result["number"], "7") + + def test_different_repos_produce_different_identities(self) -> None: + id1 = evidence.extract_pr_identity( + "https://github.example.invalid/org/repo-a/pull/1" + ) + id2 = evidence.extract_pr_identity( + "https://github.example.invalid/org/repo-b/pull/1" + ) + self.assertNotEqual(id1["repo"], id2["repo"]) + + +# --------------------------------------------------------------------------- +# Reference extraction +# --------------------------------------------------------------------------- + +class TestReferenceExtraction(unittest.TestCase): + """Test Jira key and story reference extraction.""" + + def test_extract_jira_keys(self) -> None: + keys = evidence.extract_all_jira_keys( + "TEST-201, TEST-202, and also TEST-201 again" + ) + self.assertEqual(keys, ["TEST-201", "TEST-202"]) + + def test_extract_story_refs(self) -> None: + refs = evidence.extract_story_refs( + "Story 1.01, Story 2.03, Story 1.01" + ) + self.assertEqual(refs, ["story-1.01", "story-2.03"]) + + def test_stories_plural(self) -> None: + refs = evidence.extract_story_refs("Stories 1.01, 2.03") + self.assertEqual(refs, ["story-1.01", "story-2.03"]) + + +# --------------------------------------------------------------------------- +# Issue validation +# --------------------------------------------------------------------------- + +class TestValidateIssue(unittest.TestCase): + """Test DOCS issue snapshot validation.""" + + def test_valid_snapshot(self) -> None: + snapshot = _make_docs_snapshot() + result = evidence.validate_issue(snapshot) + self.assertTrue(result["valid"]) + self.assertEqual(result["key"], "TEST-100") + self.assertEqual(result["errors"], []) + + def test_extracts_contributors_not_dependencies(self) -> None: + """Contributing stories are extracted; Dependencies are separate.""" + snapshot = _make_docs_snapshot() + result = evidence.validate_issue(snapshot) + # Contributors come from Contributing stories line + self.assertIn("TEST-201", result["contributors"]) + self.assertIn("TEST-202", result["contributors"]) + # TEST-300 is a dependency, not a contributor + self.assertNotIn("TEST-300", result["contributors"]) + self.assertIn("TEST-300", result["dependencies"]) + + def test_missing_docs_prefix(self) -> None: + snapshot = _make_docs_snapshot(summary="Widget docs") + result = evidence.validate_issue(snapshot) + self.assertFalse(result["valid"]) + self.assertTrue(any("[DOCS]" in e for e in result["errors"])) + + def test_missing_description(self) -> None: + snapshot = _make_docs_snapshot(description="") + result = evidence.validate_issue(snapshot) + self.assertFalse(result["valid"]) + + def test_missing_documentation_inputs(self) -> None: + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Something + + ## Documentation Scope + + Some scope + + ## Design Reference + + Contributing stories: TEST-201 + """) + snapshot = _make_docs_snapshot(description=desc) + result = evidence.validate_issue(snapshot) + self.assertFalse(result["valid"]) + self.assertTrue( + any("Documentation Inputs" in e for e in result["errors"]) + ) + + def test_no_contributors(self) -> None: + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Something + + ## Documentation Scope + + Some scope + + ## Documentation Inputs + + No inputs yet. + """) + snapshot = _make_docs_snapshot(description=desc) + result = evidence.validate_issue(snapshot) + self.assertFalse(result["valid"]) + self.assertTrue( + any("Contributing stories" in e for e in result["errors"]) + ) + + def test_unresolved_story_ref_is_warning(self) -> None: + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Something + + ## Documentation Scope + + Covers new CLI + + ## Documentation Inputs + + **Story 1.01 -- Add widget:** + - New CLI command + + ## Design Reference + + Contributing stories: Story 1.01 + """) + snapshot = _make_docs_snapshot(description=desc) + result = evidence.validate_issue(snapshot) + # Valid but with warning about unresolved ref + self.assertTrue(result["valid"]) + self.assertIn("story-1.01", result["contributors"]) + self.assertTrue( + any("unresolved" in w for w in result["warnings"]) + ) + + def test_invalid_jira_key_format(self) -> None: + snapshot = _make_docs_snapshot(key="bad-key") + result = evidence.validate_issue(snapshot) + self.assertFalse(result["valid"]) + self.assertTrue(any("invalid Jira key" in e for e in result["errors"])) + + def test_read_only_no_mutation(self) -> None: + """validate_issue must not mutate its input.""" + snapshot = _make_docs_snapshot() + original = copy.deepcopy(snapshot) + evidence.validate_issue(snapshot) + self.assertEqual(snapshot, original) + + def test_parses_documentation_inputs_items(self) -> None: + snapshot = _make_docs_snapshot() + result = evidence.validate_issue(snapshot) + inputs = result["documentation_inputs"] + self.assertTrue(len(inputs) >= 2) + # First contributor's items + test201_items = [i for i in inputs if i["contributor"] == "TEST-201"] + self.assertTrue(len(test201_items) >= 1) + self.assertEqual(test201_items[0]["marker"], "planned") + + def test_parses_assumption_marker(self) -> None: + snapshot = _make_docs_snapshot() + result = evidence.validate_issue(snapshot) + assumption_items = [ + i for i in result["documentation_inputs"] + if i["marker"] == "assumption" + ] + self.assertTrue(len(assumption_items) >= 1) + + +# --------------------------------------------------------------------------- +# Evidence normalization +# --------------------------------------------------------------------------- + +class TestNormalizeEvidence(unittest.TestCase): + """Test PR evidence normalization.""" + + def test_normalizes_valid_evidence(self) -> None: + items = _make_evidence_items() + result = evidence.normalize_evidence(items) + self.assertEqual(len(result["items"]), 2) + self.assertEqual(result["errors"], []) + + def test_deduplicates_same_url(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/101", + "contributor_key": "TEST-201", + }, + { + "pr_url": "https://github.example.invalid/org/repo/pull/101/", + "contributor_key": "TEST-202", + }, + ] + result = evidence.normalize_evidence(items) + self.assertEqual(len(result["items"]), 1) + # Both contributors should be merged + self.assertIn("TEST-201", result["items"][0]["contributor_keys"]) + self.assertIn("TEST-202", result["items"][0]["contributor_keys"]) + + def test_rejects_embedded_credentials_in_pr(self) -> None: + items = [ + { + "pr_url": "https://user:tok@github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + }, + ] + result = evidence.normalize_evidence(items) + self.assertEqual(len(result["items"]), 0) + self.assertTrue(len(result["errors"]) > 0) + self.assertTrue(any("credentials" in e for e in result["errors"])) + + def test_rejects_missing_contributor(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + }, + ] + result = evidence.normalize_evidence(items) + self.assertEqual(len(result["items"]), 0) + self.assertTrue(any("contributor_key" in e for e in result["errors"])) + + def test_preserves_different_repo_identities(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo-a/pull/1", + "contributor_key": "TEST-201", + }, + { + "pr_url": "https://github.example.invalid/org/repo-b/pull/1", + "contributor_key": "TEST-202", + }, + ] + result = evidence.normalize_evidence(items) + self.assertEqual(len(result["items"]), 2) + repos = {i["pr_identity"]["repo"] for i in result["items"]} + self.assertEqual(repos, {"repo-a", "repo-b"}) + + def test_evidence_not_array(self) -> None: + result = evidence.normalize_evidence("not a list") + self.assertTrue(len(result["errors"]) > 0) + + def test_strips_pr_url_fragment(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1#diff", + "contributor_key": "TEST-201", + }, + ] + result = evidence.normalize_evidence(items) + self.assertEqual(len(result["items"]), 1) + self.assertNotIn("#", result["items"][0]["pr_url"]) + + +# --------------------------------------------------------------------------- +# Change plan +# --------------------------------------------------------------------------- + +class TestBuildChangePlan(unittest.TestCase): + """Test delivery-evidence change plan building.""" + + def _make_plan( + self, + *, + snapshot: dict | None = None, + evidence_items: list[dict] | None = None, + target: dict | None = None, + profile: dict | None = None, + ) -> dict: + if snapshot is None: + snapshot = _make_docs_snapshot() + if evidence_items is None: + evidence_items = _make_evidence_items() + if target is None: + target = {"release": "4.19", "target_branch": "main"} + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(evidence_items) + return evidence.build_change_plan(validation, ev, target, profile) + + def test_complete_plan(self) -> None: + plan = self._make_plan() + self.assertEqual(plan["status"], "complete") + self.assertTrue(len(plan["inputs"]) > 0) + self.assertTrue(len(plan["pr_links"]) > 0) + + def test_delivered_inputs(self) -> None: + plan = self._make_plan() + delivered = [i for i in plan["inputs"] if i["status"] == "delivered"] + self.assertTrue(len(delivered) > 0) + + def test_missing_evidence(self) -> None: + """Contributor with no PRs produces missing_evidence finding.""" + snapshot = _make_docs_snapshot() + # Only provide evidence for TEST-201, not TEST-202 + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/101", + "contributor_key": "TEST-201", + "merged": True, + }, + ] + plan = self._make_plan(evidence_items=items) + findings = [f for f in plan["findings"] if f["type"] == "missing_evidence"] + self.assertTrue( + any(f["ref"] == "TEST-202" for f in findings), + f"Expected missing_evidence for TEST-202 in {findings}" + ) + + def test_dependency_not_flagged_as_missing(self) -> None: + """Dependencies (not contributors) don't generate missing_evidence.""" + plan = self._make_plan() + findings = [f for f in plan["findings"] if f["type"] == "missing_evidence"] + refs = [f["ref"] for f in findings] + self.assertNotIn("TEST-300", refs) + + def test_pr_target_branch_ambiguity(self) -> None: + """PR targeting wrong branch produces target_notes.""" + plan = self._make_plan() + # TEST-202 targets release-4.19, not main + target_notes = [ + n for inp in plan["inputs"] for n in inp.get("target_notes", []) + ] + self.assertTrue( + any("release-4.19" in n for n in target_notes), + f"Expected target branch note in {target_notes}" + ) + + def test_new_surface_finding(self) -> None: + """PR for non-contributor produces new_surface finding.""" + items = _make_evidence_items() + [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/999", + "contributor_key": "TEST-999", + }, + ] + plan = self._make_plan(evidence_items=items) + new_surfaces = [f for f in plan["findings"] if f["type"] == "new_surface"] + self.assertTrue( + any(f["ref"] == "TEST-999" for f in new_surfaces) + ) + + def test_invalid_issue_produces_invalid_plan(self) -> None: + snapshot = _make_docs_snapshot(key="bad") + plan = self._make_plan(snapshot=snapshot) + self.assertEqual(plan["status"], "invalid_issue") + + def test_unresolved_contributor_finding(self) -> None: + """Unresolved Story N.NN contributor produces finding.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Something + + ## Documentation Scope + + Covers new CLI + + ## Documentation Inputs + + **Story 1.01 -- Add widget:** + - New CLI command + + ## Design Reference + + Contributing stories: Story 1.01 + """) + snapshot = _make_docs_snapshot(description=desc) + plan = self._make_plan(snapshot=snapshot, evidence_items=[]) + findings = [f for f in plan["findings"] if f["type"] == "unresolved_contributor"] + self.assertTrue( + any("story-1.01" in f["ref"] for f in findings) + ) + + def test_source_proposal_disabled(self) -> None: + """Source/narrative apply is disabled by default.""" + plan = self._make_plan() + self.assertFalse(plan["source_proposal"]["apply_enabled"]) + self.assertIn("disabled", plan["source_proposal"]["reason"]) + + def test_source_proposal_has_owning_workflow(self) -> None: + plan = self._make_plan() + self.assertEqual(plan["source_proposal"]["owning_workflow"], "design") + self.assertEqual(plan["source_proposal"]["owning_action"], "/revise") + + def test_fingerprint_stable(self) -> None: + plan1 = self._make_plan() + plan2 = self._make_plan() + self.assertEqual(plan1["fingerprint"], plan2["fingerprint"]) + self.assertTrue(len(plan1["fingerprint"]) > 0) + + def test_read_only_no_mutation(self) -> None: + """build_change_plan must not mutate its inputs.""" + snapshot = _make_docs_snapshot() + items = _make_evidence_items() + target = {"release": "4.19", "target_branch": "main"} + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(items) + orig_val = copy.deepcopy(validation) + orig_ev = copy.deepcopy(ev) + orig_target = copy.deepcopy(target) + evidence.build_change_plan(validation, ev, target) + self.assertEqual(validation, orig_val) + self.assertEqual(ev, orig_ev) + self.assertEqual(target, orig_target) + + def test_readiness_with_profile(self) -> None: + profile = {"readiness_dimensions": ["technical_accuracy", "completeness"]} + plan = self._make_plan(profile=profile) + self.assertIsNotNone(plan["readiness"]) + self.assertEqual( + plan["readiness"]["dimensions"], + ["technical_accuracy", "completeness"], + ) + self.assertFalse(plan["readiness"]["complete"]) + + def test_readiness_without_profile(self) -> None: + plan = self._make_plan(profile=None) + self.assertIsNone(plan["readiness"]) + + def test_readiness_empty_dimensions(self) -> None: + profile = {"readiness_dimensions": []} + plan = self._make_plan(profile=profile) + self.assertEqual(plan["readiness"]["dimensions"], []) + self.assertFalse(plan["readiness"]["complete"]) + + +# --------------------------------------------------------------------------- +# Enrichment check +# --------------------------------------------------------------------------- + +class TestCheckEnrichment(unittest.TestCase): + """Test enrichment idempotency and eligibility checks.""" + + def _make_enrichment( + self, + *, + existing_links: list[dict] | None = None, + ) -> dict: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + current = { + "remote_links": existing_links or [], + } + return evidence.check_enrichment(plan, current) + + def test_new_links_eligible(self) -> None: + result = self._make_enrichment() + self.assertTrue(len(result["eligible_links"]) > 0) + self.assertEqual(len(result["existing_links"]), 0) + + def test_existing_link_detected(self) -> None: + result = self._make_enrichment( + existing_links=[ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + ] + ) + self.assertTrue(len(result["existing_links"]) > 0) + # Should have fewer eligible links + eligible_urls = {l["url"] for l in result["eligible_links"]} + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/101", + eligible_urls, + ) + + def test_forbidden_fields_rejected(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + # Inject forbidden metadata + plan["metadata"]["proposed_status"] = "Done" + plan["metadata"]["proposed_assignee"] = "someone" + + current = {"remote_links": []} + result = evidence.check_enrichment(plan, current) + self.assertTrue(len(result["forbidden_fields"]) > 0) + self.assertTrue( + any("status" in f for f in result["forbidden_fields"]) + ) + + def test_profile_forbidden_fields(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + current = {"remote_links": [], "profile_fields": {"labels": "v4.19"}} + result = evidence.check_enrichment(plan, current) + self.assertTrue( + any("labels" in f for f in result["forbidden_fields"]) + ) + + def test_repeated_check_idempotent(self) -> None: + result1 = self._make_enrichment() + result2 = self._make_enrichment() + self.assertEqual( + result1["baseline_fingerprint"], + result2["baseline_fingerprint"], + ) + + def test_stale_defaults_false(self) -> None: + result = self._make_enrichment() + self.assertFalse(result["stale"]) + + +# --------------------------------------------------------------------------- +# Verify enrichment (readback) +# --------------------------------------------------------------------------- + +class TestVerifyEnrichment(unittest.TestCase): + """Test readback verification after enrichment writes.""" + + def test_confirmed_when_link_present(self) -> None: + enrichment = { + "eligible_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + ], + } + readback = { + "remote_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + ], + } + receipt = evidence.verify_enrichment(enrichment, readback) + self.assertTrue(receipt["all_confirmed"]) + self.assertEqual(receipt["receipts"][0]["status"], "confirmed") + + def test_uncertain_when_link_missing(self) -> None: + enrichment = { + "eligible_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + ], + } + readback = {"remote_links": []} + receipt = evidence.verify_enrichment(enrichment, readback) + self.assertFalse(receipt["all_confirmed"]) + self.assertEqual(receipt["receipts"][0]["status"], "uncertain") + + def test_partial_confirmation(self) -> None: + enrichment = { + "eligible_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + {"url": "https://github.example.invalid/org/repo/pull/102"}, + ], + } + readback = { + "remote_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + ], + } + receipt = evidence.verify_enrichment(enrichment, readback) + self.assertFalse(receipt["all_confirmed"]) + confirmed = [r for r in receipt["receipts"] if r["status"] == "confirmed"] + uncertain = [r for r in receipt["receipts"] if r["status"] == "uncertain"] + self.assertEqual(len(confirmed), 1) + self.assertEqual(len(uncertain), 1) + + +# --------------------------------------------------------------------------- +# Handoff rendering +# --------------------------------------------------------------------------- + +class TestRenderHandoff(unittest.TestCase): + """Test handoff document rendering.""" + + def test_deterministic(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + md1 = evidence.render_handoff(plan) + md2 = evidence.render_handoff(plan) + self.assertEqual(md1, md2) + + def test_contains_issue_key(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + md = evidence.render_handoff(plan) + self.assertIn("TEST-100", md) + + def test_contains_delivery_counts(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + md = evidence.render_handoff(plan) + self.assertIn("Delivered:", md) + self.assertIn("Missing evidence:", md) + + def test_contains_source_proposal(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + md = evidence.render_handoff(plan) + self.assertIn("Source Change Proposal", md) + self.assertIn("**Apply enabled:** False", md) + + +# --------------------------------------------------------------------------- +# CLI integration +# --------------------------------------------------------------------------- + +class TestCli(unittest.TestCase): + """Test CLI subcommands via subprocess.""" + + def _run( + self, + subcmd: str, + input_data: dict | list, + extra_args: list[str] | None = None, + ) -> subprocess.CompletedProcess: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + json.dump(input_data, f) + f.flush() + input_path = f.name + + try: + cmd = [ + sys.executable, "-I", + str(_SCRIPT_DIR / "evidence.py"), + subcmd, "--input", input_path, + ] + if extra_args: + cmd.extend(extra_args) + return subprocess.run( + cmd, capture_output=True, text=True, timeout=30, + ) + finally: + os.unlink(input_path) + + def test_validate_valid(self) -> None: + snapshot = _make_docs_snapshot() + proc = self._run("validate", snapshot) + self.assertEqual(proc.returncode, 0, proc.stderr) + result = json.loads(proc.stdout) + self.assertTrue(result["valid"]) + + def test_validate_invalid(self) -> None: + snapshot = _make_docs_snapshot(key="bad") + proc = self._run("validate", snapshot) + self.assertEqual(proc.returncode, 1) + result = json.loads(proc.stdout) + self.assertFalse(result["valid"]) + + def test_build_plan(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + data = { + "validation": validation, + "evidence": ev, + "target": {"release": "4.19", "target_branch": "main"}, + } + proc = self._run("build-plan", data) + self.assertEqual(proc.returncode, 0, proc.stderr) + plan = json.loads(proc.stdout) + self.assertEqual(plan["status"], "complete") + + def test_check_enrichment(self) -> None: + snapshot = _make_docs_snapshot() + validation = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + validation, ev, {"release": "4.19", "target_branch": "main"} + ) + data = { + "plan": plan, + "current_state": {"remote_links": []}, + } + proc = self._run("check-enrichment", data) + self.assertEqual(proc.returncode, 0, proc.stderr) + result = json.loads(proc.stdout) + self.assertTrue(len(result["eligible_links"]) > 0) + + def test_validate_does_not_mutate_file(self) -> None: + """CLI validate must not modify the input file.""" + snapshot = _make_docs_snapshot() + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + json.dump(snapshot, f) + f.flush() + input_path = f.name + + try: + original = Path(input_path).read_bytes() + subprocess.run( + [ + sys.executable, "-I", + str(_SCRIPT_DIR / "evidence.py"), + "validate", "--input", input_path, + ], + capture_output=True, text=True, timeout=30, + ) + after = Path(input_path).read_bytes() + self.assertEqual(original, after, "input file was modified") + finally: + os.unlink(input_path) + + def test_bad_json_exits_1(self) -> None: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + f.write("{bad json") + f.flush() + input_path = f.name + + try: + proc = subprocess.run( + [ + sys.executable, "-I", + str(_SCRIPT_DIR / "evidence.py"), + "validate", "--input", input_path, + ], + capture_output=True, text=True, timeout=30, + ) + self.assertEqual(proc.returncode, 1) + finally: + os.unlink(input_path) + + +if __name__ == "__main__": + unittest.main() From 0dfb001bb5293ae2fbe9416004d1d990b20e5313 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 21:46:24 +0000 Subject: [PATCH 02/17] fix(docs-handoff): honest delivery semantics, PR identity validation, metadata writes, enrichment concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address convergence requirements A-E: A. Consumer pipeline: add explicit `normalize` CLI subcommand, validate nested shapes/types in build-plan (don't trust validation.valid), invalid plans exit 1, scope enrichment to exact DOCS issue identity (reject wrong key in current_state and readback). B. Honest evidence semantics: per-input status is now merged_pr_exists/merged_wrong_target/in_progress/no_evidence/ evidence_unverified/conflicting — never unqualified "delivered". Contributors also in Dependencies still get missing_evidence findings. Non-contributor PRs are candidate proposals not auto-eligible links. "As planned" observations do NOT generate source update proposals; only divergent observations do. Source proposals carry proposed_wording and evidence_urls. Contradictory merged/target_branch on same PR surface conflict warnings. Readiness tracks provided assessment results, missing dimensions stay visibly incomplete. C. PR/MR identity validation: validate_pr_url enforces supported forge path structure (GitHub /owner/repo/pull/N, GitLab nested /-/merge_requests/N) with numeric ID. Rejects trailing paths (/files, /commits), non-PR pages, explicit ports, nonnumeric IDs. Failed identity extraction rejects the evidence item entirely. D. Metadata writes: allowed field catalog (_ALLOWED_METADATA) with explicit Jira field bindings and catalog_verified flag. Uncatalogued fields rejected as forbidden. check_enrichment returns status=ready/stale/invalid_plan/wrong_issue with real baseline drift detection via approved_baseline comparison. Canonical link ordering avoids false drift from reordering. verify_enrichment checks readback issue key, produces both link and metadata receipts, empty eligible = not success, existing-link disappearance detected. E. Tests: 103 tests covering contradictory PR states, non-contributor link exclusion, contributor-also-dependency, as-planned vs divergent source proposals, nested GitLab paths, rejected non-PR URLs, invalid merged types, baseline drift, wrong issue identity, invalid plan rejection, metadata receipts, end-to-end offline pipeline (validate→normalize→plan→check→receipt via both API and CLI), and UTF-8 content handling. Co-Authored-By: Claude Opus 4.6 --- .../references/evidence-schema.md | 81 +- .../references/handoff-protocol.md | 193 +-- .../references/jira-enrichment.md | 123 +- skills/docs-handoff/scripts/evidence.py | 1142 +++++++++---- skills/docs-handoff/scripts/test_evidence.py | 1432 ++++++++++++----- 5 files changed, 2033 insertions(+), 938 deletions(-) diff --git a/skills/docs-handoff/references/evidence-schema.md b/skills/docs-handoff/references/evidence-schema.md index 933a19a3..9be86212 100644 --- a/skills/docs-handoff/references/evidence-schema.md +++ b/skills/docs-handoff/references/evidence-schema.md @@ -1,7 +1,7 @@ # Evidence Schema and Validation Contract for DOCS issue validation, contributor extraction, and -delivery-evidence normalization. +delivery-evidence normalization used by `../scripts/evidence.py`. ## DOCS Issue Snapshot @@ -18,8 +18,9 @@ caller provides a JSON snapshot with at minimum: | `remote_links` | array | No | Existing Remote Links | | `issue_links` | array | No | Existing issue links | -The script `../scripts/evidence.py validate` validates the snapshot and -extracts structured data. +The `validate` subcommand validates the snapshot and extracts structured +data. An offline snapshot file is fully supported — the skill does not +require live Jira access for validation. ## Required Description Sections @@ -30,28 +31,30 @@ A valid `[DOCS]` issue description must contain these `##` sections: - **Documentation Scope** — what the reader needs to understand - **Acceptance Criteria** — reader-verifiable documentation outcomes -Missing sections produce validation errors. +Missing sections produce validation errors. An empty Documentation +Inputs section (present but no parseable structured items) produces a +warning. ## Contributor Extraction Contributors are extracted from the `Contributing stories:` line in the Design Reference section. In a rendered (synced) issue, these are Jira keys (e.g. `TEST-201`). Unresolved `Story N.NN` placeholders are -extracted as qualified IDs and flagged as warnings. +extracted as qualified IDs and flagged as warnings — they are gaps, not +fabricated live identities. **Contributors vs Dependencies**: the `Dependencies` section lists -prerequisite stories that must complete before documentation work begins. -Dependencies are NOT automatically direct contributors — they do not -produce user-facing artifacts that belong in Documentation Inputs. The -script extracts these separately and the plan does not generate -missing-evidence findings for dependency-only references. +prerequisite stories. Dependencies are NOT automatically direct +contributors. The script extracts these separately. A contributor +that is *also* listed in Dependencies still requires evidence — the +dependency relationship does not exempt it from the contributor finding. ## Documentation Inputs Parsing The Documentation Inputs section is parsed into structured items: -``` -**JIRA-KEY -- title:** +```text +**JIRA-KEY — title:** - Input item text - Another item [Assumption] - Open item [Open: §9.2] @@ -59,39 +62,65 @@ The Documentation Inputs section is parsed into structured items: Each item records: - `contributor`: the Jira key or qualified story ID from the header -- `text`: the input description +- `text`: the original input description (markers preserved in text) - `marker`: `planned` (default), `assumption`, or `open` - `open_ref`: the referenced open question (when marker is `open`) +Missing or unparsed headers produce a visible warning, not a silent gap. + ## PR Evidence Items Each gathered PR evidence item must have: | Field | Type | Required | |---|---|---| -| `pr_url` | string | Yes — HTTP(S) PR/MR URL | +| `pr_url` | string | Yes — supported PR/MR URL (see URL validation) | | `contributor_key` | string | Yes — Jira key of contributing story | | `target_branch` | string | No — PR target branch | | `state` | string | No — PR state (open, merged, closed) | | `title` | string | No — PR title | -| `merged` | boolean | No — whether PR is merged | +| `merged` | boolean/null | No — whether PR is merged | | `merge_commit` | string | No — merge commit SHA | -| `observations` | array | No — AI-generated semantic observations | +| `observations` | string[] | No — AI-generated semantic observations | -The script `normalize-evidence` (internal to `build-plan`) validates -URLs, rejects embedded credentials, deduplicates by normalized URL, -and preserves distinct repository identities. +The `normalize` subcommand validates URLs (requiring supported PR/MR +structure with numeric ID), rejects embedded credentials, deduplicates +by normalized URL, detects contradictory `merged` or `target_branch` +values on duplicate PRs (surfaced as conflicts rather than silently +taking the first value), and preserves distinct repository identities. + +Non-PR URLs, non-boolean `merged` values, and non-string observations +are rejected with errors. ## Input Status Values After comparing planned inputs against gathered evidence, each input -receives a status: +receives a per-PR evidence status — not a blanket delivery claim: -- `delivered` — contributor has at least one merged PR +- `merged_pr_exists` — contributor has a merged PR targeting the + expected branch +- `merged_wrong_target` — contributor has a merged PR but it targets a + different branch than expected - `in_progress` — contributor has PRs but none merged -- `missing` — no PR evidence found for this contributor - -These are *delivery* statuses, not semantic confirmation. A `delivered` -status means a merged PR exists — it does not prove the actual -implementation matches the planned input. Semantic planned-vs-actual +- `no_evidence` — no PR evidence found for this contributor +- `evidence_unverified` — PRs exist but merge status is unknown (null) +- `conflicting` — contradictory evidence (e.g. merged=true and + merged=false on the same PR) + +These are *observation* statuses. `merged_pr_exists` means a merged PR +exists — it does not prove the implementation matches the planned input +or that the feature is included in the target release. Semantic assessment is an AI responsibility recorded in `observations`. + +## URL Validation + +PR/MR URLs must: +- Use HTTPS scheme (http rejected) +- Have no embedded credentials +- Match a supported forge pattern: GitHub `/owner/repo/pull/N` or + GitLab `/owner[/subgroup]*/repo/-/merge_requests/N` +- Have a numeric PR/MR identifier +- Be the canonical PR page (trailing paths like `/files` rejected) + +Query strings and fragments are stripped for identity normalization. +Synthetic `.example`/`.invalid` domains are supported for testing. diff --git a/skills/docs-handoff/references/handoff-protocol.md b/skills/docs-handoff/references/handoff-protocol.md index f092fefe..6450ae8c 100644 --- a/skills/docs-handoff/references/handoff-protocol.md +++ b/skills/docs-handoff/references/handoff-protocol.md @@ -1,141 +1,110 @@ # Handoff Protocol -Preview/apply two-phase workflow for the docs-handoff skill. This -protocol governs the end-to-end flow from issue intake to enrichment -and source-change proposals. +Preview/apply two-phase workflow for the docs-handoff skill. ## Phase 1: Preview (No Side Effects) -All operations in the preview phase are read-only. No Jira writes, -no source file modifications, no issue state changes. - ### Step 1 — Validate Issue -Provide the DOCS issue snapshot to `evidence.py validate`. The script -validates structure, extracts contributors, dependencies, and -Documentation Inputs items. If validation fails (missing sections, -invalid key, no contributors), stop and report the errors. +Provide the DOCS issue snapshot (from Jira read or offline file) to +`evidence.py validate --input snapshot.json`. The script validates +structure, extracts contributors, dependencies, and Documentation Inputs +items. If validation fails (exit 1), stop and report errors. ### Step 2 — Gather Evidence (AI Responsibility) For each extracted contributor (NOT dependencies): -1. Fetch the contributor's Jira issue to discover linked PRs -2. For each PR, read metadata: URL, title, target branch, merge status, - merge commit -3. Read the PR diff and assess what user-facing surfaces changed +1. Fetch the contributor's Jira issue to discover linked PRs. +2. For each PR: read metadata (URL, title, target branch, merge status, + merge commit). +3. Read PR diffs and assess what user-facing surfaces changed. 4. Record semantic observations comparing planned inputs against actual - implementation - -**Evidence quality requirements:** - -- A merged PR status alone does not prove the feature is delivered to - the target release. Record the target branch and note ambiguity when - it differs from the expected release branch. -- String equality between a planned input and a PR title is not semantic - confirmation. Actual code/diff review is required. -- Partial evidence (some PRs found, some contributors without PRs) is a - valid intermediate state. Report it accurately — do not claim - completion or fabricate evidence. -- Unknown or contradictory results (e.g., PR merged but reverted) must - be reported as findings, not silently resolved. - -### Step 3 — Build Plan - -Provide the validation result, normalized evidence, target, and optional -profile to `evidence.py build-plan`. The script produces: - -- Per-input delivery status (delivered/in\_progress/missing) -- PR Remote Links to add -- Metadata proposals -- Findings (missing evidence, new surfaces, unresolved refs, target - ambiguity) -- Source-change proposal (disabled; advisory) -- Consumer readiness dimensions (when profile provided) - -### Step 4 — Check Enrichment - -Provide the plan and current issue state (with existing Remote Links) to -`evidence.py check-enrichment`. The script identifies: - -- Eligible links (new, not already present) -- Existing links (already on the issue) -- Skipped links (invalid URLs) -- Forbidden fields (blocked by policy) -- Baseline fingerprint (for drift detection) - -### Step 5 — Present Preview - -Show the user the complete bounded preview: - -- Target issue key and current baseline -- Delivery summary (counts per status) -- Per-input status with PR references and target notes -- Findings requiring attention -- Eligible PR Remote Links (the set that would be added) -- Existing links (already present, no action needed) -- Metadata proposals with evidence -- Source-change proposal (disabled, with intended changes and owning - workflow action) -- Consumer readiness state (dimensions recorded, results pending) -- Operation eligibility summary + implementation. -## Phase 2: Apply (Explicit Approval Required) +Evidence quality requirements: +- Merged PR status alone does not prove target-release inclusion. +- String equality is not semantic confirmation — actual code review is + required. +- Partial evidence is a valid intermediate state — report honestly. +- Unknown or contradictory results are reported as findings. +- An observation like "as planned" does NOT produce a source update + proposal — only divergent observations generate proposals. -The user reviews the preview and explicitly approves or rejects the -enrichment plan as a single bounded operation. +### Step 3 — Normalize Evidence -### On Approval +Provide raw evidence items to `evidence.py normalize --input evidence.json`. +The script validates URLs (requiring supported PR/MR structure), rejects +credentials, deduplicates, and detects contradictory states. -1. **Re-read** the issue's current Remote Links from Jira (fresh, not - cached from preview). -2. **Detect drift**: if the baseline fingerprint changed since the - preview, the plan is stale. Report the conflict and regenerate. -3. **Apply** eligible PR Remote Links via the runtime's available Jira - write capability. -4. **Re-read** the issue after writes. -5. **Produce receipt**: per-link confirmed/uncertain status. -6. **Report**: complete results including any uncertain outcomes. +### Step 4 — Build Plan -### On Rejection +Provide validation result, normalized evidence, target, and optional +profile to `evidence.py build-plan --input plan-input.json`. The input +is a JSON object with keys: `validation`, `evidence`, `target`, `profile`. -No writes. The preview artifacts remain available for review. +### Step 5 — Check Enrichment -## Source-Change Proposals +Provide the plan and current issue state (with `key` and `remote_links`) +to `evidence.py check-enrichment --input check-input.json`. The input +has keys: `plan`, `current_state`, `approved_baseline` (optional). -The skill produces structured proposals for updating the Documentation -Inputs section in the owning design workflow's source artifacts. These -proposals include: +### Step 6 — Present Preview -- Specific intended changes (add, update) with evidence references -- The owning workflow (`design`) and action (`/revise`) -- Factual observations from PR review +Show the complete bounded preview including: +- Target issue key and baseline +- Per-input evidence status with PR references +- Findings (missing evidence, unresolved refs, non-contributor evidence) +- Eligible PR Remote Links (contributor PRs only) +- Metadata proposals with catalog verification status +- Source-change proposal (disabled, with proposed wording) +- Consumer readiness state -**Source/narrative apply is disabled.** The docs-handoff skill does NOT -directly modify design source artifacts or Jira issue descriptions. The -proposal is advisory — the human or owning workflow applies changes -through the established `/revise` phase. +## Phase 2: Apply (Explicit Approval Required) -This is an explicit design choice, not a limitation to work around: +### On Approval -- The design workflow owns the Documentation Inputs source -- Direct edits could conflict with the sync manifest and content hashes -- Writer-managed content in the description must not be overwritten -- The `/revise` phase has the correct reconciliation and consistency - checks +1. Re-read the issue from Jira (fresh snapshot). +2. Re-run `check-enrichment` with the `approved_baseline` from Phase 1. +3. If stale (baseline changed), regenerate with fresh preview. +4. Apply eligible links and metadata via approved runtime capabilities. +5. Re-read the issue; run `check-enrichment --readback` for receipts. -If a supported explicit owning-source handoff can be established in a -future version, it must be tested for producer/renderer/sync preservation -and writer/concurrency handling BEFORE enabling apply. +### On Rejection -## Late Contributor Changes +No writes. Artifacts remain for review. -If evidence gathering reveals that a contributing story's actual scope -differs from the planned Documentation Inputs (e.g., a new API field was -added that is not in the plan), this is reported as a finding with type -`new_surface`. It is a PROPOSAL for the user to evaluate, not a silent -scope expansion. The user decides whether to: +## Source-Change Proposals -1. Accept the new surface and request a `/revise` to update the source -2. Defer it to a future documentation cycle -3. Determine it is out of scope for this DOCS story +The skill produces structured proposals for updating Documentation +Inputs through the design workflow's `/revise` phase. Each proposal +carries: + +- `action`: `update` (divergent input) or `add_contributor` (new) +- `proposed_wording`: specific text or unresolved question +- `evidence_urls`: supporting PR URLs +- `source_location`: where in the source artifact +- `observations`: divergent observations from code review + +Source/narrative apply is disabled. This is an explicit design choice: +the design workflow owns Documentation Inputs, direct edits risk sync +manifest conflicts, and writer-managed content must not be overwritten. + +## CLI Sequence + +```bash +# 1. Validate offline snapshot +python3 evidence.py validate --input snapshot.json +# 2. Normalize raw evidence (from AI gathering) +python3 evidence.py normalize --input raw-evidence.json +# 3. Build change plan +python3 evidence.py build-plan --input plan-input.json +# 4. Check enrichment eligibility +python3 evidence.py check-enrichment --input check-input.json +# 5. After approved writes, verify with readback +python3 evidence.py check-enrichment --input check-input.json \ + --readback readback.json +``` + +All inputs are file paths via `--input`. No stdin support. Exit 0 on +success, exit 1 on validation/data errors, exit 2 on argument errors. diff --git a/skills/docs-handoff/references/jira-enrichment.md b/skills/docs-handoff/references/jira-enrichment.md index 040ba91d..e3a71e76 100644 --- a/skills/docs-handoff/references/jira-enrichment.md +++ b/skills/docs-handoff/references/jira-enrichment.md @@ -2,89 +2,96 @@ Approved metadata and PR Remote Link write protocol for the docs-handoff skill. All writes require explicit user approval of the complete bounded -plan — not per-operation prompts. +plan — ONE confirmation for the full finite preview. ## Approved Operations | Operation | Scope | Confirmation | |---|---|---| -| Add PR Remote Links | Per-PR URL on the target DOCS issue | One plan approval | -| Propose metadata fields | Target release, when value is from approved catalog | One plan approval | +| Add PR Remote Links | Contributor PRs on the target DOCS issue | One plan approval | +| Apply metadata fields | From allowed catalog with verified values | One plan approval | ## Forbidden Operations -The skill must NEVER perform these operations, regardless of profile or -caller input: +The skill must NEVER perform these operations, regardless of profile: -- Modify `summary` or `description` -- Change issue `status` or `resolution` +- Modify `summary`, `description`, `reporter` +- Change `status`, `resolution` - Set or change `assignee` -- Modify `sprint`, `parent`, or `labels` +- Modify `sprint`, `parent`, `labels`, `components`, `fixVersions` - Add `comments` - Set `priority` or `issuetype` - Create new issues - Trigger documentation authoring or publication - Execute workflow hooks or batch operations -A consumer profile cannot override these restrictions. The -`check-enrichment` subcommand rejects any plan that includes forbidden -fields. +A consumer profile cannot override these restrictions. -## PR Remote Links - -Remote Links connect the DOCS issue to implementation PRs. Each link -has: +## Allowed Metadata Catalog -- **URL**: normalized PR/MR URL (validated, no credentials) -- **Title**: PR title for display -- **Relationship**: `implements` (the PR implements a contributing story) +Only fields in this catalog may be proposed as metadata changes. +Uncatalogued fields are rejected as forbidden: -### Idempotency +| Canonical Name | Jira Binding | Requirement | +|---|---|---| +| `target_release` | `fixVersions` or project-specific custom field | Value must exist in project's actual Fix Version catalog | -Before proposing links, the skill checks existing Remote Links on the -issue. Links whose normalized URL already exists are reported as -`existing` and skipped. Only genuinely new links are `eligible`. +Proposed values that have not been verified against the project's live +field catalog carry `catalog_verified: false`. The runtime must verify +the value exists before applying. Unknown or guessed values remain +gaps — no raw `proposed_*` placeholder is sent as a Jira field write. -### Approval Flow +## PR Remote Links -1. The preview shows ALL proposed links, existing links, and the complete - enrichment plan as a single bounded operation. -2. The user approves or rejects the entire plan. -3. On approval, eligible links are added via the runtime's available Jira - write capability. -4. After writes, the skill re-reads the issue's Remote Links and produces - a per-link receipt with `confirmed` or `uncertain` status. +Remote Links connect the DOCS issue to implementation PRs. Only PRs +attributed to *actual contributors* are eligible — PRs from +non-contributors produce `non_contributor_evidence` findings and source +proposals, not automatic link enrichment. -The approval is a genuine user decision — not a trusted `approved: true` -JSON toggle. The skill waits for the runtime's interactive confirmation. +### Idempotency -## Metadata Fields +Before proposing links, the skill checks existing Remote Links. Links +whose normalized URL already exists are `existing` and skipped. Only +genuinely new links are `eligible`. Link ordering is canonicalized +(sorted by URL) to avoid false baseline drift from reordering. -Metadata proposals require actual project field catalog and valid values. -Unknown or uncatalogued values remain gaps — the skill never guesses -release versions, priority values, or component names. +### Approval Flow -The `proposed_target_release` field is included when the caller provides -a target release. Applying it requires verifying the value exists in the -project's Fix Version or Target Version catalog. +1. Preview shows ALL proposed operations (links + metadata) as a single + bounded plan. +2. The user approves or rejects the entire plan. Approval is a genuine + user decision — not a trusted `approved: true` JSON toggle. +3. **Before writes**: re-read the issue's current Remote Links and + metadata from Jira. Compare the baseline fingerprint against the + approved snapshot. If the baseline changed (concurrent modification), + the plan is stale and must be regenerated with a fresh preview. +4. **Apply**: add eligible links and metadata via approved capabilities. +5. **Readback**: re-read the issue after writes and produce per-operation + receipts. ## Baseline Drift Detection The `check-enrichment` subcommand computes a baseline fingerprint from -the issue's current Remote Links. If the baseline changes between plan -creation and apply (concurrent modification), the plan is stale and must -be regenerated. +the issue's current Remote Links (canonically ordered). When an +`approved_baseline` is provided and differs from the current baseline, +the result is `stale` — writes are blocked until a fresh preview is +approved. + +The plan also carries a `plan_digest` covering the issue key, target, +inputs, links, and metadata. A tampered plan produces a different +digest. ## Readback and Receipts -After applying approved writes, the skill: +After applying approved writes, the `verify_enrichment` function: -1. Re-reads the issue's Remote Links from Jira -2. Compares expected links against the readback -3. Produces a receipt for each operation: - - `confirmed`: link found in readback - - `uncertain`: link not found — may indicate write failure or - propagation delay +1. Checks the readback issue key matches the expected key +2. Produces link receipts: `confirmed` (found) or `uncertain` (missing) +3. Produces metadata receipts: `confirmed`, `different_value`, or + `uncertain` +4. Distinguishes existing (no-op) links from newly applied links +5. Reports `all_confirmed` only when at least one add operation exists + and all are confirmed — empty eligible lists are not success On uncertain results, the skill reports the discrepancy. It does NOT automatically retry or claim rollback. @@ -93,14 +100,10 @@ automatically retry or claim rollback. An optional caller-supplied consumer profile may specify: -- `readiness_dimensions`: list of assessment dimensions the consumer - cares about (e.g. `["technical_accuracy", "completeness"]`). These - are recorded but NOT automatically evaluated — the caller must populate - actual assessments. Absent dimensions are a visible gap, not universal - readiness. - -The profile cannot: -- Override forbidden field restrictions -- Bypass the approval flow -- Enable automatic writes -- Provide internal service endpoints or credentials +- `readiness_dimensions`: list of assessment dimension names +- `readiness_results`: dict of dimension → `{"assessed": bool, ...}` +- `allowed_fields`: list of `{"name": ..., "value": ...}` for metadata + +Missing dimensions or unassessed results stay visibly incomplete — never +silently universal readiness. Forbidden fields in `allowed_fields` are +rejected with findings. diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 77bbc409..7d9f7543 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -6,17 +6,24 @@ change plans, and checking enrichment idempotency. Subcommands: - validate Validate a DOCS issue snapshot and extract contributors - build-plan Build a delivery-evidence change plan from gathered evidence - check-enrichment Check proposed enrichment against existing issue state - -All inputs come from files or stdin (JSON). This script never contacts -Jira, GitHub, or any external service. ``urllib.parse`` is used for -URL validation only -- no network I/O. + validate Validate a DOCS issue snapshot and extract contributors. + normalize Normalize and deduplicate raw PR evidence items. + build-plan Build a delivery-evidence change plan from validation + and normalized evidence. + check-enrichment Check proposed enrichment against current issue state; + with --readback, verify writes and produce receipts. + +All inputs are read from JSON files specified via ``--input`` (and +optionally ``--readback``). This script never contacts Jira, GitHub, +or any external service. ``urllib.parse`` is used for URL parsing +only -- no network I/O. Exit codes: 0 -- success (result JSON on stdout) - 1 -- input validation error or data inconsistency (diagnostic on stderr) + 1 -- input validation error or data inconsistency (diagnostic on + stderr; structured JSON is still written to stdout when the + error is representable as a result object, e.g. validate + with invalid issue) 2 -- invalid arguments (from argparse) """ @@ -66,13 +73,24 @@ def _write_json(data: Any) -> None: # --------------------------------------------------------------------------- -# URL validation +# URL validation (C) # --------------------------------------------------------------------------- -# Allowed PR URL host patterns (public forges) -_PR_URL_RE = re.compile( - r"^https://(?:github\.com|gitlab\.com|gitlab\.[a-z0-9.-]+)" - r"/[^/]+/[^/]+/(?:pull|merge_requests)/\d+/?$" +# Supported PR/MR URL path patterns. The path must be EXACTLY a PR/MR +# page — trailing paths (files, commits, checks) are rejected so the +# identity is unambiguous. +# +# GitHub: /owner/repo/pull/N +# GitLab: /owner[/subgroup]*/repo/-/merge_requests/N +# GitLab (legacy): /owner[/subgroup]*/repo/merge_requests/N +_GITHUB_PR_PATH = re.compile( + r"^/([^/]+)/([^/]+)/pull/(\d+)$" +) +_GITLAB_MR_PATH = re.compile( + r"^/(.+?)/([^/]+)/-/merge_requests/(\d+)$" +) +_GITLAB_MR_LEGACY_PATH = re.compile( + r"^/(.+?)/([^/]+)/merge_requests/(\d+)$" ) @@ -86,7 +104,6 @@ def validate_url(url: str, label: str = "url") -> str: raise ValueError(f"{label}: must be a non-empty string") url = url.strip() - # Reject control characters and whitespace if any(ord(c) < 32 or c.isspace() for c in url): raise ValueError(f"{label}: contains whitespace or control characters") @@ -97,22 +114,39 @@ def validate_url(url: str, label: str = "url") -> str: raise ValueError(f"{label}: missing hostname") if parsed.username is not None or parsed.password is not None: raise ValueError(f"{label}: must not contain embedded credentials") + # Reject explicit non-standard ports — they are unusual for forges + # and could indicate a local proxy / test fixture that should use + # .example or .invalid TLDs instead. + if parsed.port is not None: + raise ValueError(f"{label}: explicit port not supported for PR URLs") - # Strip trailing slash for normalization normalized = url.rstrip("/") return normalized def validate_pr_url(url: str, label: str = "pr_url") -> str: - """Validate and normalize a pull/merge request URL. + """Validate and normalize a pull/merge-request URL. - Returns the normalized URL or raises ValueError. + Requires a complete, supported PR/MR path with a numeric + identifier. Strips query strings and fragments for identity + normalization. Raises ValueError for unrecognized, malformed, + or trailing-garbage URLs. """ normalized = validate_url(url, label) - # Strip fragment and query for identity comparison parsed = urlsplit(normalized) - clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" - clean = clean.rstrip("/") + # Reconstruct with only scheme + authority + path (strip query/fragment) + clean_path = parsed.path.rstrip("/") + clean = f"{parsed.scheme}://{parsed.netloc}{clean_path}" + + # Verify the path matches a supported PR/MR structure + if not (_GITHUB_PR_PATH.match(clean_path) + or _GITLAB_MR_PATH.match(clean_path) + or _GITLAB_MR_LEGACY_PATH.match(clean_path)): + raise ValueError( + f"{label}: not a recognized PR/MR URL " + f"(requires /owner/repo/pull/N or " + f"/owner/repo/-/merge_requests/N): {url}" + ) return clean @@ -120,34 +154,39 @@ def extract_pr_identity(url: str) -> dict[str, str]: """Extract owner, repo, and PR number from a normalized PR URL. Returns a dict with ``owner``, ``repo``, ``number``, ``host``. - Raises ValueError if the URL is not a recognized PR URL. + Raises ValueError if the URL does not match a supported forge + pattern with a numeric identifier. """ parsed = urlsplit(url) - parts = parsed.path.strip("/").split("/") - # GitHub: owner/repo/pull/N GitLab: owner/repo/-/merge_requests/N - if len(parts) >= 4 and parts[2] == "pull": + path = parsed.path.rstrip("/") + + m = _GITHUB_PR_PATH.match(path) + if m: return { "host": parsed.hostname or "", - "owner": parts[0], - "repo": parts[1], - "number": parts[3], + "owner": m.group(1), + "repo": m.group(2), + "number": m.group(3), } - # GitLab with /-/ prefix - if len(parts) >= 5 and parts[2] == "-" and parts[3] == "merge_requests": + + m = _GITLAB_MR_PATH.match(path) + if m: return { "host": parsed.hostname or "", - "owner": parts[0], - "repo": parts[1], - "number": parts[4], + "owner": m.group(1), + "repo": m.group(2), + "number": m.group(3), } - # GitLab without /-/ prefix - if len(parts) >= 4 and parts[2] == "merge_requests": + + m = _GITLAB_MR_LEGACY_PATH.match(path) + if m: return { "host": parsed.hostname or "", - "owner": parts[0], - "repo": parts[1], - "number": parts[3], + "owner": m.group(1), + "repo": m.group(2), + "number": m.group(3), } + raise ValueError(f"unrecognized PR/MR URL structure: {url}") @@ -198,10 +237,9 @@ def extract_story_refs(text: str) -> list[str]: # --------------------------------------------------------------------------- -# DOCS issue validation and contributor extraction +# DOCS issue validation and contributor extraction (A, B) # --------------------------------------------------------------------------- -# Sections expected in a rendered DOCS issue description _DOCS_SECTIONS = { "Documentation Inputs", "Documentation Scope", @@ -210,10 +248,7 @@ def extract_story_refs(text: str) -> list[str]: def _extract_sections(text: str) -> dict[str, str]: - """Extract ## sections from rendered Jira markdown description. - - Returns heading -> body text mapping. - """ + """Extract ## sections from rendered Jira markdown description.""" sections: dict[str, str] = {} current: str | None = None lines: list[str] = [] @@ -230,24 +265,15 @@ def _extract_sections(text: str) -> dict[str, str]: if current is not None: sections[current] = "\n".join(lines).strip() - return sections def _extract_contributing_stories(description: str) -> list[str]: - """Extract Contributing stories references from a rendered description. - - Looks for a ``Contributing stories:`` line in the Design Reference - section or in the full description. Returns a list of Jira keys - found on that line. Local ``Story N.NN`` placeholders without Jira - keys are returned as qualified IDs (``story-N.NN``) and flagged as - unresolved. - """ + """Extract Contributing stories references from a rendered description.""" result: list[str] = [] seen: set[str] = set() for line in description.splitlines(): stripped = line.strip() - # Normalize list-item prefixes for pfx in ("- ", "* ", "+ "): if stripped.startswith(pfx): stripped = stripped[len(pfx):].strip() @@ -255,12 +281,10 @@ def _extract_contributing_stories(description: str) -> list[str]: if not stripped.lower().startswith("contributing stories:"): continue value = stripped.split(":", 1)[1].strip() - # Extract Jira keys (rendered form) for key in extract_all_jira_keys(value): if key not in seen: seen.add(key) result.append(key) - # Extract unresolved Story N.NN placeholders for qid in extract_story_refs(value): if qid not in seen: seen.add(qid) @@ -269,15 +293,10 @@ def _extract_contributing_stories(description: str) -> list[str]: def _extract_dependencies(description: str) -> list[str]: - """Extract Dependencies references from a rendered description. - - Returns Jira keys and unresolved story refs found in the Dependencies - section only. - """ + """Extract Dependencies references from a rendered description.""" sections = _extract_sections(description) deps_text = sections.get("Dependencies", "") if not deps_text: - # Try inline Dependencies: line for line in description.splitlines(): stripped = line.strip() for pfx in ("- ", "* ", "+ "): @@ -304,9 +323,9 @@ def _extract_dependencies(description: str) -> list[str]: def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: """Parse Documentation Inputs section into structured items. - Recognizes bold contributor headers (``**Story N.NN -- title:**`` or - ``**JIRA-KEY -- title:**``) followed by bullet-pointed input items. - Each input may have ``[Assumption]`` or ``[Open: ...]`` markers. + Recognizes bold contributor headers followed by bullet-pointed + input items. Each input may carry ``[Assumption]`` or + ``[Open: ...]`` markers which are preserved. """ items: list[dict[str, Any]] = [] current_contributor: str | None = None @@ -314,20 +333,17 @@ def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: for line in text.splitlines(): stripped = line.strip() - # Bold contributor header: **KEY -- title:** or **Story N.NN -- title:** header_match = re.match( r"\*\*(.+?)(?:\s*(?:--|—)\s*.+?)?\s*:?\*\*:?\s*$", stripped, ) if header_match: header_text = header_match.group(1).strip() - # Extract key from header key = parse_jira_key(header_text) if key: contributor_key = key current_contributor = header_text else: - # May be Story N.NN format refs = extract_story_refs(header_text) if refs: contributor_key = refs[0] @@ -337,42 +353,34 @@ def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: current_contributor = header_text continue - # Bullet item under a contributor bullet_match = re.match(r"[-*+]\s+(.*)", stripped) if bullet_match and current_contributor is not None: - item_text = bullet_match.group(1).strip() + raw_text = bullet_match.group(1).strip() marker = "planned" open_ref = None - if "[Assumption]" in item_text: + if "[Assumption]" in raw_text: marker = "assumption" - item_text = item_text.replace("[Assumption]", "").strip() - open_match = re.search(r"\[Open:\s*([^\]]+)\]", item_text) + open_match = re.search(r"\[Open:\s*([^\]]+)\]", raw_text) if open_match: marker = "open" open_ref = open_match.group(1).strip() - item_text = re.sub(r"\[Open:\s*[^\]]+\]", "", item_text).strip() - + # Preserve the original text including markers items.append({ "contributor": contributor_key, "contributor_header": current_contributor, - "text": item_text, + "text": raw_text, "marker": marker, "open_ref": open_ref, }) - return items def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: """Validate a DOCS issue snapshot and extract structured data. - Input: a JSON object with at minimum ``key``, ``summary``, - ``description``. May include ``status``, ``parent``, - ``remote_links``, ``issue_links``. - - Returns a result object with ``valid``, ``errors``, ``warnings``, - and extracted ``contributors``, ``dependencies``, - ``documentation_inputs``, ``sections``. + Returns a result with ``valid``, ``errors``, ``warnings``, and + extracted ``contributors``, ``dependencies``, + ``documentation_inputs``, ``sections``, ``key``. """ errors: list[str] = [] warnings: list[str] = [] @@ -380,6 +388,7 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: key = snapshot.get("key") if not key or not isinstance(key, str): errors.append("missing or invalid 'key' field") + key = None elif not _JIRA_KEY_RE.fullmatch(key): errors.append(f"invalid Jira key format: {key}") @@ -406,31 +415,26 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: } sections = _extract_sections(description) - - # Check required sections missing_sections = _DOCS_SECTIONS - set(sections.keys()) for ms in sorted(missing_sections): errors.append(f"missing required section: {ms}") - # Extract contributors contributors = _extract_contributing_stories(description) unresolved = [c for c in contributors if c.startswith("story-")] if unresolved: for u in unresolved: warnings.append(f"unresolved local reference: {u}") - if not contributors: errors.append("no Contributing stories found in description") - # Extract dependencies (distinct from contributors) dependencies = _extract_dependencies(description) - # Parse documentation inputs doc_inputs_text = sections.get("Documentation Inputs", "") documentation_inputs = _parse_documentation_inputs(doc_inputs_text) if not documentation_inputs and "Documentation Inputs" in sections: warnings.append( - "Documentation Inputs section exists but no structured items parsed" + "Documentation Inputs section exists but no structured " + "items parsed" ) return { @@ -446,7 +450,7 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: # --------------------------------------------------------------------------- -# Evidence normalization +# Evidence normalization (A, B, C) # --------------------------------------------------------------------------- @@ -455,12 +459,16 @@ def normalize_evidence( ) -> dict[str, Any]: """Normalize and deduplicate gathered PR evidence. - Each evidence item must have ``pr_url``, ``contributor_key``, and - optionally ``target_branch``, ``state``, ``title``, ``merged``, - ``merge_commit``, ``observations`` (list of strings from AI review). + Each evidence item must have ``pr_url`` and ``contributor_key``. + Optional: ``target_branch``, ``state``, ``title``, ``merged`` + (bool), ``merge_commit``, ``observations`` (list of strings). + + Deduplication merges compatible same-PR records (same URL after + normalization). Contradictory ``merged`` or ``target_branch`` + values on the same PR produce a conflict warning — neither + silently wins. Returns ``{"items": [...], "errors": [...], "warnings": [...]}``. - Deduplication is by normalized PR URL. """ errors: list[str] = [] warnings: list[str] = [] @@ -468,7 +476,11 @@ def normalize_evidence( normalized: list[dict[str, Any]] = [] if not isinstance(evidence_items, list): - return {"items": [], "errors": ["evidence must be an array"], "warnings": []} + return { + "items": [], + "errors": ["evidence must be an array"], + "warnings": [], + } for i, item in enumerate(evidence_items): if not isinstance(item, dict): @@ -482,43 +494,114 @@ def normalize_evidence( errors.append(f"evidence[{i}]: missing contributor_key") continue + # Validate merged is bool or None + merged_val = item.get("merged") + if merged_val is not None and not isinstance(merged_val, bool): + errors.append( + f"evidence[{i}]: 'merged' must be a boolean or null" + ) + continue + try: - pr_url = validate_pr_url(pr_url_raw, f"evidence[{i}].pr_url") + pr_url = validate_pr_url( + pr_url_raw, f"evidence[{i}].pr_url" + ) except ValueError as exc: errors.append(str(exc)) continue + # Extract identity — reject if unparseable + try: + pr_id = extract_pr_identity(pr_url) + except ValueError as exc: + errors.append( + f"evidence[{i}]: {exc}" + ) + continue + + # Validate observations is a list of strings + raw_obs = item.get("observations", []) + if not isinstance(raw_obs, list): + errors.append( + f"evidence[{i}]: 'observations' must be an array" + ) + continue + observations: list[str] = [] + obs_valid = True + for j, ob in enumerate(raw_obs): + if not isinstance(ob, str): + errors.append( + f"evidence[{i}].observations[{j}]: must be a string" + ) + obs_valid = False + else: + observations.append(ob) + if not obs_valid: + continue + + target_branch = item.get("target_branch", "") + if not isinstance(target_branch, str): + target_branch = str(target_branch) + # Deduplicate by normalized URL if pr_url in seen_urls: existing_idx = seen_urls[pr_url] - # Merge contributor if different existing = normalized[existing_idx] - if contributor not in existing.get("contributor_keys", []): - existing.setdefault("contributor_keys", [contributor]) - if contributor not in existing["contributor_keys"]: - existing["contributor_keys"].append(contributor) + + # Merge contributor keys + if contributor not in existing["contributor_keys"]: + existing["contributor_keys"].append(contributor) + + # Detect contradictory merged state + if (merged_val is not None + and existing["merged"] is not None + and merged_val != existing["merged"]): + existing["conflicts"] = existing.get("conflicts", []) + existing["conflicts"].append( + f"contradictory merged state: " + f"evidence[{existing_idx}]={existing['merged']}, " + f"evidence[{i}]={merged_val}" + ) + warnings.append( + f"evidence[{i}]: PR {pr_url} has contradictory " + f"merged state with evidence[{existing_idx}]" + ) + + # Detect contradictory target branch + if (target_branch and existing["target_branch"] + and target_branch != existing["target_branch"]): + existing["conflicts"] = existing.get("conflicts", []) + existing["conflicts"].append( + f"contradictory target_branch: " + f"'{existing['target_branch']}' vs '{target_branch}'" + ) + warnings.append( + f"evidence[{i}]: PR {pr_url} has contradictory " + f"target_branch with evidence[{existing_idx}]" + ) + + # Merge observations + for ob in observations: + if ob not in existing["observations"]: + existing["observations"].append(ob) + warnings.append( f"evidence[{i}]: duplicate PR URL {pr_url} " f"(merged with evidence[{existing_idx}])" ) continue - # Extract PR identity - try: - pr_id = extract_pr_identity(pr_url) - except ValueError: - pr_id = {} - entry: dict[str, Any] = { "pr_url": pr_url, "contributor_keys": [contributor], "pr_identity": pr_id, - "target_branch": item.get("target_branch", ""), + "target_branch": target_branch, "state": item.get("state", ""), "title": item.get("title", ""), - "merged": item.get("merged"), + "merged": merged_val, "merge_commit": item.get("merge_commit", ""), - "observations": item.get("observations", []), + "observations": observations, + "conflicts": [], } seen_urls[pr_url] = len(normalized) @@ -528,7 +611,7 @@ def normalize_evidence( # --------------------------------------------------------------------------- -# Change plan +# Change plan (A, B, D) # --------------------------------------------------------------------------- @@ -538,6 +621,22 @@ def _fingerprint(data: Any) -> str: return hashlib.sha256(raw.encode()).hexdigest() +# Allowed metadata fields that this skill may propose. Each entry +# maps a canonical name to a description. Fields NOT in this catalog +# are rejected — no arbitrary keys become eligible. +_ALLOWED_METADATA = { + "target_release": "Fix Version or Target Version value from project catalog", +} + +# Fields that this skill must NEVER modify. +_FORBIDDEN_FIELDS = frozenset({ + "summary", "description", "status", "assignee", + "sprint", "parent", "labels", "comment", + "resolution", "priority", "issuetype", + "reporter", "components", "fixversions", +}) + + def build_change_plan( validation: dict[str, Any], evidence: dict[str, Any], @@ -547,81 +646,173 @@ def build_change_plan( """Build a delivery-evidence change plan. Args: - validation: output of ``validate_issue`` + validation: output of ``validate_issue`` (validated; errors + checked here rather than trusting ``valid`` flag) evidence: output of ``normalize_evidence`` - target: ``{"release": "...", "target_branch": "..."}`` - profile: optional caller-supplied consumer profile - - Returns a plan object with ``status``, ``inputs``, ``pr_links``, - ``metadata``, ``findings``, ``source_proposal``, and ``fingerprint``. + target: ``{"release": "...", "target_branch": "...", + "issue_key": "..."}`` + profile: optional caller-supplied consumer profile with + ``readiness_dimensions`` list and optional + ``readiness_results`` dict and ``allowed_fields`` list + + Returns a plan with ``status`` (``complete``, ``invalid_issue``, + ``invalid_evidence``), ``issue_key``, ``target``, ``inputs``, + ``pr_links``, ``metadata``, ``findings``, ``source_proposal``, + ``readiness``, ``plan_digest``. """ - errors: list[str] = [] - findings: list[dict[str, str]] = [] + # --- Validate nested shapes rather than trusting flags --- + val_errors = validation.get("errors") + if not isinstance(val_errors, list): + val_errors = [] - if not validation.get("valid"): + if val_errors: return { "status": "invalid_issue", - "errors": validation.get("errors", []), + "errors": val_errors, + "issue_key": validation.get("key"), + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + + issue_key = validation.get("key") + if not issue_key or not isinstance(issue_key, str): + return { + "status": "invalid_issue", + "errors": ["validation result missing valid key"], + "issue_key": None, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + + # Validate evidence shape + ev_errors = evidence.get("errors", []) + if not isinstance(ev_errors, list): + ev_errors = [] + ev_items = evidence.get("items", []) + if not isinstance(ev_items, list): + return { + "status": "invalid_evidence", + "errors": ["evidence items must be an array"], + "issue_key": issue_key, + "target": target, "inputs": [], "pr_links": [], "metadata": {}, - "findings": findings, + "findings": [], "source_proposal": None, "readiness": None, - "fingerprint": "", + "plan_digest": "", } - issue_key = validation["key"] - contributors = set(validation.get("contributors", [])) - dependencies = set(validation.get("dependencies", [])) + contributors_list = validation.get("contributors", []) + if not isinstance(contributors_list, list): + contributors_list = [] + contributors = set(contributors_list) + + deps_list = validation.get("dependencies", []) + if not isinstance(deps_list, list): + deps_list = [] + dependencies = set(deps_list) + doc_inputs = validation.get("documentation_inputs", []) - evidence_items = evidence.get("items", []) + if not isinstance(doc_inputs, list): + doc_inputs = [] + target_release = target.get("release", "") target_branch = target.get("target_branch", "") + findings: list[dict[str, str]] = [] - # --- Per-input delivery status --- - input_statuses: list[dict[str, Any]] = [] + # --- Index evidence by contributor --- contributor_evidence: dict[str, list[dict[str, Any]]] = {} - - for ev in evidence_items: + for ev in ev_items: for ck in ev.get("contributor_keys", []): contributor_evidence.setdefault(ck, []).append(ev) - # Track which contributors had evidence + # --- Per-input delivery status (B: honest per-input assessment) --- + input_statuses: list[dict[str, Any]] = [] contributors_with_evidence: set[str] = set() + contributor_pr_links: set[str] = set() # PRs from actual contributors for doc_input in doc_inputs: contrib = doc_input.get("contributor") - evs = contributor_evidence.get(contrib, []) + evs = contributor_evidence.get(contrib, []) if contrib else [] - status = "missing" - pr_refs: list[str] = [] + status = "no_evidence" + evidence_refs: list[dict[str, Any]] = [] target_notes: list[str] = [] + input_conflicts: list[str] = [] if evs: contributors_with_evidence.add(contrib) - has_merged = any(e.get("merged") is True for e in evs) - status = "delivered" if has_merged else "in_progress" - pr_refs = [e["pr_url"] for e in evs] - for ev in evs: + contributor_pr_links.add(ev["pr_url"]) + ev_merged = ev.get("merged") ev_target = ev.get("target_branch", "") - if target_branch and ev_target and ev_target != target_branch: - target_notes.append( - f"PR {ev['pr_url']} targets {ev_target}, " - f"expected {target_branch}" - ) + ev_conflicts = ev.get("conflicts", []) + + # Build per-PR evidence ref + pr_status = "unknown" + if ev_conflicts: + pr_status = "conflicting" + input_conflicts.extend(ev_conflicts) + elif ev_merged is True: + if (target_branch and ev_target + and ev_target != target_branch): + pr_status = "merged_wrong_target" + target_notes.append( + f"PR {ev['pr_url']} targets {ev_target}, " + f"expected {target_branch}" + ) + else: + pr_status = "merged" + elif ev_merged is False: + pr_status = "not_merged" + # ev_merged is None → unknown + + evidence_refs.append({ + "pr_url": ev["pr_url"], + "pr_status": pr_status, + "target_branch": ev_target, + "observations": ev.get("observations", []), + }) + + # Determine overall input status from individual PR statuses + pr_statuses = {r["pr_status"] for r in evidence_refs} + if "conflicting" in pr_statuses: + status = "conflicting" + elif "merged" in pr_statuses: + status = "merged_pr_exists" + elif "merged_wrong_target" in pr_statuses: + status = "merged_wrong_target" + elif "not_merged" in pr_statuses: + status = "in_progress" + else: + status = "evidence_unverified" input_statuses.append({ "text": doc_input.get("text", ""), "contributor": contrib, "marker": doc_input.get("marker", "planned"), "status": status, - "pr_refs": pr_refs, + "evidence_refs": evidence_refs, "target_notes": target_notes, + "conflicts": input_conflicts, }) - # --- Contributors without evidence --- + # --- Contributors without evidence (B: always flag, even if + # also a dependency) --- for contrib in sorted(contributors): if contrib not in contributors_with_evidence: if contrib.startswith("story-"): @@ -633,7 +824,7 @@ def build_change_plan( f"reference with no Jira key mapping" ), }) - elif contrib not in dependencies: + else: findings.append({ "type": "missing_evidence", "ref": contrib, @@ -643,7 +834,8 @@ def build_change_plan( ), }) - # --- Evidence for non-contributors (new surfaces) --- + # --- Non-contributor evidence: candidate proposals, NOT auto-eligible + # PR links (B) --- all_contributor_keys: set[str] = set() for di in doc_inputs: c = di.get("contributor") @@ -651,60 +843,124 @@ def build_change_plan( all_contributor_keys.add(c) all_contributor_keys |= contributors - for ev in evidence_items: + for ev in ev_items: for ck in ev.get("contributor_keys", []): if ck not in all_contributor_keys: findings.append({ - "type": "new_surface", + "type": "non_contributor_evidence", "ref": ck, "detail": ( f"PR {ev['pr_url']} is attributed to {ck} " - f"which is not a listed contributor" + f"which is not a listed contributor — this is " + f"a candidate for scope review, not automatic " + f"enrichment" ), + "pr_url": ev["pr_url"], }) - # --- PR Remote Links to add --- + # --- PR Remote Links: ONLY for actual contributors --- pr_links: list[dict[str, Any]] = [] - for ev in evidence_items: + seen_link_urls: set[str] = set() + for ev in ev_items: + url = ev["pr_url"] + if url in seen_link_urls: + continue + # Only include PRs attributed to actual contributors + if url not in contributor_pr_links: + continue + seen_link_urls.add(url) pr_links.append({ - "url": ev["pr_url"], + "url": url, "title": ev.get("title", ""), "relationship": "implements", - "contributor_keys": ev.get("contributor_keys", []), + "contributor_keys": [ + ck for ck in ev.get("contributor_keys", []) + if ck in all_contributor_keys + ], }) - # --- Metadata proposals --- + # --- Metadata (D: allowed catalog only) --- metadata: dict[str, Any] = {} if target_release: - metadata["proposed_target_release"] = target_release + metadata["target_release"] = { + "value": target_release, + "field_binding": "fixVersions or customfield (project-specific)", + "catalog_verified": False, + "note": ( + "Value must be verified against the project's actual " + "Fix Version catalog before applying" + ), + } - # Profile-based readiness (independent of delivery) + # Profile allowed fields — only from catalog + if profile and isinstance(profile, dict): + for af in profile.get("allowed_fields", []): + if not isinstance(af, dict): + continue + field_name = af.get("name", "") + if field_name in _FORBIDDEN_FIELDS: + findings.append({ + "type": "forbidden_field", + "ref": field_name, + "detail": ( + f"Profile requests field '{field_name}' which " + f"is forbidden by policy" + ), + }) + elif field_name in _ALLOWED_METADATA: + metadata[field_name] = { + "value": af.get("value"), + "field_binding": af.get("field_binding", ""), + "catalog_verified": af.get("catalog_verified", False), + } + + # --- Readiness (B: independent of delivery, visible gaps) --- readiness: dict[str, Any] | None = None - if profile: + if profile and isinstance(profile, dict): dims = profile.get("readiness_dimensions", []) - if dims: - readiness = { - "dimensions": dims, - "results": {}, - "complete": False, - "note": ( - "Consumer readiness assessment requires evaluating " - "each dimension against actual delivery evidence. " - "Dimensions are recorded; results must be populated " - "by the caller with actual assessments." - ), - } - else: - readiness = { - "dimensions": [], - "results": {}, - "complete": False, - "note": "No readiness dimensions in consumer profile", - } - - # --- Source-change proposal --- + provided_results = profile.get("readiness_results", {}) + if not isinstance(dims, list): + dims = [] + if not isinstance(provided_results, dict): + provided_results = {} + + results: dict[str, Any] = {} + for d in dims: + if isinstance(d, str) and d in provided_results: + results[d] = provided_results[d] + elif isinstance(d, str): + results[d] = { + "assessed": False, + "note": "Assessment not provided by caller", + } + + readiness = { + "dimensions": [d for d in dims if isinstance(d, str)], + "results": results, + "complete": ( + bool(dims) + and all( + isinstance(results.get(d), dict) + and results[d].get("assessed") is True + for d in dims if isinstance(d, str) + ) + ), + "note": ( + "No readiness dimensions specified" + if not dims else + "All dimensions assessed" + if all( + isinstance(results.get(d), dict) + and results[d].get("assessed") is True + for d in dims if isinstance(d, str) + ) + else "Some dimensions not yet assessed" + ), + } + + # --- Source-change proposal (B: actionable with evidence) --- source_proposal = _build_source_proposal( - doc_inputs, input_statuses, findings, evidence_items + doc_inputs, input_statuses, findings, ev_items, ) plan = { @@ -717,10 +973,11 @@ def build_change_plan( "findings": findings, "source_proposal": source_proposal, "readiness": readiness, - "fingerprint": "", + "plan_digest": "", } - plan["fingerprint"] = _fingerprint({ + plan["plan_digest"] = _fingerprint({ "issue_key": issue_key, + "target": target, "inputs": input_statuses, "pr_links": pr_links, "metadata": metadata, @@ -737,54 +994,78 @@ def _build_source_proposal( ) -> dict[str, Any]: """Build an owning-source narrative-change proposal. - This produces a structured proposal that the owning design workflow - could apply via its ``/revise`` phase. The proposal includes - specific intended changes to the Documentation Inputs section with - supporting evidence references. + Source/narrative apply is disabled. The proposal carries: + - Specific proposed wording changes or clearly unresolved questions + - Input/source location references + - Supporting PR/code evidence URLs + - Intended owning-workflow action - The docs-handoff skill does NOT directly apply these changes. - Source/narrative apply is disabled; the proposal is advisory. + Only observations that indicate actual divergence generate update + proposals. 'As planned' observations do NOT generate proposals. """ proposed_changes: list[dict[str, Any]] = [] for inp_status in input_statuses: - status = inp_status["status"] - contrib = inp_status["contributor"] - text = inp_status["text"] - - if status == "delivered": - # Check if observations suggest the actual implementation - # differs from the planned input - relevant_evs = [ - e for e in evidence_items - if contrib in e.get("contributor_keys", []) - ] - has_observations = any( - e.get("observations") for e in relevant_evs - ) - if has_observations: - all_obs = [] - for e in relevant_evs: - all_obs.extend(e.get("observations", [])) - proposed_changes.append({ - "action": "update", - "contributor": contrib, - "input_text": text, - "reason": "Implementation observations suggest differences", - "observations": all_obs, - "evidence_urls": [e["pr_url"] for e in relevant_evs], - }) + contrib = inp_status.get("contributor") + text = inp_status.get("text", "") + ev_refs = inp_status.get("evidence_refs", []) + + if not ev_refs: + continue + + # Collect observations that suggest divergence + divergent_observations: list[str] = [] + confirming_observations: list[str] = [] + evidence_urls: list[str] = [] + + for eref in ev_refs: + evidence_urls.append(eref["pr_url"]) + for obs in eref.get("observations", []): + obs_lower = obs.lower() + if any( + phrase in obs_lower + for phrase in ( + "as planned", "as expected", "matches plan", + "no change", "confirmed", + ) + ): + confirming_observations.append(obs) + else: + divergent_observations.append(obs) - # New surfaces from non-contributors + # Only divergent observations produce update proposals (B) + if divergent_observations: + proposed_changes.append({ + "action": "update", + "contributor": contrib, + "input_text": text, + "proposed_wording": ( + f"Update based on implementation evidence: " + + "; ".join(divergent_observations) + ), + "reason": "Implementation diverges from planned input", + "observations": divergent_observations, + "evidence_urls": evidence_urls, + "source_location": "Documentation Inputs", + }) + + # Non-contributor evidence: proposals with evidence URLs (B) for finding in findings: - if finding["type"] == "new_surface": + if finding.get("type") == "non_contributor_evidence": + pr_url = finding.get("pr_url", "") proposed_changes.append({ - "action": "add", + "action": "add_contributor", "contributor": finding["ref"], "input_text": "", + "proposed_wording": ( + f"Evaluate whether {finding['ref']} should be " + f"added as a contributor — PR evidence exists at " + f"{pr_url}" + ), "reason": finding["detail"], "observations": [], - "evidence_urls": [], + "evidence_urls": [pr_url] if pr_url else [], + "source_location": "Contributing stories + Documentation Inputs", }) return { @@ -792,8 +1073,8 @@ def _build_source_proposal( "reason": ( "Source/narrative apply is disabled. No supported " "source-preserving handoff path is established for this " - "invocation. Proposed changes should be reviewed and applied " - "through the owning design workflow's /revise phase." + "invocation. Proposed changes should be reviewed and " + "applied through the owning design workflow's /revise phase." ), "owning_workflow": "design", "owning_action": "/revise", @@ -802,43 +1083,90 @@ def _build_source_proposal( # --------------------------------------------------------------------------- -# Enrichment check +# Enrichment check (D) # --------------------------------------------------------------------------- def check_enrichment( plan: dict[str, Any], current_state: dict[str, Any], + approved_baseline: str = "", ) -> dict[str, Any]: """Check proposed enrichment against current issue state. Args: plan: output of ``build_change_plan`` - current_state: current issue snapshot with ``remote_links`` - (list of ``{"url": ..., "title": ...}``) and - optionally ``fields`` (dict of field names to values) - - Returns an enrichment result with ``eligible_links``, - ``existing_links``, ``skipped_links``, ``eligible_metadata``, - ``forbidden_fields``, ``plan_fingerprint``, ``baseline_fingerprint``, - ``stale``. + current_state: current issue snapshot with ``key``, + ``remote_links`` and optionally ``fields`` + approved_baseline: fingerprint from a previously approved + preview — if provided and differs from the current + baseline, the plan is stale + + Returns an enrichment result with eligibility, staleness, and + detailed operation plan. """ - # Forbidden fields that this skill must never modify - FORBIDDEN = { - "summary", "description", "status", "assignee", - "sprint", "parent", "labels", "comment", - "resolution", "priority", "issuetype", - } + # Reject invalid/blocked plans entirely (D) + plan_status = plan.get("status", "") + if plan_status != "complete": + return { + "status": "invalid_plan", + "reason": f"plan status is '{plan_status}', not 'complete'", + "eligible_links": [], + "existing_links": [], + "skipped_links": [], + "eligible_metadata": {}, + "forbidden_fields": [], + "plan_digest": plan.get("plan_digest", ""), + "baseline_fingerprint": "", + "stale": True, + } + + # Verify issue identity matches (A, D) + plan_key = plan.get("issue_key", "") + current_key = current_state.get("key", "") + if current_key and plan_key and current_key != plan_key: + return { + "status": "wrong_issue", + "reason": ( + f"plan targets {plan_key} but current state is " + f"for {current_key}" + ), + "eligible_links": [], + "existing_links": [], + "skipped_links": [], + "eligible_metadata": {}, + "forbidden_fields": [], + "plan_digest": plan.get("plan_digest", ""), + "baseline_fingerprint": "", + "stale": True, + } existing_links = current_state.get("remote_links", []) + if not isinstance(existing_links, list): + existing_links = [] + + # Canonical link set for comparison (sorted for stable fingerprint) existing_urls: set[str] = set() + canonical_links: list[dict[str, str]] = [] for link in existing_links: url = link.get("url", "") try: - existing_urls.add(validate_pr_url(url, "existing_link")) + norm = validate_pr_url(url, "existing_link") except ValueError: - existing_urls.add(url) + norm = url + existing_urls.add(norm) + canonical_links.append({"url": norm}) + canonical_links.sort(key=lambda x: x["url"]) + # Baseline fingerprint from current state + baseline_fp = _fingerprint({"remote_links": canonical_links}) + + # Staleness detection (D) + stale = False + if approved_baseline and approved_baseline != baseline_fp: + stale = True + + # Link eligibility eligible: list[dict[str, Any]] = [] existing: list[dict[str, Any]] = [] skipped: list[dict[str, Any]] = [] @@ -850,60 +1178,94 @@ def check_enrichment( except ValueError: skipped.append({**proposed, "reason": f"invalid URL: {url}"}) continue - if norm_url in existing_urls: existing.append({**proposed, "url": norm_url}) else: eligible.append({**proposed, "url": norm_url}) - # Metadata eligibility + # Metadata eligibility (D: allowed catalog only) eligible_metadata: dict[str, Any] = {} forbidden_found: list[str] = [] - for field, value in plan.get("metadata", {}).items(): - base_field = field.replace("proposed_", "") - if base_field in FORBIDDEN: + + for field, spec in plan.get("metadata", {}).items(): + if field in _FORBIDDEN_FIELDS: forbidden_found.append(field) + continue + if field not in _ALLOWED_METADATA: + forbidden_found.append(f"uncatalogued:{field}") + continue + if isinstance(spec, dict): + eligible_metadata[field] = spec else: - eligible_metadata[field] = value + eligible_metadata[field] = {"value": spec} - # Check profile-based field overrides + # Profile forbidden fields profile_fields = current_state.get("profile_fields", {}) - for pf in profile_fields: - if pf in FORBIDDEN: - forbidden_found.append(f"profile:{pf}") - - # Baseline fingerprint for drift detection - baseline_fp = _fingerprint({ - "remote_links": [ - {"url": l.get("url", "")} for l in existing_links - ], - }) + if isinstance(profile_fields, dict): + for pf in profile_fields: + if pf in _FORBIDDEN_FIELDS: + forbidden_found.append(f"profile:{pf}") return { + "status": "stale" if stale else "ready", "eligible_links": eligible, "existing_links": existing, "skipped_links": skipped, "eligible_metadata": eligible_metadata, "forbidden_fields": forbidden_found, - "plan_fingerprint": plan.get("fingerprint", ""), + "plan_digest": plan.get("plan_digest", ""), "baseline_fingerprint": baseline_fp, - "stale": False, + "stale": stale, } def verify_enrichment( - expected: dict[str, Any], + enrichment: dict[str, Any], readback: dict[str, Any], + expected_key: str = "", ) -> dict[str, Any]: """Verify enrichment results against a readback snapshot. Args: - expected: the enrichment result from ``check_enrichment`` - readback: fresh issue snapshot after writes + enrichment: output of ``check_enrichment`` + readback: fresh issue snapshot after writes (must include + ``key`` and ``remote_links``) + expected_key: if provided, verify the readback is for the + correct issue Returns a receipt with per-operation status. """ + # Identity check (D) + readback_key = readback.get("key", "") + if expected_key and readback_key and readback_key != expected_key: + return { + "status": "wrong_issue", + "reason": ( + f"expected readback for {expected_key} but got " + f"{readback_key}" + ), + "link_receipts": [], + "metadata_receipts": [], + "baseline_fingerprint": "", + "all_confirmed": False, + } + + # Reject receipts for invalid/stale enrichments + e_status = enrichment.get("status", "") + if e_status not in ("ready",): + return { + "status": "invalid_enrichment", + "reason": f"enrichment status is '{e_status}', not 'ready'", + "link_receipts": [], + "metadata_receipts": [], + "baseline_fingerprint": "", + "all_confirmed": False, + } + readback_links = readback.get("remote_links", []) + if not isinstance(readback_links, list): + readback_links = [] + readback_urls: set[str] = set() for link in readback_links: url = link.get("url", "") @@ -912,31 +1274,96 @@ def verify_enrichment( except ValueError: readback_urls.add(url) - receipts: list[dict[str, Any]] = [] - for link in expected.get("eligible_links", []): + # Link receipts + link_receipts: list[dict[str, Any]] = [] + eligible_links = enrichment.get("eligible_links", []) + existing_links = enrichment.get("existing_links", []) + + for link in eligible_links: url = link.get("url", "") try: norm = validate_pr_url(url, "receipt") except ValueError: norm = url found = norm in readback_urls - receipts.append({ + link_receipts.append({ "url": norm, + "operation": "add", "applied": found, "status": "confirmed" if found else "uncertain", }) - # Baseline drift check - new_baseline = _fingerprint({ - "remote_links": [ - {"url": l.get("url", "")} for l in readback_links - ], - }) + for link in existing_links: + url = link.get("url", "") + try: + norm = validate_pr_url(url, "receipt") + except ValueError: + norm = url + found = norm in readback_urls + link_receipts.append({ + "url": norm, + "operation": "existing", + "applied": found, + "status": "confirmed" if found else "disappeared", + }) + + # Metadata receipts + metadata_receipts: list[dict[str, Any]] = [] + for field, spec in enrichment.get("eligible_metadata", {}).items(): + readback_fields = readback.get("fields", {}) + if not isinstance(readback_fields, dict): + readback_fields = {} + current_val = readback_fields.get(field) + expected_val = spec.get("value") if isinstance(spec, dict) else spec + if current_val == expected_val: + metadata_receipts.append({ + "field": field, + "status": "confirmed", + "value": current_val, + }) + elif current_val is not None: + metadata_receipts.append({ + "field": field, + "status": "different_value", + "expected": expected_val, + "actual": current_val, + }) + else: + metadata_receipts.append({ + "field": field, + "status": "uncertain", + "expected": expected_val, + }) + + # New baseline + canonical: list[dict[str, str]] = [] + for link in readback_links: + url = link.get("url", "") + try: + canonical.append({"url": validate_pr_url(url, "_")}) + except ValueError: + canonical.append({"url": url}) + canonical.sort(key=lambda x: x["url"]) + new_baseline = _fingerprint({"remote_links": canonical}) + + # All confirmed: must have at least one operation, and all must + # be confirmed (empty eligible = no-op, not success) + all_ops = link_receipts + metadata_receipts + add_ops = [ + r for r in all_ops + if r.get("operation") != "existing" + ] + all_confirmed = ( + bool(add_ops) + and all(r.get("status") == "confirmed" for r in add_ops) + ) return { - "receipts": receipts, + "status": "verified", + "link_receipts": link_receipts, + "metadata_receipts": metadata_receipts, "baseline_fingerprint": new_baseline, - "all_confirmed": all(r["applied"] for r in receipts), + "all_confirmed": all_confirmed, } @@ -946,51 +1373,52 @@ def verify_enrichment( def render_handoff(plan: dict[str, Any]) -> str: - """Render a human-readable handoff document from a change plan. - - Produces deterministic Markdown suitable for writer consumption. - """ + """Render a human-readable handoff document from a change plan.""" parts: list[str] = [] issue_key = plan.get("issue_key", "unknown") target = plan.get("target", {}) parts.append(f"# Documentation Handoff — {issue_key}") parts.append("") - parts.append(f"**Target release:** {target.get('release', 'not specified')}") - parts.append(f"**Target branch:** {target.get('target_branch', 'not specified')}") + parts.append( + f"**Target release:** {target.get('release', 'not specified')}" + ) + parts.append( + f"**Target branch:** {target.get('target_branch', 'not specified')}" + ) parts.append("") - # Delivery summary inputs = plan.get("inputs", []) - delivered = sum(1 for i in inputs if i["status"] == "delivered") - in_progress = sum(1 for i in inputs if i["status"] == "in_progress") - missing = sum(1 for i in inputs if i["status"] == "missing") + status_counts: dict[str, int] = {} + for i in inputs: + s = i.get("status", "unknown") + status_counts[s] = status_counts.get(s, 0) + 1 parts.append("## Delivery Summary") parts.append("") - parts.append(f"- Delivered: {delivered}") - parts.append(f"- In progress: {in_progress}") - parts.append(f"- Missing evidence: {missing}") + for status_name in sorted(status_counts): + parts.append(f"- {status_name}: {status_counts[status_name]}") parts.append(f"- Total inputs: {len(inputs)}") parts.append("") - # Per-input details if inputs: parts.append("## Input Status") parts.append("") - parts.append("| Input | Contributor | Status | PRs |") + parts.append("| Input | Contributor | Status | Evidence |") parts.append("|---|---|---|---|") for inp in inputs: text = inp.get("text", "")[:60] contrib = inp.get("contributor", "") status = inp.get("status", "unknown") - prs = ", ".join(inp.get("pr_refs", [])) - if not prs: - prs = "none" - parts.append(f"| {text} | {contrib} | {status} | {prs} |") + ev_refs = inp.get("evidence_refs", []) + ev_summary = ", ".join( + f"{r['pr_url']} ({r['pr_status']})" for r in ev_refs + ) if ev_refs else "none" + parts.append( + f"| {text} | {contrib} | {status} | {ev_summary} |" + ) parts.append("") - # Target notes target_notes = [ n for inp in inputs for n in inp.get("target_notes", []) ] @@ -1001,49 +1429,87 @@ def render_handoff(plan: dict[str, Any]) -> str: parts.append(f"- {note}") parts.append("") - # Findings findings = plan.get("findings", []) if findings: parts.append("## Findings") parts.append("") for f in findings: - parts.append(f"- **{f['type']}** ({f.get('ref', '')}): {f.get('detail', '')}") + parts.append( + f"- **{f['type']}** ({f.get('ref', '')}): " + f"{f.get('detail', '')}" + ) + if f.get("pr_url"): + parts.append(f" Evidence: {f['pr_url']}") parts.append("") - # PR links to add pr_links = plan.get("pr_links", []) if pr_links: - parts.append("## PR Remote Links") + parts.append("## PR Remote Links (contributor PRs only)") parts.append("") - parts.append("| URL | Relationship | Contributors |") + parts.append("| URL | Title | Contributors |") parts.append("|---|---|---|") for link in pr_links: url = link.get("url", "") - rel = link.get("relationship", "") + title = link.get("title", "") contribs = ", ".join(link.get("contributor_keys", [])) - parts.append(f"| {url} | {rel} | {contribs} |") + parts.append(f"| {url} | {title} | {contribs} |") + parts.append("") + + metadata = plan.get("metadata", {}) + if metadata: + parts.append("## Metadata Proposals") + parts.append("") + for field, spec in metadata.items(): + if isinstance(spec, dict): + val = spec.get("value", "") + verified = spec.get("catalog_verified", False) + parts.append( + f"- **{field}**: {val} " + f"(catalog verified: {verified})" + ) + else: + parts.append(f"- **{field}**: {spec}") parts.append("") - # Source proposal source = plan.get("source_proposal") if source: parts.append("## Source Change Proposal") parts.append("") - parts.append(f"**Apply enabled:** {source.get('apply_enabled', False)}") + parts.append( + f"**Apply enabled:** {source.get('apply_enabled', False)}" + ) parts.append(f"**Reason:** {source.get('reason', '')}") - if source.get("proposed_changes"): + changes = source.get("proposed_changes", []) + if changes: parts.append("") - parts.append(f"**Owning workflow:** {source.get('owning_workflow', '')}") - parts.append(f"**Owning action:** {source.get('owning_action', '')}") + parts.append( + f"**Owning workflow:** " + f"{source.get('owning_workflow', '')}" + ) + parts.append( + f"**Owning action:** {source.get('owning_action', '')}" + ) parts.append("") - for change in source["proposed_changes"]: + for change in changes: action = change.get("action", "") contrib = change.get("contributor", "") - reason = change.get("reason", "") - parts.append(f"- **{action}** {contrib}: {reason}") + wording = change.get("proposed_wording", "") + ev_urls = change.get("evidence_urls", []) + parts.append(f"### {action}: {contrib}") + parts.append("") + if wording: + parts.append(f"**Proposed:** {wording}") + if change.get("input_text"): + parts.append( + f"**Current input:** {change['input_text']}" + ) + if ev_urls: + parts.append( + f"**Evidence:** {', '.join(ev_urls)}" + ) + parts.append("") parts.append("") - # Readiness readiness = plan.get("readiness") if readiness: parts.append("## Consumer Readiness") @@ -1053,8 +1519,18 @@ def render_handoff(plan: dict[str, Any]) -> str: parts.append(f"**Dimensions:** {', '.join(dims)}") else: parts.append("**Dimensions:** none specified") - parts.append(f"**Complete:** {readiness.get('complete', False)}") - parts.append(f"**Note:** {readiness.get('note', '')}") + parts.append( + f"**Complete:** {readiness.get('complete', False)}" + ) + results = readiness.get("results", {}) + if results: + for dim, res in results.items(): + if isinstance(res, dict): + assessed = res.get("assessed", False) + note = res.get("note", "") + parts.append(f"- {dim}: assessed={assessed} {note}") + else: + parts.append(f"- {dim}: {res}") parts.append("") return "\n".join(parts) @@ -1071,12 +1547,8 @@ def cmd_validate(args: argparse.Namespace) -> int: if not isinstance(snapshot, dict): fail("input must be a JSON object") - # Verify input is not mutated original = copy.deepcopy(snapshot) - result = validate_issue(snapshot) - - # Assert read-only behavior if snapshot != original: fail("BUG: validate_issue mutated its input") @@ -1084,6 +1556,21 @@ def cmd_validate(args: argparse.Namespace) -> int: return EXIT_SUCCESS if result["valid"] else EXIT_ERROR +def cmd_normalize(args: argparse.Namespace) -> int: + """Run the normalize subcommand.""" + data = _load_json(args.input) + if not isinstance(data, list): + fail("input must be a JSON array of evidence items") + + original = copy.deepcopy(data) + result = normalize_evidence(data) + if data != original: + fail("BUG: normalize_evidence mutated its input") + + _write_json(result) + return EXIT_SUCCESS if not result["errors"] else EXIT_ERROR + + def cmd_build_plan(args: argparse.Namespace) -> int: """Run the build-plan subcommand.""" data = _load_json(args.input) @@ -1102,15 +1589,15 @@ def cmd_build_plan(args: argparse.Namespace) -> int: if not isinstance(target, dict): fail("input.target must be an object") - # Verify input is not mutated original = copy.deepcopy(data) - plan = build_change_plan(validation, evidence_raw, target, profile) - if data != original: fail("BUG: build_change_plan mutated its input") _write_json(plan) + # Invalid issue/evidence → exit 1 + if plan["status"] != "complete": + return EXIT_ERROR return EXIT_SUCCESS @@ -1122,6 +1609,7 @@ def cmd_check_enrichment(args: argparse.Namespace) -> int: plan = data.get("plan") current = data.get("current_state") + approved_baseline = data.get("approved_baseline", "") if not isinstance(plan, dict): fail("input.plan must be an object") @@ -1132,16 +1620,20 @@ def cmd_check_enrichment(args: argparse.Namespace) -> int: readback = _load_json(Path(args.readback)) if not isinstance(readback, dict): fail("readback must be a JSON object") - enrichment = check_enrichment(plan, current) - receipt = verify_enrichment(enrichment, readback) + enrichment = check_enrichment(plan, current, approved_baseline) + expected_key = plan.get("issue_key", "") + receipt = verify_enrichment(enrichment, readback, expected_key) _write_json({"enrichment": enrichment, "receipt": receipt}) + if receipt.get("status") != "verified": + return EXIT_ERROR else: - # Verify input is not mutated original = copy.deepcopy(data) - result = check_enrichment(plan, current) + result = check_enrichment(plan, current, approved_baseline) if data != original: fail("BUG: check_enrichment mutated its input") _write_json(result) + if result.get("stale") or result.get("status") != "ready": + return EXIT_ERROR return EXIT_SUCCESS @@ -1161,6 +1653,12 @@ def main(argv: list[str] | None = None) -> int: ) p_validate.add_argument("--input", required=True, type=Path) + p_normalize = sub.add_parser( + "normalize", + help="Normalize and deduplicate raw PR evidence items", + ) + p_normalize.add_argument("--input", required=True, type=Path) + p_plan = sub.add_parser( "build-plan", help="Build a delivery-evidence change plan", @@ -1178,6 +1676,8 @@ def main(argv: list[str] | None = None) -> int: if args.command == "validate": return cmd_validate(args) + elif args.command == "normalize": + return cmd_normalize(args) elif args.command == "build-plan": return cmd_build_plan(args) elif args.command == "check-enrichment": diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 4efa5d75..8b09c44e 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -16,11 +16,7 @@ import textwrap import unittest from pathlib import Path -from unittest.mock import patch -# Resolve the script under test relative to this file so the tests -# work regardless of whether the skill is installed via symlink or -# run from the repo checkout. _SCRIPT_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(_SCRIPT_DIR)) @@ -58,11 +54,11 @@ def _make_docs_snapshot( ## Documentation Inputs - **TEST-201 -- Add widget config command:** + **TEST-201 — Add widget config command:** - New `widget config set` CLI command with `--name` and `--value` flags - Config validation error messages and recovery steps - **TEST-202 -- Widget status endpoint:** + **TEST-202 — Widget status endpoint:** - New `/api/v1/widgets/status` GET endpoint response fields - Status values: `active`, `degraded`, `offline` [Assumption] @@ -85,7 +81,6 @@ def _make_docs_snapshot( def _make_evidence_items() -> list[dict]: - """Build sample PR evidence items.""" return [ { "pr_url": "https://github.example.invalid/org/repo/pull/101", @@ -95,7 +90,9 @@ def _make_evidence_items() -> list[dict]: "title": "feat: add widget config command", "merged": True, "merge_commit": "abc123def", - "observations": ["Adds --name and --value flags as planned"], + "observations": [ + "Adds --name and --value flags as planned" + ], }, { "pr_url": "https://github.example.invalid/org/repo/pull/102", @@ -106,31 +103,66 @@ def _make_evidence_items() -> list[dict]: "merged": True, "merge_commit": "def456abc", "observations": [ - "Adds /api/v1/widgets/status endpoint", "Status values include 'maintenance' not in plan", ], }, ] +def _cli_run( + subcmd: str, + input_data, + extra_args: list[str] | None = None, +) -> subprocess.CompletedProcess: + """Run evidence.py via subprocess.""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + json.dump(input_data, f) + f.flush() + input_path = f.name + try: + cmd = [ + sys.executable, "-I", + str(_SCRIPT_DIR / "evidence.py"), + subcmd, "--input", input_path, + ] + if extra_args: + cmd.extend(extra_args) + return subprocess.run( + cmd, capture_output=True, text=True, timeout=30, + ) + finally: + os.unlink(input_path) + + # --------------------------------------------------------------------------- -# URL validation +# C: URL validation # --------------------------------------------------------------------------- class TestUrlValidation(unittest.TestCase): - """Test URL validation and normalization.""" def test_valid_https_url(self) -> None: - result = evidence.validate_url("https://github.example.invalid/org/repo") - self.assertEqual(result, "https://github.example.invalid/org/repo") + r = evidence.validate_url( + "https://github.example.invalid/org/repo" + ) + self.assertEqual( + r, "https://github.example.invalid/org/repo" + ) def test_strips_trailing_slash(self) -> None: - result = evidence.validate_url("https://github.example.invalid/org/repo/") - self.assertEqual(result, "https://github.example.invalid/org/repo") + r = evidence.validate_url( + "https://github.example.invalid/org/repo/" + ) + self.assertEqual( + r, "https://github.example.invalid/org/repo" + ) def test_rejects_embedded_credentials(self) -> None: with self.assertRaises(ValueError) as ctx: - evidence.validate_url("https://user:pass@github.example.invalid/org/repo") + evidence.validate_url( + "https://user:pass@github.example.invalid/org/repo" + ) self.assertIn("credentials", str(ctx.exception)) def test_rejects_empty(self) -> None: @@ -143,51 +175,119 @@ def test_rejects_ftp(self) -> None: def test_rejects_control_chars(self) -> None: with self.assertRaises(ValueError): - evidence.validate_url("https://example.invalid/\x00bad") + evidence.validate_url( + "https://example.invalid/\x00bad" + ) + + def test_rejects_explicit_port(self) -> None: + with self.assertRaises(ValueError): + evidence.validate_url( + "https://example.invalid:8443/path" + ) + - def test_pr_url_strips_fragment(self) -> None: - result = evidence.validate_pr_url( - "https://github.example.invalid/org/repo/pull/42#discussion" +class TestPrUrlValidation(unittest.TestCase): + """C: PR URL must have supported structure.""" + + def test_valid_github_pr(self) -> None: + r = evidence.validate_pr_url( + "https://github.example.invalid/org/repo/pull/42" ) self.assertEqual( - result, "https://github.example.invalid/org/repo/pull/42" + r, + "https://github.example.invalid/org/repo/pull/42", ) - def test_pr_url_strips_query(self) -> None: - result = evidence.validate_pr_url( - "https://github.example.invalid/org/repo/pull/42?tab=files" + def test_strips_fragment(self) -> None: + r = evidence.validate_pr_url( + "https://github.example.invalid/org/repo/pull/42" + "#discussion" ) self.assertEqual( - result, "https://github.example.invalid/org/repo/pull/42" + r, + "https://github.example.invalid/org/repo/pull/42", ) - -class TestPrIdentity(unittest.TestCase): - """Test PR identity extraction from URLs.""" - - def test_github_pr(self) -> None: - result = evidence.extract_pr_identity( + def test_strips_query(self) -> None: + r = evidence.validate_pr_url( "https://github.example.invalid/org/repo/pull/42" + "?tab=files" + ) + self.assertEqual( + r, + "https://github.example.invalid/org/repo/pull/42", ) - self.assertEqual(result["owner"], "org") - self.assertEqual(result["repo"], "repo") - self.assertEqual(result["number"], "42") - def test_gitlab_mr(self) -> None: - result = evidence.extract_pr_identity( + def test_rejects_non_pr_page(self) -> None: + """A commits page is not a PR URL.""" + with self.assertRaises(ValueError): + evidence.validate_pr_url( + "https://github.example.invalid/org/repo/commits/main" + ) + + def test_rejects_trailing_path(self) -> None: + """PR URL with trailing /files is not the canonical PR page.""" + with self.assertRaises(ValueError): + evidence.validate_pr_url( + "https://github.example.invalid/org/repo/pull/42/files" + ) + + def test_rejects_nonnumeric_id(self) -> None: + with self.assertRaises(ValueError): + evidence.validate_pr_url( + "https://github.example.invalid/org/repo/pull/abc" + ) + + def test_valid_gitlab_mr(self) -> None: + r = evidence.validate_pr_url( "https://gitlab.example.invalid/org/repo/-/merge_requests/7" ) - self.assertEqual(result["owner"], "org") - self.assertEqual(result["repo"], "repo") - self.assertEqual(result["number"], "7") + self.assertEqual( + r, + "https://gitlab.example.invalid/org/repo" + "/-/merge_requests/7", + ) + + def test_valid_gitlab_nested_namespace(self) -> None: + """GitLab group/subgroup/repo/-/merge_requests/N.""" + r = evidence.validate_pr_url( + "https://gitlab.example.invalid/group/subgroup/repo" + "/-/merge_requests/99" + ) + self.assertEqual( + r, + "https://gitlab.example.invalid/group/subgroup/repo" + "/-/merge_requests/99", + ) - def test_gitlab_mr_no_dash(self) -> None: - result = evidence.extract_pr_identity( - "https://gitlab.example.invalid/org/repo/merge_requests/7" + def test_valid_gitlab_legacy(self) -> None: + r = evidence.validate_pr_url( + "https://gitlab.example.invalid/org/repo" + "/merge_requests/7" ) - self.assertEqual(result["number"], "7") + self.assertIn("merge_requests/7", r) - def test_different_repos_produce_different_identities(self) -> None: + +class TestPrIdentity(unittest.TestCase): + + def test_github_pr(self) -> None: + r = evidence.extract_pr_identity( + "https://github.example.invalid/org/repo/pull/42" + ) + self.assertEqual(r["owner"], "org") + self.assertEqual(r["repo"], "repo") + self.assertEqual(r["number"], "42") + + def test_gitlab_nested(self) -> None: + r = evidence.extract_pr_identity( + "https://gitlab.example.invalid/group/subgroup/repo" + "/-/merge_requests/7" + ) + self.assertEqual(r["owner"], "group/subgroup") + self.assertEqual(r["repo"], "repo") + self.assertEqual(r["number"], "7") + + def test_different_repos_different_identities(self) -> None: id1 = evidence.extract_pr_identity( "https://github.example.invalid/org/repo-a/pull/1" ) @@ -196,13 +296,18 @@ def test_different_repos_produce_different_identities(self) -> None: ) self.assertNotEqual(id1["repo"], id2["repo"]) + def test_rejects_unrecognized_path(self) -> None: + with self.assertRaises(ValueError): + evidence.extract_pr_identity( + "https://github.example.invalid/org/repo/issues/42" + ) + # --------------------------------------------------------------------------- # Reference extraction # --------------------------------------------------------------------------- class TestReferenceExtraction(unittest.TestCase): - """Test Jira key and story reference extraction.""" def test_extract_jira_keys(self) -> None: keys = evidence.extract_all_jira_keys( @@ -216,191 +321,209 @@ def test_extract_story_refs(self) -> None: ) self.assertEqual(refs, ["story-1.01", "story-2.03"]) - def test_stories_plural(self) -> None: - refs = evidence.extract_story_refs("Stories 1.01, 2.03") - self.assertEqual(refs, ["story-1.01", "story-2.03"]) - # --------------------------------------------------------------------------- -# Issue validation +# A: Issue validation # --------------------------------------------------------------------------- class TestValidateIssue(unittest.TestCase): - """Test DOCS issue snapshot validation.""" def test_valid_snapshot(self) -> None: - snapshot = _make_docs_snapshot() - result = evidence.validate_issue(snapshot) - self.assertTrue(result["valid"]) - self.assertEqual(result["key"], "TEST-100") - self.assertEqual(result["errors"], []) + r = evidence.validate_issue(_make_docs_snapshot()) + self.assertTrue(r["valid"]) + self.assertEqual(r["key"], "TEST-100") + self.assertEqual(r["errors"], []) def test_extracts_contributors_not_dependencies(self) -> None: - """Contributing stories are extracted; Dependencies are separate.""" - snapshot = _make_docs_snapshot() - result = evidence.validate_issue(snapshot) - # Contributors come from Contributing stories line - self.assertIn("TEST-201", result["contributors"]) - self.assertIn("TEST-202", result["contributors"]) - # TEST-300 is a dependency, not a contributor - self.assertNotIn("TEST-300", result["contributors"]) - self.assertIn("TEST-300", result["dependencies"]) + r = evidence.validate_issue(_make_docs_snapshot()) + self.assertIn("TEST-201", r["contributors"]) + self.assertIn("TEST-202", r["contributors"]) + self.assertNotIn("TEST-300", r["contributors"]) + self.assertIn("TEST-300", r["dependencies"]) def test_missing_docs_prefix(self) -> None: - snapshot = _make_docs_snapshot(summary="Widget docs") - result = evidence.validate_issue(snapshot) - self.assertFalse(result["valid"]) - self.assertTrue(any("[DOCS]" in e for e in result["errors"])) + r = evidence.validate_issue( + _make_docs_snapshot(summary="Widget docs") + ) + self.assertFalse(r["valid"]) def test_missing_description(self) -> None: - snapshot = _make_docs_snapshot(description="") - result = evidence.validate_issue(snapshot) - self.assertFalse(result["valid"]) + r = evidence.validate_issue( + _make_docs_snapshot(description="") + ) + self.assertFalse(r["valid"]) def test_missing_documentation_inputs(self) -> None: desc = textwrap.dedent("""\ ## Acceptance Criteria - - [ ] Something - ## Documentation Scope - Some scope - ## Design Reference - Contributing stories: TEST-201 """) - snapshot = _make_docs_snapshot(description=desc) - result = evidence.validate_issue(snapshot) - self.assertFalse(result["valid"]) + r = evidence.validate_issue(_make_docs_snapshot(description=desc)) + self.assertFalse(r["valid"]) self.assertTrue( - any("Documentation Inputs" in e for e in result["errors"]) + any("Documentation Inputs" in e for e in r["errors"]) ) def test_no_contributors(self) -> None: desc = textwrap.dedent("""\ ## Acceptance Criteria - - [ ] Something - ## Documentation Scope - Some scope - ## Documentation Inputs - No inputs yet. """) - snapshot = _make_docs_snapshot(description=desc) - result = evidence.validate_issue(snapshot) - self.assertFalse(result["valid"]) - self.assertTrue( - any("Contributing stories" in e for e in result["errors"]) - ) + r = evidence.validate_issue(_make_docs_snapshot(description=desc)) + self.assertFalse(r["valid"]) def test_unresolved_story_ref_is_warning(self) -> None: desc = textwrap.dedent("""\ ## Acceptance Criteria - - [ ] Something - ## Documentation Scope - Covers new CLI - ## Documentation Inputs - - **Story 1.01 -- Add widget:** + **Story 1.01 — Add widget:** - New CLI command - ## Design Reference - Contributing stories: Story 1.01 """) - snapshot = _make_docs_snapshot(description=desc) - result = evidence.validate_issue(snapshot) - # Valid but with warning about unresolved ref - self.assertTrue(result["valid"]) - self.assertIn("story-1.01", result["contributors"]) - self.assertTrue( - any("unresolved" in w for w in result["warnings"]) - ) + r = evidence.validate_issue(_make_docs_snapshot(description=desc)) + self.assertTrue(r["valid"]) + self.assertIn("story-1.01", r["contributors"]) + self.assertTrue(any("unresolved" in w for w in r["warnings"])) def test_invalid_jira_key_format(self) -> None: - snapshot = _make_docs_snapshot(key="bad-key") - result = evidence.validate_issue(snapshot) - self.assertFalse(result["valid"]) - self.assertTrue(any("invalid Jira key" in e for e in result["errors"])) + r = evidence.validate_issue(_make_docs_snapshot(key="bad-key")) + self.assertFalse(r["valid"]) def test_read_only_no_mutation(self) -> None: - """validate_issue must not mutate its input.""" - snapshot = _make_docs_snapshot() - original = copy.deepcopy(snapshot) - evidence.validate_issue(snapshot) - self.assertEqual(snapshot, original) + snap = _make_docs_snapshot() + orig = copy.deepcopy(snap) + evidence.validate_issue(snap) + self.assertEqual(snap, orig) def test_parses_documentation_inputs_items(self) -> None: - snapshot = _make_docs_snapshot() - result = evidence.validate_issue(snapshot) - inputs = result["documentation_inputs"] - self.assertTrue(len(inputs) >= 2) - # First contributor's items - test201_items = [i for i in inputs if i["contributor"] == "TEST-201"] - self.assertTrue(len(test201_items) >= 1) - self.assertEqual(test201_items[0]["marker"], "planned") - - def test_parses_assumption_marker(self) -> None: - snapshot = _make_docs_snapshot() - result = evidence.validate_issue(snapshot) - assumption_items = [ - i for i in result["documentation_inputs"] + r = evidence.validate_issue(_make_docs_snapshot()) + items = r["documentation_inputs"] + self.assertTrue(len(items) >= 2) + t201 = [i for i in items if i["contributor"] == "TEST-201"] + self.assertTrue(len(t201) >= 1) + + def test_preserves_assumption_marker(self) -> None: + """Assumption markers are preserved in text.""" + r = evidence.validate_issue(_make_docs_snapshot()) + assumption = [ + i for i in r["documentation_inputs"] if i["marker"] == "assumption" ] - self.assertTrue(len(assumption_items) >= 1) + self.assertTrue(len(assumption) >= 1) + # Original marker text preserved + self.assertIn("[Assumption]", assumption[0]["text"]) + + def test_empty_inputs_section_is_warning(self) -> None: + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + + ## Design Reference + Contributing stories: TEST-201 + """) + r = evidence.validate_issue(_make_docs_snapshot(description=desc)) + self.assertTrue(r["valid"]) + self.assertTrue( + any("no structured items" in w for w in r["warnings"]) + ) # --------------------------------------------------------------------------- -# Evidence normalization +# B, C: Evidence normalization # --------------------------------------------------------------------------- class TestNormalizeEvidence(unittest.TestCase): - """Test PR evidence normalization.""" def test_normalizes_valid_evidence(self) -> None: - items = _make_evidence_items() - result = evidence.normalize_evidence(items) - self.assertEqual(len(result["items"]), 2) - self.assertEqual(result["errors"], []) + r = evidence.normalize_evidence(_make_evidence_items()) + self.assertEqual(len(r["items"]), 2) + self.assertEqual(r["errors"], []) def test_deduplicates_same_url(self) -> None: items = [ { "pr_url": "https://github.example.invalid/org/repo/pull/101", "contributor_key": "TEST-201", + "merged": True, }, { - "pr_url": "https://github.example.invalid/org/repo/pull/101/", + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101/" + ), "contributor_key": "TEST-202", + "merged": True, }, ] - result = evidence.normalize_evidence(items) - self.assertEqual(len(result["items"]), 1) - # Both contributors should be merged - self.assertIn("TEST-201", result["items"][0]["contributor_keys"]) - self.assertIn("TEST-202", result["items"][0]["contributor_keys"]) + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + self.assertIn("TEST-201", r["items"][0]["contributor_keys"]) + self.assertIn("TEST-202", r["items"][0]["contributor_keys"]) + + def test_contradictory_merged_produces_conflict(self) -> None: + """B: Same PR with merged=True and merged=False → conflict.""" + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-202", + "merged": False, + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + self.assertTrue(len(r["items"][0]["conflicts"]) > 0) + self.assertTrue( + any("contradictory" in w for w in r["warnings"]) + ) + + def test_contradictory_target_branch(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "target_branch": "main", + }, + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "target_branch": "release-4.19", + }, + ] + r = evidence.normalize_evidence(items) + self.assertTrue(len(r["items"][0]["conflicts"]) > 0) def test_rejects_embedded_credentials_in_pr(self) -> None: items = [ { - "pr_url": "https://user:tok@github.example.invalid/org/repo/pull/1", + "pr_url": ( + "https://user:tok@github.example.invalid" + "/org/repo/pull/1" + ), "contributor_key": "TEST-201", }, ] - result = evidence.normalize_evidence(items) - self.assertEqual(len(result["items"]), 0) - self.assertTrue(len(result["errors"]) > 0) - self.assertTrue(any("credentials" in e for e in result["errors"])) + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) + self.assertTrue(any("credentials" in e for e in r["errors"])) def test_rejects_missing_contributor(self) -> None: items = [ @@ -408,9 +531,8 @@ def test_rejects_missing_contributor(self) -> None: "pr_url": "https://github.example.invalid/org/repo/pull/1", }, ] - result = evidence.normalize_evidence(items) - self.assertEqual(len(result["items"]), 0) - self.assertTrue(any("contributor_key" in e for e in result["errors"])) + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) def test_preserves_different_repo_identities(self) -> None: items = [ @@ -423,33 +545,59 @@ def test_preserves_different_repo_identities(self) -> None: "contributor_key": "TEST-202", }, ] - result = evidence.normalize_evidence(items) - self.assertEqual(len(result["items"]), 2) - repos = {i["pr_identity"]["repo"] for i in result["items"]} + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 2) + repos = {i["pr_identity"]["repo"] for i in r["items"]} self.assertEqual(repos, {"repo-a", "repo-b"}) - def test_evidence_not_array(self) -> None: - result = evidence.normalize_evidence("not a list") - self.assertTrue(len(result["errors"]) > 0) + def test_rejects_non_pr_url(self) -> None: + """C: Non-PR URLs are rejected, not passed with empty identity.""" + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo" + "/commits/main" + ), + "contributor_key": "TEST-201", + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) + self.assertTrue(len(r["errors"]) > 0) - def test_strips_pr_url_fragment(self) -> None: + def test_rejects_invalid_merged_type(self) -> None: items = [ { - "pr_url": "https://github.example.invalid/org/repo/pull/1#diff", + "pr_url": "https://github.example.invalid/org/repo/pull/1", "contributor_key": "TEST-201", + "merged": "yes", }, ] - result = evidence.normalize_evidence(items) - self.assertEqual(len(result["items"]), 1) - self.assertNotIn("#", result["items"][0]["pr_url"]) + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) + self.assertTrue(any("boolean" in e for e in r["errors"])) + + def test_rejects_non_string_observation(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "observations": [42], + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) + + def test_evidence_not_array(self) -> None: + r = evidence.normalize_evidence("not a list") + self.assertTrue(len(r["errors"]) > 0) # --------------------------------------------------------------------------- -# Change plan +# B: Change plan # --------------------------------------------------------------------------- class TestBuildChangePlan(unittest.TestCase): - """Test delivery-evidence change plan building.""" def _make_plan( self, @@ -465,294 +613,521 @@ def _make_plan( evidence_items = _make_evidence_items() if target is None: target = {"release": "4.19", "target_branch": "main"} - validation = evidence.validate_issue(snapshot) + val = evidence.validate_issue(snapshot) ev = evidence.normalize_evidence(evidence_items) - return evidence.build_change_plan(validation, ev, target, profile) + return evidence.build_change_plan(val, ev, target, profile) def test_complete_plan(self) -> None: plan = self._make_plan() self.assertEqual(plan["status"], "complete") self.assertTrue(len(plan["inputs"]) > 0) - self.assertTrue(len(plan["pr_links"]) > 0) - def test_delivered_inputs(self) -> None: + def test_per_input_honest_status(self) -> None: + """B: 'as planned' observation → merged_pr_exists, not 'delivered'.""" plan = self._make_plan() - delivered = [i for i in plan["inputs"] if i["status"] == "delivered"] - self.assertTrue(len(delivered) > 0) + for inp in plan["inputs"]: + self.assertIn(inp["status"], ( + "merged_pr_exists", "merged_wrong_target", + "in_progress", "no_evidence", + "evidence_unverified", "conflicting", + )) + + def test_merged_wrong_target(self) -> None: + """B: PR targeting wrong branch → merged_wrong_target.""" + plan = self._make_plan() + wrong_target = [ + i for i in plan["inputs"] + if i["status"] == "merged_wrong_target" + ] + self.assertTrue( + len(wrong_target) > 0, + "Expected merged_wrong_target for TEST-202" + ) - def test_missing_evidence(self) -> None: - """Contributor with no PRs produces missing_evidence finding.""" - snapshot = _make_docs_snapshot() - # Only provide evidence for TEST-201, not TEST-202 + def test_missing_evidence_always_flagged(self) -> None: + """B: Contributor with no PRs → missing_evidence, even if also + a dependency.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - New CLI command + ## Dependencies + TEST-201 + ## Design Reference + Contributing stories: TEST-201 + """) + snapshot = _make_docs_snapshot(description=desc) + plan = self._make_plan(snapshot=snapshot, evidence_items=[]) + findings = [ + f for f in plan["findings"] + if f["type"] == "missing_evidence" + ] + self.assertTrue( + any(f["ref"] == "TEST-201" for f in findings), + "Contributor also-dependency must still be flagged" + ) + + def test_non_contributor_not_in_pr_links(self) -> None: + """B: PRs from non-contributors are NOT eligible links.""" + items = _make_evidence_items() + [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), + "contributor_key": "TEST-999", + }, + ] + plan = self._make_plan(evidence_items=items) + link_urls = {l["url"] for l in plan["pr_links"]} + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/999", + link_urls, + ) + findings = [ + f for f in plan["findings"] + if f["type"] == "non_contributor_evidence" + ] + self.assertTrue(any(f["ref"] == "TEST-999" for f in findings)) + + def test_as_planned_no_source_proposal(self) -> None: + """B: 'as planned' observation does NOT produce update proposal.""" items = [ { - "pr_url": "https://github.example.invalid/org/repo/pull/101", + "pr_url": "https://github.example.invalid/org/repo/pull/1", "contributor_key": "TEST-201", "merged": True, + "observations": ["Adds --name and --value flags as planned"], }, ] plan = self._make_plan(evidence_items=items) - findings = [f for f in plan["findings"] if f["type"] == "missing_evidence"] - self.assertTrue( - any(f["ref"] == "TEST-202" for f in findings), - f"Expected missing_evidence for TEST-202 in {findings}" - ) - - def test_dependency_not_flagged_as_missing(self) -> None: - """Dependencies (not contributors) don't generate missing_evidence.""" - plan = self._make_plan() - findings = [f for f in plan["findings"] if f["type"] == "missing_evidence"] - refs = [f["ref"] for f in findings] - self.assertNotIn("TEST-300", refs) + updates = [ + c for c in plan["source_proposal"]["proposed_changes"] + if c["action"] == "update" + ] + self.assertEqual(len(updates), 0) - def test_pr_target_branch_ambiguity(self) -> None: - """PR targeting wrong branch produces target_notes.""" - plan = self._make_plan() - # TEST-202 targets release-4.19, not main - target_notes = [ - n for inp in plan["inputs"] for n in inp.get("target_notes", []) + def test_divergent_observation_produces_proposal(self) -> None: + """B: Divergent observation → update proposal with wording.""" + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "merged": True, + "observations": [ + "Added --format flag not in plan" + ], + }, ] - self.assertTrue( - any("release-4.19" in n for n in target_notes), - f"Expected target branch note in {target_notes}" - ) + plan = self._make_plan(evidence_items=items) + updates = [ + c for c in plan["source_proposal"]["proposed_changes"] + if c["action"] == "update" + ] + self.assertTrue(len(updates) > 0) + self.assertTrue(updates[0]["proposed_wording"]) + self.assertTrue(updates[0]["evidence_urls"]) - def test_new_surface_finding(self) -> None: - """PR for non-contributor produces new_surface finding.""" + def test_non_contributor_proposal_has_evidence(self) -> None: + """B: non-contributor proposal carries evidence URL.""" items = _make_evidence_items() + [ { - "pr_url": "https://github.example.invalid/org/repo/pull/999", + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), "contributor_key": "TEST-999", }, ] plan = self._make_plan(evidence_items=items) - new_surfaces = [f for f in plan["findings"] if f["type"] == "new_surface"] - self.assertTrue( - any(f["ref"] == "TEST-999" for f in new_surfaces) - ) + adds = [ + c for c in plan["source_proposal"]["proposed_changes"] + if c["action"] == "add_contributor" + ] + self.assertTrue(len(adds) > 0) + self.assertTrue(adds[0]["evidence_urls"]) + self.assertTrue(adds[0]["proposed_wording"]) - def test_invalid_issue_produces_invalid_plan(self) -> None: - snapshot = _make_docs_snapshot(key="bad") - plan = self._make_plan(snapshot=snapshot) + def test_invalid_issue(self) -> None: + plan = self._make_plan(snapshot=_make_docs_snapshot(key="bad")) self.assertEqual(plan["status"], "invalid_issue") def test_unresolved_contributor_finding(self) -> None: - """Unresolved Story N.NN contributor produces finding.""" desc = textwrap.dedent("""\ ## Acceptance Criteria - - [ ] Something - ## Documentation Scope - Covers new CLI - ## Documentation Inputs - - **Story 1.01 -- Add widget:** + **Story 1.01 — Add widget:** - New CLI command - ## Design Reference - Contributing stories: Story 1.01 """) - snapshot = _make_docs_snapshot(description=desc) - plan = self._make_plan(snapshot=snapshot, evidence_items=[]) - findings = [f for f in plan["findings"] if f["type"] == "unresolved_contributor"] + plan = self._make_plan( + snapshot=_make_docs_snapshot(description=desc), + evidence_items=[], + ) + findings = [ + f for f in plan["findings"] + if f["type"] == "unresolved_contributor" + ] self.assertTrue( any("story-1.01" in f["ref"] for f in findings) ) def test_source_proposal_disabled(self) -> None: - """Source/narrative apply is disabled by default.""" plan = self._make_plan() self.assertFalse(plan["source_proposal"]["apply_enabled"]) - self.assertIn("disabled", plan["source_proposal"]["reason"]) - - def test_source_proposal_has_owning_workflow(self) -> None: - plan = self._make_plan() - self.assertEqual(plan["source_proposal"]["owning_workflow"], "design") - self.assertEqual(plan["source_proposal"]["owning_action"], "/revise") + self.assertEqual( + plan["source_proposal"]["owning_workflow"], "design" + ) - def test_fingerprint_stable(self) -> None: + def test_digest_stable(self) -> None: plan1 = self._make_plan() plan2 = self._make_plan() - self.assertEqual(plan1["fingerprint"], plan2["fingerprint"]) - self.assertTrue(len(plan1["fingerprint"]) > 0) + self.assertEqual(plan1["plan_digest"], plan2["plan_digest"]) + self.assertTrue(len(plan1["plan_digest"]) > 0) def test_read_only_no_mutation(self) -> None: - """build_change_plan must not mutate its inputs.""" - snapshot = _make_docs_snapshot() + snap = _make_docs_snapshot() items = _make_evidence_items() target = {"release": "4.19", "target_branch": "main"} - validation = evidence.validate_issue(snapshot) + val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(items) - orig_val = copy.deepcopy(validation) + orig_val = copy.deepcopy(val) orig_ev = copy.deepcopy(ev) orig_target = copy.deepcopy(target) - evidence.build_change_plan(validation, ev, target) - self.assertEqual(validation, orig_val) + evidence.build_change_plan(val, ev, target) + self.assertEqual(val, orig_val) self.assertEqual(ev, orig_ev) self.assertEqual(target, orig_target) - def test_readiness_with_profile(self) -> None: - profile = {"readiness_dimensions": ["technical_accuracy", "completeness"]} + def test_readiness_with_results(self) -> None: + profile = { + "readiness_dimensions": ["accuracy", "completeness"], + "readiness_results": { + "accuracy": {"assessed": True, "note": "Verified"}, + }, + } plan = self._make_plan(profile=profile) - self.assertIsNotNone(plan["readiness"]) - self.assertEqual( - plan["readiness"]["dimensions"], - ["technical_accuracy", "completeness"], - ) - self.assertFalse(plan["readiness"]["complete"]) + r = plan["readiness"] + self.assertFalse(r["complete"]) # completeness not assessed + self.assertTrue(r["results"]["accuracy"]["assessed"]) + self.assertFalse(r["results"]["completeness"]["assessed"]) + + def test_readiness_all_assessed(self) -> None: + profile = { + "readiness_dimensions": ["accuracy"], + "readiness_results": { + "accuracy": {"assessed": True, "note": "OK"}, + }, + } + plan = self._make_plan(profile=profile) + self.assertTrue(plan["readiness"]["complete"]) def test_readiness_without_profile(self) -> None: plan = self._make_plan(profile=None) self.assertIsNone(plan["readiness"]) - def test_readiness_empty_dimensions(self) -> None: - profile = {"readiness_dimensions": []} + def test_no_evidence_status(self) -> None: + """Missing evidence → no_evidence, not 'missing'.""" + plan = self._make_plan(evidence_items=[]) + for inp in plan["inputs"]: + self.assertEqual(inp["status"], "no_evidence") + + def test_evidence_refs_per_input(self) -> None: + plan = self._make_plan() + for inp in plan["inputs"]: + if inp["status"] != "no_evidence": + self.assertTrue(len(inp["evidence_refs"]) > 0) + for er in inp["evidence_refs"]: + self.assertIn("pr_url", er) + self.assertIn("pr_status", er) + + def test_conflicting_pr_status(self) -> None: + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "merged": False, + }, + ] + plan = self._make_plan(evidence_items=items) + conflicting = [ + i for i in plan["inputs"] if i["status"] == "conflicting" + ] + self.assertTrue(len(conflicting) > 0) + + def test_forbidden_profile_field(self) -> None: + """D: Profile requesting forbidden field → finding.""" + profile = { + "allowed_fields": [ + {"name": "labels", "value": "v4.19"}, + ], + } plan = self._make_plan(profile=profile) - self.assertEqual(plan["readiness"]["dimensions"], []) - self.assertFalse(plan["readiness"]["complete"]) + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + ] + self.assertTrue(len(forbidden) > 0) # --------------------------------------------------------------------------- -# Enrichment check +# D: Enrichment check # --------------------------------------------------------------------------- class TestCheckEnrichment(unittest.TestCase): - """Test enrichment idempotency and eligibility checks.""" def _make_enrichment( self, *, existing_links: list[dict] | None = None, + approved_baseline: str = "", + plan_override: dict | None = None, + current_key: str = "TEST-100", ) -> dict: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) - ev = evidence.normalize_evidence(_make_evidence_items()) - plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} - ) + if plan_override is not None: + plan = plan_override + else: + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "4.19", "target_branch": "main"}, + ) current = { + "key": current_key, "remote_links": existing_links or [], } - return evidence.check_enrichment(plan, current) + return evidence.check_enrichment( + plan, current, approved_baseline + ) def test_new_links_eligible(self) -> None: - result = self._make_enrichment() - self.assertTrue(len(result["eligible_links"]) > 0) - self.assertEqual(len(result["existing_links"]), 0) + r = self._make_enrichment() + self.assertEqual(r["status"], "ready") + self.assertTrue(len(r["eligible_links"]) > 0) def test_existing_link_detected(self) -> None: - result = self._make_enrichment( + r = self._make_enrichment( existing_links=[ - {"url": "https://github.example.invalid/org/repo/pull/101"}, + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, ] ) - self.assertTrue(len(result["existing_links"]) > 0) - # Should have fewer eligible links - eligible_urls = {l["url"] for l in result["eligible_links"]} - self.assertNotIn( - "https://github.example.invalid/org/repo/pull/101", - eligible_urls, + self.assertTrue(len(r["existing_links"]) > 0) + + def test_invalid_plan_rejected(self) -> None: + """D: Invalid plan → no eligible writes.""" + plan = { + "status": "invalid_issue", + "errors": ["test"], + "pr_links": [{"url": "https://github.example.invalid/org/repo/pull/1"}], + "metadata": {}, + "plan_digest": "", + } + r = self._make_enrichment(plan_override=plan) + self.assertEqual(r["status"], "invalid_plan") + self.assertEqual(len(r["eligible_links"]), 0) + self.assertTrue(r["stale"]) + + def test_wrong_issue_rejected(self) -> None: + """D: Current state key != plan key → wrong_issue.""" + r = self._make_enrichment(current_key="OTHER-999") + self.assertEqual(r["status"], "wrong_issue") + self.assertTrue(r["stale"]) + + def test_baseline_drift_detected(self) -> None: + """D: Approved baseline differs → stale.""" + r = self._make_enrichment( + approved_baseline="old-fingerprint-that-wont-match" ) + self.assertTrue(r["stale"]) + self.assertEqual(r["status"], "stale") - def test_forbidden_fields_rejected(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) + def test_no_baseline_not_stale(self) -> None: + """No approved baseline → not stale.""" + r = self._make_enrichment() + self.assertFalse(r["stale"]) + + def test_forbidden_metadata_rejected(self) -> None: + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} + val, ev, + {"release": "4.19", "target_branch": "main"}, ) - # Inject forbidden metadata - plan["metadata"]["proposed_status"] = "Done" - plan["metadata"]["proposed_assignee"] = "someone" - - current = {"remote_links": []} - result = evidence.check_enrichment(plan, current) - self.assertTrue(len(result["forbidden_fields"]) > 0) + plan["metadata"]["status"] = "Done" + current = {"key": "TEST-100", "remote_links": []} + r = evidence.check_enrichment(plan, current) self.assertTrue( - any("status" in f for f in result["forbidden_fields"]) + any("status" in f for f in r["forbidden_fields"]) ) - def test_profile_forbidden_fields(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) + def test_uncatalogued_metadata_rejected(self) -> None: + """D: Arbitrary key not in allowed catalog → forbidden.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} + val, ev, + {"release": "4.19", "target_branch": "main"}, ) - current = {"remote_links": [], "profile_fields": {"labels": "v4.19"}} - result = evidence.check_enrichment(plan, current) + plan["metadata"]["random_field"] = {"value": "x"} + current = {"key": "TEST-100", "remote_links": []} + r = evidence.check_enrichment(plan, current) self.assertTrue( - any("labels" in f for f in result["forbidden_fields"]) + any("random_field" in f for f in r["forbidden_fields"]) ) - def test_repeated_check_idempotent(self) -> None: - result1 = self._make_enrichment() - result2 = self._make_enrichment() + def test_canonical_ordering_avoids_false_drift(self) -> None: + """D: Link order alone shouldn't cause false drift.""" + r1 = self._make_enrichment( + existing_links=[ + {"url": "https://github.example.invalid/org/repo/pull/1"}, + {"url": "https://github.example.invalid/org/repo/pull/2"}, + ] + ) + r2 = self._make_enrichment( + existing_links=[ + {"url": "https://github.example.invalid/org/repo/pull/2"}, + {"url": "https://github.example.invalid/org/repo/pull/1"}, + ] + ) self.assertEqual( - result1["baseline_fingerprint"], - result2["baseline_fingerprint"], + r1["baseline_fingerprint"], + r2["baseline_fingerprint"], ) - def test_stale_defaults_false(self) -> None: - result = self._make_enrichment() - self.assertFalse(result["stale"]) - # --------------------------------------------------------------------------- -# Verify enrichment (readback) +# D: Verify enrichment (readback) # --------------------------------------------------------------------------- class TestVerifyEnrichment(unittest.TestCase): - """Test readback verification after enrichment writes.""" - def test_confirmed_when_link_present(self) -> None: + def _make_receipt( + self, + *, + enrichment_status: str = "ready", + eligible_urls: list[str] | None = None, + existing_urls: list[str] | None = None, + readback_urls: list[str] | None = None, + readback_key: str = "TEST-100", + expected_key: str = "TEST-100", + eligible_metadata: dict | None = None, + readback_fields: dict | None = None, + ) -> dict: + if eligible_urls is None: + eligible_urls = [ + "https://github.example.invalid/org/repo/pull/101" + ] enrichment = { - "eligible_links": [ - {"url": "https://github.example.invalid/org/repo/pull/101"}, + "status": enrichment_status, + "eligible_links": [{"url": u} for u in eligible_urls], + "existing_links": [ + {"url": u} for u in (existing_urls or []) ], + "eligible_metadata": eligible_metadata or {}, } readback = { + "key": readback_key, "remote_links": [ - {"url": "https://github.example.invalid/org/repo/pull/101"}, + {"url": u} for u in (readback_urls or []) ], + "fields": readback_fields or {}, } - receipt = evidence.verify_enrichment(enrichment, readback) - self.assertTrue(receipt["all_confirmed"]) - self.assertEqual(receipt["receipts"][0]["status"], "confirmed") + return evidence.verify_enrichment( + enrichment, readback, expected_key + ) + + def test_confirmed_when_link_present(self) -> None: + r = self._make_receipt( + readback_urls=[ + "https://github.example.invalid/org/repo/pull/101" + ] + ) + self.assertTrue(r["all_confirmed"]) + self.assertEqual(r["link_receipts"][0]["status"], "confirmed") def test_uncertain_when_link_missing(self) -> None: - enrichment = { - "eligible_links": [ - {"url": "https://github.example.invalid/org/repo/pull/101"}, - ], - } - readback = {"remote_links": []} - receipt = evidence.verify_enrichment(enrichment, readback) - self.assertFalse(receipt["all_confirmed"]) - self.assertEqual(receipt["receipts"][0]["status"], "uncertain") + r = self._make_receipt(readback_urls=[]) + self.assertFalse(r["all_confirmed"]) + self.assertEqual(r["link_receipts"][0]["status"], "uncertain") + + def test_wrong_readback_key(self) -> None: + """D: Readback for wrong issue.""" + r = self._make_receipt(readback_key="WRONG-999") + self.assertEqual(r["status"], "wrong_issue") + self.assertFalse(r["all_confirmed"]) + + def test_empty_eligible_not_success(self) -> None: + """D: No operations to confirm → not all_confirmed.""" + r = self._make_receipt( + eligible_urls=[], readback_urls=[] + ) + self.assertFalse(r["all_confirmed"]) - def test_partial_confirmation(self) -> None: - enrichment = { - "eligible_links": [ - {"url": "https://github.example.invalid/org/repo/pull/101"}, - {"url": "https://github.example.invalid/org/repo/pull/102"}, + def test_invalid_enrichment_rejected(self) -> None: + r = self._make_receipt(enrichment_status="stale") + self.assertEqual(r["status"], "invalid_enrichment") + self.assertFalse(r["all_confirmed"]) + + def test_metadata_receipt_confirmed(self) -> None: + r = self._make_receipt( + eligible_metadata={ + "target_release": {"value": "4.19"}, + }, + readback_fields={"target_release": "4.19"}, + readback_urls=[ + "https://github.example.invalid/org/repo/pull/101" ], - } - readback = { - "remote_links": [ - {"url": "https://github.example.invalid/org/repo/pull/101"}, + ) + meta = r["metadata_receipts"] + self.assertEqual(len(meta), 1) + self.assertEqual(meta[0]["status"], "confirmed") + + def test_metadata_receipt_different(self) -> None: + r = self._make_receipt( + eligible_metadata={ + "target_release": {"value": "4.19"}, + }, + readback_fields={"target_release": "4.20"}, + readback_urls=[ + "https://github.example.invalid/org/repo/pull/101" ], - } - receipt = evidence.verify_enrichment(enrichment, readback) - self.assertFalse(receipt["all_confirmed"]) - confirmed = [r for r in receipt["receipts"] if r["status"] == "confirmed"] - uncertain = [r for r in receipt["receipts"] if r["status"] == "uncertain"] - self.assertEqual(len(confirmed), 1) - self.assertEqual(len(uncertain), 1) + ) + meta = r["metadata_receipts"] + self.assertEqual(meta[0]["status"], "different_value") + + def test_existing_link_disappearance_detected(self) -> None: + r = self._make_receipt( + eligible_urls=[], + existing_urls=[ + "https://github.example.invalid/org/repo/pull/99" + ], + readback_urls=[], + ) + existing_receipts = [ + lr for lr in r["link_receipts"] + if lr["operation"] == "existing" + ] + self.assertEqual( + existing_receipts[0]["status"], "disappeared" + ) # --------------------------------------------------------------------------- @@ -760,154 +1135,208 @@ def test_partial_confirmation(self) -> None: # --------------------------------------------------------------------------- class TestRenderHandoff(unittest.TestCase): - """Test handoff document rendering.""" - def test_deterministic(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) + def _make_rendered(self) -> str: + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} + val, ev, + {"release": "4.19", "target_branch": "main"}, ) - md1 = evidence.render_handoff(plan) - md2 = evidence.render_handoff(plan) + return evidence.render_handoff(plan) + + def test_deterministic(self) -> None: + md1 = self._make_rendered() + md2 = self._make_rendered() self.assertEqual(md1, md2) def test_contains_issue_key(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) - ev = evidence.normalize_evidence(_make_evidence_items()) - plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} - ) - md = evidence.render_handoff(plan) - self.assertIn("TEST-100", md) + self.assertIn("TEST-100", self._make_rendered()) def test_contains_delivery_counts(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) - ev = evidence.normalize_evidence(_make_evidence_items()) - plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} - ) - md = evidence.render_handoff(plan) - self.assertIn("Delivered:", md) - self.assertIn("Missing evidence:", md) + md = self._make_rendered() + self.assertIn("Total inputs:", md) - def test_contains_source_proposal(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) - ev = evidence.normalize_evidence(_make_evidence_items()) - plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} - ) - md = evidence.render_handoff(plan) + def test_contains_metadata_proposals(self) -> None: + md = self._make_rendered() + self.assertIn("Metadata Proposals", md) + self.assertIn("target_release", md) + + def test_contains_source_proposal_wording(self) -> None: + """B: Source proposal shows proposed wording, not just reason.""" + md = self._make_rendered() self.assertIn("Source Change Proposal", md) self.assertIn("**Apply enabled:** False", md) + def test_contains_evidence_links(self) -> None: + md = self._make_rendered() + self.assertIn("PR Remote Links", md) + + def test_shows_per_input_evidence_status(self) -> None: + md = self._make_rendered() + self.assertIn("merged_pr_exists", md) + # --------------------------------------------------------------------------- # CLI integration # --------------------------------------------------------------------------- class TestCli(unittest.TestCase): - """Test CLI subcommands via subprocess.""" - - def _run( - self, - subcmd: str, - input_data: dict | list, - extra_args: list[str] | None = None, - ) -> subprocess.CompletedProcess: - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as f: - json.dump(input_data, f) - f.flush() - input_path = f.name - - try: - cmd = [ - sys.executable, "-I", - str(_SCRIPT_DIR / "evidence.py"), - subcmd, "--input", input_path, - ] - if extra_args: - cmd.extend(extra_args) - return subprocess.run( - cmd, capture_output=True, text=True, timeout=30, - ) - finally: - os.unlink(input_path) def test_validate_valid(self) -> None: - snapshot = _make_docs_snapshot() - proc = self._run("validate", snapshot) + proc = _cli_run("validate", _make_docs_snapshot()) self.assertEqual(proc.returncode, 0, proc.stderr) result = json.loads(proc.stdout) self.assertTrue(result["valid"]) - def test_validate_invalid(self) -> None: - snapshot = _make_docs_snapshot(key="bad") - proc = self._run("validate", snapshot) + def test_validate_invalid_exits_1(self) -> None: + proc = _cli_run("validate", _make_docs_snapshot(key="bad")) self.assertEqual(proc.returncode, 1) result = json.loads(proc.stdout) self.assertFalse(result["valid"]) + def test_normalize_valid(self) -> None: + proc = _cli_run("normalize", _make_evidence_items()) + self.assertEqual(proc.returncode, 0, proc.stderr) + result = json.loads(proc.stdout) + self.assertEqual(len(result["items"]), 2) + + def test_normalize_invalid_exits_1(self) -> None: + proc = _cli_run("normalize", [{"pr_url": "bad"}]) + self.assertEqual(proc.returncode, 1) + def test_build_plan(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) data = { - "validation": validation, + "validation": val, "evidence": ev, "target": {"release": "4.19", "target_branch": "main"}, } - proc = self._run("build-plan", data) + proc = _cli_run("build-plan", data) self.assertEqual(proc.returncode, 0, proc.stderr) plan = json.loads(proc.stdout) self.assertEqual(plan["status"], "complete") + def test_build_plan_invalid_exits_1(self) -> None: + val = evidence.validate_issue( + _make_docs_snapshot(key="bad") + ) + ev = evidence.normalize_evidence([]) + data = { + "validation": val, + "evidence": ev, + "target": {}, + } + proc = _cli_run("build-plan", data) + self.assertEqual(proc.returncode, 1) + def test_check_enrichment(self) -> None: - snapshot = _make_docs_snapshot() - validation = evidence.validate_issue(snapshot) + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "4.19", "target_branch": "main"}, + ) + data = { + "plan": plan, + "current_state": { + "key": "TEST-100", + "remote_links": [], + }, + } + proc = _cli_run("check-enrichment", data) + self.assertEqual(proc.returncode, 0, proc.stderr) + + def test_check_enrichment_stale_exits_1(self) -> None: + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "4.19", "target_branch": "main"}, + ) + data = { + "plan": plan, + "current_state": { + "key": "TEST-100", + "remote_links": [], + }, + "approved_baseline": "stale-fingerprint", + } + proc = _cli_run("check-enrichment", data) + self.assertEqual(proc.returncode, 1) + + def test_check_enrichment_with_readback(self) -> None: + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) plan = evidence.build_change_plan( - validation, ev, {"release": "4.19", "target_branch": "main"} + val, ev, + {"release": "4.19", "target_branch": "main"}, ) data = { "plan": plan, - "current_state": {"remote_links": []}, + "current_state": { + "key": "TEST-100", + "remote_links": [], + }, } - proc = self._run("check-enrichment", data) + readback = { + "key": "TEST-100", + "remote_links": [ + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, + { + "url": "https://github.example.invalid" + "/org/repo/pull/102", + }, + ], + "fields": {"target_release": "4.19"}, + } + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + json.dump(readback, f) + rb_path = f.name + try: + proc = _cli_run( + "check-enrichment", data, + extra_args=["--readback", rb_path], + ) + finally: + os.unlink(rb_path) self.assertEqual(proc.returncode, 0, proc.stderr) result = json.loads(proc.stdout) - self.assertTrue(len(result["eligible_links"]) > 0) + self.assertTrue(result["receipt"]["all_confirmed"]) def test_validate_does_not_mutate_file(self) -> None: - """CLI validate must not modify the input file.""" - snapshot = _make_docs_snapshot() + snap = _make_docs_snapshot() with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False ) as f: - json.dump(snapshot, f) + json.dump(snap, f) f.flush() - input_path = f.name - + path = f.name try: - original = Path(input_path).read_bytes() + original = Path(path).read_bytes() subprocess.run( [ sys.executable, "-I", str(_SCRIPT_DIR / "evidence.py"), - "validate", "--input", input_path, + "validate", "--input", path, ], capture_output=True, text=True, timeout=30, ) - after = Path(input_path).read_bytes() - self.assertEqual(original, after, "input file was modified") + after = Path(path).read_bytes() + self.assertEqual(original, after) finally: - os.unlink(input_path) + os.unlink(path) def test_bad_json_exits_1(self) -> None: with tempfile.NamedTemporaryFile( @@ -915,20 +1344,185 @@ def test_bad_json_exits_1(self) -> None: ) as f: f.write("{bad json") f.flush() - input_path = f.name - + path = f.name try: proc = subprocess.run( [ sys.executable, "-I", str(_SCRIPT_DIR / "evidence.py"), - "validate", "--input", input_path, + "validate", "--input", path, ], capture_output=True, text=True, timeout=30, ) self.assertEqual(proc.returncode, 1) finally: - os.unlink(input_path) + os.unlink(path) + + +# --------------------------------------------------------------------------- +# E: End-to-end offline fixture +# --------------------------------------------------------------------------- + +class TestEndToEndOffline(unittest.TestCase): + """Representative writer-facing end-to-end test with offline fixture.""" + + def test_full_pipeline_validate_normalize_plan_check_receipt( + self, + ) -> None: + """Complete pipeline: validate → normalize → plan → check → receipt.""" + # 1. Validate + snapshot = _make_docs_snapshot() + val = evidence.validate_issue(snapshot) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertIn("TEST-202", val["contributors"]) + self.assertNotIn("TEST-300", val["contributors"]) + + # 2. Normalize + raw_evidence = _make_evidence_items() + ev = evidence.normalize_evidence(raw_evidence) + self.assertEqual(len(ev["items"]), 2) + self.assertEqual(ev["errors"], []) + + # 3. Build plan + target = {"release": "4.19", "target_branch": "main"} + plan = evidence.build_change_plan(val, ev, target) + self.assertEqual(plan["status"], "complete") + self.assertEqual(plan["issue_key"], "TEST-100") + + # Verify honest per-input statuses + for inp in plan["inputs"]: + self.assertNotEqual(inp["status"], "delivered") + merged_ok = [ + i for i in plan["inputs"] + if i["status"] == "merged_pr_exists" + ] + merged_wrong = [ + i for i in plan["inputs"] + if i["status"] == "merged_wrong_target" + ] + self.assertTrue(len(merged_ok) > 0) + self.assertTrue(len(merged_wrong) > 0) + + # Only contributor PRs in links + self.assertEqual(len(plan["pr_links"]), 2) + + # Metadata has target_release with catalog_verified=False + self.assertIn("target_release", plan["metadata"]) + self.assertFalse( + plan["metadata"]["target_release"]["catalog_verified"] + ) + + # Source proposal disabled + self.assertFalse(plan["source_proposal"]["apply_enabled"]) + + # 4. Check enrichment (pre-apply) + current = {"key": "TEST-100", "remote_links": []} + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["status"], "ready") + self.assertFalse(enrichment["stale"]) + self.assertEqual(len(enrichment["eligible_links"]), 2) + baseline = enrichment["baseline_fingerprint"] + + # 5. Verify readback (post-apply) + readback = { + "key": "TEST-100", + "remote_links": [ + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, + { + "url": "https://github.example.invalid" + "/org/repo/pull/102", + }, + ], + "fields": {"target_release": "4.19"}, + } + receipt = evidence.verify_enrichment( + enrichment, readback, "TEST-100" + ) + self.assertEqual(receipt["status"], "verified") + self.assertTrue(receipt["all_confirmed"]) + self.assertEqual(len(receipt["link_receipts"]), 2) + + # 6. Re-check with new baseline (idempotent) + enrichment2 = evidence.check_enrichment( + plan, readback, baseline + ) + # All links are now existing, none eligible + self.assertEqual(len(enrichment2["eligible_links"]), 0) + self.assertEqual(len(enrichment2["existing_links"]), 2) + + # 7. Render handoff + md = evidence.render_handoff(plan) + self.assertIn("TEST-100", md) + self.assertIn("Delivery Summary", md) + self.assertIn("Metadata Proposals", md) + self.assertIn("Source Change Proposal", md) + + def test_pipeline_via_cli(self) -> None: + """Pipeline exercised through CLI subcommands.""" + # validate + snap = _make_docs_snapshot() + p1 = _cli_run("validate", snap) + self.assertEqual(p1.returncode, 0) + val = json.loads(p1.stdout) + + # normalize + p2 = _cli_run("normalize", _make_evidence_items()) + self.assertEqual(p2.returncode, 0) + ev = json.loads(p2.stdout) + + # build-plan + p3 = _cli_run("build-plan", { + "validation": val, + "evidence": ev, + "target": {"release": "4.19", "target_branch": "main"}, + }) + self.assertEqual(p3.returncode, 0) + plan = json.loads(p3.stdout) + self.assertEqual(plan["status"], "complete") + + # check-enrichment + p4 = _cli_run("check-enrichment", { + "plan": plan, + "current_state": { + "key": "TEST-100", + "remote_links": [], + }, + }) + self.assertEqual(p4.returncode, 0) + enrich = json.loads(p4.stdout) + self.assertEqual(enrich["status"], "ready") + + def test_utf8_content(self) -> None: + """UTF-8 content in description and observations.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Docs include Ünîcödé examples + ## Documentation Scope + Covers 日本語 configuration + ## Documentation Inputs + **TEST-201 — Ünîcödé config:** + - New `widget config` with —name flag + ## Design Reference + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + + items = [ + { + "pr_url": "https://github.example.invalid/org/repo/pull/1", + "contributor_key": "TEST-201", + "merged": True, + "observations": ["Adds 日本語 support"], + }, + ] + ev = evidence.normalize_evidence(items) + self.assertEqual(len(ev["errors"]), 0) if __name__ == "__main__": From 6e3e5e4c773d987da5f69f2f2ee00c42984e1754 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 22:15:12 +0000 Subject: [PATCH 03/17] fix(docs-handoff): real metadata catalog, digest verification, no prose inference Contract 1 (snapshot/baseline/identity): - check_enrichment baseline now includes metadata (fixVersions) state, not just remote_links; metadata drift between preview and apply is detected as stale - plan_digest is recomputed and verified against stored value; any post-creation tampering (injected links, altered metadata) returns invalid_plan - approved_plan_digest parameter detects plan changes since approval (e.g. from changed description, contributors, or metadata) - Missing current_state or plan issue key returns invalid, not pass - Missing readback key in verify_enrichment returns invalid Contract 2 (metadata catalog/field binding): - Add _METADATA_CATALOG_V1 mapping target_release to actual Jira fixVersions field with array type, version item schema (id, name) - Metadata entries carry jira_field, jira_field_type, catalog_entry, and before value instead of placeholder field_binding string - check_enrichment requires catalog_verified=True AND catalog_entry for metadata eligibility; unverified metadata is not eligible - verify_enrichment reads actual Jira field (fixVersions) from readback, matches by version ID from catalog item schema - Direct fixversions access blocked by _FORBIDDEN_FIELDS; only target_release alias through catalog is authorized Contract 3 (evidence error propagation): - build_change_plan rejects evidence with errors (invalid_evidence status, no eligible operations) - Issue key format validated with fullmatch, not just presence Contract 4 (no prose-keyword semantics): - _build_source_proposal no longer classifies observations as confirming/divergent based on keyword matching; all observations are display-only context for human review (action: review) - Contributor coverage computed independently of parsed inputs loop; declared contributor with evidence but no input header is not falsely flagged as missing - Documentation input header not in Contributing stories produces input_contributor_mismatch finding, not automatic eligibility 26 new contract tests covering: missing identity, plan tampering, approved digest drift, metadata catalog lifecycle, real Jira field readback, forbidden alias bypass, evidence error blocking, confirmed- text-not-excluded, arbitrary-text-no-update, input-specific mapping, input-contributor-mismatch, contributor coverage independence, full CLI pipeline with catalog metadata. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 373 ++++++-- skills/docs-handoff/scripts/test_evidence.py | 879 ++++++++++++++++++- 2 files changed, 1134 insertions(+), 118 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 7d9f7543..4957aacf 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -636,6 +636,25 @@ def _fingerprint(data: Any) -> str: "reporter", "components", "fixversions", }) +# Metadata catalog V1 — maps skill-level names to actual Jira fields. +# ``target_release`` is the only authorized access path to the Jira +# ``fixVersions`` field. Direct use of the raw Jira field name is +# blocked by ``_FORBIDDEN_FIELDS`` to prevent catalog bypass. +_METADATA_CATALOG_V1: dict[str, dict[str, Any]] = { + "target_release": { + "jira_field": "fixVersions", + "jira_field_type": "array", + "jira_item_schema": { + "type": "version", + "id_field": "id", + "name_field": "name", + }, + "description": ( + "Fix Version value from project version catalog" + ), + }, +} + def build_change_plan( validation: dict[str, Any], @@ -696,10 +715,39 @@ def build_change_plan( "plan_digest": "", } - # Validate evidence shape + if not _JIRA_KEY_RE.fullmatch(issue_key): + return { + "status": "invalid_issue", + "errors": [f"invalid Jira key format: {issue_key}"], + "issue_key": issue_key, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + + # Validate evidence shape — errors block eligible operations ev_errors = evidence.get("errors", []) if not isinstance(ev_errors, list): ev_errors = [] + if ev_errors: + return { + "status": "invalid_evidence", + "errors": ev_errors, + "issue_key": issue_key, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } ev_items = evidence.get("items", []) if not isinstance(ev_items, list): return { @@ -740,10 +788,17 @@ def build_change_plan( for ck in ev.get("contributor_keys", []): contributor_evidence.setdefault(ck, []).append(ev) + # --- Contributor coverage (independent of parsed inputs) --- + all_evidence_contributors: set[str] = set() + contributor_pr_links: set[str] = set() + for ev in ev_items: + for ck in ev.get("contributor_keys", []): + all_evidence_contributors.add(ck) + if ck in contributors: + contributor_pr_links.add(ev["pr_url"]) + # --- Per-input delivery status (B: honest per-input assessment) --- input_statuses: list[dict[str, Any]] = [] - contributors_with_evidence: set[str] = set() - contributor_pr_links: set[str] = set() # PRs from actual contributors for doc_input in doc_inputs: contrib = doc_input.get("contributor") @@ -755,9 +810,7 @@ def build_change_plan( input_conflicts: list[str] = [] if evs: - contributors_with_evidence.add(contrib) for ev in evs: - contributor_pr_links.add(ev["pr_url"]) ev_merged = ev.get("merged") ev_target = ev.get("target_branch", "") ev_conflicts = ev.get("conflicts", []) @@ -814,7 +867,7 @@ def build_change_plan( # --- Contributors without evidence (B: always flag, even if # also a dependency) --- for contrib in sorted(contributors): - if contrib not in contributors_with_evidence: + if contrib not in all_evidence_contributors: if contrib.startswith("story-"): findings.append({ "type": "unresolved_contributor", @@ -834,14 +887,21 @@ def build_change_plan( ), }) - # --- Non-contributor evidence: candidate proposals, NOT auto-eligible - # PR links (B) --- - all_contributor_keys: set[str] = set() + # --- Input/contributor alignment --- + all_contributor_keys: set[str] = set(contributors) for di in doc_inputs: c = di.get("contributor") if c: all_contributor_keys.add(c) - all_contributor_keys |= contributors + if c not in contributors: + findings.append({ + "type": "input_contributor_mismatch", + "ref": c, + "detail": ( + f"Documentation input header references {c} " + f"which is not listed in Contributing stories" + ), + }) for ev in ev_items: for ck in ev.get("contributor_keys", []): @@ -879,16 +939,19 @@ def build_change_plan( ], }) - # --- Metadata (D: allowed catalog only) --- + # --- Metadata (D: allowed catalog only, real Jira binding) --- metadata: dict[str, Any] = {} if target_release: + cat = _METADATA_CATALOG_V1["target_release"] metadata["target_release"] = { "value": target_release, - "field_binding": "fixVersions or customfield (project-specific)", + "jira_field": cat["jira_field"], + "jira_field_type": cat["jira_field_type"], "catalog_verified": False, + "catalog_entry": None, + "before": None, "note": ( - "Value must be verified against the project's actual " - "Fix Version catalog before applying" + "Requires catalog entry with version ID before apply" ), } @@ -908,10 +971,16 @@ def build_change_plan( ), }) elif field_name in _ALLOWED_METADATA: + cat = _METADATA_CATALOG_V1.get(field_name, {}) metadata[field_name] = { "value": af.get("value"), - "field_binding": af.get("field_binding", ""), - "catalog_verified": af.get("catalog_verified", False), + "jira_field": cat.get("jira_field", ""), + "jira_field_type": cat.get("jira_field_type", ""), + "catalog_verified": af.get( + "catalog_verified", False + ), + "catalog_entry": af.get("catalog_entry"), + "before": af.get("before"), } # --- Readiness (B: independent of delivery, visible gaps) --- @@ -995,13 +1064,14 @@ def _build_source_proposal( """Build an owning-source narrative-change proposal. Source/narrative apply is disabled. The proposal carries: - - Specific proposed wording changes or clearly unresolved questions + - Observations gathered per input with evidence refs - Input/source location references - Supporting PR/code evidence URLs - Intended owning-workflow action - Only observations that indicate actual divergence generate update - proposals. 'As planned' observations do NOT generate proposals. + Observations are display-only context for human review — no + automated semantic inference from free-form text. Each + observation is tied to its specific input and evidence source. """ proposed_changes: list[dict[str, Any]] = [] @@ -1013,38 +1083,26 @@ def _build_source_proposal( if not ev_refs: continue - # Collect observations that suggest divergence - divergent_observations: list[str] = [] - confirming_observations: list[str] = [] + # Collect observations — display-only, no semantic inference + observations: list[str] = [] evidence_urls: list[str] = [] for eref in ev_refs: evidence_urls.append(eref["pr_url"]) - for obs in eref.get("observations", []): - obs_lower = obs.lower() - if any( - phrase in obs_lower - for phrase in ( - "as planned", "as expected", "matches plan", - "no change", "confirmed", - ) - ): - confirming_observations.append(obs) - else: - divergent_observations.append(obs) + observations.extend(eref.get("observations", [])) - # Only divergent observations produce update proposals (B) - if divergent_observations: + # All observations are presented for human review — the skill + # does not classify them as confirming or divergent + if observations: proposed_changes.append({ - "action": "update", + "action": "review", "contributor": contrib, "input_text": text, - "proposed_wording": ( - f"Update based on implementation evidence: " - + "; ".join(divergent_observations) + "proposed_wording": None, + "reason": ( + "Implementation observations require human review" ), - "reason": "Implementation diverges from planned input", - "observations": divergent_observations, + "observations": observations, "evidence_urls": evidence_urls, "source_location": "Documentation Inputs", }) @@ -1091,6 +1149,7 @@ def check_enrichment( plan: dict[str, Any], current_state: dict[str, Any], approved_baseline: str = "", + approved_plan_digest: str = "", ) -> dict[str, Any]: """Check proposed enrichment against current issue state. @@ -1101,44 +1160,87 @@ def check_enrichment( approved_baseline: fingerprint from a previously approved preview — if provided and differs from the current baseline, the plan is stale + approved_plan_digest: digest from a previously approved plan + — if provided and differs from the current plan's digest, + the plan has changed since approval Returns an enrichment result with eligibility, staleness, and detailed operation plan. """ + _empty_result: dict[str, Any] = { + "eligible_links": [], + "existing_links": [], + "skipped_links": [], + "eligible_metadata": {}, + "forbidden_fields": [], + "plan_digest": plan.get("plan_digest", ""), + "baseline_fingerprint": "", + "stale": True, + } + # Reject invalid/blocked plans entirely (D) plan_status = plan.get("status", "") if plan_status != "complete": return { + **_empty_result, "status": "invalid_plan", - "reason": f"plan status is '{plan_status}', not 'complete'", - "eligible_links": [], - "existing_links": [], - "skipped_links": [], - "eligible_metadata": {}, - "forbidden_fields": [], - "plan_digest": plan.get("plan_digest", ""), - "baseline_fingerprint": "", - "stale": True, + "reason": ( + f"plan status is '{plan_status}', not 'complete'" + ), } - # Verify issue identity matches (A, D) + # Require both identity keys (missing = invalid, not pass) plan_key = plan.get("issue_key", "") current_key = current_state.get("key", "") - if current_key and plan_key and current_key != plan_key: + + if not plan_key: + return { + **_empty_result, + "status": "invalid_plan", + "reason": "plan missing issue key", + } + + if not current_key: + return { + **_empty_result, + "status": "invalid", + "reason": "current state missing issue key", + } + + if current_key != plan_key: return { + **_empty_result, "status": "wrong_issue", "reason": ( f"plan targets {plan_key} but current state is " f"for {current_key}" ), - "eligible_links": [], - "existing_links": [], - "skipped_links": [], - "eligible_metadata": {}, - "forbidden_fields": [], - "plan_digest": plan.get("plan_digest", ""), - "baseline_fingerprint": "", - "stale": True, + } + + # Verify plan digest integrity (detect tampering) + recomputed_digest = _fingerprint({ + "issue_key": plan.get("issue_key"), + "target": plan.get("target"), + "inputs": plan.get("inputs"), + "pr_links": plan.get("pr_links"), + "metadata": plan.get("metadata"), + }) + plan_digest = plan.get("plan_digest", "") + if plan_digest and recomputed_digest != plan_digest: + return { + **_empty_result, + "status": "invalid_plan", + "reason": ( + "plan digest mismatch — plan may have been altered" + ), + } + + # Check approved plan digest (plan changed since approval) + if approved_plan_digest and plan_digest != approved_plan_digest: + return { + **_empty_result, + "status": "stale", + "reason": "plan has changed since approval", } existing_links = current_state.get("remote_links", []) @@ -1158,8 +1260,30 @@ def check_enrichment( canonical_links.append({"url": norm}) canonical_links.sort(key=lambda x: x["url"]) - # Baseline fingerprint from current state - baseline_fp = _fingerprint({"remote_links": canonical_links}) + # Current metadata state for baseline + current_fields = current_state.get("fields", {}) + current_metadata: dict[str, Any] = {} + if isinstance(current_fields, dict): + for fname, cat in _METADATA_CATALOG_V1.items(): + jf = cat["jira_field"] + if jf in current_fields: + # Normalize for stable fingerprint (sort by id) + val = current_fields[jf] + if isinstance(val, list): + val = sorted( + val, + key=lambda x: ( + x.get("id", "") if isinstance(x, dict) + else str(x) + ), + ) + current_metadata[fname] = val + + # Baseline fingerprint from current state (links + metadata) + baseline_fp = _fingerprint({ + "remote_links": canonical_links, + "metadata": current_metadata, + }) # Staleness detection (D) stale = False @@ -1183,7 +1307,7 @@ def check_enrichment( else: eligible.append({**proposed, "url": norm_url}) - # Metadata eligibility (D: allowed catalog only) + # Metadata eligibility (D: catalog_verified + catalog_entry required) eligible_metadata: dict[str, Any] = {} forbidden_found: list[str] = [] @@ -1194,10 +1318,14 @@ def check_enrichment( if field not in _ALLOWED_METADATA: forbidden_found.append(f"uncatalogued:{field}") continue - if isinstance(spec, dict): - eligible_metadata[field] = spec - else: - eligible_metadata[field] = {"value": spec} + if not isinstance(spec, dict): + continue + # catalog_verified must be True with a catalog_entry + if not spec.get("catalog_verified", False): + continue + if not spec.get("catalog_entry"): + continue + eligible_metadata[field] = spec # Profile forbidden fields profile_fields = current_state.get("profile_fields", {}) @@ -1235,9 +1363,18 @@ def verify_enrichment( Returns a receipt with per-operation status. """ - # Identity check (D) + # Identity check (D) — missing key is invalid, not a pass readback_key = readback.get("key", "") - if expected_key and readback_key and readback_key != expected_key: + if not readback_key: + return { + "status": "invalid", + "reason": "readback missing issue key", + "link_receipts": [], + "metadata_receipts": [], + "baseline_fingerprint": "", + "all_confirmed": False, + } + if expected_key and readback_key != expected_key: return { "status": "wrong_issue", "reason": ( @@ -1307,35 +1444,54 @@ def verify_enrichment( "status": "confirmed" if found else "disappeared", }) - # Metadata receipts + # Metadata receipts — use real Jira field binding from catalog metadata_receipts: list[dict[str, Any]] = [] for field, spec in enrichment.get("eligible_metadata", {}).items(): + cat = _METADATA_CATALOG_V1.get(field, {}) + jira_field = cat.get("jira_field", field) readback_fields = readback.get("fields", {}) if not isinstance(readback_fields, dict): readback_fields = {} - current_val = readback_fields.get(field) - expected_val = spec.get("value") if isinstance(spec, dict) else spec - if current_val == expected_val: - metadata_receipts.append({ - "field": field, - "status": "confirmed", - "value": current_val, - }) - elif current_val is not None: - metadata_receipts.append({ - "field": field, - "status": "different_value", - "expected": expected_val, - "actual": current_val, - }) + current_val = readback_fields.get(jira_field) + expected_entry = ( + spec.get("catalog_entry") + if isinstance(spec, dict) else None + ) + + # For array fields (fixVersions), check entry presence by ID + if ( + cat.get("jira_field_type") == "array" + and isinstance(current_val, list) + and isinstance(expected_entry, dict) + ): + schema = cat.get("jira_item_schema", {}) + id_field = schema.get("id_field", "id") + expected_id = expected_entry.get(id_field) + found = any( + isinstance(item, dict) + and item.get(id_field) == expected_id + for item in current_val + ) if expected_id else False + status = "confirmed" if found else "uncertain" + elif current_val is not None and expected_entry is not None: + status = ( + "confirmed" if current_val == expected_entry + else "different_value" + ) + elif expected_entry is not None: + status = "uncertain" else: - metadata_receipts.append({ - "field": field, - "status": "uncertain", - "expected": expected_val, - }) + status = "uncertain" + + metadata_receipts.append({ + "field": field, + "jira_field": jira_field, + "status": status, + "expected": expected_entry, + "actual": current_val, + }) - # New baseline + # New baseline — includes metadata state (matches check_enrichment) canonical: list[dict[str, str]] = [] for link in readback_links: url = link.get("url", "") @@ -1344,7 +1500,28 @@ def verify_enrichment( except ValueError: canonical.append({"url": url}) canonical.sort(key=lambda x: x["url"]) - new_baseline = _fingerprint({"remote_links": canonical}) + + rb_fields = readback.get("fields", {}) + rb_metadata: dict[str, Any] = {} + if isinstance(rb_fields, dict): + for fname, cat in _METADATA_CATALOG_V1.items(): + jf = cat["jira_field"] + if jf in rb_fields: + val = rb_fields[jf] + if isinstance(val, list): + val = sorted( + val, + key=lambda x: ( + x.get("id", "") + if isinstance(x, dict) else str(x) + ), + ) + rb_metadata[fname] = val + + new_baseline = _fingerprint({ + "remote_links": canonical, + "metadata": rb_metadata, + }) # All confirmed: must have at least one operation, and all must # be confirmed (empty eligible = no-op, not success) @@ -1616,11 +1793,15 @@ def cmd_check_enrichment(args: argparse.Namespace) -> int: if not isinstance(current, dict): fail("input.current_state must be an object") + approved_plan_digest = data.get("approved_plan_digest", "") + if args.readback: readback = _load_json(Path(args.readback)) if not isinstance(readback, dict): fail("readback must be a JSON object") - enrichment = check_enrichment(plan, current, approved_baseline) + enrichment = check_enrichment( + plan, current, approved_baseline, approved_plan_digest, + ) expected_key = plan.get("issue_key", "") receipt = verify_enrichment(enrichment, readback, expected_key) _write_json({"enrichment": enrichment, "receipt": receipt}) @@ -1628,7 +1809,9 @@ def cmd_check_enrichment(args: argparse.Namespace) -> int: return EXIT_ERROR else: original = copy.deepcopy(data) - result = check_enrichment(plan, current, approved_baseline) + result = check_enrichment( + plan, current, approved_baseline, approved_plan_digest, + ) if data != original: fail("BUG: check_enrichment mutated its input") _write_json(result) diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 8b09c44e..feed3867 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -710,8 +710,8 @@ def test_as_planned_no_source_proposal(self) -> None: ] self.assertEqual(len(updates), 0) - def test_divergent_observation_produces_proposal(self) -> None: - """B: Divergent observation → update proposal with wording.""" + def test_observations_produce_review_proposal(self) -> None: + """B: Observations → review proposal with evidence URLs.""" items = [ { "pr_url": "https://github.example.invalid/org/repo/pull/1", @@ -723,13 +723,16 @@ def test_divergent_observation_produces_proposal(self) -> None: }, ] plan = self._make_plan(evidence_items=items) - updates = [ + reviews = [ c for c in plan["source_proposal"]["proposed_changes"] - if c["action"] == "update" + if c["action"] == "review" ] - self.assertTrue(len(updates) > 0) - self.assertTrue(updates[0]["proposed_wording"]) - self.assertTrue(updates[0]["evidence_urls"]) + self.assertTrue(len(reviews) > 0) + self.assertTrue(reviews[0]["evidence_urls"]) + self.assertIn( + "Added --format flag not in plan", + reviews[0]["observations"], + ) def test_non_contributor_proposal_has_evidence(self) -> None: """B: non-contributor proposal carries evidence URL.""" @@ -962,7 +965,8 @@ def test_no_baseline_not_stale(self) -> None: r = self._make_enrichment() self.assertFalse(r["stale"]) - def test_forbidden_metadata_rejected(self) -> None: + def test_forbidden_metadata_tamper_rejected(self) -> None: + """D: Injecting forbidden metadata into plan → invalid (tampered).""" snap = _make_docs_snapshot() val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) @@ -973,12 +977,11 @@ def test_forbidden_metadata_rejected(self) -> None: plan["metadata"]["status"] = "Done" current = {"key": "TEST-100", "remote_links": []} r = evidence.check_enrichment(plan, current) - self.assertTrue( - any("status" in f for f in r["forbidden_fields"]) - ) + self.assertEqual(r["status"], "invalid_plan") + self.assertIn("altered", r.get("reason", "")) - def test_uncatalogued_metadata_rejected(self) -> None: - """D: Arbitrary key not in allowed catalog → forbidden.""" + def test_uncatalogued_metadata_tamper_rejected(self) -> None: + """D: Injecting arbitrary key into plan metadata → invalid.""" snap = _make_docs_snapshot() val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) @@ -989,9 +992,7 @@ def test_uncatalogued_metadata_rejected(self) -> None: plan["metadata"]["random_field"] = {"value": "x"} current = {"key": "TEST-100", "remote_links": []} r = evidence.check_enrichment(plan, current) - self.assertTrue( - any("random_field" in f for f in r["forbidden_fields"]) - ) + self.assertEqual(r["status"], "invalid_plan") def test_canonical_ordering_avoids_false_drift(self) -> None: """D: Link order alone shouldn't cause false drift.""" @@ -1087,11 +1088,19 @@ def test_invalid_enrichment_rejected(self) -> None: self.assertFalse(r["all_confirmed"]) def test_metadata_receipt_confirmed(self) -> None: + """Readback uses real Jira field (fixVersions) with catalog entry.""" r = self._make_receipt( eligible_metadata={ - "target_release": {"value": "4.19"}, + "target_release": { + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "jira_field": "fixVersions", + }, + }, + readback_fields={ + "fixVersions": [{"id": "7", "name": "v1"}], }, - readback_fields={"target_release": "4.19"}, readback_urls=[ "https://github.example.invalid/org/repo/pull/101" ], @@ -1099,19 +1108,28 @@ def test_metadata_receipt_confirmed(self) -> None: meta = r["metadata_receipts"] self.assertEqual(len(meta), 1) self.assertEqual(meta[0]["status"], "confirmed") + self.assertEqual(meta[0]["jira_field"], "fixVersions") def test_metadata_receipt_different(self) -> None: + """Readback shows different version ID → different_value.""" r = self._make_receipt( eligible_metadata={ - "target_release": {"value": "4.19"}, + "target_release": { + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "jira_field": "fixVersions", + }, + }, + readback_fields={ + "fixVersions": [{"id": "99", "name": "v2"}], }, - readback_fields={"target_release": "4.20"}, readback_urls=[ "https://github.example.invalid/org/repo/pull/101" ], ) meta = r["metadata_receipts"] - self.assertEqual(meta[0]["status"], "different_value") + self.assertEqual(meta[0]["status"], "uncertain") def test_existing_link_disappearance_detected(self) -> None: r = self._make_receipt( @@ -1446,11 +1464,13 @@ def test_full_pipeline_validate_normalize_plan_check_receipt( self.assertTrue(receipt["all_confirmed"]) self.assertEqual(len(receipt["link_receipts"]), 2) - # 6. Re-check with new baseline (idempotent) + # 6. Re-check shows links as existing (idempotent) + new_baseline = receipt["baseline_fingerprint"] enrichment2 = evidence.check_enrichment( - plan, readback, baseline + plan, readback, new_baseline ) # All links are now existing, none eligible + self.assertFalse(enrichment2["stale"]) self.assertEqual(len(enrichment2["eligible_links"]), 0) self.assertEqual(len(enrichment2["existing_links"]), 2) @@ -1525,5 +1545,818 @@ def test_utf8_content(self) -> None: self.assertEqual(len(ev["errors"]), 0) +# --------------------------------------------------------------------------- +# Contract 1: Identity, snapshot, baseline, digest +# --------------------------------------------------------------------------- + +class TestSnapshotBaselineDigest(unittest.TestCase): + """Contract 1: plan_digest verification, baseline includes metadata, + missing identity fails, drift detection.""" + + def _make_plan_and_enrichment( + self, *, profile=None, current_fields=None, + ): + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + current = { + "key": "TEST-100", + "remote_links": [], + } + if current_fields is not None: + current["fields"] = current_fields + enrichment = evidence.check_enrichment(plan, current) + return plan, enrichment, current + + def test_missing_current_key_invalid(self) -> None: + """Missing current_state key → invalid, not pass.""" + plan, _, _ = self._make_plan_and_enrichment() + r = evidence.check_enrichment(plan, {"remote_links": []}) + self.assertEqual(r["status"], "invalid") + self.assertIn("missing", r["reason"]) + + def test_missing_plan_key_invalid(self) -> None: + """Missing plan issue_key → invalid_plan.""" + plan, _, _ = self._make_plan_and_enrichment() + plan_no_key = {**plan, "issue_key": ""} + # Must clear digest since we changed the plan + plan_no_key["plan_digest"] = "" + r = evidence.check_enrichment( + plan_no_key, {"key": "TEST-100", "remote_links": []} + ) + self.assertEqual(r["status"], "invalid_plan") + + def test_plan_digest_tamper_detected(self) -> None: + """Altering plan content after creation → digest mismatch.""" + plan, _, _ = self._make_plan_and_enrichment() + tampered = copy.deepcopy(plan) + tampered["pr_links"].append({ + "url": "https://github.example.invalid/x/y/pull/999", + "title": "injected", + }) + r = evidence.check_enrichment( + tampered, + {"key": "TEST-100", "remote_links": []}, + ) + self.assertEqual(r["status"], "invalid_plan") + self.assertIn("altered", r["reason"]) + + def test_approved_plan_digest_detects_changed_plan(self) -> None: + """approved_plan_digest catches a different plan (e.g. from + changed description/contributor).""" + plan1, enrichment1, current = self._make_plan_and_enrichment() + digest1 = enrichment1["plan_digest"] + baseline1 = enrichment1["baseline_fingerprint"] + + # Rebuild plan with changed contributor (affects plan digest) + desc2 = _make_docs_snapshot() + desc2["description"] = desc2["description"].replace( + "TEST-202", "TEST-203" + ) + val2 = evidence.validate_issue(desc2) + ev2 = evidence.normalize_evidence(_make_evidence_items()) + plan2 = evidence.build_change_plan( + val2, ev2, + {"release": "v1", "target_branch": "main"}, + ) + + # plan2 has different digest than plan1 + self.assertNotEqual(plan2["plan_digest"], plan1["plan_digest"]) + + # Use plan2 but approved_plan_digest from plan1 → stale + r = evidence.check_enrichment( + plan2, current, baseline1, digest1, + ) + self.assertTrue(r["stale"]) + self.assertEqual(r["status"], "stale") + + def test_metadata_in_baseline(self) -> None: + """Metadata field change in current state → baseline drift.""" + plan, _, _ = self._make_plan_and_enrichment( + current_fields={"fixVersions": []}, + ) + current_v0 = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + r1 = evidence.check_enrichment(plan, current_v0) + baseline = r1["baseline_fingerprint"] + + # Now fixVersions changed in Jira + current_v1 = { + "key": "TEST-100", + "remote_links": [], + "fields": { + "fixVersions": [{"id": "99", "name": "other"}], + }, + } + r2 = evidence.check_enrichment(plan, current_v1, baseline) + self.assertTrue(r2["stale"]) + + def test_missing_readback_key_invalid(self) -> None: + """verify_enrichment: missing readback key → invalid.""" + plan, enrichment, _ = self._make_plan_and_enrichment() + readback = {"remote_links": [], "fields": {}} + receipt = evidence.verify_enrichment( + enrichment, readback, "TEST-100" + ) + self.assertEqual(receipt["status"], "invalid") + self.assertFalse(receipt["all_confirmed"]) + + def test_reordered_links_no_false_drift(self) -> None: + """Link reordering alone does not cause false drift.""" + plan, _, _ = self._make_plan_and_enrichment() + current_a = { + "key": "TEST-100", + "remote_links": [ + {"url": "https://github.example.invalid/o/r/pull/1"}, + {"url": "https://github.example.invalid/o/r/pull/2"}, + ], + "fields": {"fixVersions": []}, + } + current_b = { + "key": "TEST-100", + "remote_links": [ + {"url": "https://github.example.invalid/o/r/pull/2"}, + {"url": "https://github.example.invalid/o/r/pull/1"}, + ], + "fields": {"fixVersions": []}, + } + r_a = evidence.check_enrichment(plan, current_a) + r_b = evidence.check_enrichment(plan, current_b) + self.assertEqual( + r_a["baseline_fingerprint"], + r_b["baseline_fingerprint"], + ) + + +# --------------------------------------------------------------------------- +# Contract 2: Metadata catalog, real field binding, before/after +# --------------------------------------------------------------------------- + +class TestMetadataCatalogV1(unittest.TestCase): + """Contract 2: real fixVersions binding, catalog_verified gates + eligibility, readback with Jira field shapes.""" + + def _make_plan(self, *, profile=None, target=None): + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + if target is None: + target = {"release": "v1", "target_branch": "main"} + return evidence.build_change_plan(val, ev, target, profile) + + def test_default_metadata_has_real_binding(self) -> None: + """target_release has jira_field=fixVersions, not placeholder.""" + plan = self._make_plan() + tr = plan["metadata"]["target_release"] + self.assertEqual(tr["jira_field"], "fixVersions") + self.assertEqual(tr["jira_field_type"], "array") + self.assertFalse(tr["catalog_verified"]) + self.assertIsNone(tr["catalog_entry"]) + + def test_uncatalogued_not_eligible(self) -> None: + """catalog_verified=False → not eligible for write.""" + plan = self._make_plan() + current = {"key": "TEST-100", "remote_links": []} + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_catalog_verified_eligible(self) -> None: + """catalog_verified=True + catalog_entry → eligible.""" + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + plan = self._make_plan(profile=profile) + tr = plan["metadata"]["target_release"] + self.assertTrue(tr["catalog_verified"]) + self.assertEqual(tr["catalog_entry"], {"id": "7", "name": "v1"}) + self.assertEqual(tr["before"], []) + + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + r = evidence.check_enrichment(plan, current) + self.assertIn("target_release", r["eligible_metadata"]) + + def test_full_metadata_lifecycle(self) -> None: + """Full lifecycle: preview → receipt with real Jira shapes.""" + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + plan = self._make_plan(profile=profile) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["status"], "ready") + self.assertIn("target_release", enrichment["eligible_metadata"]) + + # Readback with real Jira field shape + readback = { + "key": "TEST-100", + "remote_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + {"url": "https://github.example.invalid/org/repo/pull/102"}, + ], + "fields": { + "fixVersions": [{"id": "7", "name": "v1"}], + }, + } + receipt = evidence.verify_enrichment( + enrichment, readback, "TEST-100" + ) + self.assertEqual(receipt["status"], "verified") + self.assertTrue(receipt["all_confirmed"]) + meta_r = receipt["metadata_receipts"] + self.assertEqual(len(meta_r), 1) + self.assertEqual(meta_r[0]["jira_field"], "fixVersions") + self.assertEqual(meta_r[0]["status"], "confirmed") + + def test_readback_missing_field_uncertain(self) -> None: + """Readback without fixVersions → uncertain, not confirmed.""" + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + plan = self._make_plan(profile=profile) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + enrichment = evidence.check_enrichment(plan, current) + readback = { + "key": "TEST-100", + "remote_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + {"url": "https://github.example.invalid/org/repo/pull/102"}, + ], + "fields": {}, + } + receipt = evidence.verify_enrichment( + enrichment, readback, "TEST-100" + ) + self.assertFalse(receipt["all_confirmed"]) + + def test_forbidden_field_alias_blocked(self) -> None: + """Profile attempting 'fixversions' directly → forbidden.""" + profile = { + "allowed_fields": [{ + "name": "fixversions", + "value": "v1", + "catalog_verified": True, + }], + } + plan = self._make_plan(profile=profile) + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + ] + self.assertTrue( + any(f["ref"] == "fixversions" for f in forbidden) + ) + + def test_drift_after_approval(self) -> None: + """After approval, description change → stale.""" + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + plan1 = self._make_plan(profile=profile) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + e1 = evidence.check_enrichment(plan1, current) + baseline = e1["baseline_fingerprint"] + digest = e1["plan_digest"] + + # Rebuild plan with changed contributor (affects plan digest) + desc2 = _make_docs_snapshot() + desc2["description"] = desc2["description"].replace( + "TEST-202", "TEST-203" + ) + val2 = evidence.validate_issue(desc2) + ev2 = evidence.normalize_evidence(_make_evidence_items()) + plan2 = evidence.build_change_plan( + val2, ev2, + {"release": "v1", "target_branch": "main"}, + profile, + ) + e2 = evidence.check_enrichment(plan2, current, baseline, digest) + self.assertTrue(e2["stale"]) + + +# --------------------------------------------------------------------------- +# Contract 3: Evidence error propagation, key format validation +# --------------------------------------------------------------------------- + +class TestEvidenceErrorPropagation(unittest.TestCase): + """Contract 3: evidence errors block operations, invalid key + format rejected, structurally invalid inputs rejected.""" + + def test_evidence_errors_block_plan(self) -> None: + """Evidence with errors → invalid_evidence, no eligible ops.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + evidence_with_errors = { + "items": [], + "errors": ["evidence[0]: invalid PR URL"], + "warnings": [], + } + plan = evidence.build_change_plan( + val, evidence_with_errors, + {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "invalid_evidence") + self.assertEqual(plan["pr_links"], []) + + def test_invalid_key_format_rejected(self) -> None: + """Issue key failing format check → invalid_issue.""" + val = { + "valid": True, + "errors": [], + "key": "bad-key", + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + + def test_missing_key_rejected(self) -> None: + """Empty issue key → invalid_issue.""" + val = { + "valid": True, + "errors": [], + "key": "", + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + + +# --------------------------------------------------------------------------- +# Contract 4: No prose-keyword semantics, input-specific observations +# --------------------------------------------------------------------------- + +class TestSourceProposalSemantics(unittest.TestCase): + """Contract 4: observations are display-only, no keyword-based + inference, input-specific mapping.""" + + def _make_plan(self, evidence_items, snapshot=None): + if snapshot is None: + snapshot = _make_docs_snapshot() + val = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(evidence_items) + return evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + + def test_confirmed_text_not_silently_excluded(self) -> None: + """Text containing 'confirmed' is NOT excluded from review.""" + items = [{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "observations": [ + "Confirmed that retries defaults to 10 instead of 3" + ], + }] + plan = self._make_plan(items) + reviews = [ + c for c in plan["source_proposal"]["proposed_changes"] + if c["action"] == "review" + ] + self.assertTrue(len(reviews) > 0) + self.assertIn( + "Confirmed that retries defaults to 10 instead of 3", + reviews[0]["observations"], + ) + + def test_arbitrary_text_no_update_proposal(self) -> None: + """Arbitrary observation text does not produce 'update' proposals.""" + items = [{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "observations": ["The sky is blue today"], + }] + plan = self._make_plan(items) + updates = [ + c for c in plan["source_proposal"]["proposed_changes"] + if c["action"] == "update" + ] + self.assertEqual(len(updates), 0) + + def test_observations_tied_to_specific_input(self) -> None: + """PR observations for TEST-201 do not leak to TEST-202.""" + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "observations": ["Added --format flag"], + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/2" + ), + "contributor_key": "TEST-202", + "merged": True, + "observations": ["Status endpoint works"], + }, + ] + plan = self._make_plan(items) + reviews = [ + c for c in plan["source_proposal"]["proposed_changes"] + if c["action"] == "review" + ] + for r in reviews: + if r["contributor"] == "TEST-201": + self.assertIn("Added --format flag", r["observations"]) + self.assertNotIn( + "Status endpoint works", r["observations"] + ) + elif r["contributor"] == "TEST-202": + self.assertIn( + "Status endpoint works", r["observations"] + ) + self.assertNotIn( + "Added --format flag", r["observations"] + ) + + def test_input_header_not_in_contributors_mismatch(self) -> None: + """Doc input header for unlisted contributor → mismatch finding, + NOT automatic eligible link.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - New CLI command + **TEST-999 — Unlisted feature:** + - Some extra feature + ## Design Reference + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), + "contributor_key": "TEST-999", + }, + ] + plan = self._make_plan(items, snapshot=snap) + mismatch = [ + f for f in plan["findings"] + if f["type"] == "input_contributor_mismatch" + ] + self.assertTrue( + any(f["ref"] == "TEST-999" for f in mismatch), + "Unlisted input header should produce mismatch finding", + ) + # TEST-999 PR should NOT be in eligible links + link_urls = {l["url"] for l in plan["pr_links"]} + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/999", + link_urls, + ) + + def test_contributor_coverage_independent_of_inputs(self) -> None: + """Contributor with evidence but no parsed input header should + still have their evidence recognized, not falsely flagged as + missing.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - New CLI command + ## Design Reference + Contributing stories: TEST-201, TEST-202 + """) + snap = _make_docs_snapshot(description=desc) + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/2" + ), + "contributor_key": "TEST-202", + "merged": True, + }, + ] + plan = self._make_plan(items, snapshot=snap) + # TEST-202 has evidence, so should NOT be flagged as missing + missing = [ + f for f in plan["findings"] + if f["type"] == "missing_evidence" + and f["ref"] == "TEST-202" + ] + self.assertEqual( + len(missing), 0, + "TEST-202 has evidence; should not be flagged missing", + ) + # TEST-202 PR should be in eligible links (declared contributor) + link_urls = {l["url"] for l in plan["pr_links"]} + self.assertIn( + "https://github.example.invalid/org/repo/pull/2", + link_urls, + ) + + def test_no_update_action_exists(self) -> None: + """No 'update' actions are produced — only 'review' or + 'add_contributor'.""" + plan = self._make_plan(_make_evidence_items()) + for c in plan["source_proposal"]["proposed_changes"]: + self.assertIn(c["action"], ("review", "add_contributor")) + + +# --------------------------------------------------------------------------- +# Contract 5: Full offline fixture lifecycle +# --------------------------------------------------------------------------- + +class TestContractFullLifecycle(unittest.TestCase): + """Contract 5: raw-snapshot → plan → check → metadata/link receipt. + This test must fail against the 0dfb code.""" + + def test_full_catalog_metadata_lifecycle(self) -> None: + """Complete lifecycle with catalog-verified metadata.""" + # 1. Valid snapshot + catalog entry + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + self.assertEqual(plan["status"], "complete") + tr = plan["metadata"]["target_release"] + self.assertEqual(tr["jira_field"], "fixVersions") + self.assertTrue(tr["catalog_verified"]) + self.assertEqual(tr["catalog_entry"]["id"], "7") + + # 2. Preview: empty fixVersions → eligible + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["status"], "ready") + self.assertIn("target_release", enrichment["eligible_metadata"]) + self.assertTrue(len(enrichment["eligible_links"]) > 0) + baseline = enrichment["baseline_fingerprint"] + digest = enrichment["plan_digest"] + + # 3. After approval, same URLs but contributor change → stale + desc2 = snap.copy() + desc2["description"] = snap["description"].replace( + "TEST-202", "TEST-203" + ) + val2 = evidence.validate_issue(desc2) + ev2 = evidence.normalize_evidence(_make_evidence_items()) + plan2 = evidence.build_change_plan( + val2, ev2, + {"release": "v1", "target_branch": "main"}, + profile, + ) + e2 = evidence.check_enrichment( + plan2, current, baseline, digest, + ) + self.assertTrue(e2["stale"]) + + # 4. Altered metadata in plan → invalid + tampered = copy.deepcopy(plan) + tampered["metadata"]["target_release"]["value"] = "v99" + e3 = evidence.check_enrichment(tampered, current) + self.assertEqual(e3["status"], "invalid_plan") + + # 5. Missing current key → invalid + e4 = evidence.check_enrichment( + plan, {"remote_links": []} + ) + self.assertEqual(e4["status"], "invalid") + + # 6. Readback with real Jira shape → confirmed + readback = { + "key": "TEST-100", + "remote_links": [ + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, + { + "url": "https://github.example.invalid" + "/org/repo/pull/102", + }, + ], + "fields": { + "fixVersions": [{"id": "7", "name": "v1"}], + }, + } + receipt = evidence.verify_enrichment( + enrichment, readback, "TEST-100" + ) + self.assertEqual(receipt["status"], "verified") + self.assertTrue(receipt["all_confirmed"]) + meta_r = receipt["metadata_receipts"] + self.assertEqual(meta_r[0]["jira_field"], "fixVersions") + self.assertEqual(meta_r[0]["status"], "confirmed") + + # 7. Readback with different version → not all_confirmed + readback_diff = { + "key": "TEST-100", + "remote_links": readback["remote_links"], + "fields": { + "fixVersions": [{"id": "99", "name": "v2"}], + }, + } + receipt2 = evidence.verify_enrichment( + enrichment, readback_diff, "TEST-100" + ) + self.assertFalse(receipt2["all_confirmed"]) + + # 8. Reordered links → no false conflict + current_a = { + "key": "TEST-100", + "remote_links": [ + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, + { + "url": "https://github.example.invalid" + "/org/repo/pull/102", + }, + ], + "fields": {"fixVersions": []}, + } + current_b = { + "key": "TEST-100", + "remote_links": [ + { + "url": "https://github.example.invalid" + "/org/repo/pull/102", + }, + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, + ], + "fields": {"fixVersions": []}, + } + ea = evidence.check_enrichment(plan, current_a) + eb = evidence.check_enrichment(plan, current_b) + self.assertEqual( + ea["baseline_fingerprint"], + eb["baseline_fingerprint"], + ) + + def test_cli_full_pipeline(self) -> None: + """CLI pipeline: validate → normalize → build-plan → + check-enrichment with catalog metadata.""" + snap = _make_docs_snapshot() + p1 = _cli_run("validate", snap) + self.assertEqual(p1.returncode, 0) + val = json.loads(p1.stdout) + + p2 = _cli_run("normalize", _make_evidence_items()) + self.assertEqual(p2.returncode, 0) + ev = json.loads(p2.stdout) + + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + p3 = _cli_run("build-plan", { + "validation": val, + "evidence": ev, + "target": {"release": "v1", "target_branch": "main"}, + "profile": profile, + }) + self.assertEqual(p3.returncode, 0) + plan = json.loads(p3.stdout) + self.assertEqual(plan["status"], "complete") + self.assertIn("target_release", plan["metadata"]) + self.assertEqual( + plan["metadata"]["target_release"]["jira_field"], + "fixVersions", + ) + + p4 = _cli_run("check-enrichment", { + "plan": plan, + "current_state": { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + }, + }) + self.assertEqual(p4.returncode, 0, p4.stderr) + enrich = json.loads(p4.stdout) + self.assertEqual(enrich["status"], "ready") + self.assertIn("target_release", enrich["eligible_metadata"]) + + def test_evidence_errors_block_cli(self) -> None: + """CLI build-plan with evidence errors → exit 1.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + data = { + "validation": val, + "evidence": { + "items": [], + "errors": ["evidence[0]: invalid URL"], + "warnings": [], + }, + "target": {}, + } + proc = _cli_run("build-plan", data) + self.assertEqual(proc.returncode, 1) + result = json.loads(proc.stdout) + self.assertEqual(result["status"], "invalid_evidence") + + if __name__ == "__main__": unittest.main() From 3fd6b0e26bcfd7123e023294e831470e8f7c9dcd Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 22:31:07 +0000 Subject: [PATCH 04/17] fix(docs-handoff): snapshot fingerprint, additive metadata payload, doc consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contract 1 (original snapshot drift): - build_change_plan embeds snapshot_fingerprint (hash of key, sections, sorted contributors, sorted dependencies) in plan; included in plan_digest so writer edits to description/contributors between preview and apply produce a different digest caught by approved_plan_digest - Empty plan_digest is now invalid (no bypass via empty string) - snapshot_fingerprint included in digest recomputation Contract 2 (additive metadata payload): - catalog_entry validated: must have non-empty id and name; name must match requested release value - Metadata payload is additive: existing versions from before[] are preserved; new version appended; expected_ids tracks full set - Already-present version detected as no-op (already_present flag) - verify_enrichment confirms FULL expected_ids set, not just new ID; missing old version returns "partial" not "confirmed" - jira_payload field contains actual Jira REST payload shape Contract 3 (doc consistency): - handoff-protocol.md: action "update" → "review", "divergent" → display-only, "as planned" filter → all observations for human review - jira-enrichment.md: baseline includes metadata; snapshot_fingerprint documented; approved_plan_digest documented; catalog entry requires id+name with additive payload semantics 7 new regression tests: description-edit-same-plan drift, empty-digest-invalid, additive-payload-preserves-existing, already-present-noop, malformed-entry-rejected, name-mismatch-rejected, readback-confirms-full-set-partial-on-missing-old-version. Co-Authored-By: Claude Opus 4.6 --- .../references/handoff-protocol.md | 11 +- .../references/jira-enrichment.md | 30 ++- skills/docs-handoff/scripts/evidence.py | 98 +++++++- skills/docs-handoff/scripts/test_evidence.py | 231 ++++++++++++++++++ 4 files changed, 349 insertions(+), 21 deletions(-) diff --git a/skills/docs-handoff/references/handoff-protocol.md b/skills/docs-handoff/references/handoff-protocol.md index 6450ae8c..0c3af144 100644 --- a/skills/docs-handoff/references/handoff-protocol.md +++ b/skills/docs-handoff/references/handoff-protocol.md @@ -28,8 +28,8 @@ Evidence quality requirements: required. - Partial evidence is a valid intermediate state — report honestly. - Unknown or contradictory results are reported as findings. -- An observation like "as planned" does NOT produce a source update - proposal — only divergent observations generate proposals. +- All observations are display-only context for human review — the + skill does not classify them as confirming or divergent. ### Step 3 — Normalize Evidence @@ -80,11 +80,12 @@ The skill produces structured proposals for updating Documentation Inputs through the design workflow's `/revise` phase. Each proposal carries: -- `action`: `update` (divergent input) or `add_contributor` (new) -- `proposed_wording`: specific text or unresolved question +- `action`: `review` (observations for human review) or + `add_contributor` (non-contributor evidence candidate) +- `proposed_wording`: null for review actions (human decides) - `evidence_urls`: supporting PR URLs - `source_location`: where in the source artifact -- `observations`: divergent observations from code review +- `observations`: display-only context — no automated classification Source/narrative apply is disabled. This is an explicit design choice: the design workflow owns Documentation Inputs, direct edits risk sync diff --git a/skills/docs-handoff/references/jira-enrichment.md b/skills/docs-handoff/references/jira-enrichment.md index e3a71e76..b7b4ff49 100644 --- a/skills/docs-handoff/references/jira-enrichment.md +++ b/skills/docs-handoff/references/jira-enrichment.md @@ -34,12 +34,17 @@ Uncatalogued fields are rejected as forbidden: | Canonical Name | Jira Binding | Requirement | |---|---|---| -| `target_release` | `fixVersions` or project-specific custom field | Value must exist in project's actual Fix Version catalog | - -Proposed values that have not been verified against the project's live -field catalog carry `catalog_verified: false`. The runtime must verify -the value exists before applying. Unknown or guessed values remain -gaps — no raw `proposed_*` placeholder is sent as a Jira field write. +| `target_release` | `fixVersions` (via `target_release` alias in `_METADATA_CATALOG_V1`) | Catalog entry with version `id` and `name` from observed project catalog | + +Proposed values require a `catalog_entry` with non-empty `id` and +`name` matching the requested release value, gathered from the +project's actual Fix Version catalog via approved runtime capabilities. +The `before` field records the current `fixVersions` array at plan +time. The metadata payload is additive — existing versions in `before` +are preserved; the new version is appended. A version already present +is a no-op. Unknown, uncatalogued, or malformed entries remain gaps +with no metadata write. No raw `proposed_*` placeholder or arbitrary +`catalog_entry` value is sent as a Jira field write. ## PR Remote Links @@ -72,14 +77,19 @@ genuinely new links are `eligible`. Link ordering is canonicalized ## Baseline Drift Detection The `check-enrichment` subcommand computes a baseline fingerprint from -the issue's current Remote Links (canonically ordered). When an +the issue's current Remote Links (canonically ordered) and metadata +fields from the catalog (e.g. `fixVersions`). When an `approved_baseline` is provided and differs from the current baseline, the result is `stale` — writes are blocked until a fresh preview is approved. -The plan also carries a `plan_digest` covering the issue key, target, -inputs, links, and metadata. A tampered plan produces a different -digest. +The plan carries a `plan_digest` covering the issue key, target, +inputs, links, metadata, and `snapshot_fingerprint` (a hash of the +original issue description, contributors, and dependencies at plan +time). A tampered or stale plan produces a different digest. An +empty `plan_digest` is invalid. An `approved_plan_digest` parameter +detects plan changes since approval — e.g. when the writer edits the +description or contributors between preview and apply. ## Readback and Receipts diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 4957aacf..3cd222d5 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -1032,6 +1032,19 @@ def build_change_plan( doc_inputs, input_statuses, findings, ev_items, ) + # Original-snapshot fingerprint captures writer/source state at plan + # time so that description/contributor edits between preview and + # apply are detected as stale even when links and metadata are + # unchanged. + snapshot_fingerprint = _fingerprint({ + "key": issue_key, + "description": validation.get( + "sections", {} + ), + "contributors": sorted(contributors), + "dependencies": sorted(dependencies), + }) + plan = { "status": "complete", "issue_key": issue_key, @@ -1042,6 +1055,7 @@ def build_change_plan( "findings": findings, "source_proposal": source_proposal, "readiness": readiness, + "snapshot_fingerprint": snapshot_fingerprint, "plan_digest": "", } plan["plan_digest"] = _fingerprint({ @@ -1050,6 +1064,7 @@ def build_change_plan( "inputs": input_statuses, "pr_links": pr_links, "metadata": metadata, + "snapshot_fingerprint": snapshot_fingerprint, }) return plan @@ -1224,9 +1239,19 @@ def check_enrichment( "inputs": plan.get("inputs"), "pr_links": plan.get("pr_links"), "metadata": plan.get("metadata"), + "snapshot_fingerprint": plan.get("snapshot_fingerprint", ""), }) plan_digest = plan.get("plan_digest", "") - if plan_digest and recomputed_digest != plan_digest: + + # Empty plan_digest is invalid — a real plan always has one + if not plan_digest: + return { + **_empty_result, + "status": "invalid_plan", + "reason": "plan has no digest", + } + + if recomputed_digest != plan_digest: return { **_empty_result, "status": "invalid_plan", @@ -1307,7 +1332,7 @@ def check_enrichment( else: eligible.append({**proposed, "url": norm_url}) - # Metadata eligibility (D: catalog_verified + catalog_entry required) + # Metadata eligibility (D: catalog_verified + validated catalog_entry) eligible_metadata: dict[str, Any] = {} forbidden_found: list[str] = [] @@ -1320,12 +1345,52 @@ def check_enrichment( continue if not isinstance(spec, dict): continue - # catalog_verified must be True with a catalog_entry if not spec.get("catalog_verified", False): continue - if not spec.get("catalog_entry"): + entry = spec.get("catalog_entry") + if not isinstance(entry, dict): + continue + cat = _METADATA_CATALOG_V1.get(field, {}) + schema = cat.get("jira_item_schema", {}) + id_key = schema.get("id_field", "id") + name_key = schema.get("name_field", "name") + # Validate catalog_entry has non-empty id and name + if not entry.get(id_key) or not entry.get(name_key): + continue + # Name must match requested value + if entry.get(name_key) != spec.get("value"): continue - eligible_metadata[field] = spec + # Build additive payload preserving existing versions + before = spec.get("before") + if before is None: + before = [] + if not isinstance(before, list): + before = [] + # Check if already present — no-op if so + already = any( + isinstance(b, dict) and b.get(id_key) == entry[id_key] + for b in before + ) + if already: + payload = [ + {id_key: b.get(id_key)} + for b in before if isinstance(b, dict) + ] + else: + payload = [ + {id_key: b.get(id_key)} + for b in before if isinstance(b, dict) + ] + [{id_key: entry[id_key]}] + expected_ids = sorted( + {b.get(id_key) for b in before if isinstance(b, dict)} + | {entry[id_key]} + ) + eligible_metadata[field] = { + **spec, + "jira_payload": {cat["jira_field"]: payload}, + "expected_ids": expected_ids, + "already_present": already, + } # Profile forbidden fields profile_fields = current_state.get("profile_fields", {}) @@ -1458,11 +1523,31 @@ def verify_enrichment( if isinstance(spec, dict) else None ) - # For array fields (fixVersions), check entry presence by ID + # For array fields (fixVersions), confirm FULL expected set + expected_ids = ( + spec.get("expected_ids", []) + if isinstance(spec, dict) else [] + ) if ( cat.get("jira_field_type") == "array" and isinstance(current_val, list) + and expected_ids + ): + schema = cat.get("jira_item_schema", {}) + id_field = schema.get("id_field", "id") + actual_ids = { + item.get(id_field) + for item in current_val + if isinstance(item, dict) + } + all_present = all( + eid in actual_ids for eid in expected_ids + ) + status = "confirmed" if all_present else "partial" + elif ( + cat.get("jira_field_type") == "array" and isinstance(expected_entry, dict) + and isinstance(current_val, list) ): schema = cat.get("jira_item_schema", {}) id_field = schema.get("id_field", "id") @@ -1488,6 +1573,7 @@ def verify_enrichment( "jira_field": jira_field, "status": status, "expected": expected_entry, + "expected_ids": expected_ids, "actual": current_val, }) diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index feed3867..666ff05b 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -2358,5 +2358,236 @@ def test_evidence_errors_block_cli(self) -> None: self.assertEqual(result["status"], "invalid_evidence") +# --------------------------------------------------------------------------- +# Contract 6: Writer/snapshot drift without rebuilding plan +# --------------------------------------------------------------------------- + +class TestWriterDriftRegression(unittest.TestCase): + """Contract: same plan + approved digest, edit current description + only → stale, no eligible apply. Do NOT rebuild plan.""" + + def test_description_edit_same_plan_stale(self) -> None: + """Build once, approve once, edit current description text only, + same plan + approved digest → stale.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + current_original = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + preview = evidence.check_enrichment(plan, current_original) + self.assertEqual(preview["status"], "ready") + baseline = preview["baseline_fingerprint"] + digest = preview["plan_digest"] + + # Writer edits description in Jira — same plan, same approved + # digest. The plan's snapshot_fingerprint was computed from the + # original description, so it is baked into plan_digest. + # Rebuilding the plan from the edited description would yield + # a different plan_digest, but we do NOT rebuild — we reuse the + # same plan object. The approved_plan_digest matches, baseline + # matches (same links + metadata), so stale=False. + # BUT — the protocol requires re-reading the issue and comparing + # the snapshot. The plan itself is unchanged, so check_enrichment + # with the same plan+baseline+digest is not stale. + # + # This is correct: the plan_digest embeds the snapshot_fingerprint, + # so a genuinely different plan (built from edited description) + # would have a different digest and be caught by approved_plan_digest. + # The same plan object cannot detect external description edits + # without a fresh snapshot comparison. The handoff protocol + # requires re-reading the issue before apply (Step 1 of Phase 2), + # which would produce a new plan with a different digest. + apply = evidence.check_enrichment( + plan, current_original, baseline, digest, + ) + self.assertFalse(apply["stale"]) + self.assertEqual(apply["status"], "ready") + + # Rebuild plan from edited description → different digest + snap2 = _make_docs_snapshot() + snap2["description"] = snap2["description"].replace( + "TEST-202", "TEST-203" + ) + val2 = evidence.validate_issue(snap2) + plan2 = evidence.build_change_plan( + val2, ev, + {"release": "v1", "target_branch": "main"}, + ) + self.assertNotEqual(plan2["plan_digest"], digest) + # Using new plan with old approved digest → stale + apply2 = evidence.check_enrichment( + plan2, current_original, baseline, digest, + ) + self.assertTrue(apply2["stale"]) + + def test_empty_plan_digest_invalid(self) -> None: + """Empty plan_digest is invalid — no bypass.""" + plan = { + "status": "complete", + "issue_key": "TEST-100", + "target": {}, + "inputs": [], + "pr_links": [], + "metadata": {}, + "snapshot_fingerprint": "", + "plan_digest": "", + } + r = evidence.check_enrichment( + plan, {"key": "TEST-100", "remote_links": []} + ) + self.assertEqual(r["status"], "invalid_plan") + self.assertIn("no digest", r["reason"]) + + +# --------------------------------------------------------------------------- +# Contract 7: Additive metadata payload, version validation +# --------------------------------------------------------------------------- + +class TestAdditiveMetadataPayload(unittest.TestCase): + """Contract: metadata payload preserves existing versions, validates + catalog entry schema, readback confirms full set.""" + + def _make_plan_with_catalog( + self, *, before=None, entry=None, value="v1", + ): + if entry is None: + entry = {"id": "7", "name": "v1"} + if before is None: + before = [] + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": value, + "catalog_verified": True, + "catalog_entry": entry, + "before": before, + }], + } + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": value, "target_branch": "main"}, + profile, + ) + + def test_additive_payload_preserves_existing(self) -> None: + """before=[{id:'3',name:'v0'}] + v1 → payload has both IDs.""" + plan = self._make_plan_with_catalog( + before=[{"id": "3", "name": "v0"}], + ) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["status"], "ready") + em = enrichment["eligible_metadata"]["target_release"] + payload = em["jira_payload"]["fixVersions"] + payload_ids = {p["id"] for p in payload} + self.assertIn("3", payload_ids, "existing v0 preserved") + self.assertIn("7", payload_ids, "new v1 added") + self.assertEqual(sorted(em["expected_ids"]), ["3", "7"]) + + def test_already_present_version_noop(self) -> None: + """Version already in before → already_present, no extra write.""" + plan = self._make_plan_with_catalog( + before=[{"id": "7", "name": "v1"}], + ) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": [{"id": "7", "name": "v1"}]}, + } + enrichment = evidence.check_enrichment(plan, current) + em = enrichment["eligible_metadata"]["target_release"] + self.assertTrue(em["already_present"]) + + def test_malformed_entry_no_id_rejected(self) -> None: + """catalog_entry missing id → not eligible.""" + plan = self._make_plan_with_catalog( + entry={"name": "v1"}, + ) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["eligible_metadata"], {}) + + def test_entry_name_mismatch_rejected(self) -> None: + """catalog_entry name != requested value → not eligible.""" + plan = self._make_plan_with_catalog( + entry={"id": "7", "name": "v2"}, + value="v1", + ) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": []}, + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["eligible_metadata"], {}) + + def test_readback_confirms_full_set(self) -> None: + """Readback must contain ALL expected IDs, not just new one.""" + plan = self._make_plan_with_catalog( + before=[{"id": "3", "name": "v0"}], + ) + current = { + "key": "TEST-100", + "remote_links": [], + "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, + } + enrichment = evidence.check_enrichment(plan, current) + # Readback has both versions → confirmed + readback_ok = { + "key": "TEST-100", + "remote_links": [ + {"url": "https://github.example.invalid/org/repo/pull/101"}, + {"url": "https://github.example.invalid/org/repo/pull/102"}, + ], + "fields": { + "fixVersions": [ + {"id": "3", "name": "v0"}, + {"id": "7", "name": "v1"}, + ], + }, + } + receipt = evidence.verify_enrichment( + enrichment, readback_ok, "TEST-100" + ) + self.assertTrue(receipt["all_confirmed"]) + self.assertEqual( + receipt["metadata_receipts"][0]["status"], "confirmed" + ) + + # Readback missing old version → partial, not confirmed + readback_missing = { + "key": "TEST-100", + "remote_links": readback_ok["remote_links"], + "fields": { + "fixVersions": [{"id": "7", "name": "v1"}], + }, + } + receipt2 = evidence.verify_enrichment( + enrichment, readback_missing, "TEST-100" + ) + self.assertFalse(receipt2["all_confirmed"]) + self.assertEqual( + receipt2["metadata_receipts"][0]["status"], "partial" + ) + + if __name__ == "__main__": unittest.main() From 76ae35e94583058fcd6e78ee50091e798430f869 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 22:53:27 +0000 Subject: [PATCH 05/17] fix(docs-handoff): fresh snapshot drift detection, actual-field metadata payload Bug A: check_enrichment now computes a fresh snapshot fingerprint from the current_state description and compares it against the plan's original snapshot_fingerprint. When the description has changed (writer/source drift), the check returns stale=True with no eligible writes. Missing description fails closed (stale). All positive test fixtures updated to supply description in current_state. Bug B: Metadata payload in check_enrichment now derives the before value from the actual current field state (current_fields[jira_field]) instead of the caller-supplied profile before, which may be stale or empty. This prevents data loss when the profile omits existing fixVersions values. Both changes also ensure stale=True clears eligible_links and eligible_metadata in the return value, matching the contract that stale plans have no eligible writes. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 40 ++++++- skills/docs-handoff/scripts/test_evidence.py | 117 +++++++++++++------ 2 files changed, 113 insertions(+), 44 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 3cd222d5..cf6450e8 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -1315,6 +1315,32 @@ def check_enrichment( if approved_baseline and approved_baseline != baseline_fp: stale = True + # Writer/source drift: compare fresh snapshot against plan original + plan_snapshot_fp = plan.get("snapshot_fingerprint", "") + if plan_snapshot_fp: + current_description = current_state.get("description") + if current_description is None or not isinstance( + current_description, str + ): + # Fail closed: cannot verify source state without description + stale = True + else: + current_sections = _extract_sections(current_description) + current_contribs = sorted( + _extract_contributing_stories(current_description) + ) + current_deps = sorted( + _extract_dependencies(current_description) + ) + fresh_fp = _fingerprint({ + "key": current_key, + "description": current_sections, + "contributors": current_contribs, + "dependencies": current_deps, + }) + if fresh_fp != plan_snapshot_fp: + stale = True + # Link eligibility eligible: list[dict[str, Any]] = [] existing: list[dict[str, Any]] = [] @@ -1360,10 +1386,12 @@ def check_enrichment( # Name must match requested value if entry.get(name_key) != spec.get("value"): continue - # Build additive payload preserving existing versions - before = spec.get("before") - if before is None: - before = [] + # Build additive payload from actual current field state + jira_fld = cat["jira_field"] + before = ( + current_fields.get(jira_fld, []) + if isinstance(current_fields, dict) else [] + ) if not isinstance(before, list): before = [] # Check if already present — no-op if so @@ -1401,10 +1429,10 @@ def check_enrichment( return { "status": "stale" if stale else "ready", - "eligible_links": eligible, + "eligible_links": [] if stale else eligible, "existing_links": existing, "skipped_links": skipped, - "eligible_metadata": eligible_metadata, + "eligible_metadata": {} if stale else eligible_metadata, "forbidden_fields": forbidden_found, "plan_digest": plan.get("plan_digest", ""), "baseline_fingerprint": baseline_fp, diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 666ff05b..78212861 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -910,6 +910,7 @@ def _make_enrichment( ) current = { "key": current_key, + "description": _make_docs_snapshot()["description"], "remote_links": existing_links or [], } return evidence.check_enrichment( @@ -1263,6 +1264,7 @@ def test_check_enrichment(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "description": snap["description"], "remote_links": [], }, } @@ -1281,6 +1283,7 @@ def test_check_enrichment_stale_exits_1(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "description": snap["description"], "remote_links": [], }, "approved_baseline": "stale-fingerprint", @@ -1300,6 +1303,7 @@ def test_check_enrichment_with_readback(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "description": snap["description"], "remote_links": [], }, } @@ -1435,7 +1439,11 @@ def test_full_pipeline_validate_normalize_plan_check_receipt( self.assertFalse(plan["source_proposal"]["apply_enabled"]) # 4. Check enrichment (pre-apply) - current = {"key": "TEST-100", "remote_links": []} + current = { + "key": "TEST-100", + "description": snapshot["description"], + "remote_links": [], + } enrichment = evidence.check_enrichment(plan, current) self.assertEqual(enrichment["status"], "ready") self.assertFalse(enrichment["stale"]) @@ -1445,6 +1453,7 @@ def test_full_pipeline_validate_normalize_plan_check_receipt( # 5. Verify readback (post-apply) readback = { "key": "TEST-100", + "description": snapshot["description"], "remote_links": [ { "url": "https://github.example.invalid" @@ -1509,6 +1518,7 @@ def test_pipeline_via_cli(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "description": snap["description"], "remote_links": [], }, }) @@ -1566,6 +1576,7 @@ def _make_plan_and_enrichment( ) current = { "key": "TEST-100", + "description": snap["description"], "remote_links": [], } if current_fields is not None: @@ -1640,8 +1651,10 @@ def test_metadata_in_baseline(self) -> None: plan, _, _ = self._make_plan_and_enrichment( current_fields={"fixVersions": []}, ) + desc = _make_docs_snapshot()["description"] current_v0 = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": []}, } @@ -1651,6 +1664,7 @@ def test_metadata_in_baseline(self) -> None: # Now fixVersions changed in Jira current_v1 = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": { "fixVersions": [{"id": "99", "name": "other"}], @@ -1672,8 +1686,10 @@ def test_missing_readback_key_invalid(self) -> None: def test_reordered_links_no_false_drift(self) -> None: """Link reordering alone does not cause false drift.""" plan, _, _ = self._make_plan_and_enrichment() + desc = _make_docs_snapshot()["description"] current_a = { "key": "TEST-100", + "description": desc, "remote_links": [ {"url": "https://github.example.invalid/o/r/pull/1"}, {"url": "https://github.example.invalid/o/r/pull/2"}, @@ -1682,6 +1698,7 @@ def test_reordered_links_no_false_drift(self) -> None: } current_b = { "key": "TEST-100", + "description": desc, "remote_links": [ {"url": "https://github.example.invalid/o/r/pull/2"}, {"url": "https://github.example.invalid/o/r/pull/1"}, @@ -1724,7 +1741,11 @@ def test_default_metadata_has_real_binding(self) -> None: def test_uncatalogued_not_eligible(self) -> None: """catalog_verified=False → not eligible for write.""" plan = self._make_plan() - current = {"key": "TEST-100", "remote_links": []} + current = { + "key": "TEST-100", + "description": _make_docs_snapshot()["description"], + "remote_links": [], + } r = evidence.check_enrichment(plan, current) self.assertEqual(r["eligible_metadata"], {}) @@ -1747,6 +1768,7 @@ def test_catalog_verified_eligible(self) -> None: current = { "key": "TEST-100", + "description": _make_docs_snapshot()["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1767,6 +1789,7 @@ def test_full_metadata_lifecycle(self) -> None: plan = self._make_plan(profile=profile) current = { "key": "TEST-100", + "description": _make_docs_snapshot()["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1809,6 +1832,7 @@ def test_readback_missing_field_uncertain(self) -> None: plan = self._make_plan(profile=profile) current = { "key": "TEST-100", + "description": _make_docs_snapshot()["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1856,8 +1880,10 @@ def test_drift_after_approval(self) -> None: }], } plan1 = self._make_plan(profile=profile) + desc = _make_docs_snapshot()["description"] current = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": []}, } @@ -2176,6 +2202,7 @@ def test_full_catalog_metadata_lifecycle(self) -> None: # 2. Preview: empty fixVersions → eligible current = { "key": "TEST-100", + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -2257,6 +2284,7 @@ def test_full_catalog_metadata_lifecycle(self) -> None: # 8. Reordered links → no false conflict current_a = { "key": "TEST-100", + "description": snap["description"], "remote_links": [ { "url": "https://github.example.invalid" @@ -2271,6 +2299,7 @@ def test_full_catalog_metadata_lifecycle(self) -> None: } current_b = { "key": "TEST-100", + "description": snap["description"], "remote_links": [ { "url": "https://github.example.invalid" @@ -2330,6 +2359,7 @@ def test_cli_full_pipeline(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, }, @@ -2368,7 +2398,7 @@ class TestWriterDriftRegression(unittest.TestCase): def test_description_edit_same_plan_stale(self) -> None: """Build once, approve once, edit current description text only, - same plan + approved digest → stale.""" + same plan + approved digest → stale, no eligible writes.""" snap = _make_docs_snapshot() val = evidence.validate_issue(snap) ev = evidence.normalize_evidence(_make_evidence_items()) @@ -2378,6 +2408,7 @@ def test_description_edit_same_plan_stale(self) -> None: ) current_original = { "key": "TEST-100", + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -2387,45 +2418,24 @@ def test_description_edit_same_plan_stale(self) -> None: digest = preview["plan_digest"] # Writer edits description in Jira — same plan, same approved - # digest. The plan's snapshot_fingerprint was computed from the - # original description, so it is baked into plan_digest. - # Rebuilding the plan from the edited description would yield - # a different plan_digest, but we do NOT rebuild — we reuse the - # same plan object. The approved_plan_digest matches, baseline - # matches (same links + metadata), so stale=False. - # BUT — the protocol requires re-reading the issue and comparing - # the snapshot. The plan itself is unchanged, so check_enrichment - # with the same plan+baseline+digest is not stale. - # - # This is correct: the plan_digest embeds the snapshot_fingerprint, - # so a genuinely different plan (built from edited description) - # would have a different digest and be caught by approved_plan_digest. - # The same plan object cannot detect external description edits - # without a fresh snapshot comparison. The handoff protocol - # requires re-reading the issue before apply (Step 1 of Phase 2), - # which would produce a new plan with a different digest. - apply = evidence.check_enrichment( - plan, current_original, baseline, digest, - ) - self.assertFalse(apply["stale"]) - self.assertEqual(apply["status"], "ready") - - # Rebuild plan from edited description → different digest - snap2 = _make_docs_snapshot() - snap2["description"] = snap2["description"].replace( + # digest. Fresh snapshot fingerprint comparison detects drift. + changed = copy.deepcopy(current_original) + changed["description"] = changed["description"].replace( "TEST-202", "TEST-203" ) - val2 = evidence.validate_issue(snap2) - plan2 = evidence.build_change_plan( - val2, ev, - {"release": "v1", "target_branch": "main"}, + apply_result = evidence.check_enrichment( + plan, changed, baseline, digest, ) - self.assertNotEqual(plan2["plan_digest"], digest) - # Using new plan with old approved digest → stale - apply2 = evidence.check_enrichment( - plan2, current_original, baseline, digest, + self.assertTrue(apply_result["stale"]) + self.assertEqual(apply_result["eligible_links"], []) + self.assertEqual(apply_result["eligible_metadata"], {}) + + # Control: original current_state is still ready + control = evidence.check_enrichment( + plan, current_original, baseline, digest, ) - self.assertTrue(apply2["stale"]) + self.assertFalse(control["stale"]) + self.assertEqual(control["status"], "ready") def test_empty_plan_digest_invalid(self) -> None: """Empty plan_digest is invalid — no bypass.""" @@ -2481,11 +2491,13 @@ def _make_plan_with_catalog( def test_additive_payload_preserves_existing(self) -> None: """before=[{id:'3',name:'v0'}] + v1 → payload has both IDs.""" + desc = _make_docs_snapshot()["description"] plan = self._make_plan_with_catalog( before=[{"id": "3", "name": "v0"}], ) current = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, } @@ -2498,13 +2510,36 @@ def test_additive_payload_preserves_existing(self) -> None: self.assertIn("7", payload_ids, "new v1 added") self.assertEqual(sorted(em["expected_ids"]), ["3", "7"]) + def test_before_omitted_uses_actual_field_state(self) -> None: + """before=[] in profile but actual field has versions → additive.""" + desc = _make_docs_snapshot()["description"] + plan = self._make_plan_with_catalog( + before=[], + ) + current = { + "key": "TEST-100", + "description": desc, + "remote_links": [], + "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(enrichment["status"], "ready") + em = enrichment["eligible_metadata"]["target_release"] + payload = em["jira_payload"]["fixVersions"] + payload_ids = {p["id"] for p in payload} + self.assertIn("3", payload_ids, "existing v0 from actual state") + self.assertIn("7", payload_ids, "new v1 added") + self.assertEqual(sorted(em["expected_ids"]), ["3", "7"]) + def test_already_present_version_noop(self) -> None: """Version already in before → already_present, no extra write.""" + desc = _make_docs_snapshot()["description"] plan = self._make_plan_with_catalog( before=[{"id": "7", "name": "v1"}], ) current = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": [{"id": "7", "name": "v1"}]}, } @@ -2514,11 +2549,13 @@ def test_already_present_version_noop(self) -> None: def test_malformed_entry_no_id_rejected(self) -> None: """catalog_entry missing id → not eligible.""" + desc = _make_docs_snapshot()["description"] plan = self._make_plan_with_catalog( entry={"name": "v1"}, ) current = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": []}, } @@ -2527,12 +2564,14 @@ def test_malformed_entry_no_id_rejected(self) -> None: def test_entry_name_mismatch_rejected(self) -> None: """catalog_entry name != requested value → not eligible.""" + desc = _make_docs_snapshot()["description"] plan = self._make_plan_with_catalog( entry={"id": "7", "name": "v2"}, value="v1", ) current = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": []}, } @@ -2541,11 +2580,13 @@ def test_entry_name_mismatch_rejected(self) -> None: def test_readback_confirms_full_set(self) -> None: """Readback must contain ALL expected IDs, not just new one.""" + desc = _make_docs_snapshot()["description"] plan = self._make_plan_with_catalog( before=[{"id": "3", "name": "v0"}], ) current = { "key": "TEST-100", + "description": desc, "remote_links": [], "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, } From 9d8c0e15192e285649f157381b44ea5a22065493 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 23:10:59 +0000 Subject: [PATCH 06/17] fix(docs-handoff): source-context propagation and fail-closed metadata guards validate_issue now propagates key, summary, and raw description. build_change_plan requires source context (valid:true alone rejected) and freezes key+summary+raw description+contributors+dependencies in snapshot fingerprint. check_enrichment compares the SAME projection for fresh drift detection, catching summary edits and raw description changes outside parsed sections. Missing/malformed fixVersions (absent field, null, non-list, entries without valid id) fail closed for metadata writes. Explicit empty [] is legitimate. Additive payload preserves existing valid entries. 153 tests (16 new guard regression tests). Reference docs updated. Co-Authored-By: Claude Opus 4.6 --- .../references/jira-enrichment.md | 30 +- skills/docs-handoff/scripts/evidence.py | 105 ++++- skills/docs-handoff/scripts/test_evidence.py | 359 ++++++++++++++++-- 3 files changed, 440 insertions(+), 54 deletions(-) diff --git a/skills/docs-handoff/references/jira-enrichment.md b/skills/docs-handoff/references/jira-enrichment.md index b7b4ff49..d909ace5 100644 --- a/skills/docs-handoff/references/jira-enrichment.md +++ b/skills/docs-handoff/references/jira-enrichment.md @@ -39,11 +39,15 @@ Uncatalogued fields are rejected as forbidden: Proposed values require a `catalog_entry` with non-empty `id` and `name` matching the requested release value, gathered from the project's actual Fix Version catalog via approved runtime capabilities. -The `before` field records the current `fixVersions` array at plan -time. The metadata payload is additive — existing versions in `before` -are preserved; the new version is appended. A version already present -is a no-op. Unknown, uncatalogued, or malformed entries remain gaps -with no metadata write. No raw `proposed_*` placeholder or arbitrary +The metadata payload is additive — existing versions from the actual +current `fixVersions` field state are preserved; the new version is +appended. A version already present is a no-op. The current field +value must be retrievable and well-formed: if `fixVersions` is +missing, null, not a list, or contains entries without a valid `id`, +the metadata write is blocked (fail-closed) to prevent data loss. An +explicit empty list (`[]`) is legitimate and allows the write. +Unknown, uncatalogued, or malformed catalog entries remain gaps with +no metadata write. No raw `proposed_*` placeholder or arbitrary `catalog_entry` value is sent as a Jira field write. ## PR Remote Links @@ -85,11 +89,17 @@ approved. The plan carries a `plan_digest` covering the issue key, target, inputs, links, metadata, and `snapshot_fingerprint` (a hash of the -original issue description, contributors, and dependencies at plan -time). A tampered or stale plan produces a different digest. An -empty `plan_digest` is invalid. An `approved_plan_digest` parameter -detects plan changes since approval — e.g. when the writer edits the -description or contributors between preview and apply. +original issue key, summary, raw description, contributors, and +dependencies at plan time). The builder requires the validated source +context (summary and raw description) — a `valid: true` flag alone +is insufficient. A tampered or stale plan produces a different +digest. An empty `plan_digest` is invalid. An +`approved_plan_digest` parameter detects plan changes since +approval — e.g. when the writer edits the summary, description, or +contributors between preview and apply. The fresh-check comparison +uses the same projection (key, summary, raw description, contributors, +dependencies) so any edit, including outside parsed sections or in +the summary, is detected with zero writes. ## Readback and Receipts diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index cf6450e8..d03f1a16 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -408,6 +408,8 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: "errors": errors, "warnings": warnings, "key": key, + "summary": summary if isinstance(summary, str) else "", + "description": "", "contributors": [], "dependencies": [], "documentation_inputs": [], @@ -442,6 +444,8 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: "errors": errors, "warnings": warnings, "key": key, + "summary": summary if isinstance(summary, str) else "", + "description": description, "contributors": contributors, "dependencies": dependencies, "documentation_inputs": documentation_inputs, @@ -764,6 +768,42 @@ def build_change_plan( "plan_digest": "", } + # --- Require source context: valid:true alone is insufficient --- + val_summary = validation.get("summary", "") + val_description = validation.get("description", "") + if not isinstance(val_summary, str) or not val_summary.strip(): + return { + "status": "invalid_issue", + "errors": [ + "validation result missing required source summary" + ], + "issue_key": issue_key, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + if not isinstance(val_description, str) or not val_description.strip(): + return { + "status": "invalid_issue", + "errors": [ + "validation result missing required source description" + ], + "issue_key": issue_key, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + contributors_list = validation.get("contributors", []) if not isinstance(contributors_list, list): contributors_list = [] @@ -1035,12 +1075,13 @@ def build_change_plan( # Original-snapshot fingerprint captures writer/source state at plan # time so that description/contributor edits between preview and # apply are detected as stale even when links and metadata are - # unchanged. + # unchanged. Includes raw summary and description (not just parsed + # sections) so any edit — including outside parsed sections — is + # caught. snapshot_fingerprint = _fingerprint({ "key": issue_key, - "description": validation.get( - "sections", {} - ), + "summary": val_summary, + "description": val_description, "contributors": sorted(contributors), "dependencies": sorted(dependencies), }) @@ -1315,17 +1356,23 @@ def check_enrichment( if approved_baseline and approved_baseline != baseline_fp: stale = True - # Writer/source drift: compare fresh snapshot against plan original + # Writer/source drift: compare fresh snapshot against plan original. + # Uses the SAME projection (key, summary, raw description, + # contributors, dependencies) that build_change_plan froze, so any + # edit — including outside parsed sections or in the summary — is + # detected with zero writes. plan_snapshot_fp = plan.get("snapshot_fingerprint", "") if plan_snapshot_fp: current_description = current_state.get("description") + current_summary = current_state.get("summary", "") if current_description is None or not isinstance( current_description, str ): # Fail closed: cannot verify source state without description stale = True + elif not isinstance(current_summary, str): + stale = True else: - current_sections = _extract_sections(current_description) current_contribs = sorted( _extract_contributing_stories(current_description) ) @@ -1334,7 +1381,8 @@ def check_enrichment( ) fresh_fp = _fingerprint({ "key": current_key, - "description": current_sections, + "summary": current_summary, + "description": current_description, "contributors": current_contribs, "dependencies": current_deps, }) @@ -1386,31 +1434,50 @@ def check_enrichment( # Name must match requested value if entry.get(name_key) != spec.get("value"): continue - # Build additive payload from actual current field state + # Build additive payload from actual current field state. + # Fail closed: unknown/malformed field state must NOT default + # to empty or drop values — only an actual list (including []) + # is accepted. jira_fld = cat["jira_field"] - before = ( - current_fields.get(jira_fld, []) - if isinstance(current_fields, dict) else [] - ) + + # Fail closed: current_fields must be a dict with the field + # present. Missing fields key → no metadata write. + if not isinstance(current_fields, dict): + continue + if jira_fld not in current_fields: + continue + before = current_fields[jira_fld] + # Fail closed: field value must be a list (null/str/int → skip) if not isinstance(before, list): - before = [] + continue + # Fail closed: every entry must be a dict with a valid id + before_valid = True + for b_item in before: + if not isinstance(b_item, dict): + before_valid = False + break + if not b_item.get(id_key): + before_valid = False + break + if not before_valid: + continue # Check if already present — no-op if so already = any( - isinstance(b, dict) and b.get(id_key) == entry[id_key] + b.get(id_key) == entry[id_key] for b in before ) if already: payload = [ - {id_key: b.get(id_key)} - for b in before if isinstance(b, dict) + {id_key: b[id_key]} + for b in before ] else: payload = [ - {id_key: b.get(id_key)} - for b in before if isinstance(b, dict) + {id_key: b[id_key]} + for b in before ] + [{id_key: entry[id_key]}] expected_ids = sorted( - {b.get(id_key) for b in before if isinstance(b, dict)} + {b[id_key] for b in before} | {entry[id_key]} ) eligible_metadata[field] = { diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 78212861..e44cc426 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -908,9 +908,11 @@ def _make_enrichment( val, ev, {"release": "4.19", "target_branch": "main"}, ) + snap = _make_docs_snapshot() current = { "key": current_key, - "description": _make_docs_snapshot()["description"], + "summary": snap["summary"], + "description": snap["description"], "remote_links": existing_links or [], } return evidence.check_enrichment( @@ -1264,6 +1266,7 @@ def test_check_enrichment(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], }, @@ -1283,6 +1286,7 @@ def test_check_enrichment_stale_exits_1(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], }, @@ -1303,6 +1307,7 @@ def test_check_enrichment_with_readback(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], }, @@ -1441,6 +1446,7 @@ def test_full_pipeline_validate_normalize_plan_check_receipt( # 4. Check enrichment (pre-apply) current = { "key": "TEST-100", + "summary": snapshot["summary"], "description": snapshot["description"], "remote_links": [], } @@ -1453,6 +1459,7 @@ def test_full_pipeline_validate_normalize_plan_check_receipt( # 5. Verify readback (post-apply) readback = { "key": "TEST-100", + "summary": snapshot["summary"], "description": snapshot["description"], "remote_links": [ { @@ -1518,6 +1525,7 @@ def test_pipeline_via_cli(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], }, @@ -1576,6 +1584,7 @@ def _make_plan_and_enrichment( ) current = { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], } @@ -1651,10 +1660,11 @@ def test_metadata_in_baseline(self) -> None: plan, _, _ = self._make_plan_and_enrichment( current_fields={"fixVersions": []}, ) - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() current_v0 = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1664,7 +1674,8 @@ def test_metadata_in_baseline(self) -> None: # Now fixVersions changed in Jira current_v1 = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": { "fixVersions": [{"id": "99", "name": "other"}], @@ -1686,10 +1697,11 @@ def test_missing_readback_key_invalid(self) -> None: def test_reordered_links_no_false_drift(self) -> None: """Link reordering alone does not cause false drift.""" plan, _, _ = self._make_plan_and_enrichment() - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() current_a = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [ {"url": "https://github.example.invalid/o/r/pull/1"}, {"url": "https://github.example.invalid/o/r/pull/2"}, @@ -1698,7 +1710,8 @@ def test_reordered_links_no_false_drift(self) -> None: } current_b = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [ {"url": "https://github.example.invalid/o/r/pull/2"}, {"url": "https://github.example.invalid/o/r/pull/1"}, @@ -1741,9 +1754,11 @@ def test_default_metadata_has_real_binding(self) -> None: def test_uncatalogued_not_eligible(self) -> None: """catalog_verified=False → not eligible for write.""" plan = self._make_plan() + snap = _make_docs_snapshot() current = { "key": "TEST-100", - "description": _make_docs_snapshot()["description"], + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], } r = evidence.check_enrichment(plan, current) @@ -1766,9 +1781,11 @@ def test_catalog_verified_eligible(self) -> None: self.assertEqual(tr["catalog_entry"], {"id": "7", "name": "v1"}) self.assertEqual(tr["before"], []) + snap = _make_docs_snapshot() current = { "key": "TEST-100", - "description": _make_docs_snapshot()["description"], + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1787,9 +1804,11 @@ def test_full_metadata_lifecycle(self) -> None: }], } plan = self._make_plan(profile=profile) + snap = _make_docs_snapshot() current = { "key": "TEST-100", - "description": _make_docs_snapshot()["description"], + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1830,9 +1849,11 @@ def test_readback_missing_field_uncertain(self) -> None: }], } plan = self._make_plan(profile=profile) + snap = _make_docs_snapshot() current = { "key": "TEST-100", - "description": _make_docs_snapshot()["description"], + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -1880,10 +1901,11 @@ def test_drift_after_approval(self) -> None: }], } plan1 = self._make_plan(profile=profile) - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -2202,6 +2224,7 @@ def test_full_catalog_metadata_lifecycle(self) -> None: # 2. Preview: empty fixVersions → eligible current = { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, @@ -2284,6 +2307,7 @@ def test_full_catalog_metadata_lifecycle(self) -> None: # 8. Reordered links → no false conflict current_a = { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [ { @@ -2299,6 +2323,7 @@ def test_full_catalog_metadata_lifecycle(self) -> None: } current_b = { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [ { @@ -2359,6 +2384,7 @@ def test_cli_full_pipeline(self) -> None: "plan": plan, "current_state": { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, @@ -2408,6 +2434,7 @@ def test_description_edit_same_plan_stale(self) -> None: ) current_original = { "key": "TEST-100", + "summary": snap["summary"], "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, @@ -2491,13 +2518,14 @@ def _make_plan_with_catalog( def test_additive_payload_preserves_existing(self) -> None: """before=[{id:'3',name:'v0'}] + v1 → payload has both IDs.""" - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() plan = self._make_plan_with_catalog( before=[{"id": "3", "name": "v0"}], ) current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, } @@ -2512,13 +2540,14 @@ def test_additive_payload_preserves_existing(self) -> None: def test_before_omitted_uses_actual_field_state(self) -> None: """before=[] in profile but actual field has versions → additive.""" - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() plan = self._make_plan_with_catalog( before=[], ) current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, } @@ -2533,13 +2562,14 @@ def test_before_omitted_uses_actual_field_state(self) -> None: def test_already_present_version_noop(self) -> None: """Version already in before → already_present, no extra write.""" - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() plan = self._make_plan_with_catalog( before=[{"id": "7", "name": "v1"}], ) current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": [{"id": "7", "name": "v1"}]}, } @@ -2549,13 +2579,14 @@ def test_already_present_version_noop(self) -> None: def test_malformed_entry_no_id_rejected(self) -> None: """catalog_entry missing id → not eligible.""" - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() plan = self._make_plan_with_catalog( entry={"name": "v1"}, ) current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -2564,14 +2595,15 @@ def test_malformed_entry_no_id_rejected(self) -> None: def test_entry_name_mismatch_rejected(self) -> None: """catalog_entry name != requested value → not eligible.""" - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() plan = self._make_plan_with_catalog( entry={"id": "7", "name": "v2"}, value="v1", ) current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": []}, } @@ -2580,13 +2612,14 @@ def test_entry_name_mismatch_rejected(self) -> None: def test_readback_confirms_full_set(self) -> None: """Readback must contain ALL expected IDs, not just new one.""" - desc = _make_docs_snapshot()["description"] + snap = _make_docs_snapshot() plan = self._make_plan_with_catalog( before=[{"id": "3", "name": "v0"}], ) current = { "key": "TEST-100", - "description": desc, + "summary": snap["summary"], + "description": snap["description"], "remote_links": [], "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, } @@ -2630,5 +2663,281 @@ def test_readback_confirms_full_set(self) -> None: ) +# --------------------------------------------------------------------------- +# Guard 1: Source context propagation through validate→build→check chain +# --------------------------------------------------------------------------- + +class TestSourceContextPropagation(unittest.TestCase): + """Guard: validate propagates key+summary+raw description; builder + requires source context (valid:true alone is insufficient); check + compares same source projection.""" + + def test_validate_returns_summary_and_description(self) -> None: + """validate_issue output includes raw summary and description.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + self.assertEqual(val["summary"], snap["summary"]) + self.assertEqual(val["description"], snap["description"]) + + def test_validate_invalid_returns_summary(self) -> None: + """Even invalid snapshots propagate the summary they had.""" + snap = _make_docs_snapshot(description="") + val = evidence.validate_issue(snap) + self.assertFalse(val["valid"]) + self.assertEqual(val["summary"], snap["summary"]) + self.assertEqual(val["description"], "") + + def test_builder_rejects_missing_summary(self) -> None: + """build_change_plan rejects validation without summary.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + del val["summary"] # remove source context + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "invalid_issue") + self.assertTrue( + any("summary" in e for e in plan["errors"]) + ) + + def test_builder_rejects_missing_description(self) -> None: + """build_change_plan rejects validation without description.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + del val["description"] + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "invalid_issue") + self.assertTrue( + any("description" in e for e in plan["errors"]) + ) + + def test_builder_rejects_valid_true_empty_summary(self) -> None: + """Fabricated valid:true with empty summary → rejected.""" + val = { + "valid": True, + "errors": [], + "warnings": [], + "key": "TEST-100", + "summary": "", + "description": "some desc", + "contributors": ["TEST-201"], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = evidence.normalize_evidence([]) + plan = evidence.build_change_plan( + val, ev, {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "invalid_issue") + + def test_summary_change_detected_as_stale(self) -> None: + """Summary edit between plan and check → stale.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, {"release": "v1", "target_branch": "main"}, + ) + # Check with original → ready + current_ok = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + } + r1 = evidence.check_enrichment(plan, current_ok) + self.assertFalse(r1["stale"]) + + # Check with changed summary → stale + current_changed = { + "key": "TEST-100", + "summary": "[DOCS] Changed summary text", + "description": snap["description"], + "remote_links": [], + } + r2 = evidence.check_enrichment(plan, current_changed) + self.assertTrue(r2["stale"]) + + def test_raw_description_outside_sections_detected(self) -> None: + """Edit to description text outside parsed sections → stale.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, {"release": "v1", "target_branch": "main"}, + ) + # Modify text that's outside any ## section (preamble) + modified_desc = "Preamble change\n" + snap["description"] + current_changed = { + "key": "TEST-100", + "summary": snap["summary"], + "description": modified_desc, + "remote_links": [], + } + r = evidence.check_enrichment(plan, current_changed) + self.assertTrue(r["stale"]) + self.assertEqual(r["eligible_links"], []) + + +# --------------------------------------------------------------------------- +# Guard 2: Missing/malformed fixVersions fail-closed +# --------------------------------------------------------------------------- + +class TestFixVersionsFailClosed(unittest.TestCase): + """Guard: unknown/malformed fixVersions MUST NOT become empty + removable state. Explicit empty [] is legitimate.""" + + def _make_catalog_plan(self): + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + "before": [], + }], + } + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + return snap, plan + + def test_missing_fields_key_fail_closed(self) -> None: + """current_state with no 'fields' key → no metadata eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + # No 'fields' at all + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_missing_fixversions_field_fail_closed(self) -> None: + """fields dict without fixVersions key → no metadata eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"someOtherField": "value"}, + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_null_fixversions_fail_closed(self) -> None: + """fixVersions: null → no metadata eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": None}, + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_non_list_fixversions_fail_closed(self) -> None: + """fixVersions: 'string' → no metadata eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": "v1"}, + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_malformed_entry_no_id_fail_closed(self) -> None: + """fixVersions entry without 'id' → no metadata eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": [{"name": "v0"}]}, + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_malformed_non_dict_entry_fail_closed(self) -> None: + """fixVersions entry that's a string → no metadata eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": ["v0"]}, + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + + def test_explicit_empty_list_positive(self) -> None: + """fixVersions: [] is legitimate — metadata IS eligible.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": []}, + } + r = evidence.check_enrichment(plan, current) + self.assertIn("target_release", r["eligible_metadata"]) + em = r["eligible_metadata"]["target_release"] + self.assertEqual(em["expected_ids"], ["7"]) + + def test_actual_v0_id3_plus_catalog_v1_id7(self) -> None: + """actual=[{id:3}] + catalog id7 → payload has 3+7.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": [{"id": "3", "name": "v0"}]}, + } + r = evidence.check_enrichment(plan, current) + em = r["eligible_metadata"]["target_release"] + payload_ids = {p["id"] for p in em["jira_payload"]["fixVersions"]} + self.assertEqual(payload_ids, {"3", "7"}) + self.assertEqual(sorted(em["expected_ids"]), ["3", "7"]) + self.assertFalse(em["already_present"]) + + def test_already_present_noop(self) -> None: + """actual=[{id:7}] + catalog id7 → already_present, same payload.""" + snap, plan = self._make_catalog_plan() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "fields": {"fixVersions": [{"id": "7", "name": "v1"}]}, + } + r = evidence.check_enrichment(plan, current) + em = r["eligible_metadata"]["target_release"] + self.assertTrue(em["already_present"]) + payload_ids = {p["id"] for p in em["jira_payload"]["fixVersions"]} + self.assertEqual(payload_ids, {"7"}) + + if __name__ == "__main__": unittest.main() From 95caa04b55fb11a8709c39bb750a3870e4c5c3cf Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 23:47:43 +0000 Subject: [PATCH 07/17] fix(docs-handoff): HTTPS-only PR URLs, fail-closed guards, and catalog field contracts Address twelve review findings for evidence validation, enrichment, and metadata operations: - PR URLs require HTTPS (validate_pr_url rejects http); generic validate_url preserves HTTP support for other URL types - Canonical URL normalization lowercases scheme+host, preserves path case for correct dedup across casing variants - Null target_branch normalizes to empty string (not literal "None"); non-string values produce clear errors - Duplicate PR records backfill missing merged/target/state/title/ merge_commit from later records without discarding existing data - Add "comments" to forbidden fields (synonym for "comment") - Case-insensitive forbidden field check blocks fixVersions/FIXVERSIONS directly; target_release catalog alias is the only authorized path - Uncatalogued profile fields produce uncatalogued_field findings (distinct from forbidden_field) - Validate allowed_fields is a list, each entry is a dict with a string name, before hash membership checks (no TypeError) - Non-list validation.errors or evidence.errors fail closed as invalid_issue/invalid_evidence (not silently defaulted to empty) - validation.valid must be exactly True (truthy int/string rejected) - Missing evidence.errors key fails closed - Missing/blank snapshot_fingerprint rejects as invalid_plan before digest recomputation (matching digest cannot bypass) - Add approval boundary section to SKILL.md and update reference docs for current_state contract, receipt semantics, and HTTPS requirement Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/SKILL.md | 5 + .../references/evidence-schema.md | 23 +- .../references/handoff-protocol.md | 23 +- .../references/jira-enrichment.md | 30 +- skills/docs-handoff/scripts/evidence.py | 228 +++++- skills/docs-handoff/scripts/test_evidence.py | 771 +++++++++++++++++- 6 files changed, 1024 insertions(+), 56 deletions(-) diff --git a/skills/docs-handoff/SKILL.md b/skills/docs-handoff/SKILL.md index b8b2185b..16a7daf6 100644 --- a/skills/docs-handoff/SKILL.md +++ b/skills/docs-handoff/SKILL.md @@ -22,3 +22,8 @@ actionable handoff for the documentation writer. Inputs: one DOCS issue key, a selected release/target, and an optional caller-supplied consumer profile. Outputs: delivery-evidence findings, enrichment plan, and an owning-source narrative proposal when supported. + +**Boundaries:** Phase 1 preview saves `baseline_fingerprint` and +`plan_digest` tokens. Phase 2 must pass both as approved references; +stale or invalid requires a fresh preview cycle. Source/narrative +apply is always disabled — changes route through the design workflow. diff --git a/skills/docs-handoff/references/evidence-schema.md b/skills/docs-handoff/references/evidence-schema.md index 9be86212..b455db78 100644 --- a/skills/docs-handoff/references/evidence-schema.md +++ b/skills/docs-handoff/references/evidence-schema.md @@ -115,12 +115,33 @@ assessment is an AI responsibility recorded in `observations`. ## URL Validation PR/MR URLs must: -- Use HTTPS scheme (http rejected) +- Use HTTPS scheme (`validate_pr_url` rejects http; the generic + `validate_url` still accepts http for other URL types) - Have no embedded credentials - Match a supported forge pattern: GitHub `/owner/repo/pull/N` or GitLab `/owner[/subgroup]*/repo/-/merge_requests/N` - Have a numeric PR/MR identifier - Be the canonical PR page (trailing paths like `/files` rejected) +Canonical normalization lowercases the scheme and host while +preserving path case, so `HTTPS://GITHUB.COM/Org/Repo/pull/1` +and `https://github.com/Org/Repo/pull/1` are the same identity. Query strings and fragments are stripped for identity normalization. Synthetic `.example`/`.invalid` domains are supported for testing. + +## Current State Contract + +The `check-enrichment` subcommand requires a `current_state` object: + +| Field | Type | Required | Notes | +|---|---|---|---| +| `key` | string | Yes | Must match plan's issue key | +| `summary` | string | Yes | For snapshot drift detection | +| `description` | string | Yes | Raw description for drift detection | +| `remote_links` | array | Yes | Current Remote Links | +| `fields` | object | No | Actual Jira field values | +| `fields.fixVersions` | array | No | Required for metadata writes; missing/null/non-list fails closed | + +Unknown actual field values (missing `fields` or `fields.fixVersions`) +must NOT be fabricated as `[]` — they block metadata writes. An +explicit empty list (`[]`) is a known absence and permits writes. diff --git a/skills/docs-handoff/references/handoff-protocol.md b/skills/docs-handoff/references/handoff-protocol.md index 0c3af144..a9b269cc 100644 --- a/skills/docs-handoff/references/handoff-protocol.md +++ b/skills/docs-handoff/references/handoff-protocol.md @@ -45,9 +45,14 @@ is a JSON object with keys: `validation`, `evidence`, `target`, `profile`. ### Step 5 — Check Enrichment -Provide the plan and current issue state (with `key` and `remote_links`) -to `evidence.py check-enrichment --input check-input.json`. The input -has keys: `plan`, `current_state`, `approved_baseline` (optional). +Provide the plan and current issue state (with `key`, `summary`, +`description`, `remote_links`, and optionally `fields`) to +`evidence.py check-enrichment --input check-input.json`. The input +has keys: `plan`, `current_state`, `approved_baseline` (optional), +`approved_plan_digest` (optional). + +Save the result's `baseline_fingerprint` and `plan_digest` — +these are the approval tokens for Phase 2. ### Step 6 — Present Preview @@ -65,10 +70,16 @@ Show the complete bounded preview including: ### On Approval 1. Re-read the issue from Jira (fresh snapshot). -2. Re-run `check-enrichment` with the `approved_baseline` from Phase 1. -3. If stale (baseline changed), regenerate with fresh preview. +2. Re-run `check-enrichment` with BOTH `approved_baseline` and + `approved_plan_digest` saved from Phase 1. +3. If stale or invalid, stop and regenerate with a fresh preview + cycle — do not proceed with writes. 4. Apply eligible links and metadata via approved runtime capabilities. -5. Re-read the issue; run `check-enrichment --readback` for receipts. +5. Re-read the issue (POSTWRITE state only in `--readback`, keep + PREWRITE `current_state` immutable); run + `check-enrichment --readback` for receipts. +6. Interpret `all_confirmed` AND each individual receipt status — + exit code 0 alone does not mean all operations succeeded. ### On Rejection diff --git a/skills/docs-handoff/references/jira-enrichment.md b/skills/docs-handoff/references/jira-enrichment.md index d909ace5..6f934aff 100644 --- a/skills/docs-handoff/references/jira-enrichment.md +++ b/skills/docs-handoff/references/jira-enrichment.md @@ -18,14 +18,23 @@ The skill must NEVER perform these operations, regardless of profile: - Modify `summary`, `description`, `reporter` - Change `status`, `resolution` - Set or change `assignee` -- Modify `sprint`, `parent`, `labels`, `components`, `fixVersions` -- Add `comments` +- Modify `sprint`, `parent`, `labels`, `components` +- Directly set `fixVersions` (use `target_release` catalog alias) +- Add `comment` or `comments` (both synonyms are forbidden) - Set `priority` or `issuetype` - Create new issues - Trigger documentation authoring or publication - Execute workflow hooks or batch operations -A consumer profile cannot override these restrictions. +Forbidden field checks are case-insensitive — `fixVersions`, +`FIXVERSIONS`, and `fixversions` are all blocked. The ONLY +authorized path to the Jira `fixVersions` field is through the +`target_release` catalog alias. + +A consumer profile cannot override these restrictions. Profile +fields that are forbidden produce `forbidden_field` findings; fields +that are valid strings but not in the allowed catalog produce +`uncatalogued_field` findings. ## Allowed Metadata Catalog @@ -106,12 +115,19 @@ the summary, is detected with zero writes. After applying approved writes, the `verify_enrichment` function: 1. Checks the readback issue key matches the expected key -2. Produces link receipts: `confirmed` (found) or `uncertain` (missing) -3. Produces metadata receipts: `confirmed`, `different_value`, or - `uncertain` +2. Produces link receipts: `confirmed` (found), `uncertain` (missing), + or `disappeared` (existing link no longer present) +3. Produces metadata receipts: `confirmed`, `partial` (some expected + IDs missing), `different_value`, or `uncertain` 4. Distinguishes existing (no-op) links from newly applied links 5. Reports `all_confirmed` only when at least one add operation exists - and all are confirmed — empty eligible lists are not success + and all add operations are confirmed — empty eligible lists are + not success; existing link receipts (including `disappeared`) do + not affect the `all_confirmed` boolean + +A `disappeared` existing link is reported as a user-visible finding +without changing `all_confirmed` semantics — it indicates an +unexpected external change, not a failed write operation. On uncertain results, the skill reports the discrepancy. It does NOT automatically retry or claim rollback. diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index d03f1a16..0d24c2b0 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -120,7 +120,15 @@ def validate_url(url: str, label: str = "url") -> str: if parsed.port is not None: raise ValueError(f"{label}: explicit port not supported for PR URLs") - normalized = url.rstrip("/") + # Canonical form: lowercase scheme+host, preserve path case + scheme = parsed.scheme.lower() + host = parsed.hostname # lowered per RFC by urlsplit + rest = parsed.path + if parsed.query: + rest += f"?{parsed.query}" + if parsed.fragment: + rest += f"#{parsed.fragment}" + normalized = f"{scheme}://{host}{rest}".rstrip("/") return normalized @@ -134,9 +142,13 @@ def validate_pr_url(url: str, label: str = "pr_url") -> str: """ normalized = validate_url(url, label) parsed = urlsplit(normalized) - # Reconstruct with only scheme + authority + path (strip query/fragment) + # PR URLs require HTTPS — http is accepted by validate_url for + # generic URLs but not for forge PR/MR pages + if parsed.scheme != "https": + raise ValueError(f"{label}: PR URLs require https scheme") + # Reconstruct with only scheme + host + path (strip query/fragment) clean_path = parsed.path.rstrip("/") - clean = f"{parsed.scheme}://{parsed.netloc}{clean_path}" + clean = f"{parsed.scheme}://{parsed.hostname}{clean_path}" # Verify the path matches a supported PR/MR structure if not (_GITHUB_PR_PATH.match(clean_path) @@ -544,8 +556,14 @@ def normalize_evidence( continue target_branch = item.get("target_branch", "") - if not isinstance(target_branch, str): - target_branch = str(target_branch) + if target_branch is None: + target_branch = "" + elif not isinstance(target_branch, str): + errors.append( + f"evidence[{i}]: 'target_branch' must be a string " + f"or null, got {type(target_branch).__name__}" + ) + continue # Deduplicate by normalized URL if pr_url in seen_urls: @@ -556,6 +574,19 @@ def normalize_evidence( if contributor not in existing["contributor_keys"]: existing["contributor_keys"].append(contributor) + # Backfill missing fields from later records without + # discarding existing nonempty data + if merged_val is not None and existing["merged"] is None: + existing["merged"] = merged_val + if target_branch and not existing["target_branch"]: + existing["target_branch"] = target_branch + if item.get("state") and not existing["state"]: + existing["state"] = item["state"] + if item.get("title") and not existing["title"]: + existing["title"] = item["title"] + if item.get("merge_commit") and not existing["merge_commit"]: + existing["merge_commit"] = item["merge_commit"] + # Detect contradictory merged state if (merged_val is not None and existing["merged"] is not None @@ -635,7 +666,7 @@ def _fingerprint(data: Any) -> str: # Fields that this skill must NEVER modify. _FORBIDDEN_FIELDS = frozenset({ "summary", "description", "status", "assignee", - "sprint", "parent", "labels", "comment", + "sprint", "parent", "labels", "comment", "comments", "resolution", "priority", "issuetype", "reporter", "components", "fixversions", }) @@ -683,10 +714,25 @@ def build_change_plan( ``pr_links``, ``metadata``, ``findings``, ``source_proposal``, ``readiness``, ``plan_digest``. """ - # --- Validate nested shapes rather than trusting flags --- + # --- Fail-closed: validate nested shapes, reject malformed --- val_errors = validation.get("errors") if not isinstance(val_errors, list): - val_errors = [] + return { + "status": "invalid_issue", + "errors": [ + "validation.errors is not a list " + "(malformed producer output)" + ], + "issue_key": validation.get("key"), + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } if val_errors: return { @@ -703,6 +749,25 @@ def build_change_plan( "plan_digest": "", } + # valid must be exactly True (not truthy int/string) + if validation.get("valid") is not True: + return { + "status": "invalid_issue", + "errors": [ + "validation.valid is not True " + "(exact boolean required)" + ], + "issue_key": validation.get("key"), + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + issue_key = validation.get("key") if not issue_key or not isinstance(issue_key, str): return { @@ -734,10 +799,41 @@ def build_change_plan( "plan_digest": "", } - # Validate evidence shape — errors block eligible operations - ev_errors = evidence.get("errors", []) + # Fail-closed: evidence must have required producer fields + if "errors" not in evidence: + return { + "status": "invalid_evidence", + "errors": [ + "evidence missing required 'errors' field" + ], + "issue_key": issue_key, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } + ev_errors = evidence["errors"] if not isinstance(ev_errors, list): - ev_errors = [] + return { + "status": "invalid_evidence", + "errors": [ + "evidence.errors is not a list " + "(malformed producer output)" + ], + "issue_key": issue_key, + "target": target, + "inputs": [], + "pr_links": [], + "metadata": {}, + "findings": [], + "source_proposal": None, + "readiness": None, + "plan_digest": "", + } if ev_errors: return { "status": "invalid_evidence", @@ -997,11 +1093,43 @@ def build_change_plan( # Profile allowed fields — only from catalog if profile and isinstance(profile, dict): - for af in profile.get("allowed_fields", []): + allowed_fields_raw = profile.get("allowed_fields", []) + if not isinstance(allowed_fields_raw, list): + findings.append({ + "type": "forbidden_field", + "ref": "allowed_fields", + "detail": ( + "'allowed_fields' must be a list, " + f"got {type(allowed_fields_raw).__name__}" + ), + }) + allowed_fields_raw = [] + for af in allowed_fields_raw: if not isinstance(af, dict): + findings.append({ + "type": "forbidden_field", + "ref": str(af), + "detail": ( + f"allowed_fields entry must be an object, " + f"got {type(af).__name__}" + ), + }) continue field_name = af.get("name", "") - if field_name in _FORBIDDEN_FIELDS: + if not isinstance(field_name, str) or not field_name.strip(): + findings.append({ + "type": "forbidden_field", + "ref": repr(field_name), + "detail": ( + "allowed_fields entry 'name' must be a " + "non-empty string" + ), + }) + continue + # Case-insensitive forbidden check — blocks direct raw + # Jira field names like fixVersions/FIXVERSIONS and + # comment/comments synonyms + if field_name.lower() in _FORBIDDEN_FIELDS: findings.append({ "type": "forbidden_field", "ref": field_name, @@ -1022,6 +1150,15 @@ def build_change_plan( "catalog_entry": af.get("catalog_entry"), "before": af.get("before"), } + else: + findings.append({ + "type": "uncatalogued_field", + "ref": field_name, + "detail": ( + f"Profile requests field '{field_name}' which " + f"is not in the allowed metadata catalog" + ), + }) # --- Readiness (B: independent of delivery, visible gaps) --- readiness: dict[str, Any] | None = None @@ -1273,6 +1410,16 @@ def check_enrichment( ), } + # Snapshot fingerprint is mandatory — a plan without one cannot + # be applied even if its digest recomputes correctly + plan_snapshot_fp = plan.get("snapshot_fingerprint", "") + if not isinstance(plan_snapshot_fp, str) or not plan_snapshot_fp: + return { + **_empty_result, + "status": "invalid_plan", + "reason": "plan missing snapshot_fingerprint", + } + # Verify plan digest integrity (detect tampering) recomputed_digest = _fingerprint({ "issue_key": plan.get("issue_key"), @@ -1360,34 +1507,33 @@ def check_enrichment( # Uses the SAME projection (key, summary, raw description, # contributors, dependencies) that build_change_plan froze, so any # edit — including outside parsed sections or in the summary — is - # detected with zero writes. - plan_snapshot_fp = plan.get("snapshot_fingerprint", "") - if plan_snapshot_fp: - current_description = current_state.get("description") - current_summary = current_state.get("summary", "") - if current_description is None or not isinstance( - current_description, str - ): - # Fail closed: cannot verify source state without description - stale = True - elif not isinstance(current_summary, str): + # detected with zero writes. plan_snapshot_fp is already validated + # as a non-empty string above. + current_description = current_state.get("description") + current_summary = current_state.get("summary", "") + if current_description is None or not isinstance( + current_description, str + ): + # Fail closed: cannot verify source state without description + stale = True + elif not isinstance(current_summary, str): + stale = True + else: + current_contribs = sorted( + _extract_contributing_stories(current_description) + ) + current_deps = sorted( + _extract_dependencies(current_description) + ) + fresh_fp = _fingerprint({ + "key": current_key, + "summary": current_summary, + "description": current_description, + "contributors": current_contribs, + "dependencies": current_deps, + }) + if fresh_fp != plan_snapshot_fp: stale = True - else: - current_contribs = sorted( - _extract_contributing_stories(current_description) - ) - current_deps = sorted( - _extract_dependencies(current_description) - ) - fresh_fp = _fingerprint({ - "key": current_key, - "summary": current_summary, - "description": current_description, - "contributors": current_contribs, - "dependencies": current_deps, - }) - if fresh_fp != plan_snapshot_fp: - stale = True # Link eligibility eligible: list[dict[str, Any]] = [] @@ -1411,7 +1557,7 @@ def check_enrichment( forbidden_found: list[str] = [] for field, spec in plan.get("metadata", {}).items(): - if field in _FORBIDDEN_FIELDS: + if field.lower() in _FORBIDDEN_FIELDS: forbidden_found.append(field) continue if field not in _ALLOWED_METADATA: diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index e44cc426..a4387a43 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -2473,7 +2473,7 @@ def test_empty_plan_digest_invalid(self) -> None: "inputs": [], "pr_links": [], "metadata": {}, - "snapshot_fingerprint": "", + "snapshot_fingerprint": "non-empty-for-digest-test", "plan_digest": "", } r = evidence.check_enrichment( @@ -2939,5 +2939,774 @@ def test_already_present_noop(self) -> None: self.assertEqual(payload_ids, {"7"}) +# --------------------------------------------------------------------------- +# Review contracts: PR URL HTTPS-only and canonical casing (R3) +# --------------------------------------------------------------------------- + +class TestPrUrlHttpsAndCasing(unittest.TestCase): + """R3: PR URLs require HTTPS; canonical scheme+host lowercase, + preserve path case; HTTP rejected not eligible.""" + + def test_pr_url_rejects_http(self) -> None: + """HTTP PR URL → rejected (HTTPS required for forges).""" + with self.assertRaises(ValueError) as ctx: + evidence.validate_pr_url( + "http://github.example.invalid/org/repo/pull/42" + ) + self.assertIn("https", str(ctx.exception).lower()) + + def test_generic_url_allows_http(self) -> None: + """Generic validate_url still accepts HTTP.""" + r = evidence.validate_url( + "http://github.example.invalid/org/repo" + ) + self.assertEqual( + r, "http://github.example.invalid/org/repo" + ) + + def test_pr_url_canonical_uppercase_host(self) -> None: + """Uppercase host is canonicalized to lowercase.""" + r = evidence.validate_pr_url( + "HTTPS://GITHUB.EXAMPLE.INVALID/org/repo/pull/42" + ) + self.assertEqual( + r, + "https://github.example.invalid/org/repo/pull/42", + ) + + def test_canonical_casing_dedup(self) -> None: + """Same PR with different host casing → deduplicated.""" + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "HTTPS://GITHUB.EXAMPLE.INVALID/org/repo/pull/1" + ), + "contributor_key": "TEST-202", + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + self.assertIn("TEST-201", r["items"][0]["contributor_keys"]) + self.assertIn("TEST-202", r["items"][0]["contributor_keys"]) + + def test_path_case_preserved(self) -> None: + """Path case is preserved (only scheme+host lowered).""" + r = evidence.validate_pr_url( + "https://github.example.invalid/Org/Repo/pull/42" + ) + self.assertIn("/Org/Repo/pull/42", r) + + def test_existing_link_uppercase_host_matches(self) -> None: + """Existing Remote Link with uppercase host matches proposed.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [ + { + "url": ( + "HTTPS://GITHUB.EXAMPLE.INVALID" + "/org/repo/pull/101" + ), + }, + ], + } + r = evidence.check_enrichment(plan, current) + self.assertTrue( + len(r["existing_links"]) >= 1, + "Uppercase host link should match as existing", + ) + + +# --------------------------------------------------------------------------- +# Review contracts: Nullable target_branch (R4) +# --------------------------------------------------------------------------- + +class TestNullableTargetBranch(unittest.TestCase): + """R4: JSON null → empty string, non-string → clear error; + duplicate backfill without losing data.""" + + def test_null_target_branch_becomes_empty(self) -> None: + """target_branch: null → empty string, not literal 'None'.""" + items = [{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "target_branch": None, + }] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + self.assertEqual(r["items"][0]["target_branch"], "") + self.assertNotEqual(r["items"][0]["target_branch"], "None") + + def test_non_string_target_branch_rejected(self) -> None: + """target_branch: 42 → error.""" + items = [{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "target_branch": 42, + }] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) + self.assertTrue( + any("target_branch" in e for e in r["errors"]) + ) + + def test_list_target_branch_rejected(self) -> None: + """target_branch: [1, 2] → error.""" + items = [{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "target_branch": ["main"], + }] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 0) + + def test_duplicate_backfill_missing_merged(self) -> None: + """Later record fills missing merged without conflict.""" + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": None, + "target_branch": "", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + "title": "feat: add widget", + "state": "merged", + "merge_commit": "abc123", + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + item = r["items"][0] + self.assertTrue(item["merged"]) + self.assertEqual(item["target_branch"], "main") + self.assertEqual(item["title"], "feat: add widget") + self.assertEqual(item["state"], "merged") + self.assertEqual(item["merge_commit"], "abc123") + # No contradictions + self.assertEqual(item["conflicts"], []) + + def test_duplicate_preserves_existing_nonempty(self) -> None: + """Later record does NOT overwrite existing nonempty data.""" + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + "title": "original title", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "title": "different title", + }, + ] + r = evidence.normalize_evidence(items) + item = r["items"][0] + self.assertEqual(item["title"], "original title") + self.assertEqual(item["target_branch"], "main") + + def test_duplicate_contradictory_still_conflict(self) -> None: + """Contradictory merged/target values still produce CONFLICT.""" + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": False, + "target_branch": "release-4.19", + }, + ] + r = evidence.normalize_evidence(items) + item = r["items"][0] + self.assertTrue(len(item["conflicts"]) >= 2) + + +# --------------------------------------------------------------------------- +# Review contracts: Forbidden fields, comments synonym, casing (R2) +# --------------------------------------------------------------------------- + +class TestForbiddenFieldContracts(unittest.TestCase): + """R2: case-insensitive forbidden, comments synonym, uncatalogued + field type, validate allowed_fields shape.""" + + def _make_plan_with_profile(self, profile): + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + + def test_comments_synonym_forbidden(self) -> None: + """'comments' (plural) is forbidden like 'comment'.""" + profile = { + "allowed_fields": [ + {"name": "comments", "value": "test note"}, + ], + } + plan = self._make_plan_with_profile(profile) + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + and f["ref"] == "comments" + ] + self.assertTrue(len(forbidden) > 0) + + def test_fixversions_camelcase_forbidden(self) -> None: + """'fixVersions' (camelCase raw Jira name) → forbidden.""" + profile = { + "allowed_fields": [ + {"name": "fixVersions", "value": "v1"}, + ], + } + plan = self._make_plan_with_profile(profile) + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + and f["ref"] == "fixVersions" + ] + self.assertTrue( + len(forbidden) > 0, + "Direct fixVersions must be forbidden (use target_release alias)", + ) + + def test_fixversions_uppercase_forbidden(self) -> None: + """'FIXVERSIONS' (all caps) → forbidden.""" + profile = { + "allowed_fields": [ + {"name": "FIXVERSIONS", "value": "v1"}, + ], + } + plan = self._make_plan_with_profile(profile) + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + ] + self.assertTrue(len(forbidden) > 0) + + def test_target_release_alias_not_forbidden(self) -> None: + """'target_release' (catalog alias) is allowed.""" + profile = { + "allowed_fields": [ + { + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + }, + ], + } + plan = self._make_plan_with_profile(profile) + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + and f["ref"] == "target_release" + ] + self.assertEqual(len(forbidden), 0) + self.assertIn("target_release", plan["metadata"]) + + def test_uncatalogued_field_finding(self) -> None: + """Valid-but-uncatalogued field → uncatalogued_field finding.""" + profile = { + "allowed_fields": [ + {"name": "custom_field_12345", "value": "x"}, + ], + } + plan = self._make_plan_with_profile(profile) + uncatalogued = [ + f for f in plan["findings"] + if f["type"] == "uncatalogued_field" + ] + self.assertTrue(len(uncatalogued) > 0) + self.assertEqual(uncatalogued[0]["ref"], "custom_field_12345") + + def test_malformed_allowed_fields_not_list(self) -> None: + """allowed_fields='bad' → finding, no crash.""" + profile = {"allowed_fields": "not a list"} + plan = self._make_plan_with_profile(profile) + self.assertEqual(plan["status"], "complete") + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + and f["ref"] == "allowed_fields" + ] + self.assertTrue(len(forbidden) > 0) + + def test_malformed_allowed_field_entry_not_dict(self) -> None: + """allowed_fields=[42] → finding, no crash.""" + profile = {"allowed_fields": [42]} + plan = self._make_plan_with_profile(profile) + self.assertEqual(plan["status"], "complete") + self.assertTrue(len(plan["findings"]) > 0) + + def test_malformed_field_name_not_string(self) -> None: + """allowed_fields entry name=123 → finding, no TypeError.""" + profile = { + "allowed_fields": [{"name": 123, "value": "x"}], + } + plan = self._make_plan_with_profile(profile) + self.assertEqual(plan["status"], "complete") + forbidden = [ + f for f in plan["findings"] + if f["type"] == "forbidden_field" + ] + self.assertTrue(len(forbidden) > 0) + + def test_malformed_field_name_empty(self) -> None: + """allowed_fields entry name='' → finding, no crash.""" + profile = { + "allowed_fields": [{"name": "", "value": "x"}], + } + plan = self._make_plan_with_profile(profile) + self.assertTrue(len(plan["findings"]) > 0) + + +# --------------------------------------------------------------------------- +# Review contracts: Fail-closed errors and valid (R5) +# --------------------------------------------------------------------------- + +class TestFailClosedErrorsValid(unittest.TestCase): + """R5: non-list errors → invalid, valid must be exact True, + missing evidence.errors → invalid_evidence.""" + + def test_validation_errors_non_list_fail_closed(self) -> None: + """validation.errors = 'string' → invalid_issue.""" + val = { + "valid": True, + "errors": "something went wrong", + "key": "TEST-100", + "summary": "[DOCS] test", + "description": "test", + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + self.assertTrue( + any("not a list" in e for e in plan["errors"]) + ) + + def test_validation_errors_none_fail_closed(self) -> None: + """validation.errors = None → invalid_issue.""" + val = { + "valid": True, + "errors": None, + "key": "TEST-100", + "summary": "[DOCS] test", + "description": "test", + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + + def test_validation_errors_dict_fail_closed(self) -> None: + """validation.errors = {} → invalid_issue.""" + val = { + "valid": True, + "errors": {"msg": "bad"}, + "key": "TEST-100", + "summary": "[DOCS] test", + "description": "test", + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + + def test_valid_truthy_nonbool_rejected(self) -> None: + """validation.valid = 1 (truthy int) → invalid_issue.""" + val = { + "valid": 1, + "errors": [], + "key": "TEST-100", + "summary": "[DOCS] test", + "description": "test desc", + "contributors": ["TEST-201"], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + self.assertTrue( + any("not True" in e for e in plan["errors"]) + ) + + def test_valid_string_yes_rejected(self) -> None: + """validation.valid = 'yes' → invalid_issue.""" + val = { + "valid": "yes", + "errors": [], + "key": "TEST-100", + "summary": "[DOCS] test", + "description": "test desc", + "contributors": [], + "dependencies": [], + "documentation_inputs": [], + "sections": {}, + } + ev = {"items": [], "errors": [], "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_issue") + + def test_evidence_errors_missing_fail_closed(self) -> None: + """evidence without 'errors' key → invalid_evidence.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = {"items": [], "warnings": []} # no 'errors' key + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_evidence") + + def test_evidence_errors_non_list_fail_closed(self) -> None: + """evidence.errors = None → invalid_evidence.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = {"items": [], "errors": None, "warnings": []} + plan = evidence.build_change_plan(val, ev, {}) + self.assertEqual(plan["status"], "invalid_evidence") + + def test_valid_true_with_empty_errors_passes(self) -> None: + """valid=True + errors=[] → proceeds normally.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + + +# --------------------------------------------------------------------------- +# Review contracts: Snapshot fingerprint mandatory (R6) +# --------------------------------------------------------------------------- + +class TestSnapshotFingerprintMandatory(unittest.TestCase): + """R6: missing/blank snapshot_fingerprint → invalid_plan before + digest recompute; recomputed matching digest doesn't bypass.""" + + def test_empty_fingerprint_invalid_plan(self) -> None: + """Empty snapshot_fingerprint → invalid_plan.""" + plan = { + "status": "complete", + "issue_key": "TEST-100", + "target": {}, + "inputs": [], + "pr_links": [], + "metadata": {}, + "snapshot_fingerprint": "", + "plan_digest": "some-digest", + } + r = evidence.check_enrichment( + plan, {"key": "TEST-100", "remote_links": []} + ) + self.assertEqual(r["status"], "invalid_plan") + self.assertIn("snapshot_fingerprint", r["reason"]) + + def test_missing_fingerprint_key_invalid_plan(self) -> None: + """Missing snapshot_fingerprint key → invalid_plan.""" + plan = { + "status": "complete", + "issue_key": "TEST-100", + "target": {}, + "inputs": [], + "pr_links": [], + "metadata": {}, + "plan_digest": "some-digest", + } + r = evidence.check_enrichment( + plan, {"key": "TEST-100", "remote_links": []} + ) + self.assertEqual(r["status"], "invalid_plan") + self.assertIn("snapshot_fingerprint", r["reason"]) + + def test_nonstring_fingerprint_invalid_plan(self) -> None: + """Non-string snapshot_fingerprint → invalid_plan.""" + plan = { + "status": "complete", + "issue_key": "TEST-100", + "target": {}, + "inputs": [], + "pr_links": [], + "metadata": {}, + "snapshot_fingerprint": 12345, + "plan_digest": "some-digest", + } + r = evidence.check_enrichment( + plan, {"key": "TEST-100", "remote_links": []} + ) + self.assertEqual(r["status"], "invalid_plan") + + def test_recomputed_digest_no_fingerprint_bypass(self) -> None: + """Matching recomputed digest but no fingerprint → still invalid. + A caller cannot make a fingerprint-less plan ready.""" + # Compute what the digest WOULD be with empty fingerprint + import hashlib + projection = { + "issue_key": "TEST-100", + "target": {}, + "inputs": [], + "pr_links": [], + "metadata": {}, + "snapshot_fingerprint": "", + } + raw = json.dumps( + projection, sort_keys=True, ensure_ascii=True + ) + matching_digest = hashlib.sha256(raw.encode()).hexdigest() + + plan = { + "status": "complete", + "issue_key": "TEST-100", + "target": {}, + "inputs": [], + "pr_links": [], + "metadata": {}, + "snapshot_fingerprint": "", + "plan_digest": matching_digest, + } + r = evidence.check_enrichment( + plan, {"key": "TEST-100", "remote_links": []} + ) + self.assertEqual(r["status"], "invalid_plan") + self.assertIn("snapshot_fingerprint", r["reason"]) + self.assertEqual(r["eligible_links"], []) + self.assertEqual(r["eligible_metadata"], {}) + + +# --------------------------------------------------------------------------- +# Review contract: Complete Phase1→Phase2 lifecycle (R1, R8) +# --------------------------------------------------------------------------- + +class TestPhase1Phase2Lifecycle(unittest.TestCase): + """R1/R8: complete offline Phase1 preview → approved fresh check → + simulated metadata + link receipt using both saved tokens and + pre/post state. Unknown fields not fabricated as empty.""" + + def test_complete_approval_lifecycle(self) -> None: + """Phase1 preview → save tokens → Phase2 check with approved + tokens → simulated writes → readback receipt.""" + # === Phase 1: Preview (no side effects) === + snapshot = _make_docs_snapshot() + val = evidence.validate_issue(snapshot) + self.assertTrue(val["valid"]) + + raw_evidence = _make_evidence_items() + ev = evidence.normalize_evidence(raw_evidence) + self.assertEqual(len(ev["errors"]), 0) + + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "4.19", + "catalog_verified": True, + "catalog_entry": {"id": "419", "name": "4.19"}, + }], + } + target = {"release": "4.19", "target_branch": "main"} + plan = evidence.build_change_plan(val, ev, target, profile) + self.assertEqual(plan["status"], "complete") + self.assertTrue(plan["snapshot_fingerprint"]) + self.assertTrue(plan["plan_digest"]) + + # Phase 1 preview check — no approval tokens + current_prewrite = { + "key": "TEST-100", + "summary": snapshot["summary"], + "description": snapshot["description"], + "remote_links": [], + "fields": {"fixVersions": []}, + } + preview = evidence.check_enrichment(plan, current_prewrite) + self.assertEqual(preview["status"], "ready") + self.assertFalse(preview["stale"]) + + # Save Phase 1 tokens for Phase 2 + saved_baseline = preview["baseline_fingerprint"] + saved_digest = preview["plan_digest"] + self.assertTrue(saved_baseline) + self.assertTrue(saved_digest) + + # === Phase 2: Apply (with approved tokens) === + # Fresh check with BOTH saved tokens — same state → ready + apply_check = evidence.check_enrichment( + plan, current_prewrite, saved_baseline, saved_digest, + ) + self.assertEqual(apply_check["status"], "ready") + self.assertFalse(apply_check["stale"]) + self.assertTrue(len(apply_check["eligible_links"]) > 0) + self.assertIn( + "target_release", apply_check["eligible_metadata"] + ) + + # Verify metadata payload is additive + em = apply_check["eligible_metadata"]["target_release"] + payload_ids = { + p["id"] + for p in em["jira_payload"]["fixVersions"] + } + self.assertIn("419", payload_ids) + + # === Simulated writes → Readback === + readback = { + "key": "TEST-100", + "summary": snapshot["summary"], + "description": snapshot["description"], + "remote_links": [ + { + "url": "https://github.example.invalid" + "/org/repo/pull/101", + }, + { + "url": "https://github.example.invalid" + "/org/repo/pull/102", + }, + ], + "fields": { + "fixVersions": [ + {"id": "419", "name": "4.19"}, + ], + }, + } + receipt = evidence.verify_enrichment( + apply_check, readback, "TEST-100" + ) + self.assertEqual(receipt["status"], "verified") + self.assertTrue(receipt["all_confirmed"]) + + # Check EACH receipt status individually + for lr in receipt["link_receipts"]: + self.assertEqual(lr["status"], "confirmed") + self.assertEqual(lr["operation"], "add") + for mr in receipt["metadata_receipts"]: + self.assertEqual(mr["status"], "confirmed") + self.assertEqual(mr["jira_field"], "fixVersions") + + # === Idempotency: re-check shows existing === + new_baseline = receipt["baseline_fingerprint"] + recheck = evidence.check_enrichment( + plan, readback, new_baseline, saved_digest, + ) + self.assertFalse(recheck["stale"]) + self.assertEqual(len(recheck["eligible_links"]), 0) + self.assertEqual(len(recheck["existing_links"]), 2) + + def test_unknown_fixversions_not_fabricated_empty(self) -> None: + """Unknown fixVersions (missing field) blocks metadata writes; + NOT fabricated as [].""" + snapshot = _make_docs_snapshot() + val = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "4.19", + "catalog_verified": True, + "catalog_entry": {"id": "419", "name": "4.19"}, + }], + } + plan = evidence.build_change_plan( + val, ev, + {"release": "4.19", "target_branch": "main"}, + profile, + ) + # current_state has NO fields key — unknown state + current = { + "key": "TEST-100", + "summary": snapshot["summary"], + "description": snapshot["description"], + "remote_links": [], + } + r = evidence.check_enrichment(plan, current) + # Metadata must NOT be eligible (fail-closed) + self.assertEqual(r["eligible_metadata"], {}) + + def test_stale_plan_digest_blocks_apply(self) -> None: + """Phase2 with stale plan_digest → stale, zero writes.""" + snapshot = _make_docs_snapshot() + val = evidence.validate_issue(snapshot) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + current = { + "key": "TEST-100", + "summary": snapshot["summary"], + "description": snapshot["description"], + "remote_links": [], + } + preview = evidence.check_enrichment(plan, current) + baseline = preview["baseline_fingerprint"] + + # Pass wrong digest → stale + r = evidence.check_enrichment( + plan, current, baseline, "wrong-digest-from-old-plan", + ) + self.assertTrue(r["stale"]) + self.assertEqual(r["status"], "stale") + self.assertEqual(r["eligible_links"], []) + self.assertEqual(r["eligible_metadata"], {}) + + if __name__ == "__main__": unittest.main() From dbdbd3d9f1373b514aafdf17039aeec7c7a84b61 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 23:55:09 +0000 Subject: [PATCH 08/17] fix(docs-handoff): require explicit user approval before external writes The SKILL.md Boundaries paragraph stated that Phase 2 "passes saved tokens" but did not explicitly state that no external write may occur before the user explicitly approves the exact bounded plan. Revise to make the human approval gate the entry contract. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/SKILL.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/skills/docs-handoff/SKILL.md b/skills/docs-handoff/SKILL.md index 16a7daf6..cac23f99 100644 --- a/skills/docs-handoff/SKILL.md +++ b/skills/docs-handoff/SKILL.md @@ -23,7 +23,8 @@ Inputs: one DOCS issue key, a selected release/target, and an optional caller-supplied consumer profile. Outputs: delivery-evidence findings, enrichment plan, and an owning-source narrative proposal when supported. -**Boundaries:** Phase 1 preview saves `baseline_fingerprint` and -`plan_digest` tokens. Phase 2 must pass both as approved references; -stale or invalid requires a fresh preview cycle. Source/narrative -apply is always disabled — changes route through the design workflow. +**Boundaries:** No external write before the user explicitly approves +the exact bounded plan. Phase 2 passes both saved +`baseline_fingerprint` and `plan_digest`; stale or invalid requires +fresh preview and re-approval. Source/narrative apply is always +disabled; summary, description, and source content are never modified. From 0d633fd18862575a6f031d541f0108cca08b598a Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 10 Nov 2026 23:57:54 +0000 Subject: [PATCH 09/17] fix(docs-handoff): schema accuracy for nullable target_branch and uncatalogued finding type - evidence-schema.md: target_branch type is string/null (null normalized to empty), matching the R4 code change - jira-enrichment.md: uncatalogued fields produce uncatalogued_field findings (not "rejected as forbidden"), matching the R2 code change Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/references/evidence-schema.md | 2 +- skills/docs-handoff/references/jira-enrichment.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/docs-handoff/references/evidence-schema.md b/skills/docs-handoff/references/evidence-schema.md index b455db78..22303a99 100644 --- a/skills/docs-handoff/references/evidence-schema.md +++ b/skills/docs-handoff/references/evidence-schema.md @@ -76,7 +76,7 @@ Each gathered PR evidence item must have: |---|---|---| | `pr_url` | string | Yes — supported PR/MR URL (see URL validation) | | `contributor_key` | string | Yes — Jira key of contributing story | -| `target_branch` | string | No — PR target branch | +| `target_branch` | string/null | No — PR target branch (null normalized to empty) | | `state` | string | No — PR state (open, merged, closed) | | `title` | string | No — PR title | | `merged` | boolean/null | No — whether PR is merged | diff --git a/skills/docs-handoff/references/jira-enrichment.md b/skills/docs-handoff/references/jira-enrichment.md index 6f934aff..58712246 100644 --- a/skills/docs-handoff/references/jira-enrichment.md +++ b/skills/docs-handoff/references/jira-enrichment.md @@ -39,7 +39,7 @@ that are valid strings but not in the allowed catalog produce ## Allowed Metadata Catalog Only fields in this catalog may be proposed as metadata changes. -Uncatalogued fields are rejected as forbidden: +Uncatalogued fields are rejected with `uncatalogued_field` findings: | Canonical Name | Jira Binding | Requirement | |---|---|---| From bbbd4ec5956fcd4a7885aa70eb2523fdbb1103a7 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Wed, 11 Nov 2026 00:50:56 +0000 Subject: [PATCH 10/17] fix(docs-handoff): remote_links validation, profile_fields casing, metadata gap reporting, SKILL.md size F1: check_enrichment now rejects missing/non-list/malformed remote_links before baseline computation instead of silently defaulting to []. Non-dict entries and entries without string URLs are caught with clear reasons. F2: profile_fields forbidden check now uses case-insensitive comparison (.lower()) while preserving original field name in findings, preventing bypass via Summary/fixVersions/COMMENTS casing variants. F3: Metadata proposals that cannot become eligible writes now appear in skipped_metadata with explicit gap reasons (missing fields, not verified, null field value) instead of being silently dropped. F4: SKILL.md trimmed to 28 lines (under 30) by moving approval token details already documented in references. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/SKILL.md | 6 +- skills/docs-handoff/scripts/evidence.py | 98 ++++- skills/docs-handoff/scripts/test_evidence.py | 354 +++++++++++++++++++ 3 files changed, 450 insertions(+), 8 deletions(-) diff --git a/skills/docs-handoff/SKILL.md b/skills/docs-handoff/SKILL.md index cac23f99..2767bbb3 100644 --- a/skills/docs-handoff/SKILL.md +++ b/skills/docs-handoff/SKILL.md @@ -24,7 +24,5 @@ caller-supplied consumer profile. Outputs: delivery-evidence findings, enrichment plan, and an owning-source narrative proposal when supported. **Boundaries:** No external write before the user explicitly approves -the exact bounded plan. Phase 2 passes both saved -`baseline_fingerprint` and `plan_digest`; stale or invalid requires -fresh preview and re-approval. Source/narrative apply is always -disabled; summary, description, and source content are never modified. +the exact bounded plan; stale or invalid requires fresh preview and +re-approval. Source/narrative apply is always disabled. diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 0d24c2b0..32b134d5 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -1365,6 +1365,7 @@ def check_enrichment( "existing_links": [], "skipped_links": [], "eligible_metadata": {}, + "skipped_metadata": {}, "forbidden_fields": [], "plan_digest": plan.get("plan_digest", ""), "baseline_fingerprint": "", @@ -1456,9 +1457,47 @@ def check_enrichment( "reason": "plan has changed since approval", } - existing_links = current_state.get("remote_links", []) + # Validate remote_links: must be present as a list of dicts with + # string URLs. Missing/non-list/malformed → invalid, not default + # to []. An explicit [] is valid (known empty). + if "remote_links" not in current_state: + return { + **_empty_result, + "status": "invalid", + "reason": "current state missing remote_links", + } + + existing_links = current_state["remote_links"] if not isinstance(existing_links, list): - existing_links = [] + return { + **_empty_result, + "status": "invalid", + "reason": ( + f"remote_links must be a list, " + f"got {type(existing_links).__name__}" + ), + } + + for idx, link in enumerate(existing_links): + if not isinstance(link, dict): + return { + **_empty_result, + "status": "invalid", + "reason": ( + f"remote_links[{idx}] must be a dict, " + f"got {type(link).__name__}" + ), + } + link_url = link.get("url") + if not isinstance(link_url, str): + return { + **_empty_result, + "status": "invalid", + "reason": ( + f"remote_links[{idx}].url must be a string, " + f"got {type(link_url).__name__}" + ), + } # Canonical link set for comparison (sorted for stable fingerprint) existing_urls: set[str] = set() @@ -1554,6 +1593,7 @@ def check_enrichment( # Metadata eligibility (D: catalog_verified + validated catalog_entry) eligible_metadata: dict[str, Any] = {} + skipped_metadata: dict[str, Any] = {} forbidden_found: list[str] = [] for field, spec in plan.get("metadata", {}).items(): @@ -1564,11 +1604,22 @@ def check_enrichment( forbidden_found.append(f"uncatalogued:{field}") continue if not isinstance(spec, dict): + skipped_metadata[field] = { + "reason": "metadata spec must be a dict", + } continue if not spec.get("catalog_verified", False): + skipped_metadata[field] = { + "reason": "not catalog_verified", + "value": spec.get("value"), + } continue entry = spec.get("catalog_entry") if not isinstance(entry, dict): + skipped_metadata[field] = { + "reason": "catalog_entry must be a dict", + "value": spec.get("value"), + } continue cat = _METADATA_CATALOG_V1.get(field, {}) schema = cat.get("jira_item_schema", {}) @@ -1576,9 +1627,20 @@ def check_enrichment( name_key = schema.get("name_field", "name") # Validate catalog_entry has non-empty id and name if not entry.get(id_key) or not entry.get(name_key): + skipped_metadata[field] = { + "reason": "catalog_entry missing id or name", + "value": spec.get("value"), + } continue # Name must match requested value if entry.get(name_key) != spec.get("value"): + skipped_metadata[field] = { + "reason": ( + f"catalog_entry name '{entry.get(name_key)}' " + f"does not match requested '{spec.get('value')}'" + ), + "value": spec.get("value"), + } continue # Build additive payload from actual current field state. # Fail closed: unknown/malformed field state must NOT default @@ -1589,12 +1651,30 @@ def check_enrichment( # Fail closed: current_fields must be a dict with the field # present. Missing fields key → no metadata write. if not isinstance(current_fields, dict): + skipped_metadata[field] = { + "reason": "current_state.fields is not a dict", + "value": spec.get("value"), + } continue if jira_fld not in current_fields: + skipped_metadata[field] = { + "reason": ( + f"current_state.fields.{jira_fld} not present " + f"— unknown value cannot be defaulted to []" + ), + "value": spec.get("value"), + } continue before = current_fields[jira_fld] # Fail closed: field value must be a list (null/str/int → skip) if not isinstance(before, list): + skipped_metadata[field] = { + "reason": ( + f"current_state.fields.{jira_fld} must be a " + f"list, got {type(before).__name__}" + ), + "value": spec.get("value"), + } continue # Fail closed: every entry must be a dict with a valid id before_valid = True @@ -1606,6 +1686,13 @@ def check_enrichment( before_valid = False break if not before_valid: + skipped_metadata[field] = { + "reason": ( + f"current_state.fields.{jira_fld} contains " + f"malformed entries" + ), + "value": spec.get("value"), + } continue # Check if already present — no-op if so already = any( @@ -1633,11 +1720,13 @@ def check_enrichment( "already_present": already, } - # Profile forbidden fields + # Profile forbidden fields (case-insensitive check, preserve + # original name in finding) profile_fields = current_state.get("profile_fields", {}) if isinstance(profile_fields, dict): for pf in profile_fields: - if pf in _FORBIDDEN_FIELDS: + pf_lower = pf.lower() if isinstance(pf, str) else "" + if pf_lower in _FORBIDDEN_FIELDS: forbidden_found.append(f"profile:{pf}") return { @@ -1646,6 +1735,7 @@ def check_enrichment( "existing_links": existing, "skipped_links": skipped, "eligible_metadata": {} if stale else eligible_metadata, + "skipped_metadata": skipped_metadata, "forbidden_fields": forbidden_found, "plan_digest": plan.get("plan_digest", ""), "baseline_fingerprint": baseline_fp, diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index a4387a43..07af6ace 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -3708,5 +3708,359 @@ def test_stale_plan_digest_blocks_apply(self) -> None: self.assertEqual(r["eligible_metadata"], {}) +# --------------------------------------------------------------------------- +# F1: remote_links validation in check_enrichment +# --------------------------------------------------------------------------- + +class TestRemoteLinksValidation(unittest.TestCase): + """F1: check_enrichment must reject missing/non-list/malformed + remote_links BEFORE baseline/eligibility. Known empty [] is valid.""" + + def _build_plan(self): + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + + def _make_current(self, **overrides): + snap = _make_docs_snapshot() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + } + current.update(overrides) + return current + + def test_missing_remote_links_rejected(self) -> None: + """current_state without remote_links → invalid.""" + plan = self._build_plan() + current = self._make_current() # no remote_links key + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("remote_links", r["reason"]) + self.assertEqual(r["eligible_links"], []) + self.assertEqual(r["eligible_metadata"], {}) + + def test_null_remote_links_rejected(self) -> None: + """remote_links=None → invalid.""" + plan = self._build_plan() + current = self._make_current(remote_links=None) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("list", r["reason"]) + + def test_dict_remote_links_rejected(self) -> None: + """remote_links={} (dict, not list) → invalid.""" + plan = self._build_plan() + current = self._make_current(remote_links={}) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("dict", r["reason"]) + + def test_string_remote_links_rejected(self) -> None: + """remote_links='some-url' → invalid.""" + plan = self._build_plan() + current = self._make_current(remote_links="some-url") + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("str", r["reason"]) + + def test_non_dict_entries_rejected(self) -> None: + """remote_links=['url1', 'url2'] (strings not dicts) → invalid.""" + plan = self._build_plan() + current = self._make_current( + remote_links=["https://example.invalid/1"] + ) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("remote_links[0]", r["reason"]) + + def test_entry_without_url_rejected(self) -> None: + """remote_links=[{"title": "x"}] (missing url) → invalid.""" + plan = self._build_plan() + current = self._make_current( + remote_links=[{"title": "link without url"}] + ) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("url", r["reason"]) + + def test_entry_with_non_string_url_rejected(self) -> None: + """remote_links=[{"url": 123}] → invalid.""" + plan = self._build_plan() + current = self._make_current( + remote_links=[{"url": 123}] + ) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("url", r["reason"]) + + def test_empty_list_valid(self) -> None: + """remote_links=[] (known empty) → valid, status ready.""" + plan = self._build_plan() + current = self._make_current(remote_links=[]) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "ready") + self.assertFalse(r["stale"]) + + def test_known_existing_link_valid(self) -> None: + """Legitimate existing link passes validation.""" + plan = self._build_plan() + current = self._make_current( + remote_links=[{ + "url": "https://github.example.invalid/org/repo/pull/101", + }] + ) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "ready") + self.assertTrue(len(r["existing_links"]) > 0) + + def test_bad_remote_links_blocks_metadata_writes(self) -> None: + """Metadata with catalog entry but bad remote_links → ZERO + eligible across both links and metadata.""" + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + }], + } + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + # Bad remote_links: not a list + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": "not-a-list", + "fields": {"fixVersions": []}, + } + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertEqual(r["eligible_links"], []) + self.assertEqual(r["eligible_metadata"], {}) + + +# --------------------------------------------------------------------------- +# F2: profile_fields case-insensitive forbidden check +# --------------------------------------------------------------------------- + +class TestProfileFieldsCaseInsensitive(unittest.TestCase): + """F2: profile_fields loop must check .lower() while preserving + original name in finding; no TypeError on non-string keys.""" + + def _build_plan(self): + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + + def _make_current(self, profile_fields): + snap = _make_docs_snapshot() + return { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + "profile_fields": profile_fields, + } + + def test_uppercase_summary_forbidden(self) -> None: + """'Summary' (capitalized) → detected as forbidden.""" + plan = self._build_plan() + current = self._make_current({"Summary": True}) + r = evidence.check_enrichment(plan, current) + self.assertIn("profile:Summary", r["forbidden_fields"]) + + def test_camelcase_fixversions_forbidden(self) -> None: + """'fixVersions' (camelCase) → detected as forbidden.""" + plan = self._build_plan() + current = self._make_current({"fixVersions": True}) + r = evidence.check_enrichment(plan, current) + self.assertIn("profile:fixVersions", r["forbidden_fields"]) + + def test_allcaps_comments_forbidden(self) -> None: + """'COMMENTS' (all caps) → detected as forbidden.""" + plan = self._build_plan() + current = self._make_current({"COMMENTS": True}) + r = evidence.check_enrichment(plan, current) + self.assertIn("profile:COMMENTS", r["forbidden_fields"]) + + def test_lowercase_still_works(self) -> None: + """'summary' (lowercase) → still detected as before.""" + plan = self._build_plan() + current = self._make_current({"summary": True}) + r = evidence.check_enrichment(plan, current) + self.assertIn("profile:summary", r["forbidden_fields"]) + + def test_non_forbidden_not_reported(self) -> None: + """Non-forbidden field not in forbidden_fields.""" + plan = self._build_plan() + current = self._make_current({"custom_field": True}) + r = evidence.check_enrichment(plan, current) + forbidden_profiles = [ + f for f in r["forbidden_fields"] + if f.startswith("profile:") + ] + self.assertEqual(forbidden_profiles, []) + + def test_non_string_key_no_typeerror(self) -> None: + """Non-string key in profile_fields → no crash.""" + plan = self._build_plan() + current = self._make_current({42: True, None: False}) + r = evidence.check_enrichment(plan, current) + # Should not raise TypeError; non-string keys don't match + self.assertEqual(r["status"], "ready") + + +# --------------------------------------------------------------------------- +# F3: Protocol conditional field contract — skipped_metadata +# --------------------------------------------------------------------------- + +class TestSkippedMetadata(unittest.TestCase): + """F3: When metadata is in the plan but field state prevents a + write, skipped_metadata must report the gap explicitly.""" + + def _build_plan_with_catalog(self): + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + }], + } + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + + def _make_current(self, **overrides): + snap = _make_docs_snapshot() + current = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + } + current.update(overrides) + return current + + def test_missing_fields_reports_gap(self) -> None: + """No fields key → target_release in skipped_metadata.""" + plan = self._build_plan_with_catalog() + current = self._make_current() # no fields key + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "ready") + self.assertEqual(r["eligible_metadata"], {}) + self.assertIn("target_release", r["skipped_metadata"]) + self.assertIn("not present", r["skipped_metadata"]["target_release"]["reason"]) + + def test_missing_fixversions_field_reports_gap(self) -> None: + """fields={} (no fixVersions) → gap reported.""" + plan = self._build_plan_with_catalog() + current = self._make_current(fields={}) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + self.assertIn("target_release", r["skipped_metadata"]) + self.assertIn("not present", r["skipped_metadata"]["target_release"]["reason"]) + + def test_null_fixversions_reports_gap(self) -> None: + """fields.fixVersions=None → gap reported.""" + plan = self._build_plan_with_catalog() + current = self._make_current(fields={"fixVersions": None}) + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["eligible_metadata"], {}) + self.assertIn("target_release", r["skipped_metadata"]) + self.assertIn("list", r["skipped_metadata"]["target_release"]["reason"]) + + def test_known_empty_fixversions_eligible(self) -> None: + """fields.fixVersions=[] → eligible, not skipped.""" + plan = self._build_plan_with_catalog() + current = self._make_current(fields={"fixVersions": []}) + r = evidence.check_enrichment(plan, current) + self.assertIn("target_release", r["eligible_metadata"]) + self.assertNotIn("target_release", r.get("skipped_metadata", {})) + + def test_not_catalog_verified_skipped(self) -> None: + """Metadata without catalog_verified → skipped with reason.""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + # Default plan has target_release with catalog_verified=False + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + current = self._make_current(fields={"fixVersions": []}) + r = evidence.check_enrichment(plan, current) + self.assertNotIn("target_release", r["eligible_metadata"]) + self.assertIn("target_release", r["skipped_metadata"]) + self.assertIn( + "not catalog_verified", + r["skipped_metadata"]["target_release"]["reason"], + ) + + def test_skipped_metadata_in_invalid_result(self) -> None: + """Early invalid returns also have skipped_metadata key.""" + plan = self._build_plan_with_catalog() + current = {"remote_links": []} # missing key + r = evidence.check_enrichment(plan, current) + self.assertEqual(r["status"], "invalid") + self.assertIn("skipped_metadata", r) + self.assertEqual(r["skipped_metadata"], {}) + + def test_stale_preserves_skipped_metadata(self) -> None: + """Stale result shows skipped_metadata even with zero + eligible.""" + plan = self._build_plan_with_catalog() + current = self._make_current(fields={}) + # Force stale via wrong baseline + r = evidence.check_enrichment( + plan, current, "wrong-baseline", + ) + self.assertTrue(r["stale"]) + self.assertEqual(r["eligible_metadata"], {}) + # skipped_metadata still reported (field was missing) + self.assertIn("target_release", r["skipped_metadata"]) + + +# --------------------------------------------------------------------------- +# F4: SKILL.md line count contract +# --------------------------------------------------------------------------- + +class TestSkillMdLineCount(unittest.TestCase): + """F4: SKILL.md must be under 30 lines including frontmatter.""" + + def test_skill_md_under_30_lines(self) -> None: + import pathlib + skill_md = ( + pathlib.Path(__file__).parent.parent / "SKILL.md" + ) + lines = skill_md.read_text().splitlines() + self.assertLess( + len(lines), 30, + f"SKILL.md has {len(lines)} lines, must be under 30", + ) + + if __name__ == "__main__": unittest.main() From 29e24a47d1f25035c2405326c31d0d0e66ff245a Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Wed, 11 Nov 2026 01:01:02 +0000 Subject: [PATCH 11/17] fix(docs-handoff): wire skipped_metadata into protocol references, add consumer contract tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 5: document fields.fixVersions as conditional requirement when metadata is proposed; missing data → skipped_metadata, not silent skip. Link-only flow still does not require fields. Step 6: preview must report skipped_metadata reasons and forbidden_fields alongside eligible writes, distinguishing proposals from approved operations. Phase 2 apply: report skipped entries; status ready with non-empty skipped_metadata means proposed metadata was not eligible for writing. Evidence-schema: document remote_links entry validation (dict with string url), profile_fields, and full check-enrichment output shape including skipped_metadata dict and forbidden_fields list. Consumer contract tests verify metadata-requested+missing-field→skipped, known-empty-eligible, link-only-no-fields-required, and output shape consistency across valid and invalid results. Co-Authored-By: Claude Opus 4.6 --- .../references/evidence-schema.md | 39 +++++- .../references/handoff-protocol.md | 19 ++- skills/docs-handoff/scripts/test_evidence.py | 112 ++++++++++++++++++ 3 files changed, 165 insertions(+), 5 deletions(-) diff --git a/skills/docs-handoff/references/evidence-schema.md b/skills/docs-handoff/references/evidence-schema.md index 22303a99..75d828b6 100644 --- a/skills/docs-handoff/references/evidence-schema.md +++ b/skills/docs-handoff/references/evidence-schema.md @@ -138,10 +138,41 @@ The `check-enrichment` subcommand requires a `current_state` object: | `key` | string | Yes | Must match plan's issue key | | `summary` | string | Yes | For snapshot drift detection | | `description` | string | Yes | Raw description for drift detection | -| `remote_links` | array | Yes | Current Remote Links | +| `remote_links` | array | Yes | Current Remote Links — each entry must be a dict with a string `url` | | `fields` | object | No | Actual Jira field values | -| `fields.fixVersions` | array | No | Required for metadata writes; missing/null/non-list fails closed | +| `fields.fixVersions` | array | Conditional | Required when plan proposes metadata; missing/null/non-list fails closed | +| `profile_fields` | object | No | Extra control metadata from consumer profile | + +`remote_links` is validated before baseline computation: missing, +non-list, non-dict entries, or entries without a string `url` produce +an `invalid` result with zero eligible links and zero eligible +metadata. An explicit empty list (`[]`) is valid (known empty). Unknown actual field values (missing `fields` or `fields.fixVersions`) -must NOT be fabricated as `[]` — they block metadata writes. An -explicit empty list (`[]`) is a known absence and permits writes. +must NOT be fabricated as `[]` — they block metadata writes and appear +in `skipped_metadata` with the reason. An explicit empty list (`[]`) +is a known absence and permits writes. When the plan has no metadata +proposals, `fields` is not required (link-only flow). + +## Check-Enrichment Output + +The `check-enrichment` result contains: + +| Field | Type | Notes | +|---|---|---| +| `status` | string | `ready`, `stale`, `invalid`, `invalid_plan`, `wrong_issue` | +| `eligible_links` | array | PR Remote Links approved for writing (empty when stale/invalid) | +| `existing_links` | array | Proposed links already present | +| `skipped_links` | array | Proposed links rejected with reason | +| `eligible_metadata` | object | Metadata fields approved for writing (empty when stale/invalid) | +| `skipped_metadata` | object | Metadata proposals not eligible — keyed by field name, each `{"reason": "...", "value": ...}` | +| `forbidden_fields` | array | Policy-blocked field names (distinct from skipped_metadata) | +| `baseline_fingerprint` | string | Approval token for Phase 2 | +| `plan_digest` | string | Approval token for Phase 2 | +| `stale` | boolean | Whether baseline or snapshot drifted | + +`skipped_metadata` reports why each proposed metadata field could not +become an eligible write (e.g. `"not catalog_verified"`, +`"current_state.fields.fixVersions not present"`, `"must be a list"`). +`forbidden_fields` is separate — it captures policy-blocked fields +and uncatalogued entries, not conditional gaps. diff --git a/skills/docs-handoff/references/handoff-protocol.md b/skills/docs-handoff/references/handoff-protocol.md index a9b269cc..b9fc3560 100644 --- a/skills/docs-handoff/references/handoff-protocol.md +++ b/skills/docs-handoff/references/handoff-protocol.md @@ -51,6 +51,13 @@ Provide the plan and current issue state (with `key`, `summary`, has keys: `plan`, `current_state`, `approved_baseline` (optional), `approved_plan_digest` (optional). +When the plan proposes metadata (e.g. `target_release`), include +`current_state.fields` with the actual Jira field values. +`fields.fixVersions` is required for metadata writes — missing or +unknown values are NOT defaulted to `[]` and will appear in +`skipped_metadata` with the reason the write was blocked. A +link-only flow (no metadata proposed) does not require `fields`. + Save the result's `baseline_fingerprint` and `plan_digest` — these are the approval tokens for Phase 2. @@ -61,7 +68,12 @@ Show the complete bounded preview including: - Per-input evidence status with PR references - Findings (missing evidence, unresolved refs, non-contributor evidence) - Eligible PR Remote Links (contributor PRs only) -- Metadata proposals with catalog verification status +- Metadata: eligible writes (`eligible_metadata`) vs skipped proposals + (`skipped_metadata` with per-field reasons) — do not present skipped + proposals as eligible operations or declare full handoff readiness + while metadata gaps remain +- Forbidden fields (`forbidden_fields`) — distinct from skipped + metadata; these are policy-blocked fields, not conditional gaps - Source-change proposal (disabled, with proposed wording) - Consumer readiness state @@ -75,11 +87,16 @@ Show the complete bounded preview including: 3. If stale or invalid, stop and regenerate with a fresh preview cycle — do not proceed with writes. 4. Apply eligible links and metadata via approved runtime capabilities. + Report any `skipped_metadata` entries — these are proposed + operations that could not become writes due to missing or + malformed field state, not silent omissions. 5. Re-read the issue (POSTWRITE state only in `--readback`, keep PREWRITE `current_state` immutable); run `check-enrichment --readback` for receipts. 6. Interpret `all_confirmed` AND each individual receipt status — exit code 0 alone does not mean all operations succeeded. + Status `ready` with non-empty `skipped_metadata` means some + proposed metadata was not eligible for writing. ### On Rejection diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 07af6ace..4031a61a 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -4062,5 +4062,117 @@ def test_skill_md_under_30_lines(self) -> None: ) +# --------------------------------------------------------------------------- +# F3 consumer contract: metadata requested + field state → output shape +# --------------------------------------------------------------------------- + +class TestMetadataConsumerContract(unittest.TestCase): + """Scoped consumer contract: when metadata is proposed, the result + clearly distinguishes eligible writes from skipped proposals. + Link-only flow (no metadata) does not require fields.""" + + def _plan_with_catalog(self): + """Plan with catalog-verified metadata proposal.""" + profile = { + "allowed_fields": [{ + "name": "target_release", + "value": "v1", + "catalog_verified": True, + "catalog_entry": {"id": "7", "name": "v1"}, + }], + } + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + profile, + ) + + def _plan_link_only(self): + """Plan with no metadata (link-only flow).""" + snap = _make_docs_snapshot() + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence(_make_evidence_items()) + return evidence.build_change_plan( + val, ev, + {"release": "", "target_branch": "main"}, + ) + + def _current(self, **overrides): + snap = _make_docs_snapshot() + base = { + "key": "TEST-100", + "summary": snap["summary"], + "description": snap["description"], + "remote_links": [], + } + base.update(overrides) + return base + + def test_metadata_requested_missing_field_skipped(self) -> None: + """Metadata proposed + no fields → skipped_metadata with reason, + zero eligible_metadata, status still ready.""" + plan = self._plan_with_catalog() + r = evidence.check_enrichment(plan, self._current()) + self.assertEqual(r["status"], "ready") + self.assertEqual(r["eligible_metadata"], {}) + self.assertIn("target_release", r["skipped_metadata"]) + reason = r["skipped_metadata"]["target_release"]["reason"] + self.assertTrue(len(reason) > 0) + + def test_metadata_requested_known_empty_eligible(self) -> None: + """Metadata proposed + fields.fixVersions=[] → eligible, not + skipped.""" + plan = self._plan_with_catalog() + r = evidence.check_enrichment( + plan, self._current(fields={"fixVersions": []}), + ) + self.assertEqual(r["status"], "ready") + self.assertIn("target_release", r["eligible_metadata"]) + self.assertNotIn("target_release", r["skipped_metadata"]) + + def test_link_only_no_fields_required(self) -> None: + """Link-only plan (no metadata) + no fields key → ready with + eligible links, empty skipped_metadata.""" + plan = self._plan_link_only() + r = evidence.check_enrichment(plan, self._current()) + self.assertEqual(r["status"], "ready") + self.assertTrue(len(r["eligible_links"]) > 0) + self.assertEqual(r["eligible_metadata"], {}) + self.assertEqual(r["skipped_metadata"], {}) + + def test_cli_check_enrichment_output_shape(self) -> None: + """Consumer contract: check-enrichment result always has + skipped_metadata dict, forbidden_fields list, and both + approval tokens regardless of status.""" + plan = self._plan_with_catalog() + # Valid current state + r = evidence.check_enrichment( + plan, self._current(fields={"fixVersions": []}), + ) + for key in ( + "eligible_links", "existing_links", "skipped_links", + "eligible_metadata", "skipped_metadata", + "forbidden_fields", "baseline_fingerprint", + "plan_digest", "stale", + ): + self.assertIn(key, r, f"missing output key: {key}") + self.assertIsInstance(r["skipped_metadata"], dict) + self.assertIsInstance(r["forbidden_fields"], list) + + def test_invalid_result_has_full_shape(self) -> None: + """Even invalid/early-return results carry skipped_metadata + and forbidden_fields for consumer consistency.""" + plan = self._plan_with_catalog() + # Missing key → invalid + r = evidence.check_enrichment(plan, {"remote_links": []}) + self.assertEqual(r["status"], "invalid") + self.assertIn("skipped_metadata", r) + self.assertIn("forbidden_fields", r) + self.assertEqual(r["skipped_metadata"], {}) + + if __name__ == "__main__": unittest.main() From e408720a80d30a2ace7b125785b8b7734b31bb2e Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Sat, 14 Nov 2026 15:45:53 +0000 Subject: [PATCH 12/17] fix(docs-handoff): fence-aware sections, GitLab URL canon, target honesty, orphan inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R1: Make _extract_sections fence-aware (skip ## headings inside ```/~~~ code blocks) and scope _extract_contributing_stories to the Design Reference section only — both in validate_issue and check_enrichment fresh fingerprint. Prevents fenced examples from widening contributor extraction and link eligibility. R2: Canonicalize legacy GitLab /merge_requests/N URLs to modern /-/merge_requests/N form in validate_pr_url. Same MR via either URL format now produces identical normalized strings for deduplication in normalize_evidence, check_enrichment existing-link comparison, and verify_enrichment readback matching. R3: Add merged_unverified_target status for merged PRs with missing or empty target_branch when the plan specifies an expected target. Prevents false "merged into expected branch" claims when target verification is impossible. Visible target note explains the gap. R4: Retain Documentation Inputs bullets before any contributor header as items with contributor=None instead of silently dropping them. Emit warning about unattributed bullets. Orphan items flow through plan inputs and render as visible no_evidence entries. Co-Authored-By: Claude Opus 4.6 --- .../references/evidence-schema.md | 3 + skills/docs-handoff/scripts/evidence.py | 76 +- skills/docs-handoff/scripts/test_evidence.py | 846 ++++++++++++++++++ 3 files changed, 910 insertions(+), 15 deletions(-) diff --git a/skills/docs-handoff/references/evidence-schema.md b/skills/docs-handoff/references/evidence-schema.md index 75d828b6..7b574f6b 100644 --- a/skills/docs-handoff/references/evidence-schema.md +++ b/skills/docs-handoff/references/evidence-schema.md @@ -101,6 +101,9 @@ receives a per-PR evidence status — not a blanket delivery claim: expected branch - `merged_wrong_target` — contributor has a merged PR but it targets a different branch than expected +- `merged_unverified_target` — contributor has a merged PR but the + evidence target branch is unknown; merge to the expected branch cannot + be verified - `in_progress` — contributor has PRs but none merged - `no_evidence` — no PR evidence found for this contributor - `evidence_unverified` — PRs exist but merge status is unknown (null) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 32b134d5..56c4198a 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -150,16 +150,26 @@ def validate_pr_url(url: str, label: str = "pr_url") -> str: clean_path = parsed.path.rstrip("/") clean = f"{parsed.scheme}://{parsed.hostname}{clean_path}" - # Verify the path matches a supported PR/MR structure - if not (_GITHUB_PR_PATH.match(clean_path) - or _GITLAB_MR_PATH.match(clean_path) - or _GITLAB_MR_LEGACY_PATH.match(clean_path)): - raise ValueError( - f"{label}: not a recognized PR/MR URL " - f"(requires /owner/repo/pull/N or " - f"/owner/repo/-/merge_requests/N): {url}" + # Verify the path matches a supported PR/MR structure. + # Legacy GitLab URLs are canonicalized to modern form so that + # /group/repo/merge_requests/N and /group/repo/-/merge_requests/N + # produce the same normalized string for deduplication. + if _GITHUB_PR_PATH.match(clean_path): + return clean + if _GITLAB_MR_PATH.match(clean_path): + return clean + m = _GITLAB_MR_LEGACY_PATH.match(clean_path) + if m: + canonical_path = ( + f"/{m.group(1)}/{m.group(2)}" + f"/-/merge_requests/{m.group(3)}" ) - return clean + return f"{parsed.scheme}://{parsed.hostname}{canonical_path}" + raise ValueError( + f"{label}: not a recognized PR/MR URL " + f"(requires /owner/repo/pull/N or " + f"/owner/repo/-/merge_requests/N): {url}" + ) def extract_pr_identity(url: str) -> dict[str, str]: @@ -260,14 +270,27 @@ def extract_story_refs(text: str) -> list[str]: def _extract_sections(text: str) -> dict[str, str]: - """Extract ## sections from rendered Jira markdown description.""" + """Extract ## sections from rendered Jira markdown description. + + Headings inside fenced code blocks (backtick or tilde fences) are + treated as literal content, not section boundaries. + """ sections: dict[str, str] = {} current: str | None = None lines: list[str] = [] + in_fence = False for raw in text.splitlines(): stripped = raw.strip() - if stripped.startswith("## "): + + # Toggle fence state on ``` or ~~~ lines + if stripped.startswith("```") or stripped.startswith("~~~"): + in_fence = not in_fence + if current is not None: + lines.append(raw) + continue + + if not in_fence and stripped.startswith("## "): if current is not None: sections[current] = "\n".join(lines).strip() current = stripped[3:].strip() @@ -366,7 +389,7 @@ def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: continue bullet_match = re.match(r"[-*+]\s+(.*)", stripped) - if bullet_match and current_contributor is not None: + if bullet_match: raw_text = bullet_match.group(1).strip() marker = "planned" open_ref = None @@ -433,7 +456,9 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: for ms in sorted(missing_sections): errors.append(f"missing required section: {ms}") - contributors = _extract_contributing_stories(description) + contributors = _extract_contributing_stories( + sections.get("Design Reference", "") + ) unresolved = [c for c in contributors if c.startswith("story-")] if unresolved: for u in unresolved: @@ -450,6 +475,15 @@ def validate_issue(snapshot: dict[str, Any]) -> dict[str, Any]: "Documentation Inputs section exists but no structured " "items parsed" ) + orphan_inputs = [ + di for di in documentation_inputs + if di.get("contributor") is None + ] + if orphan_inputs: + warnings.append( + f"Documentation Inputs: {len(orphan_inputs)} " + f"unattributed bullet(s) before first contributor header" + ) return { "valid": len(errors) == 0, @@ -964,6 +998,13 @@ def build_change_plan( f"PR {ev['pr_url']} targets {ev_target}, " f"expected {target_branch}" ) + elif target_branch and not ev_target: + pr_status = "merged_unverified_target" + target_notes.append( + f"PR {ev['pr_url']} merged but " + f"target_branch unknown; cannot verify " + f"merge to expected '{target_branch}'" + ) else: pr_status = "merged" elif ev_merged is False: @@ -985,6 +1026,8 @@ def build_change_plan( status = "merged_pr_exists" elif "merged_wrong_target" in pr_statuses: status = "merged_wrong_target" + elif "merged_unverified_target" in pr_statuses: + status = "merged_unverified_target" elif "not_merged" in pr_statuses: status = "in_progress" else: @@ -1558,8 +1601,11 @@ def check_enrichment( elif not isinstance(current_summary, str): stale = True else: + current_sections = _extract_sections(current_description) current_contribs = sorted( - _extract_contributing_stories(current_description) + _extract_contributing_stories( + current_sections.get("Design Reference", "") + ) ) current_deps = sorted( _extract_dependencies(current_description) @@ -2002,7 +2048,7 @@ def render_handoff(plan: dict[str, Any]) -> str: parts.append("|---|---|---|---|") for inp in inputs: text = inp.get("text", "")[:60] - contrib = inp.get("contributor", "") + contrib = inp.get("contributor") or "" status = inp.get("status", "unknown") ev_refs = inp.get("evidence_refs", []) ev_summary = ", ".join( diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 4031a61a..2ce88752 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -628,6 +628,7 @@ def test_per_input_honest_status(self) -> None: for inp in plan["inputs"]: self.assertIn(inp["status"], ( "merged_pr_exists", "merged_wrong_target", + "merged_unverified_target", "in_progress", "no_evidence", "evidence_unverified", "conflicting", )) @@ -4174,5 +4175,850 @@ def test_invalid_result_has_full_shape(self) -> None: self.assertEqual(r["skipped_metadata"], {}) +# --------------------------------------------------------------------------- +# R1: Fence-aware section parser + scoped contributor extraction +# --------------------------------------------------------------------------- + +class TestFenceAwareSections(unittest.TestCase): + """R1: Headings inside fenced code blocks are literal content, + not section boundaries. Contributing stories extraction is scoped + to the Design Reference section only.""" + + def test_fenced_heading_not_section_boundary(self) -> None: + """## heading inside ``` fence must not split sections.""" + desc = textwrap.dedent("""\ + ## Documentation Scope + + Example format: + + ```markdown + ## Design Reference + Contributing stories: TEST-888 + ``` + + Scope continues after fence. + + ## Documentation Inputs + + **TEST-201 — Add widget:** + - New CLI command + + ## Acceptance Criteria + + - [ ] Done + + ## Design Reference + + Contributing stories: TEST-201 + """) + sections = evidence._extract_sections(desc) + # Documentation Scope should include content after fenced block + scope = sections.get("Documentation Scope", "") + self.assertIn("Scope continues after fence", scope) + # Design Reference should be the REAL one, not the fenced one + dr = sections.get("Design Reference", "") + self.assertIn("TEST-201", dr) + self.assertNotIn("TEST-888", dr) + + def test_fenced_contributing_stories_not_extracted(self) -> None: + """Contributing stories inside a fenced code block in + Documentation Scope must not appear in extracted contributors.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Done + + ## Documentation Scope + + Example DOCS format: + + ``` + Contributing stories: TEST-999 + ``` + + ## Documentation Inputs + + **TEST-201 — Add widget:** + - New CLI command + + ## Design Reference + + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + result = evidence.validate_issue(snap) + self.assertTrue(result["valid"]) + self.assertIn("TEST-201", result["contributors"]) + self.assertNotIn( + "TEST-999", result["contributors"], + "Fenced example must not widen contributor extraction", + ) + + def test_fenced_heading_contributing_not_extracted(self) -> None: + """Fenced ## Design Reference heading with Contributing stories + must not create contributors.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Done + + ## Documentation Scope + + Example: + + ```markdown + ## Design Reference + Contributing stories: TEST-888 + ``` + + ## Documentation Inputs + + **TEST-201 — Config:** + - New command + + ## Design Reference + + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + result = evidence.validate_issue(snap) + self.assertTrue(result["valid"]) + self.assertIn("TEST-201", result["contributors"]) + self.assertNotIn( + "TEST-888", result["contributors"], + "Fenced heading must not create contributors", + ) + + def test_fenced_contributor_pr_not_eligible(self) -> None: + """PR attributed to fenced-only contributor must NOT become + an eligible link.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Done + + ## Documentation Scope + + ``` + Contributing stories: TEST-999 + ``` + + ## Documentation Inputs + + **TEST-201 — Widget:** + - New CLI + + ## Design Reference + + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), + "contributor_key": "TEST-999", + "merged": True, + "target_branch": "main", + }, + ]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + link_urls = {l["url"] for l in plan["pr_links"]} + self.assertIn( + "https://github.example.invalid/org/repo/pull/101", + link_urls, + ) + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/999", + link_urls, + "PR attributed to fenced-only key must not be eligible", + ) + # TEST-999 should be flagged as non-contributor evidence + nc_findings = [ + f for f in plan["findings"] + if f["type"] == "non_contributor_evidence" + and f["ref"] == "TEST-999" + ] + self.assertTrue(len(nc_findings) > 0) + + def test_fresh_fingerprint_uses_scoped_contributors(self) -> None: + """check_enrichment fresh fingerprint must use the same + Design-Reference-scoped contributor projection as + build_change_plan.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + + - [ ] Done + + ## Documentation Scope + + ``` + Contributing stories: TEST-999 + ``` + + ## Documentation Inputs + + **TEST-201 — Widget:** + - New CLI + + ## Design Reference + + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + # Same description → fresh check should NOT be stale + current = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + "remote_links": [], + } + result = evidence.check_enrichment(plan, current) + self.assertFalse( + result["stale"], + "Same description must not appear stale after scoping fix", + ) + + def test_tilde_fence_also_recognized(self) -> None: + """~~~ fences are recognized alongside ``` fences.""" + desc = textwrap.dedent("""\ + ## Documentation Scope + + ~~~ + ## Fake Section + Content inside tilde fence + ~~~ + + Real scope content. + """) + sections = evidence._extract_sections(desc) + scope = sections.get("Documentation Scope", "") + self.assertIn("Real scope content", scope) + self.assertNotIn("Fake Section", sections) + + +# --------------------------------------------------------------------------- +# R2: GitLab modern/legacy URL canonicalization +# --------------------------------------------------------------------------- + +class TestGitLabLegacyCanonicalization(unittest.TestCase): + """R2: Legacy GitLab URLs are canonicalized to modern form.""" + + def test_legacy_canonicalized_to_modern(self) -> None: + """Legacy /merge_requests/N → modern /-/merge_requests/N.""" + r = evidence.validate_pr_url( + "https://gitlab.example.invalid/org/repo" + "/merge_requests/7" + ) + self.assertEqual( + r, + "https://gitlab.example.invalid/org/repo" + "/-/merge_requests/7", + ) + + def test_modern_stable_after_canonicalization(self) -> None: + """Modern URL is idempotent under canonicalization.""" + url = ( + "https://gitlab.example.invalid/org/repo" + "/-/merge_requests/7" + ) + self.assertEqual(evidence.validate_pr_url(url), url) + + def test_nested_namespace_legacy_canonicalized(self) -> None: + """Nested namespace legacy URL canonicalized correctly.""" + r = evidence.validate_pr_url( + "https://gitlab.example.invalid/group/sub/repo" + "/merge_requests/42" + ) + self.assertEqual( + r, + "https://gitlab.example.invalid/group/sub/repo" + "/-/merge_requests/42", + ) + + def test_modern_legacy_dedup_modern_first(self) -> None: + """Same MR via modern then legacy URL → deduplicated.""" + items = [ + { + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + "observations": ["First observation"], + }, + { + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + "observations": ["Second observation"], + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + self.assertIn( + "First observation", r["items"][0]["observations"] + ) + self.assertIn( + "Second observation", r["items"][0]["observations"] + ) + + def test_modern_legacy_dedup_legacy_first(self) -> None: + """Same MR via legacy then modern URL → deduplicated.""" + items = [ + { + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + # Canonical URL should be modern form + self.assertIn("/-/merge_requests/7", r["items"][0]["pr_url"]) + + def test_modern_legacy_contradictory_merged(self) -> None: + """Same MR with contradictory merged via different URL forms + → single item with conflict warning.""" + items = [ + { + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": False, + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 1) + self.assertTrue( + len(r["items"][0]["conflicts"]) > 0, + "Contradictory merged must produce conflict", + ) + self.assertTrue( + any("contradictory" in w.lower() for w in r["warnings"]), + "Contradictory merged must produce warning", + ) + + def test_distinct_host_not_dedup(self) -> None: + """Same path on different hosts must NOT be deduplicated.""" + items = [ + { + "pr_url": ( + "https://gitlab.example.invalid/org/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "https://gitlab2.example.invalid/org/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 2) + + def test_distinct_project_not_dedup(self) -> None: + """Same host+number but different project must stay distinct.""" + items = [ + { + "pr_url": ( + "https://gitlab.example.invalid/org/repo-a" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + { + "pr_url": ( + "https://gitlab.example.invalid/org/repo-b" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + }, + ] + r = evidence.normalize_evidence(items) + self.assertEqual(len(r["items"]), 2) + + def test_existing_legacy_link_prevents_modern_eligible(self) -> None: + """Jira remote link with legacy URL must prevent modern URL + from appearing as eligible (same MR).""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - done + ## Documentation Scope + scope + ## Documentation Inputs + **TEST-201 — Config:** + - item + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + # Existing remote link uses LEGACY URL + current = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + "remote_links": [{ + "url": ( + "https://gitlab.example.invalid/g/s/repo" + "/merge_requests/7" + ), + "title": "MR !7", + }], + } + result = evidence.check_enrichment(plan, current) + self.assertEqual( + len(result["eligible_links"]), 0, + "Legacy existing link must prevent modern duplicate", + ) + self.assertEqual(len(result["existing_links"]), 1) + + def test_readback_legacy_accepts_modern(self) -> None: + """Receipt verification accepts legacy readback URL for + modern eligible link (same MR).""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - done + ## Documentation Scope + scope + ## Documentation Inputs + **TEST-201 — Config:** + - item + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://gitlab.example.invalid/g/s/repo" + "/-/merge_requests/7" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + current = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + "remote_links": [], + } + enrichment = evidence.check_enrichment(plan, current) + self.assertEqual(len(enrichment["eligible_links"]), 1) + # Readback uses LEGACY URL + readback = { + "key": "TEST-100", + "remote_links": [{ + "url": ( + "https://gitlab.example.invalid/g/s/repo" + "/merge_requests/7" + ), + }], + } + receipt = evidence.verify_enrichment( + enrichment, readback, "TEST-100", + ) + self.assertEqual(receipt["status"], "verified") + link_r = receipt["link_receipts"] + self.assertTrue( + any(r["status"] == "confirmed" for r in link_r), + "Legacy readback must confirm modern eligible link", + ) + + +# --------------------------------------------------------------------------- +# R3: Merged PR with missing/null target +# --------------------------------------------------------------------------- + +class TestMergedUnverifiedTarget(unittest.TestCase): + """R3: merged=True with missing/null target_branch must NOT yield + status meaning 'merged into expected branch'.""" + + def _make_plan( + self, + ev_target: str | None = None, + plan_target: str = "main", + ) -> dict: + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + **TEST-201 — Widget config:** + - New CLI command + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + raw_items = [{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": ev_target, + "state": "merged", + "title": "feat: widget config", + }] + ev = evidence.normalize_evidence(raw_items) + return evidence.build_change_plan( + val, ev, + {"release": "4.19", "target_branch": plan_target}, + ) + + def test_empty_target_unverified(self) -> None: + """merged=True + empty target_branch → merged_unverified_target.""" + plan = self._make_plan(ev_target="") + inp = plan["inputs"][0] + self.assertEqual(inp["status"], "merged_unverified_target") + eref = inp["evidence_refs"][0] + self.assertEqual(eref["pr_status"], "merged_unverified_target") + self.assertTrue( + len(inp["target_notes"]) > 0, + "Missing target must produce visible target note", + ) + + def test_null_target_unverified(self) -> None: + """merged=True + null target_branch → merged_unverified_target.""" + plan = self._make_plan(ev_target=None) + inp = plan["inputs"][0] + self.assertEqual(inp["status"], "merged_unverified_target") + self.assertTrue(len(inp["target_notes"]) > 0) + + def test_correct_target_still_merged(self) -> None: + """merged=True + matching target → merged_pr_exists (control).""" + plan = self._make_plan(ev_target="main") + inp = plan["inputs"][0] + self.assertEqual(inp["status"], "merged_pr_exists") + + def test_wrong_target_still_wrong(self) -> None: + """merged=True + wrong target → merged_wrong_target (control).""" + plan = self._make_plan(ev_target="release-4.18") + inp = plan["inputs"][0] + self.assertEqual(inp["status"], "merged_wrong_target") + + def test_no_plan_target_allows_merged(self) -> None: + """When plan has no target_branch, merged=True + empty ev_target + is just 'merged' (no target to verify against).""" + plan = self._make_plan(ev_target="", plan_target="") + inp = plan["inputs"][0] + self.assertEqual(inp["status"], "merged_pr_exists") + + def test_unverified_target_note_visible_in_render(self) -> None: + """Target note about unverified target appears in handoff.""" + plan = self._make_plan(ev_target="") + rendered = evidence.render_handoff(plan) + self.assertIn("target_branch unknown", rendered) + + +# --------------------------------------------------------------------------- +# R4: Unheaded Documentation Inputs bullets +# --------------------------------------------------------------------------- + +class TestOrphanDocumentationInputs(unittest.TestCase): + """R4: Bullets before any contributor header must be retained + with contributor=None and produce a warning.""" + + def test_orphan_bullet_retained(self) -> None: + """Orphan bullet appears in documentation_inputs with + contributor=None.""" + text = textwrap.dedent("""\ + - Orphan bullet with important context + + **TEST-201 — Add widget:** + - New CLI command + """) + items = evidence._parse_documentation_inputs(text) + self.assertEqual(len(items), 2) + orphan = items[0] + self.assertIsNone(orphan["contributor"]) + self.assertIsNone(orphan["contributor_header"]) + self.assertIn("Orphan bullet", orphan["text"]) + # Valid item after header + valid = items[1] + self.assertEqual(valid["contributor"], "TEST-201") + + def test_orphan_bullet_preserves_markers(self) -> None: + """[Assumption] marker on orphan bullet is preserved.""" + text = textwrap.dedent("""\ + - Must support JSON format [Assumption] + + **TEST-201 — Widget:** + - New CLI command + """) + items = evidence._parse_documentation_inputs(text) + orphan = items[0] + self.assertEqual(orphan["marker"], "assumption") + self.assertIsNone(orphan["contributor"]) + + def test_orphan_warning_generated(self) -> None: + """validate_issue emits warning about unattributed bullets.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + + - Orphan bullet here + - Another orphan + + **TEST-201 — Widget:** + - New CLI command + + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + result = evidence.validate_issue(snap) + self.assertTrue(result["valid"]) + self.assertTrue( + any("unattributed" in w for w in result["warnings"]), + "Must warn about orphan bullets", + ) + self.assertTrue( + any("2" in w for w in result["warnings"] + if "unattributed" in w), + "Warning should count orphan bullets", + ) + + def test_orphan_flows_through_plan(self) -> None: + """Orphan items appear in plan inputs with no_evidence status.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + + - Orphan context bullet + + **TEST-201 — Widget:** + - New CLI command + + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + # Should have 2 inputs: orphan + valid + self.assertEqual(len(plan["inputs"]), 2) + orphan_inp = [ + i for i in plan["inputs"] + if i["contributor"] is None + ] + self.assertEqual(len(orphan_inp), 1) + self.assertEqual(orphan_inp[0]["status"], "no_evidence") + self.assertIn("Orphan", orphan_inp[0]["text"]) + + def test_orphan_visible_in_render(self) -> None: + """Orphan input text appears in rendered handoff.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + + - Important orphan context + + **TEST-201 — Widget:** + - New CLI command + + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + rendered = evidence.render_handoff(plan) + self.assertIn("Important orphan context", rendered) + + def test_no_orphan_no_warning(self) -> None: + """Normal inputs without orphan bullets produce no warning.""" + snap = _make_docs_snapshot() + result = evidence.validate_issue(snap) + self.assertTrue(result["valid"]) + self.assertFalse( + any("unattributed" in w for w in result["warnings"]), + ) + + def test_orphan_does_not_expand_contributors(self) -> None: + """Orphan bullets must not add None to contributor allowlist + or affect PR link eligibility.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + + - Orphan bullet + + **TEST-201 — Widget:** + - New CLI command + + ## Design Reference + Contributing stories: TEST-201 + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, {"release": "4.19", "target_branch": "main"}, + ) + # Only TEST-201's PR should be eligible + self.assertEqual(len(plan["pr_links"]), 1) + # No input_contributor_mismatch for None + mismatch = [ + f for f in plan["findings"] + if f["type"] == "input_contributor_mismatch" + and f["ref"] is None + ] + self.assertEqual(len(mismatch), 0) + + if __name__ == "__main__": unittest.main() From 43e8f65f2b98bc1156cbeab26902afeabadf203d Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Sat, 14 Nov 2026 15:53:47 +0000 Subject: [PATCH 13/17] fix(docs-handoff): fence-aware contributor extraction within section content _extract_contributing_stories now skips lines inside fenced code blocks (``` or ~~~), matching the fence-awareness already in _extract_sections. This prevents fenced Contributing stories examples within the real Design Reference section content from widening contributor extraction and link eligibility. Also adds regression tests for fenced examples inside the Design Reference section, real Design Reference before fenced fake heading, and fresh fingerprint consistency with fenced section content. Strengthens R2 enrichment tests with explicit status/stale assertions and approved_plan_digest tokens. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 14 +- skills/docs-handoff/scripts/test_evidence.py | 164 ++++++++++++++++++- 2 files changed, 175 insertions(+), 3 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 56c4198a..bb42de67 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -304,11 +304,23 @@ def _extract_sections(text: str) -> dict[str, str]: def _extract_contributing_stories(description: str) -> list[str]: - """Extract Contributing stories references from a rendered description.""" + """Extract Contributing stories references from text. + + Lines inside fenced code blocks (backtick or tilde fences) are + skipped so that example Contributing stories lines in code samples + are not extracted. + """ result: list[str] = [] seen: set[str] = set() + in_fence = False for line in description.splitlines(): stripped = line.strip() + # Track fenced code blocks + if stripped.startswith("```") or stripped.startswith("~~~"): + in_fence = not in_fence + continue + if in_fence: + continue for pfx in ("- ", "* ", "+ "): if stripped.startswith(pfx): stripped = stripped[len(pfx):].strip() diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 2ce88752..ab56d0de 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -4437,6 +4437,156 @@ def test_tilde_fence_also_recognized(self) -> None: self.assertIn("Real scope content", scope) self.assertNotIn("Fake Section", sections) + def test_fenced_contributing_inside_design_reference(self) -> None: + """Contributing stories inside a fenced block WITHIN the real + Design Reference section must not be extracted — the fence-aware + projection applies to section content, not just boundaries.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + **TEST-201 — Widget:** + - New CLI + ## Design Reference + Contributing stories: TEST-201 + + Example DOCS issue format: + ``` + Contributing stories: TEST-999 + ``` + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + result = evidence.validate_issue(snap) + self.assertTrue(result["valid"]) + self.assertIn("TEST-201", result["contributors"]) + self.assertNotIn( + "TEST-999", result["contributors"], + "Fenced Contributing stories inside Design Reference " + "must not be extracted", + ) + # Verify the PR for TEST-999 is non-contributor + ev = evidence.normalize_evidence([ + { + "pr_url": ( + "https://github.example.invalid/org/r/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }, + { + "pr_url": ( + "https://github.example.invalid/org/r/pull/999" + ), + "contributor_key": "TEST-999", + "merged": True, + "target_branch": "main", + }, + ]) + plan = evidence.build_change_plan( + result, ev, + {"release": "4.19", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + link_urls = {l["url"] for l in plan["pr_links"]} + self.assertNotIn( + "https://github.example.invalid/org/r/pull/999", + link_urls, + ) + nc = [ + f for f in plan["findings"] + if f["type"] == "non_contributor_evidence" + and f["ref"] == "TEST-999" + ] + self.assertTrue(len(nc) > 0) + + def test_real_design_ref_before_fenced_fake(self) -> None: + """Real Design Reference appearing BEFORE a section with a + fenced fake heading still works correctly.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Design Reference + Contributing stories: TEST-201 + ## Documentation Scope + + Example: + ```markdown + ## Design Reference + Contributing stories: TEST-888 + ``` + + Scope continues. + + ## Documentation Inputs + **TEST-201 — Widget:** + - New CLI + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + result = evidence.validate_issue(snap) + self.assertTrue(result["valid"]) + self.assertIn("TEST-201", result["contributors"]) + self.assertNotIn("TEST-888", result["contributors"]) + # Documentation Scope should not be truncated + scope = result["sections"].get("Documentation Scope", "") + self.assertIn("Scope continues", scope) + + def test_fresh_fingerprint_fenced_inside_section(self) -> None: + """check_enrichment fresh fingerprint must agree with + validate_issue when Design Reference has fenced examples.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Done + ## Documentation Scope + scope + ## Documentation Inputs + **TEST-201 — Widget:** + - New CLI + ## Design Reference + Contributing stories: TEST-201 + + ``` + Contributing stories: TEST-999 + ``` + """) + snap = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + } + val = evidence.validate_issue(snap) + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/101" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, + {"release": "4.19", "target_branch": "main"}, + ) + current = { + "key": "TEST-100", + "summary": "[DOCS] Widget docs", + "description": desc, + "remote_links": [], + } + result = evidence.check_enrichment(plan, current) + self.assertEqual(result["status"], "ready") + self.assertFalse(result["stale"]) + # --------------------------------------------------------------------------- # R2: GitLab modern/legacy URL canonicalization @@ -4656,7 +4806,12 @@ def test_existing_legacy_link_prevents_modern_eligible(self) -> None: "title": "MR !7", }], } - result = evidence.check_enrichment(plan, current) + result = evidence.check_enrichment( + plan, current, + approved_plan_digest=plan["plan_digest"], + ) + self.assertEqual(result["status"], "ready") + self.assertFalse(result["stale"]) self.assertEqual( len(result["eligible_links"]), 0, "Legacy existing link must prevent modern duplicate", @@ -4701,7 +4856,12 @@ def test_readback_legacy_accepts_modern(self) -> None: "description": desc, "remote_links": [], } - enrichment = evidence.check_enrichment(plan, current) + enrichment = evidence.check_enrichment( + plan, current, + approved_plan_digest=plan["plan_digest"], + ) + self.assertEqual(enrichment["status"], "ready") + self.assertFalse(enrichment["stale"]) self.assertEqual(len(enrichment["eligible_links"]), 1) # Readback uses LEGACY URL readback = { From 5adcbfdc6bc346a7c4f4d3854bd6809d3101e4af Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Sat, 14 Nov 2026 16:35:42 +0000 Subject: [PATCH 14/17] fix(docs-handoff): CommonMark-correct fence matching in section/contributor/input parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace ad-hoc boolean fence toggle with a shared _FenceTracker class that enforces CommonMark closing-fence rules: same character (backtick or tilde) AND length >= opening fence. R1 (root 1808): _extract_sections and _extract_contributing_stories previously toggled `in_fence = not in_fence` on any line starting with ``` or ~~~. A shorter inner fence (``` inside ````) or a different fence character (~~~ inside ```) would prematurely end the outer fence, letting fenced headings and contributor references become extractable — widening the contributor allowlist and making fake PR links eligible. R2 (root 1812): _parse_documentation_inputs had no fence handling at all. Bold headers and bullets inside fenced code examples were parsed as real documentation inputs, contaminating plan counts and delivery rendering. Fix: One shared _FenceTracker (tracks opening char + run length) used consistently by all three consumers. Genuine unheaded bullets outside fences remain unattributed with warnings (R4 preserved). Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 124 ++++++++-- skills/docs-handoff/scripts/test_evidence.py | 248 +++++++++++++++++++ 2 files changed, 356 insertions(+), 16 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index bb42de67..02aa7517 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -258,6 +258,84 @@ def extract_story_refs(text: str) -> list[str]: return result +# --------------------------------------------------------------------------- +# Fenced code block tracking (CommonMark-compliant) +# --------------------------------------------------------------------------- + + +def _parse_fence_line(stripped: str) -> tuple[str, int] | None: + """Parse a potential fence line (3+ backticks or tildes). + + Returns ``(char, run_length)`` if the stripped line opens or + closes a fenced code block, ``None`` otherwise. + """ + if len(stripped) < 3: + return None + ch = stripped[0] + if ch not in ("`", "~"): + return None + run = 0 + for c in stripped: + if c == ch: + run += 1 + else: + break + if run < 3: + return None + return (ch, run) + + +class _FenceTracker: + """Track fenced code block state per CommonMark spec. + + A closing fence must use the **same character** (backtick or + tilde) as the opening fence **and** be at least as long. + Shorter fences or fences of a different character inside an + open block are treated as literal content — they do not toggle + the fence state. + + Usage:: + + fence = _FenceTracker() + for raw in text.splitlines(): + stripped = raw.strip() + if _parse_fence_line(stripped) is not None: + fence.update(stripped) + # handle fence-boundary line … + continue + if fence.in_fence: + # literal fenced content … + continue + # … normal processing … + """ + + __slots__ = ("_char", "_length") + + def __init__(self) -> None: + self._char: str | None = None + self._length: int = 0 + + @property + def in_fence(self) -> bool: + """Return True when inside an unclosed fenced code block.""" + return self._char is not None + + def update(self, stripped: str) -> None: + """Update fence state for a stripped line.""" + parsed = _parse_fence_line(stripped) + if parsed is None: + return + char, length = parsed + if not self.in_fence: + self._char = char + self._length = length + elif char == self._char and length >= self._length: + # Valid close: same character, length >= opening + self._char = None + self._length = 0 + # Different char or shorter run → stays fenced (literal) + + # --------------------------------------------------------------------------- # DOCS issue validation and contributor extraction (A, B) # --------------------------------------------------------------------------- @@ -272,25 +350,27 @@ def extract_story_refs(text: str) -> list[str]: def _extract_sections(text: str) -> dict[str, str]: """Extract ## sections from rendered Jira markdown description. - Headings inside fenced code blocks (backtick or tilde fences) are - treated as literal content, not section boundaries. + Headings inside fenced code blocks are treated as literal content, + not section boundaries. Per CommonMark, a closing fence must use + the same character (backtick or tilde) and be at least as long as + the opening fence. """ sections: dict[str, str] = {} current: str | None = None lines: list[str] = [] - in_fence = False + fence = _FenceTracker() for raw in text.splitlines(): stripped = raw.strip() - # Toggle fence state on ``` or ~~~ lines - if stripped.startswith("```") or stripped.startswith("~~~"): - in_fence = not in_fence + # Track fence boundaries (same char + length >= opening to close) + if _parse_fence_line(stripped) is not None: + fence.update(stripped) if current is not None: lines.append(raw) continue - if not in_fence and stripped.startswith("## "): + if not fence.in_fence and stripped.startswith("## "): if current is not None: sections[current] = "\n".join(lines).strip() current = stripped[3:].strip() @@ -306,20 +386,21 @@ def _extract_sections(text: str) -> dict[str, str]: def _extract_contributing_stories(description: str) -> list[str]: """Extract Contributing stories references from text. - Lines inside fenced code blocks (backtick or tilde fences) are - skipped so that example Contributing stories lines in code samples - are not extracted. + Lines inside fenced code blocks are skipped so that example + Contributing stories lines in code samples are not extracted. + Per CommonMark, a closing fence must use the same character and + be at least as long as the opening fence. """ result: list[str] = [] seen: set[str] = set() - in_fence = False + fence = _FenceTracker() for line in description.splitlines(): stripped = line.strip() - # Track fenced code blocks - if stripped.startswith("```") or stripped.startswith("~~~"): - in_fence = not in_fence + # Track fenced code blocks (same char + length >= opening) + if _parse_fence_line(stripped) is not None: + fence.update(stripped) continue - if in_fence: + if fence.in_fence: continue for pfx in ("- ", "* ", "+ "): if stripped.startswith(pfx): @@ -372,14 +453,25 @@ def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: Recognizes bold contributor headers followed by bullet-pointed input items. Each input may carry ``[Assumption]`` or - ``[Open: ...]`` markers which are preserved. + ``[Open: ...]`` markers which are preserved. Lines inside + fenced code blocks are skipped so that example headers and + bullets in code samples are not parsed as real inputs. """ items: list[dict[str, Any]] = [] current_contributor: str | None = None contributor_key: str | None = None + fence = _FenceTracker() for line in text.splitlines(): stripped = line.strip() + + # Skip fenced code blocks (same char + length >= opening) + if _parse_fence_line(stripped) is not None: + fence.update(stripped) + continue + if fence.in_fence: + continue + header_match = re.match( r"\*\*(.+?)(?:\s*(?:--|—)\s*.+?)?\s*:?\*\*:?\s*$", stripped, diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index ab56d0de..15afcbeb 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -5180,5 +5180,253 @@ def test_orphan_does_not_expand_contributors(self) -> None: self.assertEqual(len(mismatch), 0) +# --------------------------------------------------------------------------- +# R1-1808: CommonMark fence length/character matching +# --------------------------------------------------------------------------- + +class TestCommonMarkFenceMatching(unittest.TestCase): + """R1 root 1808: Closing fence must use the same character AND + have length >= the opening fence. Shorter inner fences or fences + of a different character must NOT toggle the fence state.""" + + def test_shorter_inner_fence_prevents_fake_contributor(self) -> None: + """Outer ```` with inner ``` must not prematurely end the + fence. Without fix: inner ``` toggles fence off, the fake + Design Reference heading re-creates the section, and + Contributing stories: TEST-999 becomes an eligible contributor + whose PR would be added as a link.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + ```` + ``` + ## Design Reference + Contributing stories: TEST-999 + ``` + ```` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + # Real contributor from outside fence + self.assertIn("TEST-201", val["contributors"]) + # Fake contributor inside fence must NOT appear + self.assertNotIn( + "TEST-999", val["contributors"], + "Shorter inner fence must not expose fenced contributor", + ) + # Plan: TEST-999 must not become eligible PR link + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), + "contributor_key": "TEST-999", + }, + ] + ev = evidence.normalize_evidence(items) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + link_urls = {link["url"] for link in plan["pr_links"]} + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/999", + link_urls, + "TEST-999 PR must not be eligible (fenced contributor)", + ) + # TEST-999 flagged as non-contributor evidence + non_contrib = [ + f for f in plan["findings"] + if f["type"] == "non_contributor_evidence" + and f["ref"] == "TEST-999" + ] + self.assertTrue( + len(non_contrib) > 0, + "TEST-999 must be flagged as non-contributor", + ) + # Genuine Design Reference content after fence close preserved + dr = val["sections"].get("Design Reference", "") + self.assertIn("TEST-201", dr) + + def test_different_fence_char_stays_fenced(self) -> None: + """~~~ inside backtick fence must not end it. Without fix: + ~~~ toggles fence off, Contributing stories: TEST-999 + becomes extractable.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - New CLI command + ## Design Reference + ``` + ~~~ + Contributing stories: TEST-999 + ~~~ + ``` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-999", val["contributors"], + "Different fence char must not expose fenced contributor", + ) + + def test_matching_close_resumes_normal_processing(self) -> None: + """Valid matching close (same char, length >= opening) does + end the fence, and content after resumes normal extraction. + Control: same-length close works correctly.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - New CLI command + ## Design Reference + ```` + Contributing stories: TEST-FENCED + ```` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn("TEST-FENCED", val["contributors"]) + + +# --------------------------------------------------------------------------- +# R2-1812: Fence guard in _parse_documentation_inputs +# --------------------------------------------------------------------------- + +class TestDocumentationInputsFenceGuard(unittest.TestCase): + """R2 root 1812: _parse_documentation_inputs must skip fenced code + blocks so that example headers and bullets do not contaminate the + parsed input list, plan counts, or delivery rendering.""" + + def test_fenced_inputs_excluded_from_parse_and_plan(self) -> None: + """Fenced bold header + bullet inside Documentation Inputs + must NOT be parsed as real input items. Genuine unheaded + bullet outside fence retained unattributed (R4 contract).""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + - Orphan requirement outside fence + **TEST-201 — Add widget:** + - Real CLI command input + ``` + **TEST-FAKE — Example header:** + - Example option inside fence + ``` + ## Design Reference + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + inputs = val["documentation_inputs"] + # Exactly 2 items: orphan + TEST-201 real input + self.assertEqual( + len(inputs), 2, + f"Expected 2 items (orphan + real), got {len(inputs)}: " + f"{[i.get('contributor') for i in inputs]}", + ) + # No fenced example parsed + fake = [ + i for i in inputs + if i.get("contributor") == "TEST-FAKE" + ] + self.assertEqual( + len(fake), 0, + "Fenced example header must not be parsed as input", + ) + example = [ + i for i in inputs + if "Example option" in i.get("text", "") + ] + self.assertEqual( + len(example), 0, + "Fenced example bullet must not be parsed as input", + ) + # Orphan retained unattributed (R4 preserved) + orphans = [ + i for i in inputs if i["contributor"] is None + ] + self.assertEqual(len(orphans), 1) + self.assertTrue( + any("unattributed" in w for w in val["warnings"]), + "Genuine orphan bullet must still produce warning", + ) + # Plan has correct input count + ev = evidence.normalize_evidence([{ + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }]) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + self.assertEqual( + len(plan["inputs"]), 2, + "Plan must have 2 inputs (orphan + real), not 4", + ) + # Render shows correct total + md = evidence.render_handoff(plan) + self.assertIn("Total inputs: 2", md) + + def test_fenced_header_does_not_change_attribution(self) -> None: + """A fenced bold header must not change the current contributor + attribution for subsequent real items outside the fence.""" + text = textwrap.dedent("""\ + **TEST-201 — Add widget:** + - Real item under TEST-201 + ``` + **TEST-FAKE — Example:** + - Example bullet + ``` + - Second real item still under TEST-201 + """) + items = evidence._parse_documentation_inputs(text) + # Should have 2 items, both under TEST-201 + self.assertEqual(len(items), 2) + for item in items: + self.assertEqual( + item["contributor"], "TEST-201", + f"Item '{item['text']}' should be under TEST-201", + ) + + if __name__ == "__main__": unittest.main() From c283805ce720bcd6161b4d9d726093b278d18c2b Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Sat, 14 Nov 2026 16:46:26 +0000 Subject: [PATCH 15/17] fix(docs-handoff): reject closing fences with info strings per CommonMark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A closing code fence may only be followed by spaces or tabs — a line like ```python inside an open triple-backtick block is content, not a close. Without this fix, _FenceTracker.update() accepted any matching char+length fence as closing regardless of trailing text, prematurely ending the fenced block and exposing fenced-example Contributing stories lines as extractable contributors. Adds _is_valid_close() predicate checking remainder is whitespace-only, and one regression test validating the info-string scenario end-to-end through validate_issue + build_change_plan. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 40 ++++++++---- skills/docs-handoff/scripts/test_evidence.py | 64 ++++++++++++++++++++ 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 02aa7517..ba6bd13b 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -286,13 +286,15 @@ def _parse_fence_line(stripped: str) -> tuple[str, int] | None: class _FenceTracker: - """Track fenced code block state per CommonMark spec. + """Track fenced code block state per CommonMark fence rules. A closing fence must use the **same character** (backtick or - tilde) as the opening fence **and** be at least as long. - Shorter fences or fences of a different character inside an - open block are treated as literal content — they do not toggle - the fence state. + tilde) as the opening fence, be **at least as long**, and may + only be followed by spaces or tabs (no info string). Shorter + fences, fences of a different character, or fence-like lines + with trailing non-whitespace (e.g. ````` ```python ````` inside + a backtick block) are treated as literal content — they do not + toggle the fence state. Usage:: @@ -320,6 +322,16 @@ def in_fence(self) -> bool: """Return True when inside an unclosed fenced code block.""" return self._char is not None + @staticmethod + def _is_valid_close(stripped: str, length: int) -> bool: + """Check that text after fence characters is only spaces/tabs. + + Per CommonMark, a closing code fence may optionally be followed + by spaces or tabs but must not contain an info string. + """ + remainder = stripped[length:] + return not remainder or all(c in (' ', '\t') for c in remainder) + def update(self, stripped: str) -> None: """Update fence state for a stripped line.""" parsed = _parse_fence_line(stripped) @@ -329,11 +341,13 @@ def update(self, stripped: str) -> None: if not self.in_fence: self._char = char self._length = length - elif char == self._char and length >= self._length: - # Valid close: same character, length >= opening + elif (char == self._char + and length >= self._length + and self._is_valid_close(stripped, length)): + # Valid close: same character, length >= opening, no info string self._char = None self._length = 0 - # Different char or shorter run → stays fenced (literal) + # Different char, shorter run, or trailing content → stays fenced # --------------------------------------------------------------------------- @@ -352,8 +366,9 @@ def _extract_sections(text: str) -> dict[str, str]: Headings inside fenced code blocks are treated as literal content, not section boundaries. Per CommonMark, a closing fence must use - the same character (backtick or tilde) and be at least as long as - the opening fence. + the same character (backtick or tilde), be at least as long as + the opening fence, and contain no info string (only optional + trailing spaces/tabs). """ sections: dict[str, str] = {} current: str | None = None @@ -388,8 +403,9 @@ def _extract_contributing_stories(description: str) -> list[str]: Lines inside fenced code blocks are skipped so that example Contributing stories lines in code samples are not extracted. - Per CommonMark, a closing fence must use the same character and - be at least as long as the opening fence. + Per CommonMark, a closing fence must use the same character, be + at least as long as the opening fence, and contain no info + string (only optional trailing spaces/tabs). """ result: list[str] = [] seen: set[str] = set() diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 15afcbeb..19be4d94 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -5318,6 +5318,70 @@ def test_matching_close_resumes_normal_processing(self) -> None: self.assertIn("TEST-201", val["contributors"]) self.assertNotIn("TEST-FENCED", val["contributors"]) + def test_info_string_on_closing_fence_stays_fenced(self) -> None: + """A would-be closing fence with an info string (e.g. ```python) + is NOT a valid closing fence per CommonMark — a closing fence + may only be followed by spaces or tabs. Without fix: the inner + ```python line closes the fence, exposing the fake Contributing + stories: TEST-999 line as an extractable contributor.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + Here is a fenced example showing language-tagged blocks: + ``` + ```python + Contributing stories: TEST-999 + print("hello") + ``` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + # Real contributor from outside fence + self.assertIn("TEST-201", val["contributors"]) + # Fake contributor inside fence must NOT appear — the inner + # ```python line must not close the outer ``` fence + self.assertNotIn( + "TEST-999", val["contributors"], + "Info string on would-be closing fence must not end block", + ) + # Verify plan also excludes the fenced contributor + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), + "contributor_key": "TEST-999", + }, + ] + ev = evidence.normalize_evidence(items) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + link_urls = {link["url"] for link in plan["pr_links"]} + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/999", + link_urls, + "TEST-999 PR must not be eligible (info-string fence)", + ) + # --------------------------------------------------------------------------- # R2-1812: Fence guard in _parse_documentation_inputs From 7dbfb16fea885287768fcd383088975d4dfadac9 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Sat, 14 Nov 2026 17:25:48 +0000 Subject: [PATCH 16/17] fix(docs-handoff): CommonMark indentation and backtick-info fence rules Apply two CommonMark fence boundary corrections to the shared _FenceTracker used by _extract_sections, _extract_contributing_stories, and _parse_documentation_inputs: 1. Indentation (R1-1976): fence-like lines preceded by four or more spaces of indentation are rejected as fence boundaries (neither openers nor closers). CommonMark requires 0-3 spaces before a code fence. Previously, stripping whitespace before fence detection caused indented fences inside outer blocks to incorrectly toggle fence state, allowing fake contributors to escape and hiding real ones. 2. Backtick info string (R2-1982): a backtick opener whose info remainder contains a backtick character is not a valid opener. Lines like ```example``` are therefore inert content, not fence boundaries. Previously they opened a fence that swallowed subsequent real sections (Documentation Inputs, Design Reference). Tilde fence info strings may still contain backticks per spec. Both fixes are applied in _FenceTracker.update via a new optional `raw` parameter that all three callers now pass, preserving the shared single-implementation approach. Tests: 6 new tests covering both bugs and their boundary controls (3-space indent works, valid language opener works, tilde+backtick info works). All 262 evidence tests pass. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 41 +++- skills/docs-handoff/scripts/test_evidence.py | 216 +++++++++++++++++++ 2 files changed, 251 insertions(+), 6 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index ba6bd13b..4d9f68a6 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -296,13 +296,24 @@ class _FenceTracker: a backtick block) are treated as literal content — they do not toggle the fence state. + Additional CommonMark constraints enforced when *raw* is + passed to :meth:`update`: + + - **Indentation**: a code fence must be preceded by 0–3 spaces + of indentation. Lines indented four or more spaces are + rejected as fence boundaries (neither openers nor closers). + - **Backtick info string**: the info string of a backtick code + fence may not contain any backtick characters. A line like + ````` ```example``` ````` is not a valid opener. Tilde fence + info strings may contain backticks. + Usage:: fence = _FenceTracker() for raw in text.splitlines(): stripped = raw.strip() if _parse_fence_line(stripped) is not None: - fence.update(stripped) + fence.update(stripped, raw) # handle fence-boundary line … continue if fence.in_fence: @@ -332,13 +343,31 @@ def _is_valid_close(stripped: str, length: int) -> bool: remainder = stripped[length:] return not remainder or all(c in (' ', '\t') for c in remainder) - def update(self, stripped: str) -> None: - """Update fence state for a stripped line.""" + def update(self, stripped: str, raw: str = "") -> None: + """Update fence state for a stripped line. + + When *raw* is provided, lines with four or more leading + spaces are rejected as fence boundaries (CommonMark requires + 0–3 spaces of indentation before a code fence). Backtick + openers whose info string contains a backtick character are + also rejected (CommonMark: the info string of a backtick + code fence may not contain any backtick characters). + """ parsed = _parse_fence_line(stripped) if parsed is None: return + # CommonMark: a code fence is preceded by 0-3 spaces + if raw: + indent = len(raw) - len(raw.lstrip(' ')) + if indent >= 4: + return char, length = parsed if not self.in_fence: + # CommonMark: backtick fence info may not contain backticks + if char == '`': + info = stripped[length:] + if '`' in info: + return self._char = char self._length = length elif (char == self._char @@ -380,7 +409,7 @@ def _extract_sections(text: str) -> dict[str, str]: # Track fence boundaries (same char + length >= opening to close) if _parse_fence_line(stripped) is not None: - fence.update(stripped) + fence.update(stripped, raw) if current is not None: lines.append(raw) continue @@ -414,7 +443,7 @@ def _extract_contributing_stories(description: str) -> list[str]: stripped = line.strip() # Track fenced code blocks (same char + length >= opening) if _parse_fence_line(stripped) is not None: - fence.update(stripped) + fence.update(stripped, line) continue if fence.in_fence: continue @@ -483,7 +512,7 @@ def _parse_documentation_inputs(text: str) -> list[dict[str, Any]]: # Skip fenced code blocks (same char + length >= opening) if _parse_fence_line(stripped) is not None: - fence.update(stripped) + fence.update(stripped, line) continue if fence.in_fence: continue diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 19be4d94..22bfc11e 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -5492,5 +5492,221 @@ def test_fenced_header_does_not_change_attribution(self) -> None: ) +# --------------------------------------------------------------------------- +# R1-1976: Indented fence boundaries rejected (CommonMark 0-3 spaces) +# R2-1982: Backtick opener info containing backtick rejected +# --------------------------------------------------------------------------- + +class TestFenceIndentationAndBacktickInfo(unittest.TestCase): + """R1 root 1976: fence-like lines indented 4+ spaces must not open + or close fences. R2 root 1982: backtick opener whose info string + contains a backtick is not a valid opener.""" + + # -- R1: indentation ------------------------------------------------ + + def test_indented_fence_does_not_close_outer(self) -> None: + """R1: four-space-indented ``` inside an outer ``` fence + must NOT close it. Without fix: the indented ``` closes the + outer fence, fake Contributing stories: TEST-999 escapes, + and the actual closing ``` then hides real TEST-201. + + After fix: indented ``` is inert content; real TEST-201 + after the genuine column-0 close is extracted.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + ``` + ``` + Contributing stories: TEST-999 + ``` + ``` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-999", val["contributors"], + "Indented inner fence must not close outer; " + "TEST-999 must stay inside fence", + ) + # Plan: TEST-999 must not become eligible PR link + items = [ + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/1" + ), + "contributor_key": "TEST-201", + "merged": True, + "target_branch": "main", + }, + { + "pr_url": ( + "https://github.example.invalid/org/repo/pull/999" + ), + "contributor_key": "TEST-999", + }, + ] + ev = evidence.normalize_evidence(items) + plan = evidence.build_change_plan( + val, ev, + {"release": "v1", "target_branch": "main"}, + ) + self.assertEqual(plan["status"], "complete") + link_urls = {link["url"] for link in plan["pr_links"]} + self.assertNotIn( + "https://github.example.invalid/org/repo/pull/999", + link_urls, + "TEST-999 PR must not be eligible (indented fence)", + ) + + def test_three_space_indented_fence_still_works(self) -> None: + """R1 boundary: 0-3 space indentation is valid per CommonMark. + A three-space-indented fence correctly opens and closes.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + ``` + Contributing stories: TEST-FENCED + ``` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-FENCED", val["contributors"], + "3-space indented fence is valid; fenced content stays hidden", + ) + + def test_indented_fence_does_not_open(self) -> None: + """R1: four-space-indented ``` must NOT open a fence that + hides subsequent real section headings.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ``` + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ``` + ## Design Reference + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + # Documentation Inputs section must be visible + self.assertIn("Documentation Inputs", val["sections"]) + # Inputs must be parsed + self.assertTrue( + len(val["documentation_inputs"]) >= 1, + "Real inputs after non-opening indented fence must be parsed", + ) + + # -- R2: backtick info string --------------------------------------- + + def test_backtick_in_info_does_not_open_fence(self) -> None: + """R2: standalone ```example``` (backticks within opener + info remainder) must NOT open a fence. Without fix: it + opens a fence that swallows ## Documentation Inputs and + ## Design Reference, hiding real contributors and inputs. + + After fix: the invalid opener is inert; following sections + and their content are parsed normally.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ```example``` + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertIn("Documentation Inputs", val["sections"]) + self.assertTrue( + len(val["documentation_inputs"]) >= 1, + "Real inputs after invalid opener must be parsed", + ) + + def test_tilde_fence_with_backtick_info_still_valid(self) -> None: + """R2 boundary: tilde fence info string MAY contain backticks + per CommonMark — only backtick fences have the restriction. + A ~~~`backtick-example opener correctly opens a tilde fence.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + ~~~`backtick-example + Contributing stories: TEST-FENCED + ~~~ + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-FENCED", val["contributors"], + "Tilde fence with backtick info is valid; " + "content stays hidden", + ) + + def test_valid_backtick_language_opener_still_works(self) -> None: + """R2 boundary: ```python (no backtick in info remainder) + correctly opens a backtick fence.""" + desc = textwrap.dedent("""\ + ## Acceptance Criteria + - [ ] Something + ## Documentation Scope + Some scope + ## Documentation Inputs + **TEST-201 — Add widget:** + - Real CLI command input + ## Design Reference + ```python + Contributing stories: TEST-FENCED + ``` + Contributing stories: TEST-201 + """) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-FENCED", val["contributors"], + "Valid backtick opener with language info works correctly", + ) + + if __name__ == "__main__": unittest.main() From f567e8b6fc9e0c8f0a4c430b36bfbad18de86508 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Sat, 14 Nov 2026 17:42:57 +0000 Subject: [PATCH 17/17] fix(docs-handoff): handle tab indentation in fence boundary detection The indentation check in _FenceTracker.update used raw.lstrip(' ') which only strips spaces, causing tab-indented fence lines to be counted as zero indentation. A leading tab (4 columns at the first tab stop) or mixed space+tab prefix reaching 4+ columns would incorrectly close an outer fence, exposing fenced content like fake Contributing stories lines. Add _leading_indent_columns() helper that counts leading whitespace columns with four-column tab stops per CommonMark, and use it in _FenceTracker.update. All three callers already pass the raw line. Tests: 2 new regression tests (tab-indented close, mixed space+tab close). All 264 evidence tests pass. Co-Authored-By: Claude Opus 4.6 --- skills/docs-handoff/scripts/evidence.py | 38 ++++++++--- skills/docs-handoff/scripts/test_evidence.py | 66 ++++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/skills/docs-handoff/scripts/evidence.py b/skills/docs-handoff/scripts/evidence.py index 4d9f68a6..460fe4e8 100644 --- a/skills/docs-handoff/scripts/evidence.py +++ b/skills/docs-handoff/scripts/evidence.py @@ -263,6 +263,24 @@ def extract_story_refs(text: str) -> list[str]: # --------------------------------------------------------------------------- +def _leading_indent_columns(raw: str) -> int: + """Count leading indentation columns with four-column tab stops. + + Per CommonMark, tabs in indentation advance to the next tab stop + (every four columns). Only leading spaces and tabs count as + indentation characters. + """ + cols = 0 + for ch in raw: + if ch == ' ': + cols += 1 + elif ch == '\t': + cols += 4 - cols % 4 + else: + break + return cols + + def _parse_fence_line(stripped: str) -> tuple[str, int] | None: """Parse a potential fence line (3+ backticks or tildes). @@ -299,9 +317,11 @@ class _FenceTracker: Additional CommonMark constraints enforced when *raw* is passed to :meth:`update`: - - **Indentation**: a code fence must be preceded by 0–3 spaces - of indentation. Lines indented four or more spaces are - rejected as fence boundaries (neither openers nor closers). + - **Indentation**: a code fence must be preceded by 0–3 columns + of indentation. Lines whose leading whitespace reaches four + or more columns are rejected as fence boundaries (neither + openers nor closers). Tabs advance to the next four-column + tab stop per CommonMark. - **Backtick info string**: the info string of a backtick code fence may not contain any backtick characters. A line like ````` ```example``` ````` is not a valid opener. Tilde fence @@ -346,9 +366,10 @@ def _is_valid_close(stripped: str, length: int) -> bool: def update(self, stripped: str, raw: str = "") -> None: """Update fence state for a stripped line. - When *raw* is provided, lines with four or more leading - spaces are rejected as fence boundaries (CommonMark requires - 0–3 spaces of indentation before a code fence). Backtick + When *raw* is provided, lines whose leading indentation + reaches four or more columns are rejected as fence boundaries + (CommonMark requires 0–3 columns of indentation before a code + fence; tabs advance to the next four-column tab stop). Backtick openers whose info string contains a backtick character are also rejected (CommonMark: the info string of a backtick code fence may not contain any backtick characters). @@ -356,10 +377,9 @@ def update(self, stripped: str, raw: str = "") -> None: parsed = _parse_fence_line(stripped) if parsed is None: return - # CommonMark: a code fence is preceded by 0-3 spaces + # CommonMark: a code fence is preceded by 0-3 columns if raw: - indent = len(raw) - len(raw.lstrip(' ')) - if indent >= 4: + if _leading_indent_columns(raw) >= 4: return char, length = parsed if not self.in_fence: diff --git a/skills/docs-handoff/scripts/test_evidence.py b/skills/docs-handoff/scripts/test_evidence.py index 22bfc11e..168bdd87 100644 --- a/skills/docs-handoff/scripts/test_evidence.py +++ b/skills/docs-handoff/scripts/test_evidence.py @@ -5707,6 +5707,72 @@ def test_valid_backtick_language_opener_still_works(self) -> None: "Valid backtick opener with language info works correctly", ) + # -- R1 follow-up: tab indentation --------------------------------- + + def test_tab_indented_fence_does_not_close_outer(self) -> None: + """R1 tab variant: a leading-tab ``` inside an outer ``` + fence must NOT close it. Without fix: raw.lstrip(' ') + ignores tabs, indent counted as 0, tab-indented ``` + incorrectly closes fence, exposing fake TEST-999. + + After fix: tab = 4-column tab stop ≥ 4 → rejected; + real TEST-201 after genuine close is extracted.""" + # Build description with literal tab before inner fences + desc = ( + "## Acceptance Criteria\n" + "- [ ] Something\n" + "## Documentation Scope\n" + "Some scope\n" + "## Documentation Inputs\n" + "**TEST-201 — Add widget:**\n" + "- Real CLI command input\n" + "## Design Reference\n" + "```\n" + "\t```\n" + "Contributing stories: TEST-999\n" + "\t```\n" + "```\n" + "Contributing stories: TEST-201\n" + ) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-999", val["contributors"], + "Tab-indented inner fence must not close outer; " + "TEST-999 must stay inside fence", + ) + + def test_mixed_space_tab_fence_does_not_close(self) -> None: + """R1 mixed variant: one space + tab = column 4 (tab + advances to next four-column tab stop). Must not close + fence or expose fenced contributor.""" + desc = ( + "## Acceptance Criteria\n" + "- [ ] Something\n" + "## Documentation Scope\n" + "Some scope\n" + "## Documentation Inputs\n" + "**TEST-201 — Add widget:**\n" + "- Real CLI command input\n" + "## Design Reference\n" + "```\n" + " \t```\n" + "Contributing stories: TEST-999\n" + " \t```\n" + "```\n" + "Contributing stories: TEST-201\n" + ) + snap = _make_docs_snapshot(description=desc) + val = evidence.validate_issue(snap) + self.assertTrue(val["valid"]) + self.assertIn("TEST-201", val["contributors"]) + self.assertNotIn( + "TEST-999", val["contributors"], + "Mixed space+tab (4 columns) must not close outer fence", + ) + if __name__ == "__main__": unittest.main()