diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index b7dbb58a872..15a6427ca7b 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -59,3 +59,31 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./_site cname: pypi.riseproject.dev + + # A package is officially available once deploy has published /simple/, so + # we can now move the issue Status to "Available in RISE PyPI". + # Compares packages.txt with the project rather than diffing this push, + # so pushes the concurrency queue skipped are caught up. + sync-project-status: + needs: deploy + if: ${{ vars.RISEPROJECT_APP_CLIENT_ID != '' }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # GITHUB_TOKEN cannot write organization projects; the App needs the + # organization "Projects: read and write" permission. + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ vars.RISEPROJECT_APP_CLIENT_ID }} + private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: ci_scripts + + - name: Mark published packages in the Python Wheels project + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: python3 ci_scripts/sync_project_status.py diff --git a/ci_scripts/check_versions.py b/ci_scripts/check_versions.py index bfab4db004e..dfe192ced19 100755 --- a/ci_scripts/check_versions.py +++ b/ci_scripts/check_versions.py @@ -16,7 +16,6 @@ import json import re import shutil -import subprocess import sys import time from datetime import datetime, timedelta, timezone @@ -26,7 +25,8 @@ import requests import yaml -from update_doc import normalize_name, version_blocks +from github_issues import fetch_issues, normalize_name, oldest_open_issue +from update_doc import version_blocks ROOT_DIR = Path(__file__).resolve().parent.parent @@ -34,7 +34,6 @@ PATCHES_DIR = ROOT_DIR / "patches" PACKAGES_FILE = "ci_scripts/packages.txt" DEPRECATED_FILE = "ci_scripts/deprecated.txt" -REPO = "riseproject-dev/python-wheels" # Python versions (interpreter tags) a package must have upstream riscv64 # wheels for before we deprecate our own build. "3.14t" is the free-threaded @@ -175,31 +174,25 @@ def is_pure_python_wheel(package_info: Dict, target_version: str) -> bool: return has_wheels -def find_upstream_issue(package: str) -> Optional[str]: - """ - Find issue for a package in the wheel_builder's Upstream milestone. +_issues: Optional[Dict] = None - All issues in the upstream milestone follow the same title format: - {package} riscv64 support +def find_upstream_issue(package: str) -> Optional[str]: """ - try: - issue_title = f"{package} riscv64 support" - result = subprocess.run([ - "gh", "issue", "list", - "--search", issue_title, - "--json", "number", - "--jq", ".[0].number", - "--repo", REPO, - ], capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - return None + Number of the package's oldest open "{package} riscv64 support" issue. - return result.stdout.strip() or None - - except (subprocess.TimeoutExpired, subprocess.CalledProcessError, Exception): - return None + The issues are fetched once and reused for every package of the run. + """ + global _issues + if _issues is None: + try: + # Project fields are not needed here, and the nightly token cannot read them. + _issues = fetch_issues(with_project=False) + except RuntimeError as e: + print(f"warning: {e}", file=sys.stderr) + _issues = {} + issue = oldest_open_issue(_issues, package) + return str(issue["number"]) if issue else None def deprecate_package(package: str) -> bool: diff --git a/ci_scripts/github_issues.py b/ci_scripts/github_issues.py new file mode 100644 index 00000000000..aec4cc4b360 --- /dev/null +++ b/ci_scripts/github_issues.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Shared helpers for the " riscv64 support" issues and their status in +the "Python Wheels" project. + +Uses the ``gh`` CLI. Reading project fields needs the ``read:project`` scope, +changing them the ``project`` scope (for the GitHub App: the organization +"Projects: read and write" permission). +""" + +import json +import re +import subprocess +import sys + +OWNER = "riseproject-dev" +REPO_NAME = "python-wheels" +REPO = f"{OWNER}/{REPO_NAME}" + +# Projects > "Python Wheels - riseproject-dev/python-wheels". Node IDs rather +# than names, so that the edit works with any gh version (item-edit only +# recently learned --field/--value) and a project rename changes nothing. +PROJECT_NUMBER = 4 +PROJECT_ID = "PVT_kwDOCRlTBM4BbcwJ" +STATUS_FIELD_ID = "PVTSSF_lADOCRlTBM4BbcwJzhWNMvs" +STATUS_TODO = "Todo" +STATUS_RISE = "Available in RISE PyPI" +STATUS_UPSTREAM = "Available Upstream" +STATUS_OPTION_IDS = { + STATUS_TODO: "f75ad846", + STATUS_RISE: "47fc9ee4", + STATUS_UPSTREAM: "98236657", +} + +TITLE_RE = re.compile(r"^\s*(.+?)\s+riscv64\s+support\s*$", re.IGNORECASE) + +QUERY = """ +query($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + issues(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url state + %s + } + } + } +} +""" + +PROJECT_FIELDS = """ + projectItems(first: 10) { + nodes { + project { id } + status: fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { name } + } + } + }""" + + +def normalize_name(name): + """ + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def fetch_issues(owner=OWNER, repo=REPO_NAME, with_project=True, strict=False): + """ + Return {normalised package name: [issue, ...]} for " riscv64 support" + issues, open and closed, oldest first. + + Without the read:project scope the project fields are dropped with a + warning, unless `strict`, where that and any other GraphQL error raise + RuntimeError instead. + """ + query = QUERY % (PROJECT_FIELDS if with_project else "") + by_name = {} + cursor = None + while True: + cmd = ["gh", "api", "graphql", "-f", f"query={query}", + "-F", f"owner={owner}", "-F", f"repo={repo}"] + if cursor: + cmd += ["-F", f"cursor={cursor}"] + proc = subprocess.run(cmd, capture_output=True, text=True) + try: + data = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + data = {} + if with_project and not strict and "read:project" in proc.stderr: + print("warning: token lacks the read:project scope, project status unavailable" + " (gh auth refresh -h github.com -s read:project)", file=sys.stderr) + return fetch_issues(owner, repo, with_project=False) + if "data" not in data or data["data"] is None: + raise RuntimeError(f"gh api graphql failed: {proc.stderr.strip() or data}") + if data.get("errors"): + msgs = "; ".join(sorted({e.get("message", "") for e in data["errors"]})) + if strict: + raise RuntimeError(f"gh api graphql failed: {msgs}") + print(f"warning: {msgs}", file=sys.stderr) + issues = data["data"]["repository"]["issues"] + for issue in issues["nodes"]: + m = TITLE_RE.match(issue["title"]) + if m: + by_name.setdefault(normalize_name(m.group(1)), []).append(issue) + if not issues["pageInfo"]["hasNextPage"]: + return by_name + cursor = issues["pageInfo"]["endCursor"] + + +def oldest_open_issue(issues, package): + """The package's oldest open issue in fetch_issues() output, or None.""" + for issue in issues.get(normalize_name(package), []): + if issue["state"] == "OPEN": + return issue + return None + + +def project_status(issue): + if "projectItems" not in issue: + return "(unknown)" + for item in (issue.get("projectItems") or {}).get("nodes") or []: + if item and item["project"]["id"] == PROJECT_ID: + return (item.get("status") or {}).get("name") or "(no status)" + return "(not in project)" + + +def set_issue_status(issue_url, status): + """ + Add the issue to the project (a no-op when it is already there) and set its + Status. Raises subprocess.CalledProcessError when gh fails. + """ + item_id = subprocess.run( + ["gh", "project", "item-add", str(PROJECT_NUMBER), "--owner", OWNER, + "--url", issue_url, "--format", "json", "--jq", ".id"], + capture_output=True, text=True, check=True, + ).stdout.strip() + subprocess.run( + ["gh", "project", "item-edit", "--id", item_id, + "--project-id", PROJECT_ID, "--field-id", STATUS_FIELD_ID, + "--single-select-option-id", STATUS_OPTION_IDS[status]], + capture_output=True, text=True, check=True, + ) diff --git a/ci_scripts/sync_project_status.py b/ci_scripts/sync_project_status.py new file mode 100755 index 00000000000..605584b9d9b --- /dev/null +++ b/ci_scripts/sync_project_status.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Mark every package we publish as "Available in RISE PyPI" in the project. + +For each package in ci_scripts/packages.txt, take its oldest open +" riscv64 support" issue and, unless its Status already says it is +available (in RISE PyPI, or upstream after a deprecation), add it to the +"Python Wheels" project and set Status to "Available in RISE PyPI". + +The website workflow runs this after each deploy, so a package only flips once +its docs PR is merged and /simple/ lists it. It compares state rather than +reacting to one push, so a skipped or failed run is caught up by the next one. + +Needs a token with the ``project`` scope (the GitHub App: organization +"Projects: read and write"): + + gh auth refresh -h github.com -s project + ci_scripts/sync_project_status.py --dry-run +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +from github_issues import ( + STATUS_RISE, + STATUS_UPSTREAM, + fetch_issues, + oldest_open_issue, + project_status, + set_issue_status, +) + +PACKAGES_FILE = Path(__file__).resolve().parent / "packages.txt" + +# Each update is two mutations. Usually only a package or two change per run, +# but a backlog would otherwise trip GitHub's secondary rate limit. +PAUSE_SECONDS = 1.5 + + +def read_packages(): + return [line.strip() for line in PACKAGES_FILE.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#")] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dry-run", action="store_true", + help="Print the issues that would change, change nothing") + args = parser.parse_args() + + try: + issues = fetch_issues(strict=True) + except RuntimeError as e: + sys.exit(str(e)) + + no_issue, failed, changed = [], [], 0 + for package in read_packages(): + issue = oldest_open_issue(issues, package) + if issue is None: + no_issue.append(package) + continue + status = project_status(issue) + if status in (STATUS_RISE, STATUS_UPSTREAM): + continue + prefix = "[dry-run] would set" if args.dry_run else "[+] set" + print(f"{prefix} {package} #{issue['number']}: {status} -> {STATUS_RISE}") + if args.dry_run: + changed += 1 + continue + try: + set_issue_status(issue["url"], STATUS_RISE) + changed += 1 + time.sleep(PAUSE_SECONDS) + except subprocess.CalledProcessError as e: + print(f"::warning::{package} #{issue['number']}: {e.stderr.strip()}") + failed.append(package) + + print(f"{changed} issue(s) {'to update' if args.dry_run else 'updated'}, " + f"{len(failed)} failed, {len(no_issue)} package(s) without an open issue") + if no_issue: + print("::group::Packages without an open ' riscv64 support' issue") + print("\n".join(no_issue)) + print("::endgroup::") + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py index 47ee8457700..9d327f660f9 100644 --- a/ci_scripts/update_doc.py +++ b/ci_scripts/update_doc.py @@ -19,7 +19,6 @@ import difflib import hashlib import os -import re import subprocess import sys import zipfile @@ -29,7 +28,7 @@ import yaml -REPO = "riseproject-dev/python-wheels" +from github_issues import REPO, normalize_name CI_SCRIPTS_DIR = Path(os.path.dirname(os.path.abspath(__file__))) DOCS_DIR = (CI_SCRIPTS_DIR / ".." / "docs" / "packages").resolve() PACKAGES_FILE = CI_SCRIPTS_DIR / "packages.txt" @@ -44,13 +43,6 @@ def find_wheel_files(path): return sorted(Path(path).glob("*.whl")) -def normalize_name(name): - """ - https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization - """ - return re.sub(r"[-_.]+", "-", name).lower() - - # "License :: OSI Approved :: BSD License" -> "BSD" TROVE_LICENSE_PREFIX = "License :: " diff --git a/docs/workflows.md b/docs/workflows.md index cc16a50eb6c..da9bf07feb2 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -276,7 +276,12 @@ version is no longer pending. That push has two effects: - the package build workflow may be triggered by its YAML path, but `_setup.yml` returns `[]`, so all guarded build/publish jobs skip; - `website.yml` regenerates package pages and the static PEP 503/691 Simple API, - builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. + builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. Its + `sync-project-status` job then runs `ci_scripts/sync_project_status.py`, which + moves the open ` riscv64 support` issue of every package in + `ci_scripts/packages.txt` to *Available in RISE PyPI* in the Python Wheels + project, unless it already says *Available in RISE PyPI* or *Available Upstream*. + The issue stays open; only a deprecation closes it. The GitHub Release exists before this merge, but pip cannot discover it through the RISE index until the YAML reaches `main` and the website deployment @@ -414,7 +419,8 @@ job must grant `contents: write` and `pull-requests: write`. - `pr-checks.yml` checks commit/PR policy, validates package YAML/workflow pairing, and validates patch `Upstream-Status` headers. - `website.yml` turns `docs/packages/*.yaml` into package documentation and the - Simple API, then deploys GitHub Pages. + Simple API, deploys GitHub Pages, then marks newly published packages' + issues *Available in RISE PyPI* in the Python Wheels project. ## The RISC-V Wheels Dashboard diff --git a/skills/python-project-porting/references/pr-and-publishing.md b/skills/python-project-porting/references/pr-and-publishing.md index 66ad85af829..bc0b6d30b87 100644 --- a/skills/python-project-porting/references/pr-and-publishing.md +++ b/skills/python-project-porting/references/pr-and-publishing.md @@ -20,13 +20,17 @@ mode that makes sense): 146 already exist, titles are not always the PyPI name (`SGLang`, `LibCST`, `PyNaCl`), and duplicates are already a problem (bcrypt has four). Match on a normalised name, case-insensitively, and when several match, link the **oldest** — that is what the repo already does (#82, #84, #94). -3. **Development link**: PR -> issue. A closing keyword in the PR body also works, but for an - already-merged PR use the mutation the Development panel uses: - `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. Read it back via the PR's - `closingIssuesReferences`. -4. **Project**: add the issue to Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and set - **Status** (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* - (option `47fc9ee4`; the others are *Todo* `f75ad846` and *Available Upstream* `98236657`). +3. **Development link**: PR -> issue, on the already-merged PR, with the mutation the + Development panel uses: `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. + Read it back via the PR's `closingIssuesReferences`. Never put a closing keyword in the PR + body: the issue must stay open, only a deprecation closes it. +4. **Project**: automatic. Once the package PR is merged and the site deployed, `website.yml`'s + `sync-project-status` job (`ci_scripts/sync_project_status.py`) adds the issue to + Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and sets **Status** + (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* (option `47fc9ee4`; the + others are *Todo* `f75ad846` and *Available Upstream* `98236657`). Do not set it by hand + before the package is on `/simple/`; `ci_scripts/sync_project_status.py --dry-run` shows + what the next run will change. Needs `project` scope on the gh token (`gh auth refresh -h github.com -s project`) on top of `repo`/`workflow`. PR #390 <-> issue #405 is the reference pair to diff a new one against.