From 331af25ca0690fde2a7ce1f62e30402ae9036806 Mon Sep 17 00:00:00 2001 From: Julien Stephan Date: Mon, 26 Oct 2026 16:24:52 +0100 Subject: [PATCH] ci: mark published packages' issues "Available in RISE PyPI" A package's " riscv64 support" issue should move to "Available in RISE PyPI" in the Python Wheels project once its wheels are published. Doing it when update_doc.py opens the docs PR is too early: /simple/ is only regenerated by website.yml after that PR is merged. Add a sync-project-status job after website.yml's deploy that runs the new ci_scripts/sync_project_status.py. For every package in packages.txt it takes the oldest open issue and, unless its status is already "Available in RISE PyPI" or "Available Upstream", adds it to the project and sets the status. The issue stays open; only a deprecation closes it. It compares state instead of diffing the push, so runs dropped by the concurrency queue or failed runs are caught up by the next deploy. Move the issue and project helpers into ci_scripts/github_issues.py, shared by sync_project_status.py, check_versions.py and update_doc.py (REPO, normalize_name). check_versions.py's find_upstream_issue() now fetches the issues once per run and picks the oldest open issue whose title matches exactly, instead of the newest fuzzy search hit. The project is matched by node ID, since its title is "Python Wheels - riseproject-dev/python-wheels". Document the job, and drop the closing-keyword advice from the porting steps: the issue must stay open. Signed-off-by: Julien Stephan --- .github/workflows/website.yml | 28 ++++ ci_scripts/check_versions.py | 41 +++-- ci_scripts/github_issues.py | 144 ++++++++++++++++++ ci_scripts/sync_project_status.py | 93 +++++++++++ ci_scripts/update_doc.py | 10 +- docs/workflows.md | 10 +- .../references/pr-and-publishing.md | 18 ++- 7 files changed, 302 insertions(+), 42 deletions(-) create mode 100644 ci_scripts/github_issues.py create mode 100755 ci_scripts/sync_project_status.py diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index b7dbb58a8723..15a6427ca7b3 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -59,3 +59,31 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./_site cname: pypi.riseproject.dev + + # A package is officially available once deploy has published /simple/, so + # we can now move the issue Status to "Available in RISE PyPI". + # Compares packages.txt with the project rather than diffing this push, + # so pushes the concurrency queue skipped are caught up. + sync-project-status: + needs: deploy + if: ${{ vars.RISEPROJECT_APP_CLIENT_ID != '' }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # GITHUB_TOKEN cannot write organization projects; the App needs the + # organization "Projects: read and write" permission. + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ vars.RISEPROJECT_APP_CLIENT_ID }} + private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: ci_scripts + + - name: Mark published packages in the Python Wheels project + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: python3 ci_scripts/sync_project_status.py diff --git a/ci_scripts/check_versions.py b/ci_scripts/check_versions.py index bfab4db004e9..dfe192ced197 100755 --- a/ci_scripts/check_versions.py +++ b/ci_scripts/check_versions.py @@ -16,7 +16,6 @@ import json import re import shutil -import subprocess import sys import time from datetime import datetime, timedelta, timezone @@ -26,7 +25,8 @@ import requests import yaml -from update_doc import normalize_name, version_blocks +from github_issues import fetch_issues, normalize_name, oldest_open_issue +from update_doc import version_blocks ROOT_DIR = Path(__file__).resolve().parent.parent @@ -34,7 +34,6 @@ PATCHES_DIR = ROOT_DIR / "patches" PACKAGES_FILE = "ci_scripts/packages.txt" DEPRECATED_FILE = "ci_scripts/deprecated.txt" -REPO = "riseproject-dev/python-wheels" # Python versions (interpreter tags) a package must have upstream riscv64 # wheels for before we deprecate our own build. "3.14t" is the free-threaded @@ -175,31 +174,25 @@ def is_pure_python_wheel(package_info: Dict, target_version: str) -> bool: return has_wheels -def find_upstream_issue(package: str) -> Optional[str]: - """ - Find issue for a package in the wheel_builder's Upstream milestone. +_issues: Optional[Dict] = None - All issues in the upstream milestone follow the same title format: - {package} riscv64 support +def find_upstream_issue(package: str) -> Optional[str]: """ - try: - issue_title = f"{package} riscv64 support" - result = subprocess.run([ - "gh", "issue", "list", - "--search", issue_title, - "--json", "number", - "--jq", ".[0].number", - "--repo", REPO, - ], capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - return None + Number of the package's oldest open "{package} riscv64 support" issue. - return result.stdout.strip() or None - - except (subprocess.TimeoutExpired, subprocess.CalledProcessError, Exception): - return None + The issues are fetched once and reused for every package of the run. + """ + global _issues + if _issues is None: + try: + # Project fields are not needed here, and the nightly token cannot read them. + _issues = fetch_issues(with_project=False) + except RuntimeError as e: + print(f"warning: {e}", file=sys.stderr) + _issues = {} + issue = oldest_open_issue(_issues, package) + return str(issue["number"]) if issue else None def deprecate_package(package: str) -> bool: diff --git a/ci_scripts/github_issues.py b/ci_scripts/github_issues.py new file mode 100644 index 000000000000..aec4cc4b360e --- /dev/null +++ b/ci_scripts/github_issues.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Shared helpers for the " riscv64 support" issues and their status in +the "Python Wheels" project. + +Uses the ``gh`` CLI. Reading project fields needs the ``read:project`` scope, +changing them the ``project`` scope (for the GitHub App: the organization +"Projects: read and write" permission). +""" + +import json +import re +import subprocess +import sys + +OWNER = "riseproject-dev" +REPO_NAME = "python-wheels" +REPO = f"{OWNER}/{REPO_NAME}" + +# Projects > "Python Wheels - riseproject-dev/python-wheels". Node IDs rather +# than names, so that the edit works with any gh version (item-edit only +# recently learned --field/--value) and a project rename changes nothing. +PROJECT_NUMBER = 4 +PROJECT_ID = "PVT_kwDOCRlTBM4BbcwJ" +STATUS_FIELD_ID = "PVTSSF_lADOCRlTBM4BbcwJzhWNMvs" +STATUS_TODO = "Todo" +STATUS_RISE = "Available in RISE PyPI" +STATUS_UPSTREAM = "Available Upstream" +STATUS_OPTION_IDS = { + STATUS_TODO: "f75ad846", + STATUS_RISE: "47fc9ee4", + STATUS_UPSTREAM: "98236657", +} + +TITLE_RE = re.compile(r"^\s*(.+?)\s+riscv64\s+support\s*$", re.IGNORECASE) + +QUERY = """ +query($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + issues(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url state + %s + } + } + } +} +""" + +PROJECT_FIELDS = """ + projectItems(first: 10) { + nodes { + project { id } + status: fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { name } + } + } + }""" + + +def normalize_name(name): + """ + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def fetch_issues(owner=OWNER, repo=REPO_NAME, with_project=True, strict=False): + """ + Return {normalised package name: [issue, ...]} for " riscv64 support" + issues, open and closed, oldest first. + + Without the read:project scope the project fields are dropped with a + warning, unless `strict`, where that and any other GraphQL error raise + RuntimeError instead. + """ + query = QUERY % (PROJECT_FIELDS if with_project else "") + by_name = {} + cursor = None + while True: + cmd = ["gh", "api", "graphql", "-f", f"query={query}", + "-F", f"owner={owner}", "-F", f"repo={repo}"] + if cursor: + cmd += ["-F", f"cursor={cursor}"] + proc = subprocess.run(cmd, capture_output=True, text=True) + try: + data = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + data = {} + if with_project and not strict and "read:project" in proc.stderr: + print("warning: token lacks the read:project scope, project status unavailable" + " (gh auth refresh -h github.com -s read:project)", file=sys.stderr) + return fetch_issues(owner, repo, with_project=False) + if "data" not in data or data["data"] is None: + raise RuntimeError(f"gh api graphql failed: {proc.stderr.strip() or data}") + if data.get("errors"): + msgs = "; ".join(sorted({e.get("message", "") for e in data["errors"]})) + if strict: + raise RuntimeError(f"gh api graphql failed: {msgs}") + print(f"warning: {msgs}", file=sys.stderr) + issues = data["data"]["repository"]["issues"] + for issue in issues["nodes"]: + m = TITLE_RE.match(issue["title"]) + if m: + by_name.setdefault(normalize_name(m.group(1)), []).append(issue) + if not issues["pageInfo"]["hasNextPage"]: + return by_name + cursor = issues["pageInfo"]["endCursor"] + + +def oldest_open_issue(issues, package): + """The package's oldest open issue in fetch_issues() output, or None.""" + for issue in issues.get(normalize_name(package), []): + if issue["state"] == "OPEN": + return issue + return None + + +def project_status(issue): + if "projectItems" not in issue: + return "(unknown)" + for item in (issue.get("projectItems") or {}).get("nodes") or []: + if item and item["project"]["id"] == PROJECT_ID: + return (item.get("status") or {}).get("name") or "(no status)" + return "(not in project)" + + +def set_issue_status(issue_url, status): + """ + Add the issue to the project (a no-op when it is already there) and set its + Status. Raises subprocess.CalledProcessError when gh fails. + """ + item_id = subprocess.run( + ["gh", "project", "item-add", str(PROJECT_NUMBER), "--owner", OWNER, + "--url", issue_url, "--format", "json", "--jq", ".id"], + capture_output=True, text=True, check=True, + ).stdout.strip() + subprocess.run( + ["gh", "project", "item-edit", "--id", item_id, + "--project-id", PROJECT_ID, "--field-id", STATUS_FIELD_ID, + "--single-select-option-id", STATUS_OPTION_IDS[status]], + capture_output=True, text=True, check=True, + ) diff --git a/ci_scripts/sync_project_status.py b/ci_scripts/sync_project_status.py new file mode 100755 index 000000000000..605584b9d9bf --- /dev/null +++ b/ci_scripts/sync_project_status.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Mark every package we publish as "Available in RISE PyPI" in the project. + +For each package in ci_scripts/packages.txt, take its oldest open +" riscv64 support" issue and, unless its Status already says it is +available (in RISE PyPI, or upstream after a deprecation), add it to the +"Python Wheels" project and set Status to "Available in RISE PyPI". + +The website workflow runs this after each deploy, so a package only flips once +its docs PR is merged and /simple/ lists it. It compares state rather than +reacting to one push, so a skipped or failed run is caught up by the next one. + +Needs a token with the ``project`` scope (the GitHub App: organization +"Projects: read and write"): + + gh auth refresh -h github.com -s project + ci_scripts/sync_project_status.py --dry-run +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +from github_issues import ( + STATUS_RISE, + STATUS_UPSTREAM, + fetch_issues, + oldest_open_issue, + project_status, + set_issue_status, +) + +PACKAGES_FILE = Path(__file__).resolve().parent / "packages.txt" + +# Each update is two mutations. Usually only a package or two change per run, +# but a backlog would otherwise trip GitHub's secondary rate limit. +PAUSE_SECONDS = 1.5 + + +def read_packages(): + return [line.strip() for line in PACKAGES_FILE.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#")] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dry-run", action="store_true", + help="Print the issues that would change, change nothing") + args = parser.parse_args() + + try: + issues = fetch_issues(strict=True) + except RuntimeError as e: + sys.exit(str(e)) + + no_issue, failed, changed = [], [], 0 + for package in read_packages(): + issue = oldest_open_issue(issues, package) + if issue is None: + no_issue.append(package) + continue + status = project_status(issue) + if status in (STATUS_RISE, STATUS_UPSTREAM): + continue + prefix = "[dry-run] would set" if args.dry_run else "[+] set" + print(f"{prefix} {package} #{issue['number']}: {status} -> {STATUS_RISE}") + if args.dry_run: + changed += 1 + continue + try: + set_issue_status(issue["url"], STATUS_RISE) + changed += 1 + time.sleep(PAUSE_SECONDS) + except subprocess.CalledProcessError as e: + print(f"::warning::{package} #{issue['number']}: {e.stderr.strip()}") + failed.append(package) + + print(f"{changed} issue(s) {'to update' if args.dry_run else 'updated'}, " + f"{len(failed)} failed, {len(no_issue)} package(s) without an open issue") + if no_issue: + print("::group::Packages without an open ' riscv64 support' issue") + print("\n".join(no_issue)) + print("::endgroup::") + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py index 47ee84577003..9d327f660f9e 100644 --- a/ci_scripts/update_doc.py +++ b/ci_scripts/update_doc.py @@ -19,7 +19,6 @@ import difflib import hashlib import os -import re import subprocess import sys import zipfile @@ -29,7 +28,7 @@ import yaml -REPO = "riseproject-dev/python-wheels" +from github_issues import REPO, normalize_name CI_SCRIPTS_DIR = Path(os.path.dirname(os.path.abspath(__file__))) DOCS_DIR = (CI_SCRIPTS_DIR / ".." / "docs" / "packages").resolve() PACKAGES_FILE = CI_SCRIPTS_DIR / "packages.txt" @@ -44,13 +43,6 @@ def find_wheel_files(path): return sorted(Path(path).glob("*.whl")) -def normalize_name(name): - """ - https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization - """ - return re.sub(r"[-_.]+", "-", name).lower() - - # "License :: OSI Approved :: BSD License" -> "BSD" TROVE_LICENSE_PREFIX = "License :: " diff --git a/docs/workflows.md b/docs/workflows.md index cc16a50eb6c1..da9bf07feb28 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -276,7 +276,12 @@ version is no longer pending. That push has two effects: - the package build workflow may be triggered by its YAML path, but `_setup.yml` returns `[]`, so all guarded build/publish jobs skip; - `website.yml` regenerates package pages and the static PEP 503/691 Simple API, - builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. + builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. Its + `sync-project-status` job then runs `ci_scripts/sync_project_status.py`, which + moves the open ` riscv64 support` issue of every package in + `ci_scripts/packages.txt` to *Available in RISE PyPI* in the Python Wheels + project, unless it already says *Available in RISE PyPI* or *Available Upstream*. + The issue stays open; only a deprecation closes it. The GitHub Release exists before this merge, but pip cannot discover it through the RISE index until the YAML reaches `main` and the website deployment @@ -414,7 +419,8 @@ job must grant `contents: write` and `pull-requests: write`. - `pr-checks.yml` checks commit/PR policy, validates package YAML/workflow pairing, and validates patch `Upstream-Status` headers. - `website.yml` turns `docs/packages/*.yaml` into package documentation and the - Simple API, then deploys GitHub Pages. + Simple API, deploys GitHub Pages, then marks newly published packages' + issues *Available in RISE PyPI* in the Python Wheels project. ## The RISC-V Wheels Dashboard diff --git a/skills/python-project-porting/references/pr-and-publishing.md b/skills/python-project-porting/references/pr-and-publishing.md index 66ad85af829a..bc0b6d30b875 100644 --- a/skills/python-project-porting/references/pr-and-publishing.md +++ b/skills/python-project-porting/references/pr-and-publishing.md @@ -20,13 +20,17 @@ mode that makes sense): 146 already exist, titles are not always the PyPI name (`SGLang`, `LibCST`, `PyNaCl`), and duplicates are already a problem (bcrypt has four). Match on a normalised name, case-insensitively, and when several match, link the **oldest** — that is what the repo already does (#82, #84, #94). -3. **Development link**: PR -> issue. A closing keyword in the PR body also works, but for an - already-merged PR use the mutation the Development panel uses: - `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. Read it back via the PR's - `closingIssuesReferences`. -4. **Project**: add the issue to Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and set - **Status** (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* - (option `47fc9ee4`; the others are *Todo* `f75ad846` and *Available Upstream* `98236657`). +3. **Development link**: PR -> issue, on the already-merged PR, with the mutation the + Development panel uses: `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. + Read it back via the PR's `closingIssuesReferences`. Never put a closing keyword in the PR + body: the issue must stay open, only a deprecation closes it. +4. **Project**: automatic. Once the package PR is merged and the site deployed, `website.yml`'s + `sync-project-status` job (`ci_scripts/sync_project_status.py`) adds the issue to + Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and sets **Status** + (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* (option `47fc9ee4`; the + others are *Todo* `f75ad846` and *Available Upstream* `98236657`). Do not set it by hand + before the package is on `/simple/`; `ci_scripts/sync_project_status.py --dry-run` shows + what the next run will change. Needs `project` scope on the gh token (`gh auth refresh -h github.com -s project`) on top of `repo`/`workflow`. PR #390 <-> issue #405 is the reference pair to diff a new one against.