From 324881d9d67cf334493825d5768271dd5b0e2026 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Mon, 12 Oct 2026 21:24:16 +0000 Subject: [PATCH 01/11] Publish CUDA-specific Python wheels through Kitmaker --- .ci/gpu-cicd/context.yml | 149 ++++++++++ .github/workflows/build-and-test.yml | 6 + .github/workflows/release.yml | 128 ++++++++ .gitignore | 8 + .pre-commit-config.yaml | 14 + README.md | 30 +- docs/pypi-publishing.md | 129 ++++++++ python/CMakeLists.txt | 3 +- scripts/Dockerfile | 5 +- scripts/build_pycunls.sh | 123 ++++++-- scripts/build_pycunls_in_docker.sh | 15 + scripts/publish_pypi.sh | 427 +++++++++++++++++++++++++++ scripts/release_pycunls.sh | 25 ++ scripts/requirements-build.txt | 9 + scripts/test_pycunls.sh | 12 + scripts/test_pycunls_in_docker.sh | 7 +- 16 files changed, 1061 insertions(+), 29 deletions(-) create mode 100644 .ci/gpu-cicd/context.yml create mode 100644 .github/workflows/release.yml create mode 100644 docs/pypi-publishing.md create mode 100755 scripts/publish_pypi.sh create mode 100755 scripts/release_pycunls.sh create mode 100644 scripts/requirements-build.txt create mode 100755 scripts/test_pycunls.sh diff --git a/.ci/gpu-cicd/context.yml b/.ci/gpu-cicd/context.yml new file mode 100644 index 0000000..582be26 --- /dev/null +++ b/.ci/gpu-cicd/context.yml @@ -0,0 +1,149 @@ +project: + name: cuNLS + archetype: external_first + build_system: cmake + secondary_build_systems: + - python + test_frameworks: + - googletest + - pytest + gpu_required_for_build: false + gpu_required_for_test: true + concerns: + - python_publishing +docker: + base_image_pattern: nvidia/cuda:{cuda_version}-devel-ubuntu{ubuntu_version} + existing_dockerfile: scripts/Dockerfile + registry: '' + registry_login_secret: '' +matrix: + cuda_versions: + - 12.8.1 + - 13.2.0 + os_versions: + - ubuntu22.04 + - ubuntu24.04 + python_versions: + - '3.10' + - '3.12' + architectures: + - x86_64 +runners: + platform: github_actions + gpu_tags: + - self-hosted + - gpu + cpu_tags: + - ubuntu-latest + gpu_job_timeout_minutes: 120 + cpu_job_timeout_minutes: 15 +artifacts: + test_output_format: junit_xml + release_type: github_release + publish_wheels: true + wheel_destination: pypi +secrets: +- name: KITMAKER_TOKEN + description: Kitmaker service token scoped to the protected pypi environment + required: true +publishing: + enabled: true + legal_approval: + answer: 'yes' + date: '2026-10-12' + source: User states public binary distribution is approved and repository licensing + is correct + canonical_repository: nvidia-isaac/cuNLS + asset_source: github-release + release_workflow: .github/workflows/release.yml + release_job: release + wheel_dir: pypi-wheels + kitmaker_projects: + pycunls-cu12: 6865 + pycunls-cu13: 6866 + pic: hguillen@nvidia.com + owner_response_field: owner_email + platform: manylinux_*_x86_64 + provenance_path: pycunls/_build_provenance.json + provenance_source: python/pycunls/_build_provenance.json + package_root: python + environment: pypi + approvers: + - hguillen + publish_runner: gpu + publish_cells: + - cuda: 12.8.1 + ubuntu: '22.04' + python: '3.10' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu22.04 + - cuda: 13.2.0 + ubuntu: '22.04' + python: '3.10' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu22.04 + - cuda: 12.8.1 + ubuntu: '24.04' + python: '3.12' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu24.04 + - cuda: 13.2.0 + ubuntu: '24.04' + python: '3.12' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu24.04 + expanded_release_cells: + - cuda: 12.8.1 + ubuntu: '22.04' + python: '3.10' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu22.04 + - cuda: 13.2.0 + ubuntu: '22.04' + python: '3.10' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu22.04 + - cuda: 12.8.1 + ubuntu: '24.04' + python: '3.12' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu24.04 + - cuda: 13.2.0 + ubuntu: '24.04' + python: '3.12' + arch: x86_64 + artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu24.04 + matrix_source: .github/workflows/release.yml + matrix_source_sha256: c3dbe99aea21b42345716efe9afcd8655fe9c4731badebdcc2d8ea608ba40432 + first_version: 0.2.0 + version_source: + path: python/pyproject.toml + current: 0.2.0 + dry_run_only: true + per_release_plc: true + accepted_warnings: [] + setup_status: Input discovery passed; environment protection requires reconciliation + in the deployment playbook before activation. +discovery: + commit: fa591a6b3d606dc0fdef837b0ad23c0913a1ed89 + status: Read-only check-inputs passes with zero failures and warnings; implementation + authorized + cuda13_name: User corrected Portal distribution to pycunls-cu13 on 2026-10-12 + reviewer_setup: Owner explicitly requests manual reviewer configuration, starting + with hguillen; alexkorovko is an additional intended reviewer. Both accounts resolve. + This overrides the two-reviewer preparation default. + remote_verified: '2026-10-12: tools/projects fetch cuNLS succeeded; HEAD and origin/main + both fa591a6b3d606dc0fdef837b0ad23c0913a1ed89' + publish_runner_assumption: Reuse existing self-hosted,gpu runner selection; Kitmaker + reachability and CA trust remain for the first protected dry run. + per_release_plc_assumption: The pipeline includes the PLC checkbox conservatively; + owner has not specified per-release PLC requirements. + input_check_log: Read-only input gate passed on 2026-10-12 using the canonical template + before project edits. + environment_protection: '2026-10-12: pypi environment has no protection_rules and + no deployment_branch_policy; manual setup required.' + environment_secret: '2026-10-12: environment-scoped KITMAKER_TOKEN exists.' + actions_allowlist: '2026-10-12: github_owned_allowed=true; workflow uses only GitHub-owned + actions.' + repository_variables: '2026-10-12: no repository variables are configured; deployment + playbook sets publisher selection, enable switch, dry-run mode and PLC checklist.' diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a8b8bec..e20ffa0 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -11,6 +11,10 @@ on: Example: [{"ubuntu":"22.04","cuda":"12.8.1","archs":"75;80;86;89"}] type: string required: true + release_tag: + description: "Optional v tag for a public wheel release" + type: string + default: "" artifact_retention_days: description: "Number of days to retain uploaded artifacts" type: number @@ -40,6 +44,8 @@ jobs: config: ${{ fromJson(inputs.matrix_json) }} env: CONFIG_NAME: ${{ matrix.config.platform || 'x86_64' }}-cuda${{ matrix.config.cuda }}-ubuntu${{ matrix.config.ubuntu }} + RELEASE_TAG: ${{ inputs.release_tag }} + RELEASE_COMMIT: ${{ github.sha }} CUDA_VERSION: ${{ matrix.config.cuda }} UBUNTU_VERSION: ${{ matrix.config.ubuntu }} BASE_IMAGE: ${{ matrix.config.base_image }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9330922 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,128 @@ +name: Release Python wheels + +on: + push: + tags: ['v*'] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYPI_PUBLISH_REPOSITORY: nvidia-isaac/cuNLS + PYPI_ASSET_SOURCE: github-release + WHEEL_DIR: pypi-wheels + KITMAKER_PROJECTS: | + pycunls-cu12=6865 + pycunls-cu13=6866 + KITMAKER_PIC: hguillen@nvidia.com + PYPI_PLATFORM: manylinux_*_x86_64 + PYPI_PROVENANCE_PATH: pycunls/_build_provenance.json + # Keep the approval checkbox until the owner confirms the per-release policy. + PYPI_PER_RELEASE_PLC: ${{ vars.PYPI_PER_RELEASE_PLC || 'yes' }} + PYPI_PUBLISH_CELLS: >- + [{"cuda":"12.8.1","ubuntu":"22.04","python":"3.10","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu22.04"}, + {"cuda":"13.2.0","ubuntu":"22.04","python":"3.10","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu22.04"}, + {"cuda":"12.8.1","ubuntu":"24.04","python":"3.12","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu24.04"}, + {"cuda":"13.2.0","ubuntu":"24.04","python":"3.12","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu24.04"}] + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_COMMIT: ${{ github.sha }} + +jobs: + build-matrix: + if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v') + uses: ./.github/workflows/build-and-test.yml + with: + matrix_json: | + [ + {"ubuntu":"22.04","cuda":"12.8.1","archs":"75;80;86;89"}, + {"ubuntu":"22.04","cuda":"13.2.0","archs":"75;80;86;89;100"}, + {"ubuntu":"24.04","cuda":"12.8.1","archs":"75;80;86;89"}, + {"ubuntu":"24.04","cuda":"13.2.0","archs":"75;80;86;89;100"} + ] + release_tag: ${{ github.ref_name }} + artifact_retention_days: 30 + upload_binaries: true + timeout_minutes: 120 + + release: + if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v') + needs: build-matrix + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/download-artifact@v7 + with: + pattern: pycunls-x86_64-* + path: pypi-wheels + - run: bash scripts/release_pycunls.sh + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: nvidia-isaac/cuNLS + + pypi-prepare: + if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v') + needs: build-matrix + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/download-artifact@v7 + with: + pattern: pycunls-x86_64-* + path: pypi-wheels + - run: sh scripts/publish_pypi.sh prepare + - uses: actions/upload-artifact@v6 + with: + name: pypi-manifest + path: pypi-manifest.json + if-no-files-found: error + retention-days: 30 + + pypi-publish: + # An operator enables the job only after configuring pypi protection. + if: >- + github.repository == 'nvidia-isaac/cuNLS' && + startsWith(github.ref, 'refs/tags/v') && + vars.PYPI_PUBLISH_ENABLED == 'true' + needs: [release, pypi-prepare] + runs-on: ${{ fromJson(vars.PYPI_PUBLISH_RUNNER || '["self-hosted","gpu"]') }} + timeout-minutes: 180 + environment: pypi + concurrency: + group: pypi-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: false + env: + KITMAKER_TOKEN: ${{ secrets.KITMAKER_TOKEN }} + PYPI_DRY_RUN_ONLY: ${{ vars.PYPI_DRY_RUN_ONLY || '1' }} + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/download-artifact@v7 + with: + name: pypi-manifest + path: . + - run: sh scripts/publish_pypi.sh publish + - uses: actions/upload-artifact@v6 + if: always() + with: + name: pypi-publication-record-${{ github.run_attempt }} + path: | + pypi-manifest.json + pypi-release-uuids.txt + pypi-status-*.json + if-no-files-found: warn + retention-days: 30 diff --git a/.gitignore b/.gitignore index 32214a7..1e905b8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,11 @@ viz/ # Design notes are kept locally, not in the repository docs/design/ + +# Generated Python release evidence +python/pycunls/_build_provenance.json +pypi-manifest.json +pypi-release-uuids.txt +pypi-status-*.json +pypi-wheels/ +pypi-publication-record*/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d5ccd9..36fde26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,9 +5,23 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-clang-format rev: v15.0.7 hooks: - id: clang-format types_or: [c++, cuda] + + - repo: local + hooks: + - id: actionlint + name: Validate GitHub Actions workflows + entry: actionlint -shellcheck= + language: system + files: ^\.github/workflows/.*\.ya?ml$ + - id: shellcheck-publishing + name: Validate Python release scripts + entry: shellcheck + language: system + files: ^scripts/(publish_pypi|release_pycunls|build_pycunls|test_pycunls)\.sh$ diff --git a/README.md b/README.md index 142e3fd..f8dd8e9 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ pip wheel . --no-build-isolation --no-deps --wheel-dir ../dist #### Install the wheel ```bash -pip install ./dist/pycunls-*.whl +pip install ./dist/pycunls*.whl ``` #### Editable install for development @@ -454,3 +454,31 @@ find . -iname '*.h' -o -iname '*.cpp' | xargs clang-format -i ## License cuNLS is licensed under the [Apache License 2.0](LICENSE). Third-party license notices are in `NOTICE` and `third_party/LICENSES/`. + +## Versioned Python releases + +`release.yml` builds and tests x86_64 wheels for CUDA 12.8.1 and 13.2.0 on +Ubuntu 22.04 (Python 3.10) and 24.04 (Python 3.12). Docker builds name the +distributions `pycunls-cu12` and `pycunls-cu13`; the import remains `pycunls`. +Each wheel declares the matching CuPy dependency, carries the source commit and +CUDA version, and is repaired with auditwheel while leaving CUDA libraries external. +A matching CUDA toolkit and compatible NVIDIA driver are required at runtime. + +Before tagging `v`, update both `python/pyproject.toml` and +`python/pycunls/__init__.py` to the same public version. The build refuses a +tag/version mismatch. Local reproduction uses the existing four build/test +commands documented above; set `CUDA_VERSION`, `UBUNTU_VERSION`, and +`RELEASE_TAG` to reproduce one release cell. Use a separate output directory per cell. + +The release job attaches wheels with their original filenames. `pypi-prepare` +independently validates the selected artifacts and writes `pypi-manifest.json`. +The protected `pypi-publish` job waits for both jobs, downloads the manifest, +checks the public release bytes, and submits Kitmaker requests for both indexes. +Publication records survive failure; cancellation and automatic submission retries +are disabled. A failed batch can already be partly public. + +Configure GitHub with [the deployment playbook](docs/pypi-publishing.md). +Publishing is disabled until `PYPI_PUBLISH_ENABLED=true`; +`PYPI_DRY_RUN_ONLY` defaults to `1`. The first approved run must remain a dry run. +After review, retry only the publisher for live publication with the same tag, +commit and manifest. Do not recreate a release to retry publication. diff --git a/docs/pypi-publishing.md b/docs/pypi-publishing.md new file mode 100644 index 0000000..a6135a4 --- /dev/null +++ b/docs/pypi-publishing.md @@ -0,0 +1,129 @@ +# Configure cuNLS Python publishing + +Author: Heron Ordonez Guillen + +Run these commands after the workflow is merged into the canonical cuNLS repository. +Credentials come from the operator environment, never from this document. Public binary distribution was +confirmed legally approved by the owner on 2026-10-12. + +```bash +gh auth status +gh repo view nvidia-isaac/cuNLS --json nameWithOwner,visibility +gh api user --jq .login +gh api users/alexkorovko --jq .login +gh api repos/nvidia-isaac/cuNLS/actions/runners --paginate \ + --jq '.runners[] | {name,status,labels:[.labels[].name]}' +gh api repos/nvidia-isaac/cuNLS/actions/permissions/selected-actions +``` + +MANUAL: In GitHub → nvidia-isaac/cuNLS → Settings → Environments, create or +edit the existing `pypi` environment. Set `hguillen` as a required reviewer, +enable prevent self-review and disable administrator bypass. Restrict deployment +to tags matching `v*`. The owner requested a single initial reviewer; add +`alexkorovko` when ready. With one reviewer and self-review prevented, another +trusted account must initiate a release that this reviewer approves. + +Verify protection before storing the environment credential. + +```bash +gh api repos/nvidia-isaac/cuNLS/environments/pypi \ + --jq '{protection_rules,deployment_branch_policy}' +gh api repos/nvidia-isaac/cuNLS/environments/pypi/deployment-branch-policies \ + --jq '.branch_policies[] | {name,type}' +gh secret list --repo nvidia-isaac/cuNLS --env pypi +``` + +`KITMAKER_TOKEN` is already present in the `pypi` environment (verified 2026-10-12). +Keep that scope. To rotate it from approved credential storage, use this +idempotent command; it does not display the value. + +```bash +: "${KITMAKER_TOKEN:?Load the Kitmaker service token from approved credential storage}" +printf '%s' "$KITMAKER_TOKEN" | gh secret set KITMAKER_TOKEN \ + --repo nvidia-isaac/cuNLS --env pypi +``` + +MANUAL: Confirm the existing `[self-hosted, gpu]` runner pool has Kitmaker reachability, NVIDIA CA trust, +and POSIX sh, curl with `--fail-with-body`, jq, unzip and sha256sum. The first +protected dry run verifies reachability from the selected runner. +Portal IDs 6865/6866, names `pycunls-cu12`/`pycunls-cu13` and owner +`hguillen@nvidia.com` were verified through the read-only Portal API on 2026-10-12. + +Configure nonsecret variables; keep publication disabled during setup. + +```bash +gh variable set PYPI_PUBLISH_RUNNER --repo nvidia-isaac/cuNLS \ + --body '["self-hosted","gpu"]' +gh variable set PYPI_PUBLISH_ENABLED --repo nvidia-isaac/cuNLS --body 'false' +gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '1' +gh variable set PYPI_PER_RELEASE_PLC --repo nvidia-isaac/cuNLS --body 'yes' +``` + +MANUAL: The per-release L1 Feature PLC policy is pending owner confirmation. The +workflow includes the checkbox by default. Set `PYPI_PER_RELEASE_PLC=no` only +when the owner confirms it is not required. Each approval records IP self/peer +review completion and the selected filenames and SHA256 hashes. + +## Verify setup and the first dry run + +```bash +gh secret list --repo nvidia-isaac/cuNLS --env pypi +gh variable list --repo nvidia-isaac/cuNLS +gh api repos/nvidia-isaac/cuNLS/environments/pypi +gh api repos/nvidia-isaac/cuNLS/environments/pypi/deployment-branch-policies +``` + +MANUAL: Confirm the secret value and scope in approved credential storage, the +reviewer protections in Settings → Environments → pypi, and runner registration +in Settings → Actions → Runners. Secret values are write-only. + +After setup verification and owner approval, create the first tag from merged +`main` in dry-run mode. The input gate verified `v0.2.0` and both index versions +were absent on 2026-10-12; recheck before creating a tag. Tag creation triggers +the release workflow automatically. Run this block once for the first release; +use publisher-only retries after the producer has succeeded. + +```bash +gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '1' +gh variable set PYPI_PUBLISH_ENABLED --repo nvidia-isaac/cuNLS --body 'true' +RELEASE_COMMIT=$(gh api repos/nvidia-isaac/cuNLS/commits/main --jq .sha) +gh api --method POST repos/nvidia-isaac/cuNLS/git/refs \ + -f ref=refs/tags/v0.2.0 -f sha="$RELEASE_COMMIT" +RELEASE_RUN_ID=$(gh run list --repo nvidia-isaac/cuNLS --workflow release.yml \ + --branch v0.2.0 --event push --limit 1 --json databaseId \ + --jq '.[0].databaseId') +: "${RELEASE_RUN_ID:?No tag-triggered release run found}" +gh run watch "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --exit-status +PUBLICATION_ATTEMPT=$(gh api "repos/nvidia-isaac/cuNLS/actions/runs/$RELEASE_RUN_ID" --jq .run_attempt) +gh run download "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS \ + --name "pypi-publication-record-$PUBLICATION_ATTEMPT" --dir "pypi-publication-record-$PUBLICATION_ATTEMPT" +``` + +MANUAL: Approve the waiting `pypi` deployment after reviewing the prepare summary. +Expect all build/test cells, the public GitHub release and offline manifest to +succeed. Expect each Kitmaker dry run to reach `completed`, with no upload +request; retain the UUIDs, status JSON and manifest. This does not verify index +writes or customer installation. + +After the owner authorizes live publication, rerun only `pypi-publish`, preserving +the original tag, commit, release bytes and manifest. Job database IDs come from +`gh run view`; use the [GitHub CLI job rerun command](https://cli.github.com/manual/gh_run_rerun). + +```bash +gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '0' +PUBLISH_JOB_ID=$(gh run view "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS \ + --json jobs --jq '.jobs[] | select(.name == "pypi-publish") | .databaseId') +: "${PUBLISH_JOB_ID:?Publisher job was not found}" +gh run rerun "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --job "$PUBLISH_JOB_ID" +gh run watch "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --exit-status +``` + +MANUAL: Verify another environment approval is required and the rerun logs show +`PYPI_DRY_RUN_ONLY=0`. Confirm this behavior on the deployed GitHub instance +before live publication; a variable captured by the original run can leave it +in dry-run mode. Publication records are retained separately per run attempt. + +MANUAL: After live publication, confirm every wheel filename and SHA256 on both +PyPI and NVIDIA's public index, then test installation on each supported CUDA +variant. A partial failure requires separate review of the missing artifacts; +never resubmit the complete batch automatically. diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 19bbc8c..14d6da2 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -43,4 +43,5 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pycunls/ DESTINATION pycunls FILES_MATCHING PATTERN "*.py" PATTERN "*.pyi" - PATTERN "py.typed") + PATTERN "py.typed" + PATTERN "_build_provenance.json") diff --git a/scripts/Dockerfile b/scripts/Dockerfile index a8ef571..60fd507 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -22,15 +22,16 @@ RUN if [ -n "${UBUNTU_PORTS_MIRROR}" ]; then \ # Use the distro CMake so Ubuntu 22.04 continuously validates the 3.22 minimum. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates apt-utils g++ gcc git cmake ccache \ - software-properties-common curl \ + software-properties-common curl patchelf \ python3 python3-pip python3-dev python3-venv \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Python venv with build tools (for pycunls wheel builds) RUN python3 -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" +COPY scripts/requirements-build.txt /tmp/cunls-build-requirements.txt RUN pip install --upgrade pip && \ - pip install scikit-build-core nanobind build wheel pre-commit + pip install -r /tmp/cunls-build-requirements.txt # Sphinx docs dependencies COPY docs/sphinx/requirements.txt /tmp/cunls-docs-requirements.txt diff --git a/scripts/build_pycunls.sh b/scripts/build_pycunls.sh index 9ebba48..6f72fa3 100755 --- a/scripts/build_pycunls.sh +++ b/scripts/build_pycunls.sh @@ -1,28 +1,113 @@ #!/bin/bash -# Runs inside the Docker container to build the pycunls wheel. -# Expected mounts: -# /cunls (read-only) — source tree -# OUTPUT_DIR env or /output (read-write) — wheel output directory +# Runs inside the existing CUDA Docker image; the checkout is read-only. set -euo pipefail -WHEEL_DIR=/tmp/pycunls_wheel -mkdir -p "$WHEEL_DIR" +: "${RELEASE_COMMIT:?Pass the original checkout SHA through build_pycunls_in_docker.sh}" +ACTUAL_CUDA_VERSION=$(python3 - <<'PY' +import json +import os +import re +import subprocess +from pathlib import Path -cp -r /cunls /tmp/cunls_src -rm -rf /tmp/cunls_src/python/build /tmp/cunls_src/build_python +path = Path('/usr/local/cuda/version.json') +if path.exists(): + actual = json.loads(path.read_text())['cuda']['version'] +else: + match = re.search(r'release (\d+\.\d+)', subprocess.check_output(['nvcc', '--version'], text=True)) + if not match: + raise SystemExit('Cannot identify the installed CUDA toolkit; use a CUDA development image') + actual = match.group(1) +requested = os.environ.get('CUDA_VERSION') +if requested and requested.split('.')[:2] != actual.split('.')[:2]: + raise SystemExit(f'Requested CUDA {requested}, but the build image contains {actual}; correct CUDA_VERSION or BASE_IMAGE') +print(actual) +PY +) +export CUDA_VERSION=$ACTUAL_CUDA_VERSION +export RELEASE_COMMIT +export RELEASE_TAG=${RELEASE_TAG:-} + +WORK_DIR=$(mktemp -d) +trap 'rm -rf "$WORK_DIR"' EXIT +mkdir -p "$WORK_DIR/raw" "$WORK_DIR/repaired" +cp -r /cunls "$WORK_DIR/source" +rm -rf "$WORK_DIR/source/python/build" "$WORK_DIR/source/build_python" +cd "$WORK_DIR/source/python" + +python3 - <<'PY' +import json +import os +import re +from pathlib import Path +from packaging.version import Version + +cuda = os.environ['CUDA_VERSION'] +major = cuda.split('.')[0] +if major not in ('12', '13'): + raise SystemExit(f'Unsupported CUDA version: {cuda}; expected CUDA 12 or 13') +sha = os.environ['RELEASE_COMMIT'] +if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise SystemExit('RELEASE_COMMIT must be the full original checkout SHA') +path = Path('pyproject.toml') +text = path.read_text() +text, count = re.subn(r'^name = "pycunls"$', f'name = "pycunls-cu{major}"', text, flags=re.M) +if count != 1: + raise SystemExit('Expected one pycunls project name in pyproject.toml') +text = text.replace('cupy-cuda12x', f'cupy-cuda{major}x') +tag = os.environ['RELEASE_TAG'] +if tag: + if not tag.startswith('v'): + raise SystemExit('Release tag must start with v') + version = Version(tag[1:]) + if version.local or str(version) != tag[1:]: + raise SystemExit('Release tag must contain a normalized public PEP 440 version') + current = re.findall(r'^version = "([^"]+)"$', text, flags=re.M) + if current != [str(version)]: + raise SystemExit(f'Package version {current} differs from {tag}; update version sources before tagging') + init_version = re.findall(r'^__version__ = "([^"]+)"$', Path('pycunls/__init__.py').read_text(), flags=re.M) + if init_version != [str(version)]: + raise SystemExit(f'Import version {init_version} differs from {tag}; update version sources before tagging') +path.write_text(text) +Path('pycunls/_build_provenance.json').write_text(json.dumps({ + 'git_sha': sha, 'cuda_version': cuda, +}, sort_keys=True) + '\n') +PY -cd /tmp/cunls_src/python -# Without Ninja, scikit-build-core drives Makefiles serially unless a parallel -# level is set: build with every core. export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)} -pip wheel . --no-build-isolation --no-deps --wheel-dir "$WHEEL_DIR" +pip wheel . --no-build-isolation --no-deps --wheel-dir "$WORK_DIR/raw" + +# The system CUDA toolkit supplies these linked runtime libraries. Match SONAME +# families rather than assuming CUDA 12 and 13 use the same SONAME versions. +PLATFORM=$(python3 -c 'import os, platform; print("manylinux_" + os.confstr("CS_GNU_LIBC_VERSION").split()[1].replace(".", "_") + "_" + platform.machine())') +for wheel in "$WORK_DIR/raw"/*.whl; do + auditwheel repair "$wheel" --plat "$PLATFORM" --wheel-dir "$WORK_DIR/repaired" \ + --exclude 'libcudart.so.*' --exclude 'libcusparse.so.*' \ + --exclude 'libcublas.so.*' --exclude 'libcublasLt.so.*' \ + --exclude 'libcusolver.so.*' --exclude 'libcuda.so.*' +done + +python3 - "$WORK_DIR/repaired" <<'PY' +import json +import os +import re +import sys +import zipfile +from pathlib import Path -echo "" -echo "=========================================" -echo " Wheel built successfully" -echo "=========================================" -ls -lh "$WHEEL_DIR"/*.whl +wheels = list(Path(sys.argv[1]).glob('*.whl')) +if len(wheels) != 1: + raise SystemExit(f'Expected one repaired wheel, found {len(wheels)}') +with zipfile.ZipFile(wheels[0]) as archive: + provenance = json.loads(archive.read('pycunls/_build_provenance.json')) + if provenance != {'git_sha': os.environ['RELEASE_COMMIT'], 'cuda_version': os.environ['CUDA_VERSION']}: + raise SystemExit('Repaired wheel lost the original build provenance') + for member in archive.namelist(): + if re.search(r'(^|/)lib(cudart|cusparse|cublas|cusolver|cudss|cuda)[^/]*\.so', member): + raise SystemExit(f'CUDA runtime library must remain external: {member}') +PY OUTPUT=${OUTPUT_DIR:-/output} -cp "$WHEEL_DIR"/*.whl "$OUTPUT"/ -echo "Wheel copied to $OUTPUT/" +mkdir -p "$OUTPUT" +cp "$WORK_DIR/repaired"/*.whl "$OUTPUT/" +ls -lh "$OUTPUT"/*.whl diff --git a/scripts/build_pycunls_in_docker.sh b/scripts/build_pycunls_in_docker.sh index 8e7773d..020a878 100755 --- a/scripts/build_pycunls_in_docker.sh +++ b/scripts/build_pycunls_in_docker.sh @@ -11,6 +11,17 @@ LOCAL_OUTPUT_DIR=$(realpath -m "$LOCAL_OUTPUT_DIR") mkdir -p "$LOCAL_OUTPUT_DIR" +CHECKOUT_COMMIT=$(git rev-parse HEAD) +RELEASE_COMMIT=${RELEASE_COMMIT:-$CHECKOUT_COMMIT} +if [ "$RELEASE_COMMIT" != "$CHECKOUT_COMMIT" ]; then + echo "RELEASE_COMMIT differs from the checked-out source; build the release checkout." >&2 + exit 1 +fi +if [ -n "${RELEASE_TAG:-}" ] && ! git diff --quiet HEAD --; then + echo "Release wheels require committed source; commit the changes before building a release tag." >&2 + exit 1 +fi + DOCKERFILE=$(dirname "$(realpath "$0")")/Dockerfile DOCKER_BUILD_ARGS="" [ -n "${CUDA_VERSION:-}" ] && DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg CUDA_VERSION=$CUDA_VERSION" @@ -30,6 +41,10 @@ docker run --runtime=nvidia --gpus all --rm $TTY_FLAG \ -v "$(pwd):/cunls:ro" \ -v "$LOCAL_OUTPUT_DIR:/cunls_install" \ -e OUTPUT_DIR=/cunls_install \ + -e RELEASE_COMMIT="$RELEASE_COMMIT" \ + -e RELEASE_TAG="${RELEASE_TAG:-}" \ + -e CUDA_VERSION="${CUDA_VERSION:-}" \ + -e CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-}" \ cunls:local bash /cunls/scripts/build_pycunls.sh echo "" diff --git a/scripts/publish_pypi.sh b/scripts/publish_pypi.sh new file mode 100755 index 0000000..b35a94c --- /dev/null +++ b/scripts/publish_pypi.sh @@ -0,0 +1,427 @@ +#!/bin/sh +# TEMPLATE_VERSION=1.8.0 +set -eu +set -f + +fail() { + printf '%s: actual=%s; expected=%s; fix=%s' "$1" "$2" "$3" "$4" | tr '\n' ' ' >&2 + printf '\n' >&2 + exit 1 +} +for dependency in curl jq unzip sha256sum; do + command -v "$dependency" >/dev/null || fail "$0" "$dependency missing" 'curl jq unzip sha256sum' "install $dependency on the runner host" +done +mode=${1:-} +case "$mode" in check-inputs|prepare|publish) ;; *) fail "$0" "$mode" 'check-inputs | prepare | publish' 'select one command' ;; esac +PYPI_ASSET_SOURCE=${PYPI_ASSET_SOURCE:-} +if [ -z "$PYPI_ASSET_SOURCE" ]; then + if [ -n "${GITHUB_REPOSITORY:-}" ]; then PYPI_ASSET_SOURCE=github-release + elif [ -n "${CI_PROJECT_PATH:-}" ]; then PYPI_ASSET_SOURCE=artifactory; fi +fi +RELEASE_TAG=${RELEASE_TAG:-${GITHUB_REF_NAME:-${CI_COMMIT_TAG:-}}} +RELEASE_COMMIT=${RELEASE_COMMIT:-${GITHUB_SHA:-${CI_COMMIT_SHA:-}}} +PYPI_PLATFORM=${PYPI_PLATFORM:-manylinux_*_x86_64} +PYPI_DRY_RUN_ONLY=${PYPI_DRY_RUN_ONLY:-0} +PYPI_ENVIRONMENT=${PYPI_ENVIRONMENT:-pypi} +PYPI_PER_RELEASE_PLC=${PYPI_PER_RELEASE_PLC:-no} +PYPI_POLL_SECONDS=${PYPI_POLL_SECONDS:-10} +PYPI_TIMEOUT_SECONDS=${PYPI_TIMEOUT_SECONDS:-1800} +PYPI_INDEX_TIMEOUT_SECONDS=${PYPI_INDEX_TIMEOUT_SECONDS:-600} +PYPI_PACKAGE_ROOT=${PYPI_PACKAGE_ROOT:-.} +KITMAKER_OWNER_FIELD=${KITMAKER_OWNER_FIELD:-} +KITMAKER_API=https://kitmaker-portal.nvidia.com/api/v0 +manifest=pypi-manifest.json +# Enumerating known nonsecret inputs avoids dumping ambient credentials. +for input_name in PYPI_ASSET_SOURCE PYPI_PUBLISH_REPOSITORY RELEASE_TAG RELEASE_COMMIT WHEEL_DIR KITMAKER_PROJECTS KITMAKER_PIC PYPI_PLATFORM PYPI_PROVENANCE_PATH PYPI_PROVENANCE_SOURCE PYPI_DRY_RUN_ONLY ARTIFACTORY_BASE_URL PYPI_ENVIRONMENT PYPI_PER_RELEASE_PLC PYPI_POLL_SECONDS PYPI_TIMEOUT_SECONDS PYPI_INDEX_TIMEOUT_SECONDS PYPI_RELEASE_WORKFLOW PYPI_RELEASE_JOB PYPI_CI_SECRET_NAMES PYPI_APPROVERS PYPI_PUBLISH_RUNNER PYPI_PUBLISH_CELLS PYPI_RELEASE_CELLS PYPI_MATRIX_PATH PYPI_MATRIX_SHA256 PYPI_FIRST_VERSION PYPI_VERSION_FILE PYPI_CURRENT_VERSION PYPI_PACKAGE_ROOT KITMAKER_OWNER_FIELD; do + eval 'input_value=${'"$input_name"':-}' + printf '%s=%s\n' "$input_name" "$input_value" +done +umask 077 +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT HUP INT TERM +normalize() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[-_.][-_\.]*/-/g'; } +required() { + missing= + for input_name in "$@"; do + eval 'input_value=${'"$input_name"':-}' + [ -n "$input_value" ] || missing="$missing $input_name" + done + [ -z "$missing" ] || fail "$0" "missing:$missing" 'all required inputs set' 'export the listed names from the confirmed context' +} +repository_guard() { + case "$PYPI_ASSET_SOURCE" in + github-release) actual_repository=${GITHUB_REPOSITORY:-} ;; + artifactory) actual_repository=${CI_PROJECT_PATH:-} ;; + *) fail "$0" "$PYPI_ASSET_SOURCE" 'github-release or artifactory' 'set PYPI_ASSET_SOURCE' ;; + esac + [ "$actual_repository" = "$PYPI_PUBLISH_REPOSITORY" ] || fail "$0" "$actual_repository" "$PYPI_PUBLISH_REPOSITORY" "private copies don't publish; run in the canonical repository" +} +public_version() { + printf '%s' "$1" | jq -eRs 'test("^([0-9]+!)?[0-9]+(\\.[0-9]+)*((a|b|rc)[0-9]+)?(\\.post[0-9]+)?(\\.dev[0-9]+)?$")' >/dev/null +} +project_map() { + printf '%s\n' "${KITMAKER_PROJECTS:-}" | jq -Rsc ' + split("\n") | map(select(length > 0) | split("=")) | + if length == 0 or any(.[]; length != 2 or (.[0] | test("^[a-z0-9]+(-[a-z0-9]+)*$") | not) or (.[1] | test("^[1-9][0-9]*$") | not)) + then error("expected canonical-name=numeric-id, one per line") + elif (map(.[0]) | unique | length) != length or (map(.[1]) | unique | length) != length + then error("duplicate name or project ID") + else map({name:.[0], id:.[1]}) end' > "$scratch/map.json" +} +valid_cell_directories() { + printf '%s' "$1" | jq -e 'type == "array" and length > 0 and all(.[]; + (.cuda | type == "string") and (.artifact_dir | type == "string" and (. == "." or (test("^[A-Za-z0-9_][A-Za-z0-9_./-]*$") and (split("/") | all(.[]; . != ".." and . != "." and . != "")))))) and + (length == 1 or all(.[]; .artifact_dir != ".")) and + (map(.artifact_dir) | unique | length) == length' >/dev/null +} +# Preserve error bodies and never retry a POST or PUT automatically. +http() { + http_status=000 + http_rc=0 + : > "$scratch/response" + http_status=$(curl --fail-with-body --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 15 --max-time 120 --output "$scratch/response" --write-out '%{http_code}' "$@") || http_rc=$? + printf 'HTTP %s (curl=%s)\n' "$http_status" "$http_rc" +} +kitmaker() { + http --header "Authorization: Bearer $KITMAKER_TOKEN" --header 'Content-Type: application/json' "$@" + cat "$scratch/response" + printf '\n' + [ "$http_rc" -eq 0 ] || fail "$KITMAKER_API" "HTTP $http_status, curl $http_rc" 'successful response' 'inspect the response; check token, network and NVIDIA CA; never blindly resubmit' +} + +prepare() { + required PYPI_PUBLISH_REPOSITORY PYPI_ASSET_SOURCE RELEASE_TAG RELEASE_COMMIT WHEEL_DIR KITMAKER_PROJECTS KITMAKER_PIC PYPI_PROVENANCE_PATH PYPI_PUBLISH_CELLS + repository_guard + case "$PYPI_PROVENANCE_PATH" in /*|../*|*/../*|*..|*\**|*\?*|*\[*) fail "$0" "$PYPI_PROVENANCE_PATH" 'literal wheel-relative provenance path' 'use import_package/_build_provenance.json' ;; esac + case "$RELEASE_COMMIT" in *[!a-fA-F0-9]*|'') fail "$0" "$RELEASE_COMMIT" 'full git commit SHA' 'use the commit built by the release matrix' ;; esac + [ "${#RELEASE_COMMIT}" -eq 40 ] || fail "$0" "$RELEASE_COMMIT" '40-character git SHA' 'resolve the full release commit' + project_map || fail "$0" "$KITMAKER_PROJECTS" 'unique canonical-name=numeric-id pairs' 'correct KITMAKER_PROJECTS' + valid_cell_directories "$PYPI_PUBLISH_CELLS" || fail "$0" "$PYPI_PUBLISH_CELLS" 'selected cells with unique safe artifact_dir paths' 'record each selected cell artifact directory from the producer' + printf '%s\n' "$PYPI_PUBLISH_CELLS" > "$scratch/cells" + version=${RELEASE_TAG#v} + public_version "$version" || fail "$0" "$version" 'normalized public PEP 440 version, no +local' 'set v and rebuild all wheels' + [ -d "$WHEEL_DIR" ] || fail "$WHEEL_DIR" 'missing directory' 'downloaded build artifacts' 'download the selected wheel artifacts' + find "$WHEEL_DIR" -type f -name '*.whl' | LC_ALL=C sort > "$scratch/files" + : > "$scratch/entries" + while IFS= read -r wheel; do + filename=${wheel##*/} + stem=${filename%.whl} + platform=${stem##*-} + matched=0 + old_ifs=$IFS; IFS=. + for wheel_platform in $platform; do + # PYPI_PLATFORM is intentionally a caller-selected shell pattern. + # shellcheck disable=SC2254 + case "$wheel_platform" in $PYPI_PLATFORM) matched=1 ;; esac + done + IFS=$old_ifs + if [ "$matched" -eq 0 ]; then printf 'SKIP %s: platform=%s; selected=%s\n' "$wheel" "$platform" "$PYPI_PLATFORM"; continue; fi + fields=$(printf '%s' "$stem" | awk -F- '{print NF}') + [ "$fields" -eq 5 ] || fail "$wheel" "$filename ($fields fields)" 'five wheel filename fields; no build tag' 'remove the build tag in the producer and rebuild' + distribution=$(normalize "${stem%%-*}") + remainder=${stem#*-}; wheel_version=${remainder%%-*} + project_id=$(jq -r --arg name "$distribution" '.[] | select(.name == $name) | .id' "$scratch/map.json") + [ -n "$project_id" ] || fail "$wheel" "$distribution" "$KITMAKER_PROJECTS" 'register the distribution and add its Kitmaker project mapping' + [ "$wheel_version" = "$version" ] || fail "$wheel" "$wheel_version" "$version (no +local)" 'set the build version from the release tag and rebuild' + unzip -Z1 "$wheel" > "$scratch/members" || fail "$wheel" 'invalid ZIP' 'readable wheel archive' 'rebuild the wheel' + metadata_count=$(awk '/^[^\/]+\.dist-info\/METADATA$/ {n++} END {print n+0}' "$scratch/members") + [ "$metadata_count" -eq 1 ] || fail "$wheel" "$metadata_count METADATA entries" 'exactly one .dist-info/METADATA' 'rebuild the wheel with its packaging backend' + metadata_path=$(awk '/^[^\/]+\.dist-info\/METADATA$/ {print}' "$scratch/members") + unzip -p "$wheel" "$metadata_path" > "$scratch/metadata" || fail "$wheel" 'unreadable METADATA' 'readable core metadata' 'rebuild the wheel' + metadata_name=$(sed -n 's/^Name: //p' "$scratch/metadata" | tr -d '\r') + metadata_version=$(sed -n 's/^Version: //p' "$scratch/metadata" | tr -d '\r') + [ "$(normalize "$metadata_name")" = "$distribution" ] || fail "$wheel" "METADATA Name=$metadata_name" "$distribution" 'change package metadata before building; do not rename the wheel' + [ "$metadata_version" = "$wheel_version" ] || fail "$wheel" "METADATA Version=$metadata_version" "$wheel_version" 'align the version sources and rebuild' + case ".$platform." in *.linux_*.*) fail "$wheel" "$platform" 'repaired manylinux platform; no bare linux_*' 'run auditwheel repair excluding CUDA libraries' ;; esac + duplicate=$(jq -rs --arg file "$filename" '[.[] | select(.filename == $file)][0].path // empty' "$scratch/entries") + [ -z "$duplicate" ] || fail "$wheel" "duplicate $filename at $duplicate and $wheel" 'unique wheel filenames across selected cells' 'select one cell for each CUDA-major, Python/Ubuntu and architecture combination' + provenance_count=$(awk -v path="$PYPI_PROVENANCE_PATH" '$0 == path {n++} END {print n+0}' "$scratch/members") + [ "$provenance_count" -eq 1 ] || fail "$wheel" "$provenance_count $PYPI_PROVENANCE_PATH entries" 'one provenance file' 'write and package provenance before pip wheel' + unzip -p "$wheel" "$PYPI_PROVENANCE_PATH" > "$scratch/provenance" || fail "$wheel" 'unreadable provenance' 'JSON provenance' 'rebuild the wheel' + git_sha=$(jq -er '.git_sha | select(type == "string")' "$scratch/provenance") || fail "$wheel" 'missing or invalid git_sha' "$RELEASE_COMMIT" 'write JSON provenance before pip wheel' + [ "$git_sha" = "$RELEASE_COMMIT" ] || fail "$wheel" "git_sha=$git_sha" "$RELEASE_COMMIT" 'rebuild from the release commit' + cuda_major=$(printf '%s' "$distribution" | sed -n 's/.*-cu\([0-9][0-9]*\)$/\1/p') + if [ -n "$cuda_major" ]; then + cuda_version=$(jq -r '.cuda_version // empty' "$scratch/provenance") + [ "${cuda_version%%.*}" = "$cuda_major" ] || fail "$wheel" "cuda_version=$cuda_version" "CUDA major $cuda_major" 'build the named CUDA variant and regenerate provenance' + fi + bytes=$(wc -c < "$wheel" | tr -d ' ') + [ "$bytes" -le 1048576000 ] || fail "$wheel" "$bytes bytes" 'at most 1048576000 bytes (1000 MiB)' 'reduce the wheel size or arrange a different release strategy' + checksum=$(sha256sum "$wheel"); checksum=${checksum%% *} + jq -n --arg filename "$filename" --arg path "$wheel" --arg distribution "$distribution" --arg project_id "$project_id" --arg version "$wheel_version" --arg sha256 "$checksum" --argjson bytes "$bytes" '{filename:$filename,path:$path,distribution:$distribution,project_id:$project_id,version:$version,sha256:$sha256,bytes:$bytes}' >> "$scratch/entries" + done < "$scratch/files" + [ -s "$scratch/entries" ] || fail "$WHEEL_DIR" 'zero matching wheels' "$PYPI_PLATFORM" 'download the selected wheel artifacts and check the platform filter' + jq -r '.[] | [.artifact_dir,(.cuda | split(".")[0])] | @tsv' "$scratch/cells" > "$scratch/expected-cells" + while IFS="$(printf '\t')" read -r cell_dir cell_cuda; do + if [ "$cell_dir" = . ]; then cell_prefix="${WHEEL_DIR%/}/"; else cell_prefix="${WHEEL_DIR%/}/$cell_dir/"; fi + cell_count=$(jq -rs --arg prefix "$cell_prefix" --arg suffix "-cu$cell_cuda" '[.[] | select((.path | startswith($prefix)) and (.distribution | endswith($suffix)))] | length' "$scratch/entries") + [ "$cell_count" -eq 1 ] || fail "$cell_prefix" "$cell_count selected CUDA $cell_cuda wheels" 'exactly one finished wheel per selected matrix cell' 'restore the missing cell artifact or separate extra build outputs' + done < "$scratch/expected-cells" + jq -es --slurpfile cells "$scratch/cells" --arg root "${WHEEL_DIR%/}/" 'length == ($cells[0] | length) and all(.[]; . as $wheel | any($cells[0][]; (if .artifact_dir == "." then $root else $root + .artifact_dir + "/" end) as $prefix | $wheel.path | startswith($prefix)))' "$scratch/entries" >/dev/null || fail "$WHEEL_DIR" 'extra wheel or wheel outside selected cell directories' 'only the confirmed publish cells' 'narrow the artifact download selection' + jq -s --arg tag "$RELEASE_TAG" --arg commit "$RELEASE_COMMIT" --arg repository "$PYPI_PUBLISH_REPOSITORY" --arg source "$PYPI_ASSET_SOURCE" --arg plc "$PYPI_PER_RELEASE_PLC" '{template_version:"1.8.0",tag:$tag,commit:$commit,repository:$repository,asset_source:$source,per_release_plc:$plc,wheels:.}' "$scratch/entries" > "$manifest" + { + printf '| Wheel | Bytes | SHA256 |\n|---|---:|---|\n' + jq -r '.wheels[] | "| \(.filename) | \(.bytes) | \(.sha256) |"' "$manifest" + printf '\nApproval checklist (record in the approval comment):\n- [ ] IP self/peer review done\n- [ ] Artifacts and hashes above confirmed\n' + [ "$PYPI_PER_RELEASE_PLC" != yes ] || printf '%s\n' '- [ ] L1 Feature PLC complete' + } > "$scratch/summary" + cat "$scratch/summary" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$scratch/summary" >> "$GITHUB_STEP_SUMMARY"; fi +} + +check_inputs() { + failures=0 + result() { + printf '%s %s: %s\n' "$1" "$2" "$3" + [ "$1" != FAIL ] || failures=$((failures + 1)) + } + case "$PYPI_ASSET_SOURCE" in + github-release) cli=gh ;; artifactory) cli=glab ;; *) result FAIL PYPI_ASSET_SOURCE 'expected github-release or artifactory'; exit 1 ;; + esac + for dependency in "$cli" git; do + if command -v "$dependency" >/dev/null; then result PASS "$dependency" 'installed'; else result FAIL "$dependency" 'missing; install before checking inputs'; fi + done + [ "$failures" -eq 0 ] || exit 1 + for input_name in PYPI_PUBLISH_REPOSITORY KITMAKER_PROJECTS KITMAKER_PIC KITMAKER_TOKEN PYPI_PROVENANCE_PATH PYPI_RELEASE_WORKFLOW PYPI_RELEASE_JOB PYPI_CI_SECRET_NAMES PYPI_APPROVERS PYPI_PUBLISH_RUNNER PYPI_PUBLISH_CELLS PYPI_RELEASE_CELLS PYPI_MATRIX_PATH PYPI_MATRIX_SHA256 PYPI_FIRST_VERSION PYPI_VERSION_FILE PYPI_CURRENT_VERSION; do + eval 'input_value=${'"$input_name"':-}' + [ -n "$input_value" ] || result FAIL "$input_name" 'missing' + done + if [ "$cli" = glab ]; then + for input_name in ARTIFACTORY_BASE_URL ARTIFACTORY_TOKEN; do + eval 'input_value=${'"$input_name"':-}' + [ -n "$input_value" ] || result FAIL "$input_name" 'missing' + done + fi + if [ "${PYPI_RELEASE_JOB:-}" = new ]; then + if [ -n "${PYPI_RELEASE_WORKFLOW:-}" ] && [ ! -e "$PYPI_RELEASE_WORKFLOW" ]; then result PASS release_workflow "$PYPI_RELEASE_WORKFLOW is free for a new release job"; else result FAIL release_workflow 'new workflow path exists or is missing'; fi + elif [ -f "${PYPI_RELEASE_WORKFLOW:-}" ] && awk -v job="${PYPI_RELEASE_JOB:-}" '$0 ~ "^[ ]*" job ":[ ]*($|#)" {found=1} END {exit !found}' "$PYPI_RELEASE_WORKFLOW"; then result PASS release_workflow "$PYPI_RELEASE_WORKFLOW contains $PYPI_RELEASE_JOB" + else result FAIL release_workflow 'workflow file or exact job ID missing'; fi + if [ -f "${PYPI_MATRIX_PATH:-}" ]; then + matrix_sha=$(sha256sum "$PYPI_MATRIX_PATH"); matrix_sha=${matrix_sha%% *} + if [ "$matrix_sha" = "${PYPI_MATRIX_SHA256:-}" ]; then result PASS matrix_source "$PYPI_MATRIX_PATH sha256=$matrix_sha"; else result FAIL matrix_source 'discovery snapshot is stale; rediscover the expanded release matrix'; fi + else result FAIL matrix_source 'matrix source file missing'; fi + if valid_cell_directories "${PYPI_PUBLISH_CELLS:-null}" && printf '%s' "${PYPI_PUBLISH_CELLS:-null}" | jq -e --argjson available "${PYPI_RELEASE_CELLS:-null}" ' + type == "array" and length > 0 and ($available | type == "array") and + all(.[]; . as $cell | (.cuda | type == "string") and (.ubuntu | type == "string") and (.python | type == "string") and .arch == "x86_64" and (.artifact_dir | type == "string" and length > 0) and ($available | index($cell)) != null) and + (group_by([(.cuda | split(".")[0]),.ubuntu,.python,.arch]) | all(.[]; length == 1))' >/dev/null; then result PASS publish_cells "$PYPI_PUBLISH_CELLS" + else result FAIL publish_cells 'select existing x86_64 cells, unique by CUDA major, Ubuntu/Python and architecture, with safe distinct artifact directories'; fi + if public_version "${PYPI_FIRST_VERSION:-}"; then + tags=$(git tag --list "v$PYPI_FIRST_VERSION") + if [ -z "$tags" ]; then result PASS first_version "$PYPI_FIRST_VERSION; tag absent in synced clone"; else result FAIL first_version "tag $tags already exists"; fi + else result FAIL first_version 'expected normalized public PEP 440 version'; fi + if [ -f "${PYPI_VERSION_FILE:-}" ] && [ -w "$PYPI_VERSION_FILE" ] && [ -n "${PYPI_CURRENT_VERSION:-}" ] && grep -F "$PYPI_CURRENT_VERSION" "$PYPI_VERSION_FILE" >/dev/null; then result PASS version_source "$PYPI_VERSION_FILE contains $PYPI_CURRENT_VERSION and is writable"; else result FAIL version_source 'version source or current value missing/unwritable'; fi + source_path=${PYPI_PROVENANCE_SOURCE:-$PYPI_PACKAGE_ROOT/${PYPI_PROVENANCE_PATH:-}} + if [ -n "${PYPI_PROVENANCE_PATH:-}" ] && [ -f "$(dirname "$source_path")/__init__.py" ] && [ -z "$(git ls-files -- "$source_path")" ]; then result PASS provenance "$source_path has import package and is untracked"; else result FAIL provenance 'expected untracked provenance inside a package with __init__.py'; fi + case "$PYPI_PER_RELEASE_PLC" in yes|no) result PASS per_release_plc "$PYPI_PER_RELEASE_PLC (owner decision; no smoke test)" ;; *) result FAIL per_release_plc 'expected yes or no' ;; esac + if [ "$cli" = gh ]; then + if gh repo view --json nameWithOwner,visibility > "$scratch/repo"; then + slug=$(jq -r .nameWithOwner "$scratch/repo"); visibility=$(jq -r .visibility "$scratch/repo") + if [ "$visibility" = PUBLIC ]; then result PASS platform "GitHub $visibility"; else result FAIL platform "GitHub $visibility; public assets required"; fi + else slug=; result FAIL platform 'gh repo view failed'; fi + else + if glab repo view --output json > "$scratch/repo"; then slug=$(jq -r .path_with_namespace "$scratch/repo"); result PASS platform "GitLab $slug"; else slug=; result FAIL platform 'glab repo view failed'; fi + fi + if [ -n "$slug" ] && [ "$slug" = "${PYPI_PUBLISH_REPOSITORY:-}" ]; then result PASS canonical_repository "$slug"; else result FAIL canonical_repository "clone=$slug canonical=${PYPI_PUBLISH_REPOSITORY:-}; private copies don't publish"; fi + if [ "$slug" != "${PYPI_PUBLISH_REPOSITORY:-}" ] || [ -z "$slug" ]; then result FAIL remote_checks 'canonical repository check failed; remaining remote inputs not tested'; exit 1; fi + repo_encoded=$(printf '%s' "$slug" | jq -sRr @uri) + if project_map; then + jq -r '.[] | [.name,.id] | @tsv' "$scratch/map.json" > "$scratch/mappings" + while IFS="$(printf '\t')" read -r name id; do + major=$(printf '%s' "$name" | sed -n 's/.*-cu\([0-9][0-9]*\)$/\1/p') + if [ -n "$major" ] && printf '%s' "${PYPI_PUBLISH_CELLS:-null}" | jq -e --arg major "$major" 'any(.[]; (.cuda | split(".")[0]) == $major)' >/dev/null; then result PASS "distribution/$name" "CUDA major $major"; else result FAIL "distribution/$name" 'expected -cuNN matching a publish-cell CUDA major'; fi + http "https://pypi.org/pypi/$name/json" + case "$http_status" in 404) result PASS "pypi.org/$name" 'HTTP 404; unclaimed name' ;; 200) result WARN "pypi.org/$name" 'HTTP 200; explicitly confirm NVIDIA ownership' ;; *) cat "$scratch/response"; result FAIL "pypi.org/$name" "HTTP $http_status curl=$http_rc" ;; esac + http "https://pypi.nvidia.com/$name/" + case "$http_status" in 404) result PASS "pypi.nvidia.com/$name" 'HTTP 404; no project listing' ;; 200) result WARN "pypi.nvidia.com/$name" 'HTTP 200; explicitly confirm NVIDIA ownership' ;; *) cat "$scratch/response"; result FAIL "pypi.nvidia.com/$name" "HTTP $http_status curl=$http_rc" ;; esac + http "https://pypi.org/pypi/$name/${PYPI_FIRST_VERSION:-}/json" + if [ "$http_status" = 404 ]; then result PASS "first_version/$name" 'HTTP 404'; else result FAIL "first_version/$name" "HTTP $http_status; require unused version"; fi + done < "$scratch/mappings" + if ! printf '%s' "${PYPI_PUBLISH_CELLS:-null}" | jq -e --slurpfile mapping "$scratch/map.json" 'all(.[]; (.cuda | split(".")[0]) as $major | any($mapping[0][]; .name | endswith("-cu" + $major)))' >/dev/null; then result FAIL distributions 'each publish-cell CUDA major needs a project mapping'; fi + if [ -n "${KITMAKER_TOKEN:-}" ]; then + http --header "Authorization: Bearer $KITMAKER_TOKEN" "$KITMAKER_API/projects" + cat "$scratch/response"; printf '\n' + if [ "$http_status" = 200 ] && [ "$http_rc" -eq 0 ] && jq -e 'if type == "array" then . elif (.projects | type) == "array" then .projects else error("unknown projects response shape") end' "$scratch/response" > "$scratch/portal-projects"; then + result PASS kitmaker_token 'HTTP 200' + while IFS="$(printf '\t')" read -r name id; do + if jq -e --arg id "$id" --arg name "$name" '.[] | select((.id | tostring) == $id and .name == $name)' "$scratch/portal-projects" > "$scratch/project"; then + result PASS "kitmaker_project/$name" "id=$id, name=$name" + if [ -n "$KITMAKER_OWNER_FIELD" ]; then + owner=$(jq -r --arg path "$KITMAKER_OWNER_FIELD" 'getpath($path | split(".")) // empty' "$scratch/project") + if [ -n "$owner" ] && [ "$owner" = "${KITMAKER_PIC:-}" ]; then result PASS "pic/$name" "$owner"; else result FAIL "pic/$name" "owner=$owner, pic=${KITMAKER_PIC:-}; correct owner or confirmed response field"; fi + else result WARN "pic/$name" 'owner email field unverified; inspect response and confirm Portal owner or set KITMAKER_OWNER_FIELD'; fi + else result FAIL "kitmaker_project/$name" "id=$id missing or project name differs"; fi + done < "$scratch/mappings" + else result FAIL kitmaker_token "HTTP $http_status curl=$http_rc; 401=token, 000=network/CA, or unsupported response schema"; fi + fi + else result FAIL kitmaker_projects 'invalid name=id map'; fi + if printf '%s' "${KITMAKER_PIC:-}" | jq -eRs 'test("^[A-Za-z0-9._%+-]+@nvidia\\.com$")' >/dev/null; then result PASS pic_format "$KITMAKER_PIC"; else result FAIL pic_format 'expected owner@nvidia.com'; fi + : > "$scratch/secrets" + if [ "$cli" = gh ]; then + if gh secret list --repo "$slug" --json name > "$scratch/repo-secrets" && gh api "repos/$slug/environments" --paginate > "$scratch/environments"; then + jq -r '.[].name' "$scratch/repo-secrets" > "$scratch/secrets" + if jq -e --arg env "$PYPI_ENVIRONMENT" '.environments[] | select(.name == $env)' "$scratch/environments" >/dev/null; then + result PASS environment "$PYPI_ENVIRONMENT exists" + if gh secret list --repo "$slug" --env "$PYPI_ENVIRONMENT" --json name > "$scratch/env-secrets"; then jq -r '.[].name' "$scratch/env-secrets" >> "$scratch/secrets"; else result FAIL ci_secrets 'environment secret listing failed'; fi + else result WARN environment "$PYPI_ENVIRONMENT missing; playbook creates it"; fi + else result FAIL ci_secrets 'secret/environment API failed'; fi + else + # JSON contains variable values; filter before displaying anything. + page=1 + while :; do + if glab variable list --repo "$slug" --output json --per-page 100 --page "$page" > "$scratch/variables"; then + jq -r '.[].key' "$scratch/variables" >> "$scratch/secrets" + [ "$(jq length "$scratch/variables")" -eq 100 ] || break + page=$((page + 1)) + else result FAIL ci_secrets 'variable list failed'; break; fi + done + if glab api "projects/$repo_encoded/environments?per_page=100" > "$scratch/environments"; then + if jq -e --arg env "$PYPI_ENVIRONMENT" '.[] | select(.name == $env)' "$scratch/environments" >/dev/null; then result PASS environment "$PYPI_ENVIRONMENT exists"; else result WARN environment "$PYPI_ENVIRONMENT missing; playbook creates it"; fi + else result FAIL environment 'environment API failed'; fi + fi + for secret_name in ${PYPI_CI_SECRET_NAMES:-}; do + if grep -Fx "$secret_name" "$scratch/secrets" >/dev/null; then result PASS "ci_secret/$secret_name" 'name present; verify required environment scope in playbook'; else result WARN "ci_secret/$secret_name" 'name missing; playbook creates it'; fi + done + printf '%s\n' "${PYPI_APPROVERS:-}" | tr '[:space:]' '\n' | LC_ALL=C sort -u > "$scratch/approvers" + count=$(awk 'NF {n++} END {print n+0}' "$scratch/approvers") + [ "$count" -ge 2 ] || result FAIL approvers 'at least two distinct accounts required' + while IFS= read -r account; do + [ -n "$account" ] || continue + if [ "$cli" = gh ]; then + if gh api "users/$account" --jq .login > "$scratch/account"; then result PASS "approver/$account" "$(cat "$scratch/account")"; else result FAIL "approver/$account" 'account lookup failed'; fi + else + if glab api "users?username=$account" > "$scratch/account" && jq -e --arg account "$account" 'any(.[]; .username == $account)' "$scratch/account" >/dev/null; then result PASS "approver/$account" 'account exists'; else result FAIL "approver/$account" 'account lookup failed'; fi + fi + done < "$scratch/approvers" + : > "$scratch/workflows" + if [ -d .github/workflows ]; then find .github/workflows -type f >> "$scratch/workflows"; fi + if [ -f .gitlab-ci.yml ]; then printf '%s\n' .gitlab-ci.yml >> "$scratch/workflows"; fi + : > "$scratch/runner-labels" + while IFS= read -r workflow; do + awk '/^[ ]*(runs-on|tags):/ {active=1; match($0,/[^ ]/); indent=RSTART; print; next} active {match($0,/[^ ]/); if (RSTART > indent) print; else active=0}' "$workflow" >> "$scratch/runner-labels" + done < "$scratch/workflows" + if [ -n "${PYPI_PUBLISH_RUNNER:-}" ] && grep -F -- "$PYPI_PUBLISH_RUNNER" "$scratch/runner-labels" > "$scratch/matched-runner"; then + result PASS publish_runner "$(cat "$scratch/matched-runner")" + elif [ "$cli" = gh ]; then + if gh api "repos/$slug/actions/runners" --paginate > "$scratch/runners" && jq -e --arg label "${PYPI_PUBLISH_RUNNER:-}" 'any(.runners[].labels[]; .name == $label)' "$scratch/runners" >/dev/null; then result PASS publish_runner 'label in runners API'; else result FAIL publish_runner 'label not verified in workflow or readable runner API'; fi + else + runner_found=0 + if glab api "projects/$repo_encoded/runners?per_page=100" --paginate > "$scratch/runners"; then + jq -r '.[].id' "$scratch/runners" > "$scratch/runner-ids" + while IFS= read -r runner_id; do + if glab api "runners/$runner_id" > "$scratch/runner" && jq -e --arg tag "${PYPI_PUBLISH_RUNNER:-}" '.tag_list | index($tag) != null' "$scratch/runner" >/dev/null; then runner_found=1; fi + done < "$scratch/runner-ids" + fi + if [ "$runner_found" -eq 1 ]; then result PASS publish_runner 'tag in readable runners API'; else result FAIL publish_runner 'tag not verified in workflow or readable runner API'; fi + fi + if [ "$cli" = glab ] && [ -n "${ARTIFACTORY_BASE_URL:-}" ] && [ -n "${ARTIFACTORY_TOKEN:-}" ]; then + case "$ARTIFACTORY_BASE_URL" in + https://artifactory.nvidia.com/artifactory/*) + storage_url="https://artifactory.nvidia.com/artifactory/api/storage/${ARTIFACTORY_BASE_URL#https://artifactory.nvidia.com/artifactory/}" + http --header "Authorization: Bearer $ARTIFACTORY_TOKEN" "$storage_url" + cat "$scratch/response"; printf '\n' + if [ "$http_status" = 200 ] && [ "$http_rc" -eq 0 ]; then result PASS artifactory 'storage API HTTP 200; write permission remains for first dry run'; else result FAIL artifactory "HTTP $http_status curl=$http_rc"; fi ;; + *) result FAIL artifactory 'expected https://artifactory.nvidia.com/artifactory//' ;; + esac + fi + printf 'Input checks: %s failure(s); every WARN requires explicit recorded acceptance.\n' "$failures" + [ "$failures" -eq 0 ] +} + +poll_release() { + deadline=$(($(date +%s) + PYPI_TIMEOUT_SECONDS)) + while :; do + kitmaker "$KITMAKER_API/status/$release_uuid" + cp "$scratch/response" "pypi-status-$release_uuid.json" + state=$(jq -er '.status' "$scratch/response") || fail "$release_uuid" 'missing status' 'status JSON' 'inspect the saved response' + case "$state" in + completed) return 0 ;; + failed) + printf 'Batch outcomes (unlisted wheels may be unstarted; inspect both indexes):\n' + jq -r '.jobs[]? | "\(.status) \(.wheel_url)"' "$scratch/response" + cat "$scratch/completed" + fail "$release_uuid" 'failed or partially published batch' 'all wheels completed' 'inspect every wheel on both indexes; retain UUID; do not resubmit the batch' ;; + esac + [ "$(date +%s)" -lt "$deadline" ] || fail "$release_uuid" "$state at polling deadline" 'terminal completed state' 'inspect saved status and both indexes; do not resubmit or infer cancellation' + printf 'POLL %s: %s; waiting %s seconds\n' "$release_uuid" "$state" "$PYPI_POLL_SECONDS" + sleep "$PYPI_POLL_SECONDS" + done +} +submit_projects() { + jq -r '.wheels[].project_id' "$manifest" | LC_ALL=C sort -u > "$scratch/ids" + : > "$scratch/completed" + while IFS= read -r id; do + jq --arg id "$id" --arg pic "$KITMAKER_PIC" --argjson upload "$upload" '.wheels | map(select(.project_id == $id)) | {project_name:.[0].distribution,payload:map({pic:$pic,job_type:"wheel-release-job",url:.url,upload:$upload,mirroring_strategy:"upload-both"})}' "$scratch/publish-manifest" > "$scratch/request" + kitmaker --request POST --data-binary "@$scratch/request" "$KITMAKER_API/projects/$id/releases" + release_uuid=$(jq -er '.release_uuid | select(type == "string" and test("^[A-Za-z0-9-]+$"))' "$scratch/response") || fail "$id" 'missing release_uuid' 'release UUID' 'inspect response; do not resubmit an uncertain request' + printf '%s %s %s\n' "$id" "$upload" "$release_uuid" >> pypi-release-uuids.txt + poll_release + jq -r --arg id "$id" --arg upload "$upload" '.wheels[] | select(.project_id == $id) | "COMPLETED upload=\($upload) \(.filename)"' "$manifest" >> "$scratch/completed" + cat "$scratch/completed" + done < "$scratch/ids" +} +verify_indexes() { + deadline=$(($(date +%s) + PYPI_INDEX_TIMEOUT_SECONDS)) + while :; do + missing=0 + while IFS="$(printf '\t')" read -r name version filename checksum; do + http "https://pypi.org/pypi/$name/$version/json" + if [ "$http_status" = 200 ] && [ "$http_rc" -eq 0 ] && jq -e --arg file "$filename" --arg sum "$checksum" 'any(.urls[]; .filename == $file and .digests.sha256 == $sum)' "$scratch/response" >/dev/null; then printf 'VERIFIED pypi.org %s %s\n' "$filename" "$checksum"; else missing=$((missing + 1)); printf 'WAIT pypi.org %s %s\n' "$filename" "$checksum"; cat "$scratch/response"; fi + http "https://pypi.nvidia.com/$name/" + if [ "$http_status" = 200 ] && [ "$http_rc" -eq 0 ] && grep -F "$filename#sha256=$checksum" "$scratch/response" >/dev/null; then printf 'VERIFIED pypi.nvidia.com %s %s\n' "$filename" "$checksum"; else missing=$((missing + 1)); printf 'WAIT pypi.nvidia.com %s %s\n' "$filename" "$checksum"; cat "$scratch/response"; fi + done < "$scratch/expected" + [ "$missing" -ne 0 ] || return 0 + [ "$(date +%s)" -lt "$deadline" ] || fail "$manifest" "$missing index entries missing or hash mismatch" 'every filename and SHA256 on both indexes' 'inspect indexes and saved UUIDs; do not resubmit' + sleep "$PYPI_POLL_SECONDS" + done +} +publish() { + required PYPI_PUBLISH_REPOSITORY PYPI_ASSET_SOURCE RELEASE_TAG RELEASE_COMMIT + repository_guard + required KITMAKER_TOKEN KITMAKER_PIC + for number in "$PYPI_POLL_SECONDS" "$PYPI_TIMEOUT_SECONDS" "$PYPI_INDEX_TIMEOUT_SECONDS"; do + case "$number" in ''|*[!0-9]*) fail "$0" "$number" 'positive integer deadline/interval' 'correct polling configuration' ;; esac + [ "$number" -gt 0 ] || fail "$0" "$number" 'positive polling value' 'correct polling configuration' + done + case "$PYPI_DRY_RUN_ONLY" in 0|1) ;; *) fail "$0" "$PYPI_DRY_RUN_ONLY" '0 or 1' 'set the dry-run switch explicitly' ;; esac + [ -f "$manifest" ] || fail "$manifest" 'missing' 'manifest from successful prepare' 'download the prepare artifact' + jq -e --arg tag "$RELEASE_TAG" --arg commit "$RELEASE_COMMIT" --arg repository "$PYPI_PUBLISH_REPOSITORY" --arg source "$PYPI_ASSET_SOURCE" ' + .template_version == "1.8.0" and .tag == $tag and .commit == $commit and .repository == $repository and .asset_source == $source and + (.wheels | type == "array" and length > 0 and all(.[]; (.filename | test("^[A-Za-z0-9_.!+-]+\\.whl$")) and (.sha256 | test("^[a-f0-9]{64}$")) and (.project_id | test("^[1-9][0-9]*$"))))' "$manifest" >/dev/null || fail "$manifest" 'invalid manifest or tag/commit/repository/source mismatch' "$RELEASE_TAG $RELEASE_COMMIT $PYPI_PUBLISH_REPOSITORY $PYPI_ASSET_SOURCE" 'use the manifest from this release and template version' + jq -r '.wheels[] | [.distribution,.version,.filename,.sha256] | @tsv' "$manifest" > "$scratch/expected" + jq -r '.wheels[] | [.path,.filename,.sha256] | @tsv' "$manifest" > "$scratch/to-publish" + : > "$scratch/urls" + if [ "$PYPI_ASSET_SOURCE" = artifactory ]; then + required ARTIFACTORY_BASE_URL ARTIFACTORY_TOKEN + case "$ARTIFACTORY_BASE_URL" in https://artifactory.nvidia.com/artifactory/*) ;; *) fail "$0" "$ARTIFACTORY_BASE_URL" 'direct Artifactory base URL' 'use an approved repo and path' ;; esac + fi + while IFS="$(printf '\t')" read -r wheel filename checksum; do + if [ "$PYPI_ASSET_SOURCE" = github-release ]; then + tag_uri=$(printf '%s' "$RELEASE_TAG" | jq -sRr @uri) + file_uri=$(printf '%s' "$filename" | jq -sRr @uri) + url="https://github.com/$PYPI_PUBLISH_REPOSITORY/releases/download/$tag_uri/$file_uri" + http "$url" + if [ "$http_status" = 404 ]; then fail "$filename" 'HTTP 404' 'public GitHub release asset' "a draft release isn't readable; publish the draft before approval"; fi + [ "$http_rc" -eq 0 ] || fail "$filename" "HTTP $http_status curl=$http_rc" 'anonymous asset download' 'fix release asset visibility or URL' + actual=$(sha256sum "$scratch/response"); actual=${actual%% *} + else + [ -f "$wheel" ] || fail "$wheel" 'missing' 'original wheel artifact' 'download build artifacts with the same directory layout as prepare' + actual=$(sha256sum "$wheel"); actual=${actual%% *} + [ "$actual" = "$checksum" ] || fail "$wheel" "$actual" "$checksum" 'restore the approved artifact' + url="${ARTIFACTORY_BASE_URL%/}/$RELEASE_TAG/$RELEASE_COMMIT/$filename" + http --request PUT --header "Authorization: Bearer $ARTIFACTORY_TOKEN" --header "X-Checksum-Sha256: $checksum" --upload-file "$wheel" "$url" + cat "$scratch/response"; printf '\n' + [ "$http_rc" -eq 0 ] || fail "$wheel" "HTTP $http_status curl=$http_rc" 'successful Artifactory deployment' 'inspect deploy permissions and response; do not blindly overwrite' + actual=$(jq -er '.checksums.sha256' "$scratch/response") || fail "$wheel" 'no checksums.sha256 in PUT response' "$checksum" 'verify deployed Artifactory API contract before proceeding' + fi + [ "$actual" = "$checksum" ] || fail "$filename" "$actual" "$checksum" 'restore the exact immutable artifact; regenerate approval after any change' + jq -n --arg filename "$filename" --arg url "$url" '{filename:$filename,url:$url}' >> "$scratch/urls" + done < "$scratch/to-publish" + jq --slurpfile urls "$scratch/urls" '.wheels |= map(. as $wheel | . + ($urls[] | select(.filename == $wheel.filename)))' "$manifest" > "$scratch/publish-manifest" + upload=false + submit_projects + if [ "$PYPI_DRY_RUN_ONLY" = 1 ]; then printf 'DRY RUN complete; no Kitmaker upload submitted.\n'; return 0; fi + upload=true + submit_projects + verify_indexes +} +case "$mode" in check-inputs) check_inputs ;; prepare) prepare ;; publish) publish ;; esac diff --git a/scripts/release_pycunls.sh b/scripts/release_pycunls.sh new file mode 100755 index 0000000..b557e0c --- /dev/null +++ b/scripts/release_pycunls.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Publishes the built wheel filenames unchanged as public GitHub release assets. +set -euo pipefail + +: "${RELEASE_TAG:?Expected v}" +: "${RELEASE_COMMIT:?Expected the built checkout SHA}" +: "${GH_REPO:?Expected nvidia-isaac/cuNLS}" +[ "$GH_REPO" = nvidia-isaac/cuNLS ] || { echo "Only the canonical repository releases wheels." >&2; exit 1; } +[[ "$RELEASE_TAG" = v* ]] || { echo "Expected a v tag." >&2; exit 1; } + +shopt -s nullglob +wheels=(pypi-wheels/*/*.whl) +[ "${#wheels[@]}" -eq 4 ] || { echo "Expected four release wheels; check matrix artifacts." >&2; exit 1; } +declare -A seen +for wheel in "${wheels[@]}"; do + name=${wheel##*/} + [ -z "${seen[$name]:-}" ] || { echo "Duplicate release filename: $name" >&2; exit 1; } + seen[$name]=1 +done + +# Existing assets are immutable inputs to approval. Retry only pypi-publish +# after a dry run or a partial failure, rather than replacing release bytes. +gh release create "$RELEASE_TAG" "${wheels[@]}" --verify-tag \ + --target "$RELEASE_COMMIT" --title "cuNLS $RELEASE_TAG" \ + --notes "Built from commit $RELEASE_COMMIT. Wheels require the matching CUDA toolkit; the Python import remains pycunls." diff --git a/scripts/requirements-build.txt b/scripts/requirements-build.txt new file mode 100644 index 0000000..3e8a632 --- /dev/null +++ b/scripts/requirements-build.txt @@ -0,0 +1,9 @@ +scikit-build-core>=0.10 +nanobind>=2.4 +build +wheel +pre-commit +auditwheel +packaging +actionlint-py +shellcheck-py diff --git a/scripts/test_pycunls.sh b/scripts/test_pycunls.sh new file mode 100755 index 0000000..6b02641 --- /dev/null +++ b/scripts/test_pycunls.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Installs the exact wheel and its extras inside the CUDA image. +set -euo pipefail +shopt -s nullglob +wheels=(/cunls_install/*.whl) +if [ "${#wheels[@]}" -ne 1 ]; then + echo "Expected one wheel in /cunls_install; build into a separate directory for each matrix cell." >&2 + exit 1 +fi +pip install "${wheels[0]}[test]" +cd /cunls +python -m pytest python/tests/ --junitxml=/cunls_install/python-test-results.xml -v diff --git a/scripts/test_pycunls_in_docker.sh b/scripts/test_pycunls_in_docker.sh index a3119b8..a702e01 100755 --- a/scripts/test_pycunls_in_docker.sh +++ b/scripts/test_pycunls_in_docker.sh @@ -23,9 +23,4 @@ TTY_FLAG="" docker run --runtime=nvidia --gpus all --rm $TTY_FLAG \ -v "$(pwd):/cunls:ro" \ -v "$WHEEL_DIR:/cunls_install" \ - cunls:local bash -c ' - set -euo pipefail - pip install /cunls_install/*.whl "pycunls[test]" - cd /cunls - python -m pytest python/tests/ --junitxml=/cunls_install/python-test-results.xml -v - ' + cunls:local bash /cunls/scripts/test_pycunls.sh From 83ae001cb984d521f61c1232bfb886a1368d7e94 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Tue, 13 Oct 2026 17:11:19 +0000 Subject: [PATCH 02/11] Use auditwheel-compatible patchelf for wheel repair --- scripts/Dockerfile | 2 +- scripts/requirements-build.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 60fd507..7a0cf88 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -22,7 +22,7 @@ RUN if [ -n "${UBUNTU_PORTS_MIRROR}" ]; then \ # Use the distro CMake so Ubuntu 22.04 continuously validates the 3.22 minimum. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates apt-utils g++ gcc git cmake ccache \ - software-properties-common curl patchelf \ + software-properties-common curl \ python3 python3-pip python3-dev python3-venv \ && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/scripts/requirements-build.txt b/scripts/requirements-build.txt index 3e8a632..f4d1e7d 100644 --- a/scripts/requirements-build.txt +++ b/scripts/requirements-build.txt @@ -4,6 +4,8 @@ build wheel pre-commit auditwheel +# Ubuntu 22.04's patchelf 0.14.3 is below auditwheel's repair minimum. +patchelf>=0.14.5 packaging actionlint-py shellcheck-py From 9e66e2b9577ed077b5828cdd3d2d1d1dc264cbca Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Tue, 13 Oct 2026 17:51:02 +0000 Subject: [PATCH 03/11] Document the current Python publishing configuration --- .ci/gpu-cicd/context.yml | 55 ---------------- .github/workflows/release.yml | 2 - README.md | 11 ++-- docs/pypi-publishing.md | 121 +++++++++++++++------------------- 4 files changed, 58 insertions(+), 131 deletions(-) diff --git a/.ci/gpu-cicd/context.yml b/.ci/gpu-cicd/context.yml index 582be26..aeea4ce 100644 --- a/.ci/gpu-cicd/context.yml +++ b/.ci/gpu-cicd/context.yml @@ -50,9 +50,6 @@ publishing: enabled: true legal_approval: answer: 'yes' - date: '2026-10-12' - source: User states public binary distribution is approved and repository licensing - is correct canonical_repository: nvidia-isaac/cuNLS asset_source: github-release release_workflow: .github/workflows/release.yml @@ -68,31 +65,8 @@ publishing: provenance_source: python/pycunls/_build_provenance.json package_root: python environment: pypi - approvers: - - hguillen publish_runner: gpu publish_cells: - - cuda: 12.8.1 - ubuntu: '22.04' - python: '3.10' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu22.04 - - cuda: 13.2.0 - ubuntu: '22.04' - python: '3.10' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu22.04 - - cuda: 12.8.1 - ubuntu: '24.04' - python: '3.12' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu24.04 - - cuda: 13.2.0 - ubuntu: '24.04' - python: '3.12' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu24.04 - expanded_release_cells: - cuda: 12.8.1 ubuntu: '22.04' python: '3.10' @@ -114,36 +88,7 @@ publishing: arch: x86_64 artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu24.04 matrix_source: .github/workflows/release.yml - matrix_source_sha256: c3dbe99aea21b42345716efe9afcd8655fe9c4731badebdcc2d8ea608ba40432 - first_version: 0.2.0 version_source: path: python/pyproject.toml - current: 0.2.0 dry_run_only: true per_release_plc: true - accepted_warnings: [] - setup_status: Input discovery passed; environment protection requires reconciliation - in the deployment playbook before activation. -discovery: - commit: fa591a6b3d606dc0fdef837b0ad23c0913a1ed89 - status: Read-only check-inputs passes with zero failures and warnings; implementation - authorized - cuda13_name: User corrected Portal distribution to pycunls-cu13 on 2026-10-12 - reviewer_setup: Owner explicitly requests manual reviewer configuration, starting - with hguillen; alexkorovko is an additional intended reviewer. Both accounts resolve. - This overrides the two-reviewer preparation default. - remote_verified: '2026-10-12: tools/projects fetch cuNLS succeeded; HEAD and origin/main - both fa591a6b3d606dc0fdef837b0ad23c0913a1ed89' - publish_runner_assumption: Reuse existing self-hosted,gpu runner selection; Kitmaker - reachability and CA trust remain for the first protected dry run. - per_release_plc_assumption: The pipeline includes the PLC checkbox conservatively; - owner has not specified per-release PLC requirements. - input_check_log: Read-only input gate passed on 2026-10-12 using the canonical template - before project edits. - environment_protection: '2026-10-12: pypi environment has no protection_rules and - no deployment_branch_policy; manual setup required.' - environment_secret: '2026-10-12: environment-scoped KITMAKER_TOKEN exists.' - actions_allowlist: '2026-10-12: github_owned_allowed=true; workflow uses only GitHub-owned - actions.' - repository_variables: '2026-10-12: no repository variables are configured; deployment - playbook sets publisher selection, enable switch, dry-run mode and PLC checklist.' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9330922..c995ce3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,6 @@ env: KITMAKER_PIC: hguillen@nvidia.com PYPI_PLATFORM: manylinux_*_x86_64 PYPI_PROVENANCE_PATH: pycunls/_build_provenance.json - # Keep the approval checkbox until the owner confirms the per-release policy. PYPI_PER_RELEASE_PLC: ${{ vars.PYPI_PER_RELEASE_PLC || 'yes' }} PYPI_PUBLISH_CELLS: >- [{"cuda":"12.8.1","ubuntu":"22.04","python":"3.10","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu22.04"}, @@ -92,7 +91,6 @@ jobs: retention-days: 30 pypi-publish: - # An operator enables the job only after configuring pypi protection. if: >- github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v') && diff --git a/README.md b/README.md index f8dd8e9..9eff518 100644 --- a/README.md +++ b/README.md @@ -475,10 +475,9 @@ independently validates the selected artifacts and writes `pypi-manifest.json`. The protected `pypi-publish` job waits for both jobs, downloads the manifest, checks the public release bytes, and submits Kitmaker requests for both indexes. Publication records survive failure; cancellation and automatic submission retries -are disabled. A failed batch can already be partly public. +are disabled. Publication failure does not roll back uploaded files. -Configure GitHub with [the deployment playbook](docs/pypi-publishing.md). -Publishing is disabled until `PYPI_PUBLISH_ENABLED=true`; -`PYPI_DRY_RUN_ONLY` defaults to `1`. The first approved run must remain a dry run. -After review, retry only the publisher for live publication with the same tag, -commit and manifest. Do not recreate a release to retry publication. +The [publishing guide](docs/pypi-publishing.md) documents GitHub configuration +and release commands. `PYPI_PUBLISH_ENABLED=true` enables the publisher; +`PYPI_DRY_RUN_ONLY` defaults to `1`. Review the protected dry run before live +publication. Publisher retries use the same tag, commit and manifest. diff --git a/docs/pypi-publishing.md b/docs/pypi-publishing.md index a6135a4..287894c 100644 --- a/docs/pypi-publishing.md +++ b/docs/pypi-publishing.md @@ -1,41 +1,39 @@ -# Configure cuNLS Python publishing +# cuNLS Python publishing Author: Heron Ordonez Guillen -Run these commands after the workflow is merged into the canonical cuNLS repository. -Credentials come from the operator environment, never from this document. Public binary distribution was -confirmed legally approved by the owner on 2026-10-12. +`release.yml` builds four x86_64 wheels for `pycunls-cu12` and `pycunls-cu13`, +attaches them to a public GitHub release, and prepares a manifest containing +their filenames, source commit and SHA256 hashes. The protected `pypi-publish` +job submits the release assets to Kitmaker projects 6865 and 6866 with owner +`hguillen@nvidia.com`. Kitmaker publishes to PyPI and NVIDIA's public index. + +## GitHub configuration + +Credentials come from the operator environment. Inspect the repository, +runner pool and environment protection with: ```bash gh auth status gh repo view nvidia-isaac/cuNLS --json nameWithOwner,visibility -gh api user --jq .login -gh api users/alexkorovko --jq .login gh api repos/nvidia-isaac/cuNLS/actions/runners --paginate \ --jq '.runners[] | {name,status,labels:[.labels[].name]}' gh api repos/nvidia-isaac/cuNLS/actions/permissions/selected-actions -``` - -MANUAL: In GitHub → nvidia-isaac/cuNLS → Settings → Environments, create or -edit the existing `pypi` environment. Set `hguillen` as a required reviewer, -enable prevent self-review and disable administrator bypass. Restrict deployment -to tags matching `v*`. The owner requested a single initial reviewer; add -`alexkorovko` when ready. With one reviewer and self-review prevented, another -trusted account must initiate a release that this reviewer approves. - -Verify protection before storing the environment credential. - -```bash gh api repos/nvidia-isaac/cuNLS/environments/pypi \ --jq '{protection_rules,deployment_branch_policy}' gh api repos/nvidia-isaac/cuNLS/environments/pypi/deployment-branch-policies \ --jq '.branch_policies[] | {name,type}' gh secret list --repo nvidia-isaac/cuNLS --env pypi +gh variable list --repo nvidia-isaac/cuNLS ``` -`KITMAKER_TOKEN` is already present in the `pypi` environment (verified 2026-10-12). -Keep that scope. To rotate it from approved credential storage, use this -idempotent command; it does not display the value. +In Settings → Environments → `pypi`, configure required reviewers, prevent +self-review, disable administrator bypass, and restrict deployment to tags +matching `v*`. Reviewer membership is managed in GitHub. A release requires +approval by an account other than its initiator. + +Store `KITMAKER_TOKEN` in the `pypi` environment. This command sets or rotates +the credential without displaying its value: ```bash : "${KITMAKER_TOKEN:?Load the Kitmaker service token from approved credential storage}" @@ -43,13 +41,9 @@ printf '%s' "$KITMAKER_TOKEN" | gh secret set KITMAKER_TOKEN \ --repo nvidia-isaac/cuNLS --env pypi ``` -MANUAL: Confirm the existing `[self-hosted, gpu]` runner pool has Kitmaker reachability, NVIDIA CA trust, -and POSIX sh, curl with `--fail-with-body`, jq, unzip and sha256sum. The first -protected dry run verifies reachability from the selected runner. -Portal IDs 6865/6866, names `pycunls-cu12`/`pycunls-cu13` and owner -`hguillen@nvidia.com` were verified through the read-only Portal API on 2026-10-12. - -Configure nonsecret variables; keep publication disabled during setup. +The publisher uses `[self-hosted, gpu]` runners and requires Kitmaker access, +NVIDIA CA trust, POSIX sh, curl supporting `--fail-with-body`, jq, unzip and sha256sum. +The repository variables control runner selection, publication and approval: ```bash gh variable set PYPI_PUBLISH_RUNNER --repo nvidia-isaac/cuNLS \ @@ -59,38 +53,30 @@ gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '1' gh variable set PYPI_PER_RELEASE_PLC --repo nvidia-isaac/cuNLS --body 'yes' ``` -MANUAL: The per-release L1 Feature PLC policy is pending owner confirmation. The -workflow includes the checkbox by default. Set `PYPI_PER_RELEASE_PLC=no` only -when the owner confirms it is not required. Each approval records IP self/peer -review completion and the selected filenames and SHA256 hashes. - -## Verify setup and the first dry run - -```bash -gh secret list --repo nvidia-isaac/cuNLS --env pypi -gh variable list --repo nvidia-isaac/cuNLS -gh api repos/nvidia-isaac/cuNLS/environments/pypi -gh api repos/nvidia-isaac/cuNLS/environments/pypi/deployment-branch-policies -``` +`PYPI_PUBLISH_ENABLED=true` enables the protected publisher. With +`PYPI_DRY_RUN_ONLY=1`, Kitmaker validates the assets without uploading them. +`PYPI_PER_RELEASE_PLC=yes` includes L1 Feature PLC completion in the approval +checklist. Each approval records IP self/peer review completion and the selected +filenames and SHA256 hashes. -MANUAL: Confirm the secret value and scope in approved credential storage, the -reviewer protections in Settings → Environments → pypi, and runner registration -in Settings → Actions → Runners. Secret values are write-only. +## Release dry run -After setup verification and owner approval, create the first tag from merged -`main` in dry-run mode. The input gate verified `v0.2.0` and both index versions -were absent on 2026-10-12; recheck before creating a tag. Tag creation triggers -the release workflow automatically. Run this block once for the first release; -use publisher-only retries after the producer has succeeded. +Run the tag commands from a clean checkout of merged `main`. +`python/pyproject.toml` and `python/pycunls/__init__.py` must contain the same +public version. The wheel build enforces agreement between these versions and +the release tag. Tag creation starts the release workflow. ```bash gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '1' gh variable set PYPI_PUBLISH_ENABLED --repo nvidia-isaac/cuNLS --body 'true' -RELEASE_COMMIT=$(gh api repos/nvidia-isaac/cuNLS/commits/main --jq .sha) +RELEASE_VERSION=$(sed -n 's/^version = "\([^" ]*\)"$/\1/p' python/pyproject.toml) +: "${RELEASE_VERSION:?Package version was not found}" +RELEASE_TAG="v$RELEASE_VERSION" +RELEASE_COMMIT=$(git rev-parse HEAD) gh api --method POST repos/nvidia-isaac/cuNLS/git/refs \ - -f ref=refs/tags/v0.2.0 -f sha="$RELEASE_COMMIT" + -f ref="refs/tags/$RELEASE_TAG" -f sha="$RELEASE_COMMIT" RELEASE_RUN_ID=$(gh run list --repo nvidia-isaac/cuNLS --workflow release.yml \ - --branch v0.2.0 --event push --limit 1 --json databaseId \ + --branch "$RELEASE_TAG" --event push --limit 1 --json databaseId \ --jq '.[0].databaseId') : "${RELEASE_RUN_ID:?No tag-triggered release run found}" gh run watch "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --exit-status @@ -99,15 +85,15 @@ gh run download "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS \ --name "pypi-publication-record-$PUBLICATION_ATTEMPT" --dir "pypi-publication-record-$PUBLICATION_ATTEMPT" ``` -MANUAL: Approve the waiting `pypi` deployment after reviewing the prepare summary. -Expect all build/test cells, the public GitHub release and offline manifest to -succeed. Expect each Kitmaker dry run to reach `completed`, with no upload -request; retain the UUIDs, status JSON and manifest. This does not verify index -writes or customer installation. +Review the prepare summary and approve the waiting `pypi` deployment. +A successful dry run includes all build/test cells, the public GitHub release, +the offline manifest and a `completed` Kitmaker validation for each package. +Publication records contain the UUIDs, status JSON and manifest. + +## Live publication and recovery -After the owner authorizes live publication, rerun only `pypi-publish`, preserving -the original tag, commit, release bytes and manifest. Job database IDs come from -`gh run view`; use the [GitHub CLI job rerun command](https://cli.github.com/manual/gh_run_rerun). +After approving the dry-run results, set live mode and rerun only the publisher. +The tag, source commit, release assets and manifest stay fixed. ```bash gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '0' @@ -118,12 +104,11 @@ gh run rerun "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --job "$PUBLISH_JOB_ID" gh run watch "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --exit-status ``` -MANUAL: Verify another environment approval is required and the rerun logs show -`PYPI_DRY_RUN_ONLY=0`. Confirm this behavior on the deployed GitHub instance -before live publication; a variable captured by the original run can leave it -in dry-run mode. Publication records are retained separately per run attempt. +Approve the `pypi` deployment and check that the rerun logs show +`PYPI_DRY_RUN_ONLY=0`. A rerun that retains `1` performs validation only. +Publication records are retained separately per run attempt. -MANUAL: After live publication, confirm every wheel filename and SHA256 on both -PyPI and NVIDIA's public index, then test installation on each supported CUDA -variant. A partial failure requires separate review of the missing artifacts; -never resubmit the complete batch automatically. +Verify every wheel filename and SHA256 on both indexes, then test installation +for each supported CUDA variant. Publication failure does not roll back uploaded +files. Inspect the batch UUIDs, status JSON and both indexes before retrying. +Review missing artifacts separately and never resubmit the complete batch automatically. From f8584e57b93fc7e37342d75680409777f024cc3c Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Tue, 13 Oct 2026 20:24:49 +0000 Subject: [PATCH 04/11] Keep the Python publisher focused on release execution --- .ci/gpu-cicd/context.yml | 94 ---------------------- README.md | 27 ------- docs/pypi-publishing.md | 114 -------------------------- scripts/publish_pypi.sh | 168 +-------------------------------------- 4 files changed, 4 insertions(+), 399 deletions(-) delete mode 100644 .ci/gpu-cicd/context.yml delete mode 100644 docs/pypi-publishing.md diff --git a/.ci/gpu-cicd/context.yml b/.ci/gpu-cicd/context.yml deleted file mode 100644 index aeea4ce..0000000 --- a/.ci/gpu-cicd/context.yml +++ /dev/null @@ -1,94 +0,0 @@ -project: - name: cuNLS - archetype: external_first - build_system: cmake - secondary_build_systems: - - python - test_frameworks: - - googletest - - pytest - gpu_required_for_build: false - gpu_required_for_test: true - concerns: - - python_publishing -docker: - base_image_pattern: nvidia/cuda:{cuda_version}-devel-ubuntu{ubuntu_version} - existing_dockerfile: scripts/Dockerfile - registry: '' - registry_login_secret: '' -matrix: - cuda_versions: - - 12.8.1 - - 13.2.0 - os_versions: - - ubuntu22.04 - - ubuntu24.04 - python_versions: - - '3.10' - - '3.12' - architectures: - - x86_64 -runners: - platform: github_actions - gpu_tags: - - self-hosted - - gpu - cpu_tags: - - ubuntu-latest - gpu_job_timeout_minutes: 120 - cpu_job_timeout_minutes: 15 -artifacts: - test_output_format: junit_xml - release_type: github_release - publish_wheels: true - wheel_destination: pypi -secrets: -- name: KITMAKER_TOKEN - description: Kitmaker service token scoped to the protected pypi environment - required: true -publishing: - enabled: true - legal_approval: - answer: 'yes' - canonical_repository: nvidia-isaac/cuNLS - asset_source: github-release - release_workflow: .github/workflows/release.yml - release_job: release - wheel_dir: pypi-wheels - kitmaker_projects: - pycunls-cu12: 6865 - pycunls-cu13: 6866 - pic: hguillen@nvidia.com - owner_response_field: owner_email - platform: manylinux_*_x86_64 - provenance_path: pycunls/_build_provenance.json - provenance_source: python/pycunls/_build_provenance.json - package_root: python - environment: pypi - publish_runner: gpu - publish_cells: - - cuda: 12.8.1 - ubuntu: '22.04' - python: '3.10' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu22.04 - - cuda: 13.2.0 - ubuntu: '22.04' - python: '3.10' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu22.04 - - cuda: 12.8.1 - ubuntu: '24.04' - python: '3.12' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda12.8.1-ubuntu24.04 - - cuda: 13.2.0 - ubuntu: '24.04' - python: '3.12' - arch: x86_64 - artifact_dir: pycunls-x86_64-cuda13.2.0-ubuntu24.04 - matrix_source: .github/workflows/release.yml - version_source: - path: python/pyproject.toml - dry_run_only: true - per_release_plc: true diff --git a/README.md b/README.md index 9eff518..92bd17d 100644 --- a/README.md +++ b/README.md @@ -454,30 +454,3 @@ find . -iname '*.h' -o -iname '*.cpp' | xargs clang-format -i ## License cuNLS is licensed under the [Apache License 2.0](LICENSE). Third-party license notices are in `NOTICE` and `third_party/LICENSES/`. - -## Versioned Python releases - -`release.yml` builds and tests x86_64 wheels for CUDA 12.8.1 and 13.2.0 on -Ubuntu 22.04 (Python 3.10) and 24.04 (Python 3.12). Docker builds name the -distributions `pycunls-cu12` and `pycunls-cu13`; the import remains `pycunls`. -Each wheel declares the matching CuPy dependency, carries the source commit and -CUDA version, and is repaired with auditwheel while leaving CUDA libraries external. -A matching CUDA toolkit and compatible NVIDIA driver are required at runtime. - -Before tagging `v`, update both `python/pyproject.toml` and -`python/pycunls/__init__.py` to the same public version. The build refuses a -tag/version mismatch. Local reproduction uses the existing four build/test -commands documented above; set `CUDA_VERSION`, `UBUNTU_VERSION`, and -`RELEASE_TAG` to reproduce one release cell. Use a separate output directory per cell. - -The release job attaches wheels with their original filenames. `pypi-prepare` -independently validates the selected artifacts and writes `pypi-manifest.json`. -The protected `pypi-publish` job waits for both jobs, downloads the manifest, -checks the public release bytes, and submits Kitmaker requests for both indexes. -Publication records survive failure; cancellation and automatic submission retries -are disabled. Publication failure does not roll back uploaded files. - -The [publishing guide](docs/pypi-publishing.md) documents GitHub configuration -and release commands. `PYPI_PUBLISH_ENABLED=true` enables the publisher; -`PYPI_DRY_RUN_ONLY` defaults to `1`. Review the protected dry run before live -publication. Publisher retries use the same tag, commit and manifest. diff --git a/docs/pypi-publishing.md b/docs/pypi-publishing.md deleted file mode 100644 index 287894c..0000000 --- a/docs/pypi-publishing.md +++ /dev/null @@ -1,114 +0,0 @@ -# cuNLS Python publishing - -Author: Heron Ordonez Guillen - -`release.yml` builds four x86_64 wheels for `pycunls-cu12` and `pycunls-cu13`, -attaches them to a public GitHub release, and prepares a manifest containing -their filenames, source commit and SHA256 hashes. The protected `pypi-publish` -job submits the release assets to Kitmaker projects 6865 and 6866 with owner -`hguillen@nvidia.com`. Kitmaker publishes to PyPI and NVIDIA's public index. - -## GitHub configuration - -Credentials come from the operator environment. Inspect the repository, -runner pool and environment protection with: - -```bash -gh auth status -gh repo view nvidia-isaac/cuNLS --json nameWithOwner,visibility -gh api repos/nvidia-isaac/cuNLS/actions/runners --paginate \ - --jq '.runners[] | {name,status,labels:[.labels[].name]}' -gh api repos/nvidia-isaac/cuNLS/actions/permissions/selected-actions -gh api repos/nvidia-isaac/cuNLS/environments/pypi \ - --jq '{protection_rules,deployment_branch_policy}' -gh api repos/nvidia-isaac/cuNLS/environments/pypi/deployment-branch-policies \ - --jq '.branch_policies[] | {name,type}' -gh secret list --repo nvidia-isaac/cuNLS --env pypi -gh variable list --repo nvidia-isaac/cuNLS -``` - -In Settings → Environments → `pypi`, configure required reviewers, prevent -self-review, disable administrator bypass, and restrict deployment to tags -matching `v*`. Reviewer membership is managed in GitHub. A release requires -approval by an account other than its initiator. - -Store `KITMAKER_TOKEN` in the `pypi` environment. This command sets or rotates -the credential without displaying its value: - -```bash -: "${KITMAKER_TOKEN:?Load the Kitmaker service token from approved credential storage}" -printf '%s' "$KITMAKER_TOKEN" | gh secret set KITMAKER_TOKEN \ - --repo nvidia-isaac/cuNLS --env pypi -``` - -The publisher uses `[self-hosted, gpu]` runners and requires Kitmaker access, -NVIDIA CA trust, POSIX sh, curl supporting `--fail-with-body`, jq, unzip and sha256sum. -The repository variables control runner selection, publication and approval: - -```bash -gh variable set PYPI_PUBLISH_RUNNER --repo nvidia-isaac/cuNLS \ - --body '["self-hosted","gpu"]' -gh variable set PYPI_PUBLISH_ENABLED --repo nvidia-isaac/cuNLS --body 'false' -gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '1' -gh variable set PYPI_PER_RELEASE_PLC --repo nvidia-isaac/cuNLS --body 'yes' -``` - -`PYPI_PUBLISH_ENABLED=true` enables the protected publisher. With -`PYPI_DRY_RUN_ONLY=1`, Kitmaker validates the assets without uploading them. -`PYPI_PER_RELEASE_PLC=yes` includes L1 Feature PLC completion in the approval -checklist. Each approval records IP self/peer review completion and the selected -filenames and SHA256 hashes. - -## Release dry run - -Run the tag commands from a clean checkout of merged `main`. -`python/pyproject.toml` and `python/pycunls/__init__.py` must contain the same -public version. The wheel build enforces agreement between these versions and -the release tag. Tag creation starts the release workflow. - -```bash -gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '1' -gh variable set PYPI_PUBLISH_ENABLED --repo nvidia-isaac/cuNLS --body 'true' -RELEASE_VERSION=$(sed -n 's/^version = "\([^" ]*\)"$/\1/p' python/pyproject.toml) -: "${RELEASE_VERSION:?Package version was not found}" -RELEASE_TAG="v$RELEASE_VERSION" -RELEASE_COMMIT=$(git rev-parse HEAD) -gh api --method POST repos/nvidia-isaac/cuNLS/git/refs \ - -f ref="refs/tags/$RELEASE_TAG" -f sha="$RELEASE_COMMIT" -RELEASE_RUN_ID=$(gh run list --repo nvidia-isaac/cuNLS --workflow release.yml \ - --branch "$RELEASE_TAG" --event push --limit 1 --json databaseId \ - --jq '.[0].databaseId') -: "${RELEASE_RUN_ID:?No tag-triggered release run found}" -gh run watch "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --exit-status -PUBLICATION_ATTEMPT=$(gh api "repos/nvidia-isaac/cuNLS/actions/runs/$RELEASE_RUN_ID" --jq .run_attempt) -gh run download "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS \ - --name "pypi-publication-record-$PUBLICATION_ATTEMPT" --dir "pypi-publication-record-$PUBLICATION_ATTEMPT" -``` - -Review the prepare summary and approve the waiting `pypi` deployment. -A successful dry run includes all build/test cells, the public GitHub release, -the offline manifest and a `completed` Kitmaker validation for each package. -Publication records contain the UUIDs, status JSON and manifest. - -## Live publication and recovery - -After approving the dry-run results, set live mode and rerun only the publisher. -The tag, source commit, release assets and manifest stay fixed. - -```bash -gh variable set PYPI_DRY_RUN_ONLY --repo nvidia-isaac/cuNLS --body '0' -PUBLISH_JOB_ID=$(gh run view "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS \ - --json jobs --jq '.jobs[] | select(.name == "pypi-publish") | .databaseId') -: "${PUBLISH_JOB_ID:?Publisher job was not found}" -gh run rerun "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --job "$PUBLISH_JOB_ID" -gh run watch "$RELEASE_RUN_ID" --repo nvidia-isaac/cuNLS --exit-status -``` - -Approve the `pypi` deployment and check that the rerun logs show -`PYPI_DRY_RUN_ONLY=0`. A rerun that retains `1` performs validation only. -Publication records are retained separately per run attempt. - -Verify every wheel filename and SHA256 on both indexes, then test installation -for each supported CUDA variant. Publication failure does not roll back uploaded -files. Inspect the batch UUIDs, status JSON and both indexes before retrying. -Review missing artifacts separately and never resubmit the complete batch automatically. diff --git a/scripts/publish_pypi.sh b/scripts/publish_pypi.sh index b35a94c..024c190 100755 --- a/scripts/publish_pypi.sh +++ b/scripts/publish_pypi.sh @@ -12,8 +12,9 @@ for dependency in curl jq unzip sha256sum; do command -v "$dependency" >/dev/null || fail "$0" "$dependency missing" 'curl jq unzip sha256sum' "install $dependency on the runner host" done mode=${1:-} -case "$mode" in check-inputs|prepare|publish) ;; *) fail "$0" "$mode" 'check-inputs | prepare | publish' 'select one command' ;; esac +case "$mode" in prepare|publish) ;; *) fail "$0" "$mode" 'prepare | publish' 'select one command' ;; esac PYPI_ASSET_SOURCE=${PYPI_ASSET_SOURCE:-} +KITMAKER_PIC=${KITMAKER_PIC:-} if [ -z "$PYPI_ASSET_SOURCE" ]; then if [ -n "${GITHUB_REPOSITORY:-}" ]; then PYPI_ASSET_SOURCE=github-release elif [ -n "${CI_PROJECT_PATH:-}" ]; then PYPI_ASSET_SOURCE=artifactory; fi @@ -22,17 +23,14 @@ RELEASE_TAG=${RELEASE_TAG:-${GITHUB_REF_NAME:-${CI_COMMIT_TAG:-}}} RELEASE_COMMIT=${RELEASE_COMMIT:-${GITHUB_SHA:-${CI_COMMIT_SHA:-}}} PYPI_PLATFORM=${PYPI_PLATFORM:-manylinux_*_x86_64} PYPI_DRY_RUN_ONLY=${PYPI_DRY_RUN_ONLY:-0} -PYPI_ENVIRONMENT=${PYPI_ENVIRONMENT:-pypi} PYPI_PER_RELEASE_PLC=${PYPI_PER_RELEASE_PLC:-no} PYPI_POLL_SECONDS=${PYPI_POLL_SECONDS:-10} PYPI_TIMEOUT_SECONDS=${PYPI_TIMEOUT_SECONDS:-1800} PYPI_INDEX_TIMEOUT_SECONDS=${PYPI_INDEX_TIMEOUT_SECONDS:-600} -PYPI_PACKAGE_ROOT=${PYPI_PACKAGE_ROOT:-.} -KITMAKER_OWNER_FIELD=${KITMAKER_OWNER_FIELD:-} KITMAKER_API=https://kitmaker-portal.nvidia.com/api/v0 manifest=pypi-manifest.json # Enumerating known nonsecret inputs avoids dumping ambient credentials. -for input_name in PYPI_ASSET_SOURCE PYPI_PUBLISH_REPOSITORY RELEASE_TAG RELEASE_COMMIT WHEEL_DIR KITMAKER_PROJECTS KITMAKER_PIC PYPI_PLATFORM PYPI_PROVENANCE_PATH PYPI_PROVENANCE_SOURCE PYPI_DRY_RUN_ONLY ARTIFACTORY_BASE_URL PYPI_ENVIRONMENT PYPI_PER_RELEASE_PLC PYPI_POLL_SECONDS PYPI_TIMEOUT_SECONDS PYPI_INDEX_TIMEOUT_SECONDS PYPI_RELEASE_WORKFLOW PYPI_RELEASE_JOB PYPI_CI_SECRET_NAMES PYPI_APPROVERS PYPI_PUBLISH_RUNNER PYPI_PUBLISH_CELLS PYPI_RELEASE_CELLS PYPI_MATRIX_PATH PYPI_MATRIX_SHA256 PYPI_FIRST_VERSION PYPI_VERSION_FILE PYPI_CURRENT_VERSION PYPI_PACKAGE_ROOT KITMAKER_OWNER_FIELD; do +for input_name in PYPI_ASSET_SOURCE PYPI_PUBLISH_REPOSITORY RELEASE_TAG RELEASE_COMMIT WHEEL_DIR KITMAKER_PROJECTS KITMAKER_PIC PYPI_PLATFORM PYPI_PROVENANCE_PATH PYPI_DRY_RUN_ONLY ARTIFACTORY_BASE_URL PYPI_PER_RELEASE_PLC PYPI_POLL_SECONDS PYPI_TIMEOUT_SECONDS PYPI_INDEX_TIMEOUT_SECONDS PYPI_PUBLISH_CELLS; do eval 'input_value=${'"$input_name"':-}' printf '%s=%s\n' "$input_name" "$input_value" done @@ -169,164 +167,6 @@ prepare() { if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$scratch/summary" >> "$GITHUB_STEP_SUMMARY"; fi } -check_inputs() { - failures=0 - result() { - printf '%s %s: %s\n' "$1" "$2" "$3" - [ "$1" != FAIL ] || failures=$((failures + 1)) - } - case "$PYPI_ASSET_SOURCE" in - github-release) cli=gh ;; artifactory) cli=glab ;; *) result FAIL PYPI_ASSET_SOURCE 'expected github-release or artifactory'; exit 1 ;; - esac - for dependency in "$cli" git; do - if command -v "$dependency" >/dev/null; then result PASS "$dependency" 'installed'; else result FAIL "$dependency" 'missing; install before checking inputs'; fi - done - [ "$failures" -eq 0 ] || exit 1 - for input_name in PYPI_PUBLISH_REPOSITORY KITMAKER_PROJECTS KITMAKER_PIC KITMAKER_TOKEN PYPI_PROVENANCE_PATH PYPI_RELEASE_WORKFLOW PYPI_RELEASE_JOB PYPI_CI_SECRET_NAMES PYPI_APPROVERS PYPI_PUBLISH_RUNNER PYPI_PUBLISH_CELLS PYPI_RELEASE_CELLS PYPI_MATRIX_PATH PYPI_MATRIX_SHA256 PYPI_FIRST_VERSION PYPI_VERSION_FILE PYPI_CURRENT_VERSION; do - eval 'input_value=${'"$input_name"':-}' - [ -n "$input_value" ] || result FAIL "$input_name" 'missing' - done - if [ "$cli" = glab ]; then - for input_name in ARTIFACTORY_BASE_URL ARTIFACTORY_TOKEN; do - eval 'input_value=${'"$input_name"':-}' - [ -n "$input_value" ] || result FAIL "$input_name" 'missing' - done - fi - if [ "${PYPI_RELEASE_JOB:-}" = new ]; then - if [ -n "${PYPI_RELEASE_WORKFLOW:-}" ] && [ ! -e "$PYPI_RELEASE_WORKFLOW" ]; then result PASS release_workflow "$PYPI_RELEASE_WORKFLOW is free for a new release job"; else result FAIL release_workflow 'new workflow path exists or is missing'; fi - elif [ -f "${PYPI_RELEASE_WORKFLOW:-}" ] && awk -v job="${PYPI_RELEASE_JOB:-}" '$0 ~ "^[ ]*" job ":[ ]*($|#)" {found=1} END {exit !found}' "$PYPI_RELEASE_WORKFLOW"; then result PASS release_workflow "$PYPI_RELEASE_WORKFLOW contains $PYPI_RELEASE_JOB" - else result FAIL release_workflow 'workflow file or exact job ID missing'; fi - if [ -f "${PYPI_MATRIX_PATH:-}" ]; then - matrix_sha=$(sha256sum "$PYPI_MATRIX_PATH"); matrix_sha=${matrix_sha%% *} - if [ "$matrix_sha" = "${PYPI_MATRIX_SHA256:-}" ]; then result PASS matrix_source "$PYPI_MATRIX_PATH sha256=$matrix_sha"; else result FAIL matrix_source 'discovery snapshot is stale; rediscover the expanded release matrix'; fi - else result FAIL matrix_source 'matrix source file missing'; fi - if valid_cell_directories "${PYPI_PUBLISH_CELLS:-null}" && printf '%s' "${PYPI_PUBLISH_CELLS:-null}" | jq -e --argjson available "${PYPI_RELEASE_CELLS:-null}" ' - type == "array" and length > 0 and ($available | type == "array") and - all(.[]; . as $cell | (.cuda | type == "string") and (.ubuntu | type == "string") and (.python | type == "string") and .arch == "x86_64" and (.artifact_dir | type == "string" and length > 0) and ($available | index($cell)) != null) and - (group_by([(.cuda | split(".")[0]),.ubuntu,.python,.arch]) | all(.[]; length == 1))' >/dev/null; then result PASS publish_cells "$PYPI_PUBLISH_CELLS" - else result FAIL publish_cells 'select existing x86_64 cells, unique by CUDA major, Ubuntu/Python and architecture, with safe distinct artifact directories'; fi - if public_version "${PYPI_FIRST_VERSION:-}"; then - tags=$(git tag --list "v$PYPI_FIRST_VERSION") - if [ -z "$tags" ]; then result PASS first_version "$PYPI_FIRST_VERSION; tag absent in synced clone"; else result FAIL first_version "tag $tags already exists"; fi - else result FAIL first_version 'expected normalized public PEP 440 version'; fi - if [ -f "${PYPI_VERSION_FILE:-}" ] && [ -w "$PYPI_VERSION_FILE" ] && [ -n "${PYPI_CURRENT_VERSION:-}" ] && grep -F "$PYPI_CURRENT_VERSION" "$PYPI_VERSION_FILE" >/dev/null; then result PASS version_source "$PYPI_VERSION_FILE contains $PYPI_CURRENT_VERSION and is writable"; else result FAIL version_source 'version source or current value missing/unwritable'; fi - source_path=${PYPI_PROVENANCE_SOURCE:-$PYPI_PACKAGE_ROOT/${PYPI_PROVENANCE_PATH:-}} - if [ -n "${PYPI_PROVENANCE_PATH:-}" ] && [ -f "$(dirname "$source_path")/__init__.py" ] && [ -z "$(git ls-files -- "$source_path")" ]; then result PASS provenance "$source_path has import package and is untracked"; else result FAIL provenance 'expected untracked provenance inside a package with __init__.py'; fi - case "$PYPI_PER_RELEASE_PLC" in yes|no) result PASS per_release_plc "$PYPI_PER_RELEASE_PLC (owner decision; no smoke test)" ;; *) result FAIL per_release_plc 'expected yes or no' ;; esac - if [ "$cli" = gh ]; then - if gh repo view --json nameWithOwner,visibility > "$scratch/repo"; then - slug=$(jq -r .nameWithOwner "$scratch/repo"); visibility=$(jq -r .visibility "$scratch/repo") - if [ "$visibility" = PUBLIC ]; then result PASS platform "GitHub $visibility"; else result FAIL platform "GitHub $visibility; public assets required"; fi - else slug=; result FAIL platform 'gh repo view failed'; fi - else - if glab repo view --output json > "$scratch/repo"; then slug=$(jq -r .path_with_namespace "$scratch/repo"); result PASS platform "GitLab $slug"; else slug=; result FAIL platform 'glab repo view failed'; fi - fi - if [ -n "$slug" ] && [ "$slug" = "${PYPI_PUBLISH_REPOSITORY:-}" ]; then result PASS canonical_repository "$slug"; else result FAIL canonical_repository "clone=$slug canonical=${PYPI_PUBLISH_REPOSITORY:-}; private copies don't publish"; fi - if [ "$slug" != "${PYPI_PUBLISH_REPOSITORY:-}" ] || [ -z "$slug" ]; then result FAIL remote_checks 'canonical repository check failed; remaining remote inputs not tested'; exit 1; fi - repo_encoded=$(printf '%s' "$slug" | jq -sRr @uri) - if project_map; then - jq -r '.[] | [.name,.id] | @tsv' "$scratch/map.json" > "$scratch/mappings" - while IFS="$(printf '\t')" read -r name id; do - major=$(printf '%s' "$name" | sed -n 's/.*-cu\([0-9][0-9]*\)$/\1/p') - if [ -n "$major" ] && printf '%s' "${PYPI_PUBLISH_CELLS:-null}" | jq -e --arg major "$major" 'any(.[]; (.cuda | split(".")[0]) == $major)' >/dev/null; then result PASS "distribution/$name" "CUDA major $major"; else result FAIL "distribution/$name" 'expected -cuNN matching a publish-cell CUDA major'; fi - http "https://pypi.org/pypi/$name/json" - case "$http_status" in 404) result PASS "pypi.org/$name" 'HTTP 404; unclaimed name' ;; 200) result WARN "pypi.org/$name" 'HTTP 200; explicitly confirm NVIDIA ownership' ;; *) cat "$scratch/response"; result FAIL "pypi.org/$name" "HTTP $http_status curl=$http_rc" ;; esac - http "https://pypi.nvidia.com/$name/" - case "$http_status" in 404) result PASS "pypi.nvidia.com/$name" 'HTTP 404; no project listing' ;; 200) result WARN "pypi.nvidia.com/$name" 'HTTP 200; explicitly confirm NVIDIA ownership' ;; *) cat "$scratch/response"; result FAIL "pypi.nvidia.com/$name" "HTTP $http_status curl=$http_rc" ;; esac - http "https://pypi.org/pypi/$name/${PYPI_FIRST_VERSION:-}/json" - if [ "$http_status" = 404 ]; then result PASS "first_version/$name" 'HTTP 404'; else result FAIL "first_version/$name" "HTTP $http_status; require unused version"; fi - done < "$scratch/mappings" - if ! printf '%s' "${PYPI_PUBLISH_CELLS:-null}" | jq -e --slurpfile mapping "$scratch/map.json" 'all(.[]; (.cuda | split(".")[0]) as $major | any($mapping[0][]; .name | endswith("-cu" + $major)))' >/dev/null; then result FAIL distributions 'each publish-cell CUDA major needs a project mapping'; fi - if [ -n "${KITMAKER_TOKEN:-}" ]; then - http --header "Authorization: Bearer $KITMAKER_TOKEN" "$KITMAKER_API/projects" - cat "$scratch/response"; printf '\n' - if [ "$http_status" = 200 ] && [ "$http_rc" -eq 0 ] && jq -e 'if type == "array" then . elif (.projects | type) == "array" then .projects else error("unknown projects response shape") end' "$scratch/response" > "$scratch/portal-projects"; then - result PASS kitmaker_token 'HTTP 200' - while IFS="$(printf '\t')" read -r name id; do - if jq -e --arg id "$id" --arg name "$name" '.[] | select((.id | tostring) == $id and .name == $name)' "$scratch/portal-projects" > "$scratch/project"; then - result PASS "kitmaker_project/$name" "id=$id, name=$name" - if [ -n "$KITMAKER_OWNER_FIELD" ]; then - owner=$(jq -r --arg path "$KITMAKER_OWNER_FIELD" 'getpath($path | split(".")) // empty' "$scratch/project") - if [ -n "$owner" ] && [ "$owner" = "${KITMAKER_PIC:-}" ]; then result PASS "pic/$name" "$owner"; else result FAIL "pic/$name" "owner=$owner, pic=${KITMAKER_PIC:-}; correct owner or confirmed response field"; fi - else result WARN "pic/$name" 'owner email field unverified; inspect response and confirm Portal owner or set KITMAKER_OWNER_FIELD'; fi - else result FAIL "kitmaker_project/$name" "id=$id missing or project name differs"; fi - done < "$scratch/mappings" - else result FAIL kitmaker_token "HTTP $http_status curl=$http_rc; 401=token, 000=network/CA, or unsupported response schema"; fi - fi - else result FAIL kitmaker_projects 'invalid name=id map'; fi - if printf '%s' "${KITMAKER_PIC:-}" | jq -eRs 'test("^[A-Za-z0-9._%+-]+@nvidia\\.com$")' >/dev/null; then result PASS pic_format "$KITMAKER_PIC"; else result FAIL pic_format 'expected owner@nvidia.com'; fi - : > "$scratch/secrets" - if [ "$cli" = gh ]; then - if gh secret list --repo "$slug" --json name > "$scratch/repo-secrets" && gh api "repos/$slug/environments" --paginate > "$scratch/environments"; then - jq -r '.[].name' "$scratch/repo-secrets" > "$scratch/secrets" - if jq -e --arg env "$PYPI_ENVIRONMENT" '.environments[] | select(.name == $env)' "$scratch/environments" >/dev/null; then - result PASS environment "$PYPI_ENVIRONMENT exists" - if gh secret list --repo "$slug" --env "$PYPI_ENVIRONMENT" --json name > "$scratch/env-secrets"; then jq -r '.[].name' "$scratch/env-secrets" >> "$scratch/secrets"; else result FAIL ci_secrets 'environment secret listing failed'; fi - else result WARN environment "$PYPI_ENVIRONMENT missing; playbook creates it"; fi - else result FAIL ci_secrets 'secret/environment API failed'; fi - else - # JSON contains variable values; filter before displaying anything. - page=1 - while :; do - if glab variable list --repo "$slug" --output json --per-page 100 --page "$page" > "$scratch/variables"; then - jq -r '.[].key' "$scratch/variables" >> "$scratch/secrets" - [ "$(jq length "$scratch/variables")" -eq 100 ] || break - page=$((page + 1)) - else result FAIL ci_secrets 'variable list failed'; break; fi - done - if glab api "projects/$repo_encoded/environments?per_page=100" > "$scratch/environments"; then - if jq -e --arg env "$PYPI_ENVIRONMENT" '.[] | select(.name == $env)' "$scratch/environments" >/dev/null; then result PASS environment "$PYPI_ENVIRONMENT exists"; else result WARN environment "$PYPI_ENVIRONMENT missing; playbook creates it"; fi - else result FAIL environment 'environment API failed'; fi - fi - for secret_name in ${PYPI_CI_SECRET_NAMES:-}; do - if grep -Fx "$secret_name" "$scratch/secrets" >/dev/null; then result PASS "ci_secret/$secret_name" 'name present; verify required environment scope in playbook'; else result WARN "ci_secret/$secret_name" 'name missing; playbook creates it'; fi - done - printf '%s\n' "${PYPI_APPROVERS:-}" | tr '[:space:]' '\n' | LC_ALL=C sort -u > "$scratch/approvers" - count=$(awk 'NF {n++} END {print n+0}' "$scratch/approvers") - [ "$count" -ge 2 ] || result FAIL approvers 'at least two distinct accounts required' - while IFS= read -r account; do - [ -n "$account" ] || continue - if [ "$cli" = gh ]; then - if gh api "users/$account" --jq .login > "$scratch/account"; then result PASS "approver/$account" "$(cat "$scratch/account")"; else result FAIL "approver/$account" 'account lookup failed'; fi - else - if glab api "users?username=$account" > "$scratch/account" && jq -e --arg account "$account" 'any(.[]; .username == $account)' "$scratch/account" >/dev/null; then result PASS "approver/$account" 'account exists'; else result FAIL "approver/$account" 'account lookup failed'; fi - fi - done < "$scratch/approvers" - : > "$scratch/workflows" - if [ -d .github/workflows ]; then find .github/workflows -type f >> "$scratch/workflows"; fi - if [ -f .gitlab-ci.yml ]; then printf '%s\n' .gitlab-ci.yml >> "$scratch/workflows"; fi - : > "$scratch/runner-labels" - while IFS= read -r workflow; do - awk '/^[ ]*(runs-on|tags):/ {active=1; match($0,/[^ ]/); indent=RSTART; print; next} active {match($0,/[^ ]/); if (RSTART > indent) print; else active=0}' "$workflow" >> "$scratch/runner-labels" - done < "$scratch/workflows" - if [ -n "${PYPI_PUBLISH_RUNNER:-}" ] && grep -F -- "$PYPI_PUBLISH_RUNNER" "$scratch/runner-labels" > "$scratch/matched-runner"; then - result PASS publish_runner "$(cat "$scratch/matched-runner")" - elif [ "$cli" = gh ]; then - if gh api "repos/$slug/actions/runners" --paginate > "$scratch/runners" && jq -e --arg label "${PYPI_PUBLISH_RUNNER:-}" 'any(.runners[].labels[]; .name == $label)' "$scratch/runners" >/dev/null; then result PASS publish_runner 'label in runners API'; else result FAIL publish_runner 'label not verified in workflow or readable runner API'; fi - else - runner_found=0 - if glab api "projects/$repo_encoded/runners?per_page=100" --paginate > "$scratch/runners"; then - jq -r '.[].id' "$scratch/runners" > "$scratch/runner-ids" - while IFS= read -r runner_id; do - if glab api "runners/$runner_id" > "$scratch/runner" && jq -e --arg tag "${PYPI_PUBLISH_RUNNER:-}" '.tag_list | index($tag) != null' "$scratch/runner" >/dev/null; then runner_found=1; fi - done < "$scratch/runner-ids" - fi - if [ "$runner_found" -eq 1 ]; then result PASS publish_runner 'tag in readable runners API'; else result FAIL publish_runner 'tag not verified in workflow or readable runner API'; fi - fi - if [ "$cli" = glab ] && [ -n "${ARTIFACTORY_BASE_URL:-}" ] && [ -n "${ARTIFACTORY_TOKEN:-}" ]; then - case "$ARTIFACTORY_BASE_URL" in - https://artifactory.nvidia.com/artifactory/*) - storage_url="https://artifactory.nvidia.com/artifactory/api/storage/${ARTIFACTORY_BASE_URL#https://artifactory.nvidia.com/artifactory/}" - http --header "Authorization: Bearer $ARTIFACTORY_TOKEN" "$storage_url" - cat "$scratch/response"; printf '\n' - if [ "$http_status" = 200 ] && [ "$http_rc" -eq 0 ]; then result PASS artifactory 'storage API HTTP 200; write permission remains for first dry run'; else result FAIL artifactory "HTTP $http_status curl=$http_rc"; fi ;; - *) result FAIL artifactory 'expected https://artifactory.nvidia.com/artifactory//' ;; - esac - fi - printf 'Input checks: %s failure(s); every WARN requires explicit recorded acceptance.\n' "$failures" - [ "$failures" -eq 0 ] -} - poll_release() { deadline=$(($(date +%s) + PYPI_TIMEOUT_SECONDS)) while :; do @@ -424,4 +264,4 @@ publish() { submit_projects verify_indexes } -case "$mode" in check-inputs) check_inputs ;; prepare) prepare ;; publish) publish ;; esac +case "$mode" in prepare) prepare ;; publish) publish ;; esac From 48f7ce7fa2c41e04c150e9513f09f8a9ebee4af1 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Thu, 15 Oct 2026 17:56:02 +0000 Subject: [PATCH 05/11] Default Kitmaker publishing to dry runs --- scripts/publish_pypi.sh | 61 ++++++++++++----------------------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/scripts/publish_pypi.sh b/scripts/publish_pypi.sh index 024c190..74c345f 100755 --- a/scripts/publish_pypi.sh +++ b/scripts/publish_pypi.sh @@ -1,5 +1,5 @@ #!/bin/sh -# TEMPLATE_VERSION=1.8.0 +# TEMPLATE_VERSION=1.9.0 set -eu set -f @@ -13,27 +13,18 @@ for dependency in curl jq unzip sha256sum; do done mode=${1:-} case "$mode" in prepare|publish) ;; *) fail "$0" "$mode" 'prepare | publish' 'select one command' ;; esac -PYPI_ASSET_SOURCE=${PYPI_ASSET_SOURCE:-} +PYPI_ASSET_SOURCE=${PYPI_ASSET_SOURCE:-github-release} KITMAKER_PIC=${KITMAKER_PIC:-} -if [ -z "$PYPI_ASSET_SOURCE" ]; then - if [ -n "${GITHUB_REPOSITORY:-}" ]; then PYPI_ASSET_SOURCE=github-release - elif [ -n "${CI_PROJECT_PATH:-}" ]; then PYPI_ASSET_SOURCE=artifactory; fi -fi -RELEASE_TAG=${RELEASE_TAG:-${GITHUB_REF_NAME:-${CI_COMMIT_TAG:-}}} -RELEASE_COMMIT=${RELEASE_COMMIT:-${GITHUB_SHA:-${CI_COMMIT_SHA:-}}} +RELEASE_TAG=${RELEASE_TAG:-${GITHUB_REF_NAME:-}} +RELEASE_COMMIT=${RELEASE_COMMIT:-${GITHUB_SHA:-}} PYPI_PLATFORM=${PYPI_PLATFORM:-manylinux_*_x86_64} -PYPI_DRY_RUN_ONLY=${PYPI_DRY_RUN_ONLY:-0} +PYPI_DRY_RUN_ONLY=${PYPI_DRY_RUN_ONLY:-1} PYPI_PER_RELEASE_PLC=${PYPI_PER_RELEASE_PLC:-no} PYPI_POLL_SECONDS=${PYPI_POLL_SECONDS:-10} PYPI_TIMEOUT_SECONDS=${PYPI_TIMEOUT_SECONDS:-1800} PYPI_INDEX_TIMEOUT_SECONDS=${PYPI_INDEX_TIMEOUT_SECONDS:-600} KITMAKER_API=https://kitmaker-portal.nvidia.com/api/v0 manifest=pypi-manifest.json -# Enumerating known nonsecret inputs avoids dumping ambient credentials. -for input_name in PYPI_ASSET_SOURCE PYPI_PUBLISH_REPOSITORY RELEASE_TAG RELEASE_COMMIT WHEEL_DIR KITMAKER_PROJECTS KITMAKER_PIC PYPI_PLATFORM PYPI_PROVENANCE_PATH PYPI_DRY_RUN_ONLY ARTIFACTORY_BASE_URL PYPI_PER_RELEASE_PLC PYPI_POLL_SECONDS PYPI_TIMEOUT_SECONDS PYPI_INDEX_TIMEOUT_SECONDS PYPI_PUBLISH_CELLS; do - eval 'input_value=${'"$input_name"':-}' - printf '%s=%s\n' "$input_name" "$input_value" -done umask 077 scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT HUP INT TERM @@ -47,11 +38,8 @@ required() { [ -z "$missing" ] || fail "$0" "missing:$missing" 'all required inputs set' 'export the listed names from the confirmed context' } repository_guard() { - case "$PYPI_ASSET_SOURCE" in - github-release) actual_repository=${GITHUB_REPOSITORY:-} ;; - artifactory) actual_repository=${CI_PROJECT_PATH:-} ;; - *) fail "$0" "$PYPI_ASSET_SOURCE" 'github-release or artifactory' 'set PYPI_ASSET_SOURCE' ;; - esac + [ "$PYPI_ASSET_SOURCE" = github-release ] || fail "$0" "$PYPI_ASSET_SOURCE" 'github-release' 'only public GitHub release assets are supported' + actual_repository=${GITHUB_REPOSITORY:-} [ "$actual_repository" = "$PYPI_PUBLISH_REPOSITORY" ] || fail "$0" "$actual_repository" "$PYPI_PUBLISH_REPOSITORY" "private copies don't publish; run in the canonical repository" } public_version() { @@ -72,7 +60,7 @@ valid_cell_directories() { (length == 1 or all(.[]; .artifact_dir != ".")) and (map(.artifact_dir) | unique | length) == length' >/dev/null } -# Preserve error bodies and never retry a POST or PUT automatically. +# Preserve error bodies and never retry a POST automatically. http() { http_status=000 http_rc=0 @@ -156,7 +144,7 @@ prepare() { [ "$cell_count" -eq 1 ] || fail "$cell_prefix" "$cell_count selected CUDA $cell_cuda wheels" 'exactly one finished wheel per selected matrix cell' 'restore the missing cell artifact or separate extra build outputs' done < "$scratch/expected-cells" jq -es --slurpfile cells "$scratch/cells" --arg root "${WHEEL_DIR%/}/" 'length == ($cells[0] | length) and all(.[]; . as $wheel | any($cells[0][]; (if .artifact_dir == "." then $root else $root + .artifact_dir + "/" end) as $prefix | $wheel.path | startswith($prefix)))' "$scratch/entries" >/dev/null || fail "$WHEEL_DIR" 'extra wheel or wheel outside selected cell directories' 'only the confirmed publish cells' 'narrow the artifact download selection' - jq -s --arg tag "$RELEASE_TAG" --arg commit "$RELEASE_COMMIT" --arg repository "$PYPI_PUBLISH_REPOSITORY" --arg source "$PYPI_ASSET_SOURCE" --arg plc "$PYPI_PER_RELEASE_PLC" '{template_version:"1.8.0",tag:$tag,commit:$commit,repository:$repository,asset_source:$source,per_release_plc:$plc,wheels:.}' "$scratch/entries" > "$manifest" + jq -s --arg tag "$RELEASE_TAG" --arg commit "$RELEASE_COMMIT" --arg repository "$PYPI_PUBLISH_REPOSITORY" --arg source "$PYPI_ASSET_SOURCE" --arg plc "$PYPI_PER_RELEASE_PLC" '{template_version:"1.9.0",tag:$tag,commit:$commit,repository:$repository,asset_source:$source,per_release_plc:$plc,wheels:.}' "$scratch/entries" > "$manifest" { printf '| Wheel | Bytes | SHA256 |\n|---|---:|---|\n' jq -r '.wheels[] | "| \(.filename) | \(.bytes) | \(.sha256) |"' "$manifest" @@ -225,34 +213,19 @@ publish() { case "$PYPI_DRY_RUN_ONLY" in 0|1) ;; *) fail "$0" "$PYPI_DRY_RUN_ONLY" '0 or 1' 'set the dry-run switch explicitly' ;; esac [ -f "$manifest" ] || fail "$manifest" 'missing' 'manifest from successful prepare' 'download the prepare artifact' jq -e --arg tag "$RELEASE_TAG" --arg commit "$RELEASE_COMMIT" --arg repository "$PYPI_PUBLISH_REPOSITORY" --arg source "$PYPI_ASSET_SOURCE" ' - .template_version == "1.8.0" and .tag == $tag and .commit == $commit and .repository == $repository and .asset_source == $source and + .template_version == "1.9.0" and .tag == $tag and .commit == $commit and .repository == $repository and .asset_source == $source and (.wheels | type == "array" and length > 0 and all(.[]; (.filename | test("^[A-Za-z0-9_.!+-]+\\.whl$")) and (.sha256 | test("^[a-f0-9]{64}$")) and (.project_id | test("^[1-9][0-9]*$"))))' "$manifest" >/dev/null || fail "$manifest" 'invalid manifest or tag/commit/repository/source mismatch' "$RELEASE_TAG $RELEASE_COMMIT $PYPI_PUBLISH_REPOSITORY $PYPI_ASSET_SOURCE" 'use the manifest from this release and template version' jq -r '.wheels[] | [.distribution,.version,.filename,.sha256] | @tsv' "$manifest" > "$scratch/expected" jq -r '.wheels[] | [.path,.filename,.sha256] | @tsv' "$manifest" > "$scratch/to-publish" : > "$scratch/urls" - if [ "$PYPI_ASSET_SOURCE" = artifactory ]; then - required ARTIFACTORY_BASE_URL ARTIFACTORY_TOKEN - case "$ARTIFACTORY_BASE_URL" in https://artifactory.nvidia.com/artifactory/*) ;; *) fail "$0" "$ARTIFACTORY_BASE_URL" 'direct Artifactory base URL' 'use an approved repo and path' ;; esac - fi while IFS="$(printf '\t')" read -r wheel filename checksum; do - if [ "$PYPI_ASSET_SOURCE" = github-release ]; then - tag_uri=$(printf '%s' "$RELEASE_TAG" | jq -sRr @uri) - file_uri=$(printf '%s' "$filename" | jq -sRr @uri) - url="https://github.com/$PYPI_PUBLISH_REPOSITORY/releases/download/$tag_uri/$file_uri" - http "$url" - if [ "$http_status" = 404 ]; then fail "$filename" 'HTTP 404' 'public GitHub release asset' "a draft release isn't readable; publish the draft before approval"; fi - [ "$http_rc" -eq 0 ] || fail "$filename" "HTTP $http_status curl=$http_rc" 'anonymous asset download' 'fix release asset visibility or URL' - actual=$(sha256sum "$scratch/response"); actual=${actual%% *} - else - [ -f "$wheel" ] || fail "$wheel" 'missing' 'original wheel artifact' 'download build artifacts with the same directory layout as prepare' - actual=$(sha256sum "$wheel"); actual=${actual%% *} - [ "$actual" = "$checksum" ] || fail "$wheel" "$actual" "$checksum" 'restore the approved artifact' - url="${ARTIFACTORY_BASE_URL%/}/$RELEASE_TAG/$RELEASE_COMMIT/$filename" - http --request PUT --header "Authorization: Bearer $ARTIFACTORY_TOKEN" --header "X-Checksum-Sha256: $checksum" --upload-file "$wheel" "$url" - cat "$scratch/response"; printf '\n' - [ "$http_rc" -eq 0 ] || fail "$wheel" "HTTP $http_status curl=$http_rc" 'successful Artifactory deployment' 'inspect deploy permissions and response; do not blindly overwrite' - actual=$(jq -er '.checksums.sha256' "$scratch/response") || fail "$wheel" 'no checksums.sha256 in PUT response' "$checksum" 'verify deployed Artifactory API contract before proceeding' - fi + tag_uri=$(printf '%s' "$RELEASE_TAG" | jq -sRr @uri) + file_uri=$(printf '%s' "$filename" | jq -sRr @uri) + url="https://github.com/$PYPI_PUBLISH_REPOSITORY/releases/download/$tag_uri/$file_uri" + http "$url" + if [ "$http_status" = 404 ]; then fail "$filename" 'HTTP 404' 'public GitHub release asset' "a draft release isn't readable; publish the draft before approval"; fi + [ "$http_rc" -eq 0 ] || fail "$filename" "HTTP $http_status curl=$http_rc" 'anonymous asset download' 'fix release asset visibility or URL' + actual=$(sha256sum "$scratch/response"); actual=${actual%% *} [ "$actual" = "$checksum" ] || fail "$filename" "$actual" "$checksum" 'restore the exact immutable artifact; regenerate approval after any change' jq -n --arg filename "$filename" --arg url "$url" '{filename:$filename,url:$url}' >> "$scratch/urls" done < "$scratch/to-publish" From eb0f8e4c3b674dbb3b83877e2eae4cae6335f4c4 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Thu, 15 Oct 2026 17:56:08 +0000 Subject: [PATCH 06/11] Share wheel release selection with artifact validation --- .github/workflows/build-and-test.yml | 4 ++-- .github/workflows/release.yml | 20 +++++++------------- scripts/release_pycunls.sh | 4 +++- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e20ffa0..0f3501f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -113,10 +113,10 @@ jobs: if-no-files-found: warn - name: Upload Python wheel - if: ${{ always() && inputs.upload_binaries }} + if: inputs.upload_binaries uses: actions/upload-artifact@v6 with: name: pycunls-${{ env.CONFIG_NAME }} path: output/wheel/*.whl retention-days: ${{ inputs.artifact_retention_days }} - if-no-files-found: warn + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c995ce3..77263fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,11 +24,11 @@ env: PYPI_PLATFORM: manylinux_*_x86_64 PYPI_PROVENANCE_PATH: pycunls/_build_provenance.json PYPI_PER_RELEASE_PLC: ${{ vars.PYPI_PER_RELEASE_PLC || 'yes' }} - PYPI_PUBLISH_CELLS: >- - [{"cuda":"12.8.1","ubuntu":"22.04","python":"3.10","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu22.04"}, - {"cuda":"13.2.0","ubuntu":"22.04","python":"3.10","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu22.04"}, - {"cuda":"12.8.1","ubuntu":"24.04","python":"3.12","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu24.04"}, - {"cuda":"13.2.0","ubuntu":"24.04","python":"3.12","arch":"x86_64","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu24.04"}] + PYPI_PUBLISH_CELLS: &release_cells >- + [{"cuda":"12.8.1","ubuntu":"22.04","python":"3.10","arch":"x86_64","archs":"75;80;86;89","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu22.04"}, + {"cuda":"13.2.0","ubuntu":"22.04","python":"3.10","arch":"x86_64","archs":"75;80;86;89;100","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu22.04"}, + {"cuda":"12.8.1","ubuntu":"24.04","python":"3.12","arch":"x86_64","archs":"75;80;86;89","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu24.04"}, + {"cuda":"13.2.0","ubuntu":"24.04","python":"3.12","arch":"x86_64","archs":"75;80;86;89;100","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu24.04"}] RELEASE_TAG: ${{ github.ref_name }} RELEASE_COMMIT: ${{ github.sha }} @@ -37,13 +37,7 @@ jobs: if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v') uses: ./.github/workflows/build-and-test.yml with: - matrix_json: | - [ - {"ubuntu":"22.04","cuda":"12.8.1","archs":"75;80;86;89"}, - {"ubuntu":"22.04","cuda":"13.2.0","archs":"75;80;86;89;100"}, - {"ubuntu":"24.04","cuda":"12.8.1","archs":"75;80;86;89"}, - {"ubuntu":"24.04","cuda":"13.2.0","archs":"75;80;86;89;100"} - ] + matrix_json: *release_cells release_tag: ${{ github.ref_name }} artifact_retention_days: 30 upload_binaries: true @@ -117,7 +111,7 @@ jobs: - uses: actions/upload-artifact@v6 if: always() with: - name: pypi-publication-record-${{ github.run_attempt }} + name: pypi-publication-record-${{ github.run_id }}-${{ github.run_attempt }} path: | pypi-manifest.json pypi-release-uuids.txt diff --git a/scripts/release_pycunls.sh b/scripts/release_pycunls.sh index b557e0c..51d0f53 100755 --- a/scripts/release_pycunls.sh +++ b/scripts/release_pycunls.sh @@ -5,12 +5,14 @@ set -euo pipefail : "${RELEASE_TAG:?Expected v}" : "${RELEASE_COMMIT:?Expected the built checkout SHA}" : "${GH_REPO:?Expected nvidia-isaac/cuNLS}" +: "${PYPI_PUBLISH_CELLS:?Expected the release matrix}" [ "$GH_REPO" = nvidia-isaac/cuNLS ] || { echo "Only the canonical repository releases wheels." >&2; exit 1; } [[ "$RELEASE_TAG" = v* ]] || { echo "Expected a v tag." >&2; exit 1; } shopt -s nullglob wheels=(pypi-wheels/*/*.whl) -[ "${#wheels[@]}" -eq 4 ] || { echo "Expected four release wheels; check matrix artifacts." >&2; exit 1; } +expected=$(printf '%s' "$PYPI_PUBLISH_CELLS" | jq -er 'length | select(. > 0)') +[ "${#wheels[@]}" -eq "$expected" ] || { echo "Expected $expected release wheels; check matrix artifacts." >&2; exit 1; } declare -A seen for wheel in "${wheels[@]}"; do name=${wheel##*/} From ac6cbbb04074e0ac2c678fa064c689e4e9ffe9d3 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Thu, 15 Oct 2026 17:56:09 +0000 Subject: [PATCH 07/11] Bind release wheels to committed source and measured CUDA --- .pre-commit-config.yaml | 2 +- README.md | 7 +++++++ scripts/build_pycunls.sh | 18 +++++++++--------- scripts/build_pycunls_in_docker.sh | 26 +++++++++++++++----------- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36fde26..206bbfd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,4 +24,4 @@ repos: name: Validate Python release scripts entry: shellcheck language: system - files: ^scripts/(publish_pypi|release_pycunls|build_pycunls|test_pycunls)\.sh$ + files: ^scripts/((build|test)_pycunls(_in_docker)?|publish_pypi|release_pycunls)\.sh$ diff --git a/README.md b/README.md index 92bd17d..715fc72 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,13 @@ a container with the source mounted read-only. Intermediate build directories live inside the container and are discarded; only the final `.whl` file is written to the host output directory. +Docker wheels use the distribution name `pycunls-cu12` or `pycunls-cu13` and the +matching CuPy dependency; both use `import pycunls`. With `RELEASE_TAG=v`, +the Docker build uses the committed checkout, excluding local changes and generated +files. The tag version must match `python/pyproject.toml` and +`python/pycunls/__init__.py`. Wheels include the source commit and measured CUDA +toolkit version and are repaired with auditwheel while keeping CUDA libraries external. + #### Build the wheel locally ```bash diff --git a/scripts/build_pycunls.sh b/scripts/build_pycunls.sh index 6f72fa3..8e46ef2 100755 --- a/scripts/build_pycunls.sh +++ b/scripts/build_pycunls.sh @@ -10,16 +10,16 @@ import re import subprocess from pathlib import Path -path = Path('/usr/local/cuda/version.json') -if path.exists(): - actual = json.loads(path.read_text())['cuda']['version'] -else: - match = re.search(r'release (\d+\.\d+)', subprocess.check_output(['nvcc', '--version'], text=True)) - if not match: - raise SystemExit('Cannot identify the installed CUDA toolkit; use a CUDA development image') - actual = match.group(1) +cuda = Path('/usr/local/cuda') +match = re.search(r'release (\d+\.\d+)', subprocess.check_output([str(cuda / 'bin/nvcc'), '--version'], text=True)) +if not match: + raise SystemExit('Cannot identify the installed CUDA toolkit; use a CUDA development image') +path = cuda / 'version.json' +actual = json.loads(path.read_text())['cuda']['version'] if path.exists() else match[1] +if actual.split('.')[:2] != match[1].split('.'): + raise SystemExit(f'CUDA version.json {actual} disagrees with nvcc {match[1]}') requested = os.environ.get('CUDA_VERSION') -if requested and requested.split('.')[:2] != actual.split('.')[:2]: +if requested and not (actual == requested or actual.startswith(requested + '.')): raise SystemExit(f'Requested CUDA {requested}, but the build image contains {actual}; correct CUDA_VERSION or BASE_IMAGE') print(actual) PY diff --git a/scripts/build_pycunls_in_docker.sh b/scripts/build_pycunls_in_docker.sh index 020a878..0763228 100755 --- a/scripts/build_pycunls_in_docker.sh +++ b/scripts/build_pycunls_in_docker.sh @@ -17,19 +17,23 @@ if [ "$RELEASE_COMMIT" != "$CHECKOUT_COMMIT" ]; then echo "RELEASE_COMMIT differs from the checked-out source; build the release checkout." >&2 exit 1 fi -if [ -n "${RELEASE_TAG:-}" ] && ! git diff --quiet HEAD --; then - echo "Release wheels require committed source; commit the changes before building a release tag." >&2 - exit 1 +SOURCE_DIR=$(pwd) +if [ -n "${RELEASE_TAG:-}" ]; then + # A local release has no checkout action to remove ignored package inputs. + # Build the committed tree without deleting or packaging local edits. + SOURCE_DIR=$(mktemp -d) + trap 'rm -rf "$SOURCE_DIR"' EXIT + git archive "$CHECKOUT_COMMIT" | tar -x -C "$SOURCE_DIR" fi -DOCKERFILE=$(dirname "$(realpath "$0")")/Dockerfile -DOCKER_BUILD_ARGS="" -[ -n "${CUDA_VERSION:-}" ] && DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg CUDA_VERSION=$CUDA_VERSION" -[ -n "${UBUNTU_VERSION:-}" ] && DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg UBUNTU_VERSION=$UBUNTU_VERSION" -[ -n "${BASE_IMAGE:-}" ] && DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg BASE_IMAGE=$BASE_IMAGE" +DOCKERFILE="$SOURCE_DIR/scripts/Dockerfile" +DOCKER_BUILD_ARGS=() +[ -n "${CUDA_VERSION:-}" ] && DOCKER_BUILD_ARGS+=(--build-arg "CUDA_VERSION=$CUDA_VERSION") +[ -n "${UBUNTU_VERSION:-}" ] && DOCKER_BUILD_ARGS+=(--build-arg "UBUNTU_VERSION=$UBUNTU_VERSION") +[ -n "${BASE_IMAGE:-}" ] && DOCKER_BUILD_ARGS+=(--build-arg "BASE_IMAGE=$BASE_IMAGE") [ -n "${UBUNTU_PORTS_MIRROR:-}" ] \ - && DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg UBUNTU_PORTS_MIRROR=$UBUNTU_PORTS_MIRROR" -docker build -f "$DOCKERFILE" . --network host $DOCKER_BUILD_ARGS --tag cunls:local + && DOCKER_BUILD_ARGS+=(--build-arg "UBUNTU_PORTS_MIRROR=$UBUNTU_PORTS_MIRROR") +docker build -f "$DOCKERFILE" "$SOURCE_DIR" --network host "${DOCKER_BUILD_ARGS[@]}" --tag cunls:local TTY_FLAG="" [ -t 0 ] && TTY_FLAG="-it" @@ -38,7 +42,7 @@ TTY_FLAG="" # /cunls (ro) — source tree # /cunls_install (rw) — wheel output (host: $LOCAL_OUTPUT_DIR) docker run --runtime=nvidia --gpus all --rm $TTY_FLAG \ - -v "$(pwd):/cunls:ro" \ + -v "$SOURCE_DIR:/cunls:ro" \ -v "$LOCAL_OUTPUT_DIR:/cunls_install" \ -e OUTPUT_DIR=/cunls_install \ -e RELEASE_COMMIT="$RELEASE_COMMIT" \ From 3400887b66ead8957ffb42c5f8e3870ee0f7fa99 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Thu, 15 Oct 2026 20:06:40 +0000 Subject: [PATCH 08/11] Compare CUDA toolkit versions at measured precision --- scripts/build_pycunls.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_pycunls.sh b/scripts/build_pycunls.sh index 8e46ef2..58c22c4 100755 --- a/scripts/build_pycunls.sh +++ b/scripts/build_pycunls.sh @@ -19,7 +19,8 @@ actual = json.loads(path.read_text())['cuda']['version'] if path.exists() else m if actual.split('.')[:2] != match[1].split('.'): raise SystemExit(f'CUDA version.json {actual} disagrees with nvcc {match[1]}') requested = os.environ.get('CUDA_VERSION') -if requested and not (actual == requested or actual.startswith(requested + '.')): +# nvcc exposes major.minor when version.json has no toolkit patch version. +if requested and not (actual == requested or actual.startswith(requested + '.') or requested.startswith(actual + '.')): raise SystemExit(f'Requested CUDA {requested}, but the build image contains {actual}; correct CUDA_VERSION or BASE_IMAGE') print(actual) PY From caa2aa0abb918bdccd8a192c1c1743fc50a36413 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Mon, 19 Oct 2026 22:14:06 +0000 Subject: [PATCH 09/11] Validate wheel manifests before creating GitHub releases --- scripts/release_pycunls.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/scripts/release_pycunls.sh b/scripts/release_pycunls.sh index 51d0f53..235841c 100755 --- a/scripts/release_pycunls.sh +++ b/scripts/release_pycunls.sh @@ -5,20 +5,12 @@ set -euo pipefail : "${RELEASE_TAG:?Expected v}" : "${RELEASE_COMMIT:?Expected the built checkout SHA}" : "${GH_REPO:?Expected nvidia-isaac/cuNLS}" -: "${PYPI_PUBLISH_CELLS:?Expected the release matrix}" [ "$GH_REPO" = nvidia-isaac/cuNLS ] || { echo "Only the canonical repository releases wheels." >&2; exit 1; } [[ "$RELEASE_TAG" = v* ]] || { echo "Expected a v tag." >&2; exit 1; } -shopt -s nullglob -wheels=(pypi-wheels/*/*.whl) -expected=$(printf '%s' "$PYPI_PUBLISH_CELLS" | jq -er 'length | select(. > 0)') -[ "${#wheels[@]}" -eq "$expected" ] || { echo "Expected $expected release wheels; check matrix artifacts." >&2; exit 1; } -declare -A seen -for wheel in "${wheels[@]}"; do - name=${wheel##*/} - [ -z "${seen[$name]:-}" ] || { echo "Duplicate release filename: $name" >&2; exit 1; } - seen[$name]=1 -done +sh "$(dirname "$0")/publish_pypi.sh" prepare +validated_paths=$(jq -er '.wheels[].path' pypi-manifest.json) +mapfile -t wheels <<< "$validated_paths" # Existing assets are immutable inputs to approval. Retry only pypi-publish # after a dry run or a partial failure, rather than replacing release bytes. From e8f5355f9087d2d9b01742355452580e58dc9a90 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Mon, 19 Oct 2026 22:14:06 +0000 Subject: [PATCH 10/11] Retry transient Kitmaker status reads within the polling deadline --- scripts/publish_pypi.sh | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/publish_pypi.sh b/scripts/publish_pypi.sh index 74c345f..ef885e0 100755 --- a/scripts/publish_pypi.sh +++ b/scripts/publish_pypi.sh @@ -158,17 +158,26 @@ prepare() { poll_release() { deadline=$(($(date +%s) + PYPI_TIMEOUT_SECONDS)) while :; do - kitmaker "$KITMAKER_API/status/$release_uuid" - cp "$scratch/response" "pypi-status-$release_uuid.json" - state=$(jq -er '.status' "$scratch/response") || fail "$release_uuid" 'missing status' 'status JSON' 'inspect the saved response' - case "$state" in - completed) return 0 ;; - failed) - printf 'Batch outcomes (unlisted wheels may be unstarted; inspect both indexes):\n' - jq -r '.jobs[]? | "\(.status) \(.wheel_url)"' "$scratch/response" - cat "$scratch/completed" - fail "$release_uuid" 'failed or partially published batch' 'all wheels completed' 'inspect every wheel on both indexes; retain UUID; do not resubmit the batch' ;; - esac + http --header "Authorization: Bearer $KITMAKER_TOKEN" "$KITMAKER_API/status/$release_uuid" + cat "$scratch/response"; printf '\n' + if [ "$http_rc" -eq 0 ]; then + cp "$scratch/response" "pypi-status-$release_uuid.json" + state=$(jq -er '.status | select(type == "string" and length > 0)' "$scratch/response") || fail "$release_uuid" 'missing or invalid status' 'status JSON' 'inspect the saved response' + case "$state" in + completed) return 0 ;; + failed) + printf 'Batch outcomes (unlisted wheels may be unstarted; inspect both indexes):\n' + jq -r '.jobs[]? | "\(.status) \(.wheel_url)"' "$scratch/response" + cat "$scratch/completed" + fail "$release_uuid" 'failed or partially published batch' 'all wheels completed' 'inspect every wheel on both indexes; retain UUID; do not resubmit the batch' ;; + esac + else + case "$http_rc:$http_status" in + 5:*|6:*|7:*|18:*|28:*|52:*|55:*|56:*|22:408|22:429|22:5??) + state="transient HTTP $http_status (curl=$http_rc)" ;; + *) fail "$release_uuid" "HTTP $http_status, curl $http_rc" 'successful status response' 'inspect the response; check token and NVIDIA CA; do not resubmit' ;; + esac + fi [ "$(date +%s)" -lt "$deadline" ] || fail "$release_uuid" "$state at polling deadline" 'terminal completed state' 'inspect saved status and both indexes; do not resubmit or infer cancellation' printf 'POLL %s: %s; waiting %s seconds\n' "$release_uuid" "$state" "$PYPI_POLL_SECONDS" sleep "$PYPI_POLL_SECONDS" From e026244a3870ec9b527cc7f50c3c21240a46d136 Mon Sep 17 00:00:00 2001 From: Heron Ordonez Guillen Date: Tue, 20 Oct 2026 00:08:04 +0000 Subject: [PATCH 11/11] Back off transient PyPI release status retries --- scripts/publish_pypi.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/publish_pypi.sh b/scripts/publish_pypi.sh index ef885e0..d27aead 100755 --- a/scripts/publish_pypi.sh +++ b/scripts/publish_pypi.sh @@ -157,10 +157,14 @@ prepare() { poll_release() { deadline=$(($(date +%s) + PYPI_TIMEOUT_SECONDS)) + retry_delay=$PYPI_POLL_SECONDS + max_retry_delay=60 while :; do + poll_delay=$PYPI_POLL_SECONDS http --header "Authorization: Bearer $KITMAKER_TOKEN" "$KITMAKER_API/status/$release_uuid" cat "$scratch/response"; printf '\n' if [ "$http_rc" -eq 0 ]; then + retry_delay=$PYPI_POLL_SECONDS cp "$scratch/response" "pypi-status-$release_uuid.json" state=$(jq -er '.status | select(type == "string" and length > 0)' "$scratch/response") || fail "$release_uuid" 'missing or invalid status' 'status JSON' 'inspect the saved response' case "$state" in @@ -174,13 +178,16 @@ poll_release() { else case "$http_rc:$http_status" in 5:*|6:*|7:*|18:*|28:*|52:*|55:*|56:*|22:408|22:429|22:5??) + poll_delay=$retry_delay + [ "$poll_delay" -le "$max_retry_delay" ] || poll_delay=$max_retry_delay + retry_delay=$((poll_delay * 2)) state="transient HTTP $http_status (curl=$http_rc)" ;; *) fail "$release_uuid" "HTTP $http_status, curl $http_rc" 'successful status response' 'inspect the response; check token and NVIDIA CA; do not resubmit' ;; esac fi [ "$(date +%s)" -lt "$deadline" ] || fail "$release_uuid" "$state at polling deadline" 'terminal completed state' 'inspect saved status and both indexes; do not resubmit or infer cancellation' - printf 'POLL %s: %s; waiting %s seconds\n' "$release_uuid" "$state" "$PYPI_POLL_SECONDS" - sleep "$PYPI_POLL_SECONDS" + printf 'POLL %s: %s; waiting %s seconds\n' "$release_uuid" "$state" "$poll_delay" + sleep "$poll_delay" done } submit_projects() {