Publish CUDA 12 and 13 Python wheels through Kitmaker

.github/workflows/build-and-test.yml

@@ -11,6 +11,10 @@ on:
           Example: [{"ubuntu":"22.04","cuda":"12.8.1","archs":"75;80;86;89"}]
         type: string
         required: true
+      release_tag:
+        description: "Optional v<version> tag for a public wheel release"
+        type: string
+        default: ""
       artifact_retention_days:
         description: "Number of days to retain uploaded artifacts"
         type: number
@@ -40,6 +44,8 @@ jobs:
         config: ${{ fromJson(inputs.matrix_json) }}
     env:
       CONFIG_NAME: ${{ matrix.config.platform || 'x86_64' }}-cuda${{ matrix.config.cuda }}-ubuntu${{ matrix.config.ubuntu }}
+      RELEASE_TAG: ${{ inputs.release_tag }}
+      RELEASE_COMMIT: ${{ github.sha }}
       CUDA_VERSION: ${{ matrix.config.cuda }}
       UBUNTU_VERSION: ${{ matrix.config.ubuntu }}
       BASE_IMAGE: ${{ matrix.config.base_image }}
@@ -107,10 +113,10 @@ jobs:
           if-no-files-found: warn
 
       - name: Upload Python wheel
-        if: ${{ always() && inputs.upload_binaries }}
+        if: inputs.upload_binaries
         uses: actions/upload-artifact@v6
         with:
           name: pycunls-${{ env.CONFIG_NAME }}
           path: output/wheel/*.whl
           retention-days: ${{ inputs.artifact_retention_days }}
-          if-no-files-found: warn
+          if-no-files-found: error

.github/workflows/release.yml

@@ -0,0 +1,120 @@
+name: Release Python wheels
+
+on:
+  push:
+    tags: ['v*']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-${{ github.repository }}-${{ github.ref_name }}
+  cancel-in-progress: false
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  PYPI_PUBLISH_REPOSITORY: nvidia-isaac/cuNLS
+  PYPI_ASSET_SOURCE: github-release
+  WHEEL_DIR: pypi-wheels
+  KITMAKER_PROJECTS: |
+    pycunls-cu12=6865
+    pycunls-cu13=6866
+  KITMAKER_PIC: hguillen@nvidia.com
+  PYPI_PLATFORM: manylinux_*_x86_64
+  PYPI_PROVENANCE_PATH: pycunls/_build_provenance.json
+  PYPI_PER_RELEASE_PLC: ${{ vars.PYPI_PER_RELEASE_PLC || 'yes' }}
+  PYPI_PUBLISH_CELLS: &release_cells >-
+    [{"cuda":"12.8.1","ubuntu":"22.04","python":"3.10","arch":"x86_64","archs":"75;80;86;89","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu22.04"},
+     {"cuda":"13.2.0","ubuntu":"22.04","python":"3.10","arch":"x86_64","archs":"75;80;86;89;100","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu22.04"},
+     {"cuda":"12.8.1","ubuntu":"24.04","python":"3.12","arch":"x86_64","archs":"75;80;86;89","artifact_dir":"pycunls-x86_64-cuda12.8.1-ubuntu24.04"},
+     {"cuda":"13.2.0","ubuntu":"24.04","python":"3.12","arch":"x86_64","archs":"75;80;86;89;100","artifact_dir":"pycunls-x86_64-cuda13.2.0-ubuntu24.04"}]
+  RELEASE_TAG: ${{ github.ref_name }}
+  RELEASE_COMMIT: ${{ github.sha }}
+
+jobs:
+  build-matrix:
+    if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v')
+    uses: ./.github/workflows/build-and-test.yml
+    with:
+      matrix_json: *release_cells
+      release_tag: ${{ github.ref_name }}
+      artifact_retention_days: 30
+      upload_binaries: true
+      timeout_minutes: 120
+
+  release:
+    if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v')
+    needs: build-matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@v7
+        with:
+          pattern: pycunls-x86_64-*
+          path: pypi-wheels
+      - run: bash scripts/release_pycunls.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: nvidia-isaac/cuNLS
+
+  pypi-prepare:
+    if: github.repository == 'nvidia-isaac/cuNLS' && startsWith(github.ref, 'refs/tags/v')
+    needs: build-matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@v7
+        with:
+          pattern: pycunls-x86_64-*
+          path: pypi-wheels
+      - run: sh scripts/publish_pypi.sh prepare
+      - uses: actions/upload-artifact@v6
+        with:
+          name: pypi-manifest
+          path: pypi-manifest.json
+          if-no-files-found: error
+          retention-days: 30
+
+  pypi-publish:
+    if: >-
+      github.repository == 'nvidia-isaac/cuNLS' &&
+      startsWith(github.ref, 'refs/tags/v') &&
+      vars.PYPI_PUBLISH_ENABLED == 'true'
+    needs: [release, pypi-prepare]
+    runs-on: ${{ fromJson(vars.PYPI_PUBLISH_RUNNER || '["self-hosted","gpu"]') }}
+    timeout-minutes: 180
+    environment: pypi
+    concurrency:
+      group: pypi-${{ github.repository }}-${{ github.ref_name }}
+      cancel-in-progress: false
+    env:
+      KITMAKER_TOKEN: ${{ secrets.KITMAKER_TOKEN }}
+      PYPI_DRY_RUN_ONLY: ${{ vars.PYPI_DRY_RUN_ONLY || '1' }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@v7
+        with:
+          name: pypi-manifest
+          path: .
+      - run: sh scripts/publish_pypi.sh publish
+      - uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: pypi-publication-record-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            pypi-manifest.json
+            pypi-release-uuids.txt
+            pypi-status-*.json
+          if-no-files-found: warn
+          retention-days: 30

.gitignore

@@ -6,3 +6,11 @@ viz/
 
 # Design notes are kept locally, not in the repository
 docs/design/
+
+# Generated Python release evidence
+python/pycunls/_build_provenance.json
+pypi-manifest.json
+pypi-release-uuids.txt
+pypi-status-*.json
+pypi-wheels/
+pypi-publication-record*/

.pre-commit-config.yaml

@@ -5,9 +5,23 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+      - id: check-shebang-scripts-are-executable
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v15.0.7
     hooks:
       - id: clang-format
         types_or: [c++, cuda]
+
+  - repo: local
+    hooks:
+      - id: actionlint
+        name: Validate GitHub Actions workflows
+        entry: actionlint -shellcheck=
+        language: system
+        files: ^\.github/workflows/.*\.ya?ml$
+      - id: shellcheck-publishing
+        name: Validate Python release scripts
+        entry: shellcheck
+        language: system
+        files: ^scripts/((build|test)_pycunls(_in_docker)?|publish_pypi|release_pycunls)\.sh$

README.md

@@ -119,6 +119,13 @@ a container with the source mounted read-only. Intermediate build directories
 live inside the container and are discarded; only the final `.whl` file is
 written to the host output directory.
 
+Docker wheels use the distribution name `pycunls-cu12` or `pycunls-cu13` and the
+matching CuPy dependency; both use `import pycunls`. With `RELEASE_TAG=v<version>`,
+the Docker build uses the committed checkout, excluding local changes and generated
+files. The tag version must match `python/pyproject.toml` and
+`python/pycunls/__init__.py`. Wheels include the source commit and measured CUDA
+toolkit version and are repaired with auditwheel while keeping CUDA libraries external.
+
 #### Build the wheel locally
 
 ```bash
@@ -130,7 +137,7 @@ pip wheel . --no-build-isolation --no-deps --wheel-dir ../dist
 #### Install the wheel
 
 ```bash
-pip install ./dist/pycunls-*.whl
+pip install ./dist/pycunls*.whl
 ```
 
 #### Editable install for development

python/CMakeLists.txt

@@ -43,4 +43,5 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pycunls/ DESTINATION pycunls
   FILES_MATCHING
     PATTERN "*.py"
     PATTERN "*.pyi"
-    PATTERN "py.typed")
+    PATTERN "py.typed"
+    PATTERN "_build_provenance.json")

scripts/Dockerfile

@@ -29,8 +29,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Python venv with build tools (for pycunls wheel builds)
 RUN python3 -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
+COPY scripts/requirements-build.txt /tmp/cunls-build-requirements.txt
 RUN pip install --upgrade pip && \
-    pip install scikit-build-core nanobind build wheel pre-commit
+    pip install -r /tmp/cunls-build-requirements.txt
 
 # Sphinx docs dependencies
 COPY docs/sphinx/requirements.txt /tmp/cunls-docs-requirements.txt

scripts/build_pycunls.sh

@@ -1,28 +1,114 @@
 #!/bin/bash
-# Runs inside the Docker container to build the pycunls wheel.
-# Expected mounts:
-#   /cunls                    (read-only)  — source tree
-#   OUTPUT_DIR env or /output (read-write) — wheel output directory
+# Runs inside the existing CUDA Docker image; the checkout is read-only.
 set -euo pipefail
 
-WHEEL_DIR=/tmp/pycunls_wheel
-mkdir -p "$WHEEL_DIR"
+: "${RELEASE_COMMIT:?Pass the original checkout SHA through build_pycunls_in_docker.sh}"
+ACTUAL_CUDA_VERSION=$(python3 - <<'PY'
+import json
+import os
+import re
+import subprocess
+from pathlib import Path
 
-cp -r /cunls /tmp/cunls_src
-rm -rf /tmp/cunls_src/python/build /tmp/cunls_src/build_python
+cuda = Path('/usr/local/cuda')
+match = re.search(r'release (\d+\.\d+)', subprocess.check_output([str(cuda / 'bin/nvcc'), '--version'], text=True))
+if not match:
+    raise SystemExit('Cannot identify the installed CUDA toolkit; use a CUDA development image')
+path = cuda / 'version.json'
+actual = json.loads(path.read_text())['cuda']['version'] if path.exists() else match[1]
+if actual.split('.')[:2] != match[1].split('.'):
+    raise SystemExit(f'CUDA version.json {actual} disagrees with nvcc {match[1]}')
+requested = os.environ.get('CUDA_VERSION')
+# nvcc exposes major.minor when version.json has no toolkit patch version.
+if requested and not (actual == requested or actual.startswith(requested + '.') or requested.startswith(actual + '.')):
+    raise SystemExit(f'Requested CUDA {requested}, but the build image contains {actual}; correct CUDA_VERSION or BASE_IMAGE')
+print(actual)
+PY
+)
+export CUDA_VERSION=$ACTUAL_CUDA_VERSION
+export RELEASE_COMMIT
+export RELEASE_TAG=${RELEASE_TAG:-}
+
+WORK_DIR=$(mktemp -d)
+trap 'rm -rf "$WORK_DIR"' EXIT
+mkdir -p "$WORK_DIR/raw" "$WORK_DIR/repaired"
+cp -r /cunls "$WORK_DIR/source"
+rm -rf "$WORK_DIR/source/python/build" "$WORK_DIR/source/build_python"
+cd "$WORK_DIR/source/python"
+
+python3 - <<'PY'
+import json
+import os
+import re
+from pathlib import Path
+from packaging.version import Version
+
+cuda = os.environ['CUDA_VERSION']
+major = cuda.split('.')[0]
+if major not in ('12', '13'):
+    raise SystemExit(f'Unsupported CUDA version: {cuda}; expected CUDA 12 or 13')
+sha = os.environ['RELEASE_COMMIT']
+if not re.fullmatch(r'[0-9a-f]{40}', sha):
+    raise SystemExit('RELEASE_COMMIT must be the full original checkout SHA')
+path = Path('pyproject.toml')
+text = path.read_text()
+text, count = re.subn(r'^name = "pycunls"$', f'name = "pycunls-cu{major}"', text, flags=re.M)
+if count != 1:
+    raise SystemExit('Expected one pycunls project name in pyproject.toml')
+text = text.replace('cupy-cuda12x', f'cupy-cuda{major}x')
+tag = os.environ['RELEASE_TAG']
+if tag:
+    if not tag.startswith('v'):
+        raise SystemExit('Release tag must start with v')
+    version = Version(tag[1:])
+    if version.local or str(version) != tag[1:]:
+        raise SystemExit('Release tag must contain a normalized public PEP 440 version')
+    current = re.findall(r'^version = "([^"]+)"$', text, flags=re.M)
+    if current != [str(version)]:
+        raise SystemExit(f'Package version {current} differs from {tag}; update version sources before tagging')
+    init_version = re.findall(r'^__version__ = "([^"]+)"$', Path('pycunls/__init__.py').read_text(), flags=re.M)
+    if init_version != [str(version)]:
+        raise SystemExit(f'Import version {init_version} differs from {tag}; update version sources before tagging')
+path.write_text(text)
+Path('pycunls/_build_provenance.json').write_text(json.dumps({
+    'git_sha': sha, 'cuda_version': cuda,
+}, sort_keys=True) + '\n')
+PY
 
-cd /tmp/cunls_src/python
-# Without Ninja, scikit-build-core drives Makefiles serially unless a parallel
-# level is set: build with every core.
 export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)}
-pip wheel . --no-build-isolation --no-deps --wheel-dir "$WHEEL_DIR"
+pip wheel . --no-build-isolation --no-deps --wheel-dir "$WORK_DIR/raw"
+
+# The system CUDA toolkit supplies these linked runtime libraries. Match SONAME
+# families rather than assuming CUDA 12 and 13 use the same SONAME versions.
+PLATFORM=$(python3 -c 'import os, platform; print("manylinux_" + os.confstr("CS_GNU_LIBC_VERSION").split()[1].replace(".", "_") + "_" + platform.machine())')
+for wheel in "$WORK_DIR/raw"/*.whl; do
+  auditwheel repair "$wheel" --plat "$PLATFORM" --wheel-dir "$WORK_DIR/repaired" \
+    --exclude 'libcudart.so.*' --exclude 'libcusparse.so.*' \
+    --exclude 'libcublas.so.*' --exclude 'libcublasLt.so.*' \
+    --exclude 'libcusolver.so.*' --exclude 'libcuda.so.*'
+done
+
+python3 - "$WORK_DIR/repaired" <<'PY'
+import json
+import os
+import re
+import sys
+import zipfile
+from pathlib import Path
 
-echo ""
-echo "========================================="
-echo "  Wheel built successfully"
-echo "========================================="
-ls -lh "$WHEEL_DIR"/*.whl
+wheels = list(Path(sys.argv[1]).glob('*.whl'))
+if len(wheels) != 1:
+    raise SystemExit(f'Expected one repaired wheel, found {len(wheels)}')
+with zipfile.ZipFile(wheels[0]) as archive:
+    provenance = json.loads(archive.read('pycunls/_build_provenance.json'))
+    if provenance != {'git_sha': os.environ['RELEASE_COMMIT'], 'cuda_version': os.environ['CUDA_VERSION']}:
+        raise SystemExit('Repaired wheel lost the original build provenance')
+    for member in archive.namelist():
+        if re.search(r'(^|/)lib(cudart|cusparse|cublas|cusolver|cudss|cuda)[^/]*\.so', member):
+            raise SystemExit(f'CUDA runtime library must remain external: {member}')
+PY
 
 OUTPUT=${OUTPUT_DIR:-/output}
-cp "$WHEEL_DIR"/*.whl "$OUTPUT"/
-echo "Wheel copied to $OUTPUT/"
+mkdir -p "$OUTPUT"
+cp "$WORK_DIR/repaired"/*.whl "$OUTPUT/"
+ls -lh "$OUTPUT"/*.whl