From c8ee11d9d721dc17548c3d8b0958e21823621c6f Mon Sep 17 00:00:00 2001 From: Simon Scatton Date: Thu, 16 Jul 2026 18:14:44 +0200 Subject: [PATCH] fix(python): remove CLI from wheel The Maturin-based wheel packaging was a historical remnant from when the local gateway launch path and OpenShell CLI were coupled in one binary. The gateway and CLI now ship as standalone artifacts, so the Python distribution should contain only the SDK. Build a single platform-independent setuptools wheel, verify that it cannot contain native code or an openshell entry point, and simplify the release jobs and documentation for SDK-only PyPI installs. Signed-off-by: Simon Scatton --- .github/workflows/release-dev.yml | 119 +++-------- .github/workflows/release-tag.yml | 120 +++--------- README.md | 8 +- THIRD-PARTY-NOTICES | 7 - architecture/build.md | 21 +- deploy/docker/Dockerfile.python-wheels | 115 ----------- deploy/docker/Dockerfile.python-wheels-macos | 126 ------------ docs/about/installation.mdx | 6 + docs/get-started/quickstart.mdx | 6 - docs/get-started/tutorials/docker-compose.mdx | 8 +- flake.nix | 1 + pyproject.toml | 36 ++-- python/wheel_verifier_test.py | 143 ++++++++++++++ tasks/python.toml | 184 ++---------------- tasks/scripts/verify-python-wheel.py | 151 ++++++++++++++ tasks/test.toml | 4 +- uv.lock | 40 +--- 17 files changed, 398 insertions(+), 697 deletions(-) delete mode 100644 deploy/docker/Dockerfile.python-wheels delete mode 100644 deploy/docker/Dockerfile.python-wheels-macos create mode 100644 python/wheel_verifier_test.py create mode 100644 tasks/scripts/verify-python-wheel.py diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 222ba7a8f0..750995dd9a 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -95,91 +95,24 @@ jobs: "${REGISTRY}/${component}:${{ github.sha }}" done - build-python-wheels-linux: - name: Build Python Wheels (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - artifact: linux-amd64 - task: python:build:linux:amd64 - output_path: target/wheels/linux-amd64/*.whl - - arch: arm64 - runner: linux-arm64-cpu8 - artifact: linux-arm64 - task: python:build:linux:arm64 - output_path: target/wheels/linux-arm64/*.whl - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: dev - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Sync Python dependencies - run: uv sync - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - with: - shared-key: python-wheel-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Build Python wheels - run: | - set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }} - ls -la ${{ matrix.output_path }} - - - name: Upload wheel artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: python-wheels-${{ matrix.artifact }} - path: ${{ matrix.output_path }} - retention-days: 5 - - build-python-wheel-macos: - name: Build Python Wheel (macOS) + build-python-wheel: + name: Build Python Wheel needs: [compute-versions] runs-on: linux-amd64-cpu8 - timeout-minutes: 120 + timeout-minutes: 20 container: image: ghcr.io/nvidia/openshell/ci:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: dev + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL: ${{ needs.compute-versions.outputs.python_version }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -189,13 +122,13 @@ jobs: - name: Build Python wheel run: | set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos + mise run python:build ls -la target/wheels/*.whl - - name: Upload wheel artifacts + - name: Upload wheel artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: python-wheels-macos + name: python-wheel path: target/wheels/*.whl retention-days: 5 @@ -679,7 +612,7 @@ jobs: smoke-linux-dev-artifacts: name: Smoke Linux Dev Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheel] timeout-minutes: 20 strategy: fail-fast: false @@ -709,18 +642,10 @@ jobs: kind: rpm artifact_arch: arm64 rpm_arch: aarch64 - - name: python-wheel-amd64 + - name: python-wheel runner: linux-amd64-cpu8 image: python:3.12-slim kind: wheel - artifact_arch: amd64 - rpm_arch: x86_64 - - name: python-wheel-arm64 - runner: linux-arm64-cpu8 - image: python:3.12-slim - kind: wheel - artifact_arch: arm64 - rpm_arch: aarch64 runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -759,27 +684,29 @@ jobs: if: matrix.kind == 'wheel' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-wheels-linux-${{ matrix.artifact_arch }} + name: python-wheel path: wheel-input/ - name: Smoke Python wheel if: matrix.kind == 'wheel' run: | set -euo pipefail - pip install --no-cache-dir wheel-input/*.whl - python - <<'PY' + python -m venv /tmp/openshell-wheel-smoke + /tmp/openshell-wheel-smoke/bin/pip install --no-cache-dir wheel-input/*.whl + /tmp/openshell-wheel-smoke/bin/python - <<'PY' import importlib for name in ("openshell", "openshell._proto", "openshell.sandbox"): importlib.import_module(name) print(name, "OK") PY + test ! -e /tmp/openshell-wheel-smoke/bin/openshell # --------------------------------------------------------------------------- # Create / update the dev GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- release-dev: name: Release Dev - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-dev-artifacts] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheel, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-dev-artifacts] runs-on: linux-amd64-cpu8 timeout-minutes: 10 permissions: @@ -820,12 +747,11 @@ jobs: path: release/ merge-multiple: true - - name: Download wheel artifacts + - name: Download wheel artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: python-wheels-* + name: python-wheel path: release/ - merge-multiple: true - name: Download Debian package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -877,9 +803,14 @@ jobs: id: wheel_filenames run: | set -euo pipefail - ls -la release/*.whl - WHEEL_FILENAMES=$(ls release/*.whl | xargs -n1 basename | sort | paste -sd, -) - echo "wheel_filenames=${WHEEL_FILENAMES}" >> "$GITHUB_OUTPUT" + shopt -s nullglob + wheels=(release/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "expected exactly one Python wheel, found ${#wheels[@]}" >&2 + exit 1 + fi + wheel_filename=$(basename "${wheels[0]}") + echo "wheel_filenames=${wheel_filename}" >> "$GITHUB_OUTPUT" - name: Generate checksums run: | diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index ec95ab5ea1..a32661b9a6 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -124,93 +124,25 @@ jobs: "${REGISTRY}/${component}:${SOURCE_TAG}" done - build-python-wheels-linux: - name: Build Python Wheels (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - artifact: linux-amd64 - task: python:build:linux:amd64 - output_path: target/wheels/linux-amd64/*.whl - - arch: arm64 - runner: linux-arm64-cpu8 - artifact: linux-arm64 - task: python:build:linux:arm64 - output_path: target/wheels/linux-arm64/*.whl - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Sync Python dependencies - run: uv sync - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - with: - shared-key: python-wheel-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Build Python wheels - run: | - set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }} - ls -la ${{ matrix.output_path }} - - - name: Upload wheel artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: python-wheels-${{ matrix.artifact }} - path: ${{ matrix.output_path }} - retention-days: 5 - - build-python-wheel-macos: - name: Build Python Wheel (macOS) + build-python-wheel: + name: Build Python Wheel needs: [compute-versions] runs-on: linux-amd64-cpu8 - timeout-minutes: 120 + timeout-minutes: 20 container: image: ghcr.io/nvidia/openshell/ci:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL: ${{ needs.compute-versions.outputs.python_version }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.tag || github.ref }} fetch-depth: 0 - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -220,13 +152,13 @@ jobs: - name: Build Python wheel run: | set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos + mise run python:build ls -la target/wheels/*.whl - - name: Upload wheel artifacts + - name: Upload wheel artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: python-wheels-macos + name: python-wheel path: target/wheels/*.whl retention-days: 5 @@ -715,7 +647,7 @@ jobs: smoke-linux-release-artifacts: name: Smoke Linux Release Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheel] timeout-minutes: 20 strategy: fail-fast: false @@ -767,16 +699,6 @@ jobs: runner: linux-amd64-cpu8 image: python:3.12-slim kind: wheel - artifact_arch: amd64 - rpm_arch: x86_64 - target: x86_64-unknown-linux-gnu - - name: python-wheel-arm64 - runner: linux-arm64-cpu8 - image: python:3.12-slim - kind: wheel - artifact_arch: arm64 - rpm_arch: aarch64 - target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -840,27 +762,29 @@ jobs: if: matrix.kind == 'wheel' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-wheels-linux-${{ matrix.artifact_arch }} + name: python-wheel path: wheel-input/ - name: Smoke Python wheel if: matrix.kind == 'wheel' run: | set -euo pipefail - pip install --no-cache-dir wheel-input/*.whl - python - <<'PY' + python -m venv /tmp/openshell-wheel-smoke + /tmp/openshell-wheel-smoke/bin/pip install --no-cache-dir wheel-input/*.whl + /tmp/openshell-wheel-smoke/bin/python - <<'PY' import importlib for name in ("openshell", "openshell._proto", "openshell.sandbox"): importlib.import_module(name) print(name, "OK") PY + test ! -e /tmp/openshell-wheel-smoke/bin/openshell # --------------------------------------------------------------------------- # Create a tagged GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- release: name: Release - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-release-artifacts] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheel, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-release-artifacts] runs-on: linux-amd64-cpu8 timeout-minutes: 10 permissions: @@ -903,12 +827,11 @@ jobs: path: release/ merge-multiple: true - - name: Download wheel artifacts + - name: Download wheel artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: python-wheels-* + name: python-wheel path: release/ - merge-multiple: true - name: Download Debian package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -935,9 +858,14 @@ jobs: id: wheel_filenames run: | set -euo pipefail - ls -la release/*.whl - WHEEL_FILENAMES=$(ls release/*.whl | xargs -n1 basename | sort | paste -sd, -) - echo "wheel_filenames=${WHEEL_FILENAMES}" >> "$GITHUB_OUTPUT" + shopt -s nullglob + wheels=(release/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "expected exactly one Python wheel, found ${#wheels[@]}" >&2 + exit 1 + fi + wheel_filename=$(basename "${wheels[0]}") + echo "wheel_filenames=${wheel_filename}" >> "$GITHUB_OUTPUT" - name: Generate checksums run: | diff --git a/README.md b/README.md index 33b2736bc2..8e5957aafa 100644 --- a/README.md +++ b/README.md @@ -27,14 +27,14 @@ OpenShell is built agent-first. The project ships with agent skills for everythi curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -**From PyPI (requires [uv](https://docs.astral.sh/uv/)):** +The installer installs the latest stable release by default. To install a specific version, set `OPENSHELL_VERSION`. A [`dev` release](https://github.com/NVIDIA/OpenShell/releases/tag/dev) is also available that tracks the latest commit on `main`. + +The `openshell` package on PyPI provides the Python SDK only. It does not install the `openshell` CLI. Add the SDK to a Python project with [uv](https://docs.astral.sh/uv/): ```bash -uv tool install -U openshell +uv add openshell ``` -Both methods install the latest stable release by default. To install a specific version, set `OPENSHELL_VERSION` (binary) or pin the version with `uv tool install openshell==`. A [`dev` release](https://github.com/NVIDIA/OpenShell/releases/tag/dev) is also available that tracks the latest commit on `main`. - **Helm chart:** > **Experimental** — the Kubernetes deployment path is under active development. Expect rough edges and breaking changes. diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES index 41eda0e0a8..8125e5692f 100644 --- a/THIRD-PARTY-NOTICES +++ b/THIRD-PARTY-NOTICES @@ -14813,13 +14813,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -================================================================================ -maturin 1.11.5 -License: MIT OR Apache-2.0 --------------------------------------------------------------------------------- - -UNKNOWN - ================================================================================ packaging 26.0 License: Apache-2.0 OR BSD-2-Clause diff --git a/architecture/build.md b/architecture/build.md index e3ac5012e3..8571bfe0d4 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -11,7 +11,8 @@ OpenShell builds these main artifacts: | Artifact | Source | |---|---| | Gateway binary | `crates/openshell-server` | -| CLI package and Python SDK | `python/openshell` plus Rust binaries where packaged | +| CLI binaries and system packages | `crates/openshell-cli` plus release packaging | +| Python SDK wheel | `python/openshell` | | Gateway container image | `deploy/docker/Dockerfile.gateway` | | Supervisor container image | `deploy/docker/Dockerfile.supervisor` | | Helm chart | `deploy/helm/openshell` | @@ -133,12 +134,18 @@ do not infer from kube context. ## Python Wheel Packaging The generated protobuf/gRPC stubs under `python/openshell/_proto/` are gitignored -build outputs of `mise run python:proto`. maturin honors `.gitignore` when -collecting `python-source` files, so native builds (Linux CI, local -`pip install .`) would drop them and ship an unimportable wheel. `pyproject.toml` -pins them back in with `[tool.maturin].include` globs. The release workflows -install each Linux wheel in a clean image and import `openshell.sandbox` as a -smoke check. +build outputs of `mise run python:proto`. Setuptools includes them through the +package-data configuration in `pyproject.toml`. Release workflows build the +wheel directly and do not produce a source distribution. Setuptools SCM derives +local versions from Git and accepts the release workflow's computed version +through its distribution-specific override. + +The build produces one platform-independent `py3-none-any` wheel. A verifier +checks its tag, metadata, version, required package files, and the absence of +native files or an `openshell` executable entry point. Release workflows build +the wheel once, install it in a clean virtual environment, import the public +package modules, and confirm that installation did not create an `openshell` +command. ## CI and E2E diff --git a/deploy/docker/Dockerfile.python-wheels b/deploy/docker/Dockerfile.python-wheels deleted file mode 100644 index e93bf8f22e..0000000000 --- a/deploy/docker/Dockerfile.python-wheels +++ /dev/null @@ -1,115 +0,0 @@ -# syntax=docker/dockerfile:1.6 - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - - -ARG PYTHON_VERSION=3.12 - -FROM --platform=$BUILDPLATFORM python:${PYTHON_VERSION}-slim AS base - -ENV PATH="/root/.cargo/bin:${PATH}" - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - curl \ - gcc \ - libc6-dev \ - libclang-dev \ - pkg-config \ - libssl-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 -RUN pip install --no-cache-dir maturin - -COPY deploy/docker/cross-build.sh /usr/local/bin/ - -FROM base AS builder - -ARG TARGETARCH -ARG BUILDARCH -ARG CARGO_TARGET_CACHE_SCOPE=default - -ARG SCCACHE_MEMCACHED_ENDPOINT - -WORKDIR /build - -RUN . cross-build.sh && install_cross_toolchain && install_sccache && add_rust_target - -# Copy dependency manifests first for better caching. -COPY Cargo.toml Cargo.lock ./ -COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml -COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml -COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml -COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml -COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml -COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml -COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml -COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml -COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml -COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml -COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml -COPY crates/openshell-core/build.rs crates/openshell-core/build.rs -COPY proto/ proto/ - -# Create dummy source files to build dependencies. -RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src crates/openshell-bootstrap/src crates/openshell-tui/src && \ - echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ - echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ - touch crates/openshell-core/src/lib.rs && \ - touch crates/openshell-ocsf/src/lib.rs && \ - touch crates/openshell-providers/src/lib.rs && \ - touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-bootstrap/src/lib.rs && \ - touch crates/openshell-policy/src/lib.rs && \ - touch crates/openshell-prover/src/lib.rs && \ - touch crates/openshell-tui/src/lib.rs - -# Build dependencies only (cached unless Cargo.toml/lock changes). -# sccache uses memcached in CI or the local disk cache mount for local dev. -# The cargo-target mount gives cargo a persistent target/ dir for incremental rebuilds. -RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \ - . cross-build.sh && cargo_cross_build --release -p openshell-cli --features bundled-z3 2>/dev/null || true - -# Copy actual source code and Python packaging files. -COPY crates/ crates/ -COPY pyproject.toml README.md ./ -COPY python/ python/ - -# Touch source files to ensure they're rebuilt (not the cached dummy). -RUN touch crates/openshell-cli/src/main.rs \ - crates/openshell-cli/src/lib.rs \ - crates/openshell-bootstrap/src/lib.rs \ - crates/openshell-core/src/lib.rs \ - crates/openshell-providers/src/lib.rs \ - crates/openshell-router/src/lib.rs \ - crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ - crates/openshell-core/build.rs \ - proto/*.proto - -# Declare version ARGs here (not earlier) so the git-hash-bearing values do not -# invalidate the expensive dependency-build layers above on every commit. -ARG OPENSHELL_CARGO_VERSION -ARG OPENSHELL_IMAGE_TAG -RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \ - . cross-build.sh && \ - export_cross_env && \ - export RUSTC_WRAPPER=sccache && \ - export CARGO_BUILD_TARGET="$(rust_target)" && \ - if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ - fi && \ - maturin build --release --target "${CARGO_BUILD_TARGET}" --features bundled-z3 --out /wheels - -FROM scratch AS wheels -COPY --from=builder /wheels/*.whl / diff --git a/deploy/docker/Dockerfile.python-wheels-macos b/deploy/docker/Dockerfile.python-wheels-macos deleted file mode 100644 index 45194b0690..0000000000 --- a/deploy/docker/Dockerfile.python-wheels-macos +++ /dev/null @@ -1,126 +0,0 @@ -# syntax=docker/dockerfile:1.6 - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - - -ARG OSXCROSS_IMAGE=ghcr.io/crazy-max/osxcross:latest -ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python -ARG PYTHON_VERSION=3.12 - -FROM ${OSXCROSS_IMAGE} AS osxcross - -FROM ${PYTHON_IMAGE}:${PYTHON_VERSION}-slim AS builder - -ARG TARGETARCH -ARG CARGO_TARGET_CACHE_SCOPE=default - -ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}" -ENV LD_LIBRARY_PATH="/osxcross/lib" - -COPY --from=osxcross /osxcross /osxcross - -RUN SDKROOT="$(echo /osxcross/SDK/MacOSX*.sdk)" && ln -sfn "${SDKROOT}" /osxcross/SDK/MacOSX.sdk - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - clang \ - cmake \ - curl \ - libclang-dev \ - libssl-dev \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* - -# aws-lc-sys probes with --target=arm64-apple-macosx and clang then looks for -# arm64-apple-macosx-ld. Provide a linker alias to osxcross ld64. -RUN ln -sf /osxcross/bin/arm64-apple-darwin25.1-ld /usr/local/bin/arm64-apple-macosx-ld - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 -RUN rustup target add aarch64-apple-darwin -RUN pip install --no-cache-dir maturin - -WORKDIR /build - -ENV CC_aarch64_apple_darwin=oa64-clang -ENV CXX_aarch64_apple_darwin=oa64-clang++ -ENV AR_aarch64_apple_darwin=aarch64-apple-darwin25.1-ar -ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=oa64-clang -ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_AR=aarch64-apple-darwin25.1-ar -ENV SDKROOT=/osxcross/SDK/MacOSX.sdk -ENV MACOSX_DEPLOYMENT_TARGET=13.3 -ENV CFLAGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 -ENV CXXFLAGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 -ENV BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -isysroot\ ${SDKROOT} - -# Copy dependency manifests first for better caching. -COPY Cargo.toml Cargo.lock ./ -COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml -COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml -COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml -COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml -COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml -COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml -COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml -COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml -COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml -COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml -COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml -COPY crates/openshell-core/build.rs crates/openshell-core/build.rs -COPY proto/ proto/ - -# Create dummy source files to build dependencies. -RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src crates/openshell-bootstrap/src crates/openshell-tui/src && \ - echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ - echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ - touch crates/openshell-core/src/lib.rs && \ - touch crates/openshell-ocsf/src/lib.rs && \ - touch crates/openshell-providers/src/lib.rs && \ - touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-bootstrap/src/lib.rs && \ - touch crates/openshell-policy/src/lib.rs && \ - touch crates/openshell-prover/src/lib.rs && \ - touch crates/openshell-tui/src/lib.rs - -# Build dependencies only (cached unless Cargo.toml/lock changes). -RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-cli --features bundled-z3 2>/dev/null || true - -# Copy actual source code and Python packaging files. -COPY crates/ crates/ -COPY providers/ providers/ -COPY pyproject.toml README.md ./ -COPY python/ python/ - -# Touch source files to ensure they're rebuilt (not the cached dummy). -# Touch build.rs and proto files to force proto code regeneration when the -# cargo target cache mount retains stale OUT_DIR artifacts from prior builds. -RUN touch crates/openshell-cli/src/main.rs \ - crates/openshell-cli/src/lib.rs \ - crates/openshell-bootstrap/src/lib.rs \ - crates/openshell-core/src/lib.rs \ - crates/openshell-providers/src/lib.rs \ - crates/openshell-router/src/lib.rs \ - crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ - crates/openshell-core/build.rs \ - proto/*.proto - -# Declare version ARGs here (not earlier) so the git-hash-bearing values do not -# invalidate the expensive dependency-build layers above on every commit. -ARG OPENSHELL_CARGO_VERSION -ARG OPENSHELL_IMAGE_TAG -RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ - fi && \ - maturin build --release --target aarch64-apple-darwin --features bundled-z3 --out /wheels - -FROM scratch AS wheels -COPY --from=builder /wheels/*.whl / diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 2ac077e7b9..c9de188dec 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -20,6 +20,12 @@ The script detects your operating system and installs the OpenShell CLI and gate You can also download release artifacts directly from the [OpenShell GitHub Releases](https://github.com/NVIDIA/OpenShell/releases) page. +The `openshell` package on PyPI provides the Python SDK only. It does not install the `openshell` CLI. Add the SDK to a Python project with: + +```shell +uv add openshell +``` + Use `openshell status` to confirm the CLI can reach the gateway. ## Supported Compute Drivers diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index ebd7a9880c..8a8b82046e 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -30,12 +30,6 @@ curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | The install script uses Homebrew, RPM, or a Debian package based on your machine. It starts the local gateway server after installation. -If you prefer [uv](https://docs.astral.sh/uv/): - -```shell -uv tool install -U openshell -``` - After installing the CLI, run `openshell --help` in your terminal to view the full CLI reference. diff --git a/docs/get-started/tutorials/docker-compose.mdx b/docs/get-started/tutorials/docker-compose.mdx index 645b5ddc98..59536141f6 100644 --- a/docs/get-started/tutorials/docker-compose.mdx +++ b/docs/get-started/tutorials/docker-compose.mdx @@ -72,13 +72,9 @@ curl -sf http://localhost:8080/healthz curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -**From PyPI (any platform with [uv](https://docs.astral.sh/uv/)):** - -```shell -uv tool install -U openshell -``` - +The `openshell` package on PyPI provides the Python SDK only and does not install the CLI. + On Windows without WSL, install the CLI inside a WSL 2 distribution (for example AlmaLinux or Ubuntu) and run all `openshell` commands from that distribution. diff --git a/flake.nix b/flake.nix index 13c4857bc6..cf7ccebf3b 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,7 @@ llvmPackages.libclang # system dependency for openshell-prover z3 + uv ]; env = { diff --git a/pyproject.toml b/pyproject.toml index fb8aac17a2..247a40b412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [build-system] -requires = ["maturin>=1.5,<2.0"] -build-backend = "maturin" +requires = ["setuptools>=80", "setuptools-scm[simple]>=8"] +build-backend = "setuptools.build_meta" [project] name = "openshell" @@ -29,8 +29,6 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", - "Programming Language :: Rust", - "License :: OSI Approved :: Apache Software License", "Topic :: Security", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -51,33 +49,23 @@ dev = [ "pytest-xdist>=3.0", "ruff>=0.4", "ty>=0.0.1a6", - "maturin>=1.5,<2.0", - "setuptools-scm>=8", "grpcio-tools>=1.60", "pyelftools>=0.30", "pyyaml==6.0.2", "types-pyyaml>=6.0.12.20260518", ] -[tool.uv] -# Don't try to install the root package with uv sync - use uv pip install . instead -# This is needed because maturin bin packages don't install the binary with editable installs -# (both `uv pip install -e .` and `maturin develop` skip binary installation) -# Use `uv pip install .` (without -e) to install the openshell CLI binary -package = false +[tool.setuptools] +package-dir = { "" = "python" } +include-package-data = false -[tool.maturin] -bindings = "bin" -manifest-path = "crates/openshell-cli/Cargo.toml" -python-source = "python" -module-name = "openshell" -# _proto/ stubs are gitignored; force them into the wheel so git-aware -# maturin builds don't drop them (package-relative paths). -include = [ - "openshell/_proto/*_pb2.py", - "openshell/_proto/*_pb2_grpc.py", - "openshell/_proto/*.pyi", -] +[tool.setuptools.packages.find] +where = ["python"] +include = ["openshell*"] +namespaces = false + +[tool.setuptools.package-data] +openshell = ["py.typed", "_proto/*.py", "_proto/*.pyi"] [tool.ruff] target-version = "py311" diff --git a/python/wheel_verifier_test.py b/python/wheel_verifier_test.py new file mode 100644 index 0000000000..777f348f73 --- /dev/null +++ b/python/wheel_verifier_test.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from zipfile import ZipFile + +import pytest + + +def _load_verifier_module(): + path = Path(__file__).resolve().parents[1] / "tasks/scripts/verify-python-wheel.py" + spec = importlib.util.spec_from_file_location("openshell_wheel_verifier", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +verifier = _load_verifier_module() + + +def _wheel_files() -> set[str]: + files = { + "openshell/__init__.py", + "openshell/sandbox.py", + "openshell/py.typed", + "openshell/_proto/__init__.py", + } + for stem in ("datamodel", "inference", "openshell", "options", "sandbox"): + files.add(f"openshell/_proto/{stem}_pb2.py") + files.add(f"openshell/_proto/{stem}_pb2.pyi") + files.add(f"openshell/_proto/{stem}_pb2_grpc.py") + return files + + +def _write_wheel( + directory: Path, + *, + version: str = "1.2.3", + tag: str = "py3-none-any", + files: set[str] | None = None, + entry_points: str | None = None, +) -> Path: + wheel = directory / f"openshell-{version}-{tag}.whl" + dist_info = f"openshell-{version}.dist-info" + with ZipFile(wheel, "w") as archive: + for name in files if files is not None else _wheel_files(): + archive.writestr(name, "") + archive.writestr( + f"{dist_info}/METADATA", + f"Metadata-Version: 2.4\nName: openshell\nVersion: {version}\n", + ) + archive.writestr( + f"{dist_info}/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {tag}\n", + ) + if entry_points is not None: + archive.writestr(f"{dist_info}/entry_points.txt", entry_points) + return wheel + + +def test_accepts_pure_python_sdk_wheel(tmp_path: Path) -> None: + wheel = _write_wheel(tmp_path) + + assert verifier.verify_wheel(wheel, "1.2.3") == wheel + + +def test_requires_exactly_one_wheel_in_directory(tmp_path: Path) -> None: + _write_wheel(tmp_path, version="1.2.3") + _write_wheel(tmp_path, version="1.2.4") + + with pytest.raises(ValueError, match="expected exactly one wheel"): + verifier.verify_wheel(tmp_path) + + +def test_rejects_missing_generated_proto(tmp_path: Path) -> None: + files = _wheel_files() - {"openshell/_proto/sandbox_pb2.py"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match=r"sandbox_pb2\.py"): + verifier.verify_wheel(wheel) + + +def test_rejects_bundled_cli(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell-1.2.3.data/scripts/openshell"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains an openshell executable"): + verifier.verify_wheel(wheel) + + +def test_rejects_bundled_windows_cli(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell-1.2.3.data/scripts/openshell.exe"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains an openshell executable"): + verifier.verify_wheel(wheel) + + +def test_rejects_openshell_console_script(tmp_path: Path) -> None: + wheel = _write_wheel( + tmp_path, + entry_points="[console_scripts]\nopenshell = openshell.cli:main\n", + ) + + with pytest.raises(ValueError, match="defines an openshell console script"): + verifier.verify_wheel(wheel) + + +def test_rejects_native_extension(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell/_native.so"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains native files"): + verifier.verify_wheel(wheel) + + +def test_rejects_bytecode(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell/__pycache__/sandbox.cpython-314.pyc"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains Python bytecode"): + verifier.verify_wheel(wheel) + + +def test_rejects_platform_wheel(tmp_path: Path) -> None: + wheel = _write_wheel(tmp_path, tag="cp311-cp311-manylinux_2_28_x86_64") + + with pytest.raises(ValueError, match="not platform-independent"): + verifier.verify_wheel(wheel) + + +def test_rejects_unexpected_version(tmp_path: Path) -> None: + wheel = _write_wheel(tmp_path) + + with pytest.raises(ValueError, match=r"expected version 1\.2\.4"): + verifier.verify_wheel(wheel, "1.2.4") diff --git a/tasks/python.toml b/tasks/python.toml index 62b8b4f787..6e4530de9e 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -4,193 +4,35 @@ # Python development, build, lint, and format tasks ["python:dev"] -description = "Install Python package in development mode (builds CLI binary)" +description = "Install the Python SDK in development mode" depends = ["python:proto"] -run = "uv sync --group dev && uv pip install ." +run = "uv sync --group dev" hide = true ["build:python:wheel"] -description = "Build Python wheel with CLI binary (native)" +description = "Build the pure-Python SDK wheel" depends = ["python:proto"] -run = "uv run maturin build --release" -hide = true - -["python:build"] -description = "Alias for build:python:wheel" -depends = ["build:python:wheel"] -hide = true - -["build:python:wheel:linux"] -description = "Build a Linux Python wheel natively on the current host" -depends = ["python:proto"] -run = ''' +run = """ #!/usr/bin/env bash set -euo pipefail -WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:?Set WHEEL_OUTPUT_DIR to a per-platform wheel output directory} -EXPECTED_HOST_ARCH=${EXPECTED_HOST_ARCH:?Set EXPECTED_HOST_ARCH to amd64 or arm64} -ORIGINAL_CARGO_TOML="" - -normalize_arch() { - case "$1" in - x86_64|amd64) echo amd64 ;; - aarch64|arm64) echo arm64 ;; - *) echo "$1" ;; - esac -} - -cleanup() { - if [ -n "$ORIGINAL_CARGO_TOML" ] && [ -f "$ORIGINAL_CARGO_TOML" ]; then - cp "$ORIGINAL_CARGO_TOML" Cargo.toml - rm -f "$ORIGINAL_CARGO_TOML" - fi -} - -trap cleanup EXIT - -if [ "$(uname -s)" != "Linux" ]; then - echo "build:python:wheel:linux requires a Linux host" >&2 - exit 1 -fi - -HOST_ARCH=$(normalize_arch "$(uname -m)") -EXPECTED_HOST_ARCH=$(normalize_arch "$EXPECTED_HOST_ARCH") -if [ "$HOST_ARCH" != "$EXPECTED_HOST_ARCH" ]; then - echo "build:python:wheel:linux expected ${EXPECTED_HOST_ARCH} host, got ${HOST_ARCH}" >&2 - exit 1 -fi - -# Avoid failures when RUSTC_WRAPPER is set to an unresolved `sccache` binary. -if command -v sccache >/dev/null 2>&1; then - export RUSTC_WRAPPER="$(command -v sccache)" -else - unset RUSTC_WRAPPER || true -fi - -if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then - ORIGINAL_CARGO_TOML=$(mktemp) - cp Cargo.toml "$ORIGINAL_CARGO_TOML" - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml -fi - +WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:-target/wheels} rm -rf "$WHEEL_OUTPUT_DIR" mkdir -p "$WHEEL_OUTPUT_DIR" -uv run maturin build --release --features bundled-z3 --out "$WHEEL_OUTPUT_DIR" - -ls -la "$WHEEL_OUTPUT_DIR"/*.whl -''' -hide = true - -["build:python:wheel:linux:amd64"] -description = "Build Python wheel for Linux amd64 natively" -depends = ["EXPECTED_HOST_ARCH=amd64 WHEEL_OUTPUT_DIR=target/wheels/linux-amd64 build:python:wheel:linux"] -hide = true - -["python:build:linux:amd64"] -description = "Alias for build:python:wheel:linux:amd64" -depends = ["build:python:wheel:linux:amd64"] -hide = true - -["build:python:wheel:linux:arm64"] -description = "Build Python wheel for Linux arm64 natively" -depends = ["EXPECTED_HOST_ARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux"] -hide = true - -["python:build:linux:arm64"] -description = "Alias for build:python:wheel:linux:arm64" -depends = ["build:python:wheel:linux:arm64"] -hide = true +uv build --wheel --out-dir "$WHEEL_OUTPUT_DIR" -["build:python:wheel:macos"] -description = "Build Python wheel for macOS arm64" -depends = ["python:proto"] -run = """ -#!/usr/bin/env bash -set -euo pipefail - -if [ "$(uname -s)" = "Darwin" ] && [ -z "${OPENSHELL_CARGO_VERSION:-}" ]; then - # Avoid failures when RUSTC_WRAPPER is set to an unresolved `sccache` binary. - if command -v sccache >/dev/null 2>&1; then - export RUSTC_WRAPPER="$(command -v sccache)" - else - unset RUSTC_WRAPPER || true - fi - - mkdir -p target/wheels - - rustup target add aarch64-apple-darwin - - uv run maturin build --release --target aarch64-apple-darwin --out target/wheels -else - echo "Building macOS wheel via Docker cross-toolchain" - mise run build:python:wheel:macos:docker -fi - -ls -la target/wheels/*.whl -""" -hide = true - -["python:build:macos"] -description = "Alias for build:python:wheel:macos" -depends = ["build:python:wheel:macos"] -hide = true - -["build:python:wheel:macos:docker"] -description = "Build Python wheel for macOS arm64 from Docker" -depends = ["python:proto"] -run = """ -#!/usr/bin/env bash -set -euo pipefail - -source tasks/scripts/container-engine.sh - -sha256_16() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}' - fi -} - -sha256_16_stdin() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 | awk '{print substr($1, 1, 16)}' - fi -} - -CARGO_VERSION=${OPENSHELL_CARGO_VERSION:-} -OSXCROSS_IMAGE_REF=${OSXCROSS_IMAGE:-crazymax/osxcross:latest} -if [ -z "$CARGO_VERSION" ] && [ -n "${CI:-}" ]; then - CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo) +VERIFY_ARGS=() +if [ -n "${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL:-}" ]; then + VERIFY_ARGS+=(--expected-version "$SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL") fi - -LOCK_HASH=$(sha256_16 Cargo.lock) -RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-1.95.0} -CACHE_SCOPE_INPUT="v1|python-wheels-macos|base|${LOCK_HASH}|${RUST_SCOPE}" -CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin) - -mkdir -p target/wheels - -ce build \ - -f deploy/docker/Dockerfile.python-wheels-macos \ - --target wheels \ - --build-arg "OSXCROSS_IMAGE=${OSXCROSS_IMAGE_REF}" \ - --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ - ${CARGO_VERSION:+--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}"} \ - ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} \ - --output type=local,dest=target/wheels \ - . - -ls -la target/wheels/*macosx*arm64.whl +uv run --no-project python tasks/scripts/verify-python-wheel.py "$WHEEL_OUTPUT_DIR" "${VERIFY_ARGS[@]}" """ hide = true -["python:build:macos:docker"] -description = "Alias for build:python:wheel:macos:docker" -depends = ["build:python:wheel:macos:docker"] +["python:build"] +description = "Alias for build:python:wheel" +depends = ["build:python:wheel"] hide = true ["python:lint"] diff --git a/tasks/scripts/verify-python-wheel.py b/tasks/scripts/verify-python-wheel.py new file mode 100644 index 0000000000..6234c0cdb6 --- /dev/null +++ b/tasks/scripts/verify-python-wheel.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import sys +from configparser import ConfigParser +from configparser import Error as ConfigParserError +from email.parser import Parser +from pathlib import Path +from zipfile import BadZipFile, ZipFile + +PROTO_STEMS = ("datamodel", "inference", "openshell", "options", "sandbox") +BYTECODE_SUFFIXES = (".pyc", ".pyo") +NATIVE_SUFFIXES = (".dll", ".dylib", ".exe", ".pyd", ".so") + + +def _find_wheel(path: Path) -> Path: + if path.is_file(): + return path + + wheels = sorted(path.glob("*.whl")) + if len(wheels) != 1: + raise ValueError(f"expected exactly one wheel in {path}, found {len(wheels)}") + return wheels[0] + + +def _required_files() -> set[str]: + files = { + "openshell/__init__.py", + "openshell/sandbox.py", + "openshell/py.typed", + "openshell/_proto/__init__.py", + } + for stem in PROTO_STEMS: + files.add(f"openshell/_proto/{stem}_pb2.py") + files.add(f"openshell/_proto/{stem}_pb2.pyi") + files.add(f"openshell/_proto/{stem}_pb2_grpc.py") + return files + + +def _single_member(names: set[str], suffix: str) -> str: + matches = [name for name in names if name.endswith(suffix)] + if len(matches) != 1: + raise ValueError(f"expected exactly one {suffix} file, found {len(matches)}") + return matches[0] + + +def verify_wheel(path: Path, expected_version: str | None = None) -> Path: + wheel = _find_wheel(path) + if not wheel.name.endswith("-py3-none-any.whl"): + raise ValueError(f"wheel is not platform-independent: {wheel.name}") + + with ZipFile(wheel) as archive: + names = set(archive.namelist()) + missing = sorted(_required_files() - names) + if missing: + raise ValueError(f"wheel is missing required files: {', '.join(missing)}") + + scripts = sorted( + name + for name in names + if not name.endswith("/") + and Path(name).name in {"openshell", "openshell.exe"} + ) + if scripts: + raise ValueError( + f"wheel contains an openshell executable: {', '.join(scripts)}" + ) + + entry_points = [ + name for name in names if name.endswith(".dist-info/entry_points.txt") + ] + if len(entry_points) > 1: + raise ValueError( + f"expected at most one entry_points.txt file, found {len(entry_points)}" + ) + if entry_points: + parser = ConfigParser(interpolation=None) + try: + parser.read_string(archive.read(entry_points[0]).decode()) + except (ConfigParserError, UnicodeDecodeError) as error: + raise ValueError( + "wheel contains invalid entry point metadata" + ) from error + if parser.has_option("console_scripts", "openshell"): + raise ValueError("wheel defines an openshell console script") + + bytecode_files = sorted( + name for name in names if name.endswith(BYTECODE_SUFFIXES) + ) + if bytecode_files: + raise ValueError( + f"wheel contains Python bytecode: {', '.join(bytecode_files)}" + ) + + native_files = sorted(name for name in names if name.endswith(NATIVE_SUFFIXES)) + if native_files: + raise ValueError(f"wheel contains native files: {', '.join(native_files)}") + + metadata_name = _single_member(names, ".dist-info/METADATA") + metadata = Parser().parsestr(archive.read(metadata_name).decode()) + if metadata["Name"] != "openshell": + raise ValueError(f"unexpected distribution name: {metadata['Name']}") + + version = metadata["Version"] + filename_version = wheel.name.removesuffix("-py3-none-any.whl").removeprefix( + "openshell-" + ) + if version != filename_version: + raise ValueError( + f"wheel filename version {filename_version} does not match metadata {version}" + ) + if expected_version is not None and version != expected_version: + raise ValueError(f"expected version {expected_version}, found {version}") + + wheel_name = _single_member(names, ".dist-info/WHEEL") + wheel_metadata = Parser().parsestr(archive.read(wheel_name).decode()) + if wheel_metadata.get_all("Tag", []) != ["py3-none-any"]: + raise ValueError( + f"unexpected wheel tags: {wheel_metadata.get_all('Tag', [])}" + ) + if wheel_metadata["Root-Is-Purelib"] != "true": + raise ValueError("wheel is not marked as pure Python") + + return wheel + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify the OpenShell Python wheel") + parser.add_argument( + "path", type=Path, help="Wheel file or directory containing one wheel" + ) + parser.add_argument("--expected-version") + args = parser.parse_args() + + try: + wheel = verify_wheel(args.path, args.expected_version) + except (BadZipFile, OSError, ValueError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + print(f"Verified {wheel}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tasks/test.toml b/tasks/test.toml index a75b05cce2..e926bd3790 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -9,8 +9,8 @@ depends = ["test:rust", "test:python", "test:sbom", "test:install-sh", "test:pac ["test:docs-website"] description = "Test the docs-website sync script" -# --no-project skips the workspace (maturin) build; --with supplies pytest and -# the script's runtime dep (PyYAML), which live outside the project env. +# --no-project skips installing the OpenShell package; --with supplies pytest +# and the script's runtime dependency (PyYAML), which lives outside the project env. run = "uv run --no-project --with pytest --with pyyaml pytest tasks/scripts/sync_docs_website_test.py" ["test:sbom"] diff --git a/uv.lock b/uv.lock index 1e86bc0df2..b12d0c0fcd 100644 --- a/uv.lock +++ b/uv.lock @@ -302,30 +302,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "maturin" -version = "1.11.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/84/bfed8cc10e2d8b6656cf0f0ca6609218e6fcb45a62929f5094e1063570f7/maturin-1.11.5.tar.gz", hash = "sha256:7579cf47640fb9595a19fe83a742cbf63203f0343055c349c1cab39045a30c29", size = 226885, upload-time = "2026-01-09T11:06:13.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/6c/3443d2f8c6d4eae5fc7479cd4053542aff4c1a8566d0019d0612d241b15a/maturin-1.11.5-py3-none-linux_armv6l.whl", hash = "sha256:edd1d4d35050ea2b9ef42aa01e87fe019a1e822940346b35ccb973e0aa8f6d82", size = 8845897, upload-time = "2026-01-09T11:06:17.327Z" }, - { url = "https://files.pythonhosted.org/packages/c5/03/abf1826d8aebc0d47ef6d21bdd752d98d63ac4372ad2b115db9cd5176229/maturin-1.11.5-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2a596eab137cb3e169b97e89a739515abfa7a8755e2e5f0fc91432ef446f74f4", size = 17233855, upload-time = "2026-01-09T11:06:04.272Z" }, - { url = "https://files.pythonhosted.org/packages/90/a1/5ad62913271724035a7e4bcf796d7c95b4119317ae5f8cb034844aa99bc4/maturin-1.11.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:1c27a2eb47821edf26c75d100b3150b52dca2c1a5f074d7514af06f7a7acb9d5", size = 8881776, upload-time = "2026-01-09T11:06:10.24Z" }, - { url = "https://files.pythonhosted.org/packages/c6/66/997974b44f8d3de641281ec04fbf5b6ca821bdc8291a2fa73305978db74d/maturin-1.11.5-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:f1320dacddcd3aa84a4bdfc77ee6fdb60e4c3835c853d7eb79c09473628b0498", size = 8870347, upload-time = "2026-01-09T11:06:12.178Z" }, - { url = "https://files.pythonhosted.org/packages/58/e0/c8fa042daf0608cc2e9a59b6df3a9e287bfc7f229136f17727f4118bac2d/maturin-1.11.5-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:ffe7418834ff3b4a6c987187b7abb85ba033f4733e089d77d84e2de87057b4e7", size = 9291396, upload-time = "2026-01-09T11:06:02.05Z" }, - { url = "https://files.pythonhosted.org/packages/99/af/9d3edc8375efc8d435d5f24794bc4de234d4e743447592da970d53b31361/maturin-1.11.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:c739b243d012386902f112ea63a54a94848932b70ae3565fa5e121fd1c0200e0", size = 8827831, upload-time = "2026-01-09T11:06:19.523Z" }, - { url = "https://files.pythonhosted.org/packages/8a/12/cc341f6abbf9005f90935a4ee5dc7b30e2df7d1bb90b96d48b756b2c0ee7/maturin-1.11.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:8127d2cd25950bacbcdc8a2ec6daab1d4d27200f7d73964392680ad64d27f7f0", size = 8718895, upload-time = "2026-01-09T11:06:21.617Z" }, - { url = "https://files.pythonhosted.org/packages/76/17/654a59c66287e287373f2a0086e4fc8a23f0545a81c2bd6e324db26a5801/maturin-1.11.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:2a4e872fb78e77748217084ffeb59de565d08a86ccefdace054520aaa7b66db4", size = 11384741, upload-time = "2026-01-09T11:06:15.261Z" }, - { url = "https://files.pythonhosted.org/packages/2e/da/7118de648182971d723ea99d79c55007f96cdafc95f5322cc1ad15f6683e/maturin-1.11.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2079447967819b5cf615e5b5b99a406d662effdc8d6afd493dcd253c6afc3707", size = 9423814, upload-time = "2026-01-09T11:05:57.242Z" }, - { url = "https://files.pythonhosted.org/packages/cf/8f/be14395c6e23b19ddaa0c171e68915bdcd1ef61ad1f411739c6721196903/maturin-1.11.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:50f6c668c1d5d4d4dc1c3ffec7b4270dab493e5b2368f8e4213f4bcde6a50eea", size = 9104378, upload-time = "2026-01-09T11:05:59.835Z" }, - { url = "https://files.pythonhosted.org/packages/77/83/53ea82a2f42a03930ea5545673d11a4ef49bb886827353a701f41a5f11c4/maturin-1.11.5-py3-none-win32.whl", hash = "sha256:49f85ce6cbe478e9743ecddd6da2964afc0ded57013aa4d054256be702d23d40", size = 7738696, upload-time = "2026-01-09T11:06:06.651Z" }, - { url = "https://files.pythonhosted.org/packages/3c/41/353a26d49aa80081c514a6354d429efbecedb90d0153ec598cece3baa607/maturin-1.11.5-py3-none-win_amd64.whl", hash = "sha256:70d3e5beffb9ef9dfae5f3c1a7eeb572091505eb8cb076e9434518df1c42a73b", size = 9029838, upload-time = "2026-01-09T11:05:54.543Z" }, - { url = "https://files.pythonhosted.org/packages/15/67/c94f8f5440bc42d54113a2d99de0d6107f06b5a33f31823e52b2715d856f/maturin-1.11.5-py3-none-win_arm64.whl", hash = "sha256:9348f7f0a346108e0c96e6719be91da4470bd43c15802435e9f4157f5cca43d4", size = 7624029, upload-time = "2026-01-09T11:06:08.728Z" }, -] - [[package]] name = "openshell" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "cloudpickle" }, { name = "grpcio" }, @@ -336,7 +315,6 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "grpcio-tools" }, - { name = "maturin" }, { name = "pyelftools" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -344,7 +322,6 @@ dev = [ { name = "pytest-xdist" }, { name = "pyyaml" }, { name = "ruff" }, - { name = "setuptools-scm" }, { name = "ty" }, { name = "types-pyyaml" }, ] @@ -360,7 +337,6 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "grpcio-tools", specifier = ">=1.60" }, - { name = "maturin", specifier = ">=1.5,<2.0" }, { name = "pyelftools", specifier = ">=0.30" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=0.23" }, @@ -368,7 +344,6 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.0" }, { name = "pyyaml", specifier = "==6.0.2" }, { name = "ruff", specifier = ">=0.4" }, - { name = "setuptools-scm", specifier = ">=8" }, { name = "ty", specifier = ">=0.0.1a6" }, { name = "types-pyyaml", specifier = ">=6.0.12.20260518" }, ] @@ -550,19 +525,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, ] -[[package]] -name = "setuptools-scm" -version = "9.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7b/b1/19587742aad604f1988a8a362e660e8c3ac03adccdb71c96d86526e5eb62/setuptools_scm-9.2.2.tar.gz", hash = "sha256:1c674ab4665686a0887d7e24c03ab25f24201c213e82ea689d2f3e169ef7ef57", size = 203385, upload-time = "2025-10-19T22:08:05.608Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ea/ac2bf868899d0d2e82ef72d350d97a846110c709bacf2d968431576ca915/setuptools_scm-9.2.2-py3-none-any.whl", hash = "sha256:30e8f84d2ab1ba7cb0e653429b179395d0c33775d54807fc5f1dd6671801aef7", size = 62975, upload-time = "2025-10-19T22:08:04.007Z" }, -] - [[package]] name = "tomli" version = "2.4.1"