diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 222ba7a8f0..750995dd9a 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -95,91 +95,24 @@ jobs: "${REGISTRY}/${component}:${{ github.sha }}" done - build-python-wheels-linux: - name: Build Python Wheels (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - artifact: linux-amd64 - task: python:build:linux:amd64 - output_path: target/wheels/linux-amd64/*.whl - - arch: arm64 - runner: linux-arm64-cpu8 - artifact: linux-arm64 - task: python:build:linux:arm64 - output_path: target/wheels/linux-arm64/*.whl - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: dev - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Sync Python dependencies - run: uv sync - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - with: - shared-key: python-wheel-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Build Python wheels - run: | - set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }} - ls -la ${{ matrix.output_path }} - - - name: Upload wheel artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: python-wheels-${{ matrix.artifact }} - path: ${{ matrix.output_path }} - retention-days: 5 - - build-python-wheel-macos: - name: Build Python Wheel (macOS) + build-python-wheel: + name: Build Python Wheel needs: [compute-versions] runs-on: linux-amd64-cpu8 - timeout-minutes: 120 + timeout-minutes: 20 container: image: ghcr.io/nvidia/openshell/ci:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: dev + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL: ${{ needs.compute-versions.outputs.python_version }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -189,13 +122,13 @@ jobs: - name: Build Python wheel run: | set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos + mise run python:build ls -la target/wheels/*.whl - - name: Upload wheel artifacts + - name: Upload wheel artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: python-wheels-macos + name: python-wheel path: target/wheels/*.whl retention-days: 5 @@ -679,7 +612,7 @@ jobs: smoke-linux-dev-artifacts: name: Smoke Linux Dev Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheel] timeout-minutes: 20 strategy: fail-fast: false @@ -709,18 +642,10 @@ jobs: kind: rpm artifact_arch: arm64 rpm_arch: aarch64 - - name: python-wheel-amd64 + - name: python-wheel runner: linux-amd64-cpu8 image: python:3.12-slim kind: wheel - artifact_arch: amd64 - rpm_arch: x86_64 - - name: python-wheel-arm64 - runner: linux-arm64-cpu8 - image: python:3.12-slim - kind: wheel - artifact_arch: arm64 - rpm_arch: aarch64 runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -759,27 +684,29 @@ jobs: if: matrix.kind == 'wheel' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-wheels-linux-${{ matrix.artifact_arch }} + name: python-wheel path: wheel-input/ - name: Smoke Python wheel if: matrix.kind == 'wheel' run: | set -euo pipefail - pip install --no-cache-dir wheel-input/*.whl - python - <<'PY' + python -m venv /tmp/openshell-wheel-smoke + /tmp/openshell-wheel-smoke/bin/pip install --no-cache-dir wheel-input/*.whl + /tmp/openshell-wheel-smoke/bin/python - <<'PY' import importlib for name in ("openshell", "openshell._proto", "openshell.sandbox"): importlib.import_module(name) print(name, "OK") PY + test ! -e /tmp/openshell-wheel-smoke/bin/openshell # --------------------------------------------------------------------------- # Create / update the dev GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- release-dev: name: Release Dev - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-dev-artifacts] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheel, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-dev-artifacts] runs-on: linux-amd64-cpu8 timeout-minutes: 10 permissions: @@ -820,12 +747,11 @@ jobs: path: release/ merge-multiple: true - - name: Download wheel artifacts + - name: Download wheel artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: python-wheels-* + name: python-wheel path: release/ - merge-multiple: true - name: Download Debian package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -877,9 +803,14 @@ jobs: id: wheel_filenames run: | set -euo pipefail - ls -la release/*.whl - WHEEL_FILENAMES=$(ls release/*.whl | xargs -n1 basename | sort | paste -sd, -) - echo "wheel_filenames=${WHEEL_FILENAMES}" >> "$GITHUB_OUTPUT" + shopt -s nullglob + wheels=(release/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "expected exactly one Python wheel, found ${#wheels[@]}" >&2 + exit 1 + fi + wheel_filename=$(basename "${wheels[0]}") + echo "wheel_filenames=${wheel_filename}" >> "$GITHUB_OUTPUT" - name: Generate checksums run: | diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index ec95ab5ea1..a32661b9a6 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -124,93 +124,25 @@ jobs: "${REGISTRY}/${component}:${SOURCE_TAG}" done - build-python-wheels-linux: - name: Build Python Wheels (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - artifact: linux-amd64 - task: python:build:linux:amd64 - output_path: target/wheels/linux-amd64/*.whl - - arch: arm64 - runner: linux-arm64-cpu8 - artifact: linux-arm64 - task: python:build:linux:arm64 - output_path: target/wheels/linux-arm64/*.whl - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Sync Python dependencies - run: uv sync - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - with: - shared-key: python-wheel-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Build Python wheels - run: | - set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }} - ls -la ${{ matrix.output_path }} - - - name: Upload wheel artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: python-wheels-${{ matrix.artifact }} - path: ${{ matrix.output_path }} - retention-days: 5 - - build-python-wheel-macos: - name: Build Python Wheel (macOS) + build-python-wheel: + name: Build Python Wheel needs: [compute-versions] runs-on: linux-amd64-cpu8 - timeout-minutes: 120 + timeout-minutes: 20 container: image: ghcr.io/nvidia/openshell/ci:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL: ${{ needs.compute-versions.outputs.python_version }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.tag || github.ref }} fetch-depth: 0 - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -220,13 +152,13 @@ jobs: - name: Build Python wheel run: | set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos + mise run python:build ls -la target/wheels/*.whl - - name: Upload wheel artifacts + - name: Upload wheel artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: python-wheels-macos + name: python-wheel path: target/wheels/*.whl retention-days: 5 @@ -715,7 +647,7 @@ jobs: smoke-linux-release-artifacts: name: Smoke Linux Release Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheel] timeout-minutes: 20 strategy: fail-fast: false @@ -767,16 +699,6 @@ jobs: runner: linux-amd64-cpu8 image: python:3.12-slim kind: wheel - artifact_arch: amd64 - rpm_arch: x86_64 - target: x86_64-unknown-linux-gnu - - name: python-wheel-arm64 - runner: linux-arm64-cpu8 - image: python:3.12-slim - kind: wheel - artifact_arch: arm64 - rpm_arch: aarch64 - target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -840,27 +762,29 @@ jobs: if: matrix.kind == 'wheel' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-wheels-linux-${{ matrix.artifact_arch }} + name: python-wheel path: wheel-input/ - name: Smoke Python wheel if: matrix.kind == 'wheel' run: | set -euo pipefail - pip install --no-cache-dir wheel-input/*.whl - python - <<'PY' + python -m venv /tmp/openshell-wheel-smoke + /tmp/openshell-wheel-smoke/bin/pip install --no-cache-dir wheel-input/*.whl + /tmp/openshell-wheel-smoke/bin/python - <<'PY' import importlib for name in ("openshell", "openshell._proto", "openshell.sandbox"): importlib.import_module(name) print(name, "OK") PY + test ! -e /tmp/openshell-wheel-smoke/bin/openshell # --------------------------------------------------------------------------- # Create a tagged GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- release: name: Release - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-release-artifacts] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheel, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-release-artifacts] runs-on: linux-amd64-cpu8 timeout-minutes: 10 permissions: @@ -903,12 +827,11 @@ jobs: path: release/ merge-multiple: true - - name: Download wheel artifacts + - name: Download wheel artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: python-wheels-* + name: python-wheel path: release/ - merge-multiple: true - name: Download Debian package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -935,9 +858,14 @@ jobs: id: wheel_filenames run: | set -euo pipefail - ls -la release/*.whl - WHEEL_FILENAMES=$(ls release/*.whl | xargs -n1 basename | sort | paste -sd, -) - echo "wheel_filenames=${WHEEL_FILENAMES}" >> "$GITHUB_OUTPUT" + shopt -s nullglob + wheels=(release/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "expected exactly one Python wheel, found ${#wheels[@]}" >&2 + exit 1 + fi + wheel_filename=$(basename "${wheels[0]}") + echo "wheel_filenames=${wheel_filename}" >> "$GITHUB_OUTPUT" - name: Generate checksums run: | diff --git a/README.md b/README.md index 33b2736bc2..8e5957aafa 100644 --- a/README.md +++ b/README.md @@ -27,14 +27,14 @@ OpenShell is built agent-first. The project ships with agent skills for everythi curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -**From PyPI (requires [uv](https://docs.astral.sh/uv/)):** +The installer installs the latest stable release by default. To install a specific version, set `OPENSHELL_VERSION`. A [`dev` release](https://github.com/NVIDIA/OpenShell/releases/tag/dev) is also available that tracks the latest commit on `main`. + +The `openshell` package on PyPI provides the Python SDK only. It does not install the `openshell` CLI. Add the SDK to a Python project with [uv](https://docs.astral.sh/uv/): ```bash -uv tool install -U openshell +uv add openshell ``` -Both methods install the latest stable release by default. To install a specific version, set `OPENSHELL_VERSION` (binary) or pin the version with `uv tool install openshell==`. A [`dev` release](https://github.com/NVIDIA/OpenShell/releases/tag/dev) is also available that tracks the latest commit on `main`. - **Helm chart:** > **Experimental** — the Kubernetes deployment path is under active development. Expect rough edges and breaking changes. diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES index 41eda0e0a8..8125e5692f 100644 --- a/THIRD-PARTY-NOTICES +++ b/THIRD-PARTY-NOTICES @@ -14813,13 +14813,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -================================================================================ -maturin 1.11.5 -License: MIT OR Apache-2.0 --------------------------------------------------------------------------------- - -UNKNOWN - ================================================================================ packaging 26.0 License: Apache-2.0 OR BSD-2-Clause diff --git a/architecture/build.md b/architecture/build.md index e3ac5012e3..8571bfe0d4 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -11,7 +11,8 @@ OpenShell builds these main artifacts: | Artifact | Source | |---|---| | Gateway binary | `crates/openshell-server` | -| CLI package and Python SDK | `python/openshell` plus Rust binaries where packaged | +| CLI binaries and system packages | `crates/openshell-cli` plus release packaging | +| Python SDK wheel | `python/openshell` | | Gateway container image | `deploy/docker/Dockerfile.gateway` | | Supervisor container image | `deploy/docker/Dockerfile.supervisor` | | Helm chart | `deploy/helm/openshell` | @@ -133,12 +134,18 @@ do not infer from kube context. ## Python Wheel Packaging The generated protobuf/gRPC stubs under `python/openshell/_proto/` are gitignored -build outputs of `mise run python:proto`. maturin honors `.gitignore` when -collecting `python-source` files, so native builds (Linux CI, local -`pip install .`) would drop them and ship an unimportable wheel. `pyproject.toml` -pins them back in with `[tool.maturin].include` globs. The release workflows -install each Linux wheel in a clean image and import `openshell.sandbox` as a -smoke check. +build outputs of `mise run python:proto`. Setuptools includes them through the +package-data configuration in `pyproject.toml`. Release workflows build the +wheel directly and do not produce a source distribution. Setuptools SCM derives +local versions from Git and accepts the release workflow's computed version +through its distribution-specific override. + +The build produces one platform-independent `py3-none-any` wheel. A verifier +checks its tag, metadata, version, required package files, and the absence of +native files or an `openshell` executable entry point. Release workflows build +the wheel once, install it in a clean virtual environment, import the public +package modules, and confirm that installation did not create an `openshell` +command. ## CI and E2E diff --git a/deploy/docker/Dockerfile.python-wheels b/deploy/docker/Dockerfile.python-wheels deleted file mode 100644 index e93bf8f22e..0000000000 --- a/deploy/docker/Dockerfile.python-wheels +++ /dev/null @@ -1,115 +0,0 @@ -# syntax=docker/dockerfile:1.6 - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - - -ARG PYTHON_VERSION=3.12 - -FROM --platform=$BUILDPLATFORM python:${PYTHON_VERSION}-slim AS base - -ENV PATH="/root/.cargo/bin:${PATH}" - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - curl \ - gcc \ - libc6-dev \ - libclang-dev \ - pkg-config \ - libssl-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 -RUN pip install --no-cache-dir maturin - -COPY deploy/docker/cross-build.sh /usr/local/bin/ - -FROM base AS builder - -ARG TARGETARCH -ARG BUILDARCH -ARG CARGO_TARGET_CACHE_SCOPE=default - -ARG SCCACHE_MEMCACHED_ENDPOINT - -WORKDIR /build - -RUN . cross-build.sh && install_cross_toolchain && install_sccache && add_rust_target - -# Copy dependency manifests first for better caching. -COPY Cargo.toml Cargo.lock ./ -COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml -COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml -COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml -COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml -COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml -COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml -COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml -COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml -COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml -COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml -COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml -COPY crates/openshell-core/build.rs crates/openshell-core/build.rs -COPY proto/ proto/ - -# Create dummy source files to build dependencies. -RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src crates/openshell-bootstrap/src crates/openshell-tui/src && \ - echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ - echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ - touch crates/openshell-core/src/lib.rs && \ - touch crates/openshell-ocsf/src/lib.rs && \ - touch crates/openshell-providers/src/lib.rs && \ - touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-bootstrap/src/lib.rs && \ - touch crates/openshell-policy/src/lib.rs && \ - touch crates/openshell-prover/src/lib.rs && \ - touch crates/openshell-tui/src/lib.rs - -# Build dependencies only (cached unless Cargo.toml/lock changes). -# sccache uses memcached in CI or the local disk cache mount for local dev. -# The cargo-target mount gives cargo a persistent target/ dir for incremental rebuilds. -RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \ - . cross-build.sh && cargo_cross_build --release -p openshell-cli --features bundled-z3 2>/dev/null || true - -# Copy actual source code and Python packaging files. -COPY crates/ crates/ -COPY pyproject.toml README.md ./ -COPY python/ python/ - -# Touch source files to ensure they're rebuilt (not the cached dummy). -RUN touch crates/openshell-cli/src/main.rs \ - crates/openshell-cli/src/lib.rs \ - crates/openshell-bootstrap/src/lib.rs \ - crates/openshell-core/src/lib.rs \ - crates/openshell-providers/src/lib.rs \ - crates/openshell-router/src/lib.rs \ - crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ - crates/openshell-core/build.rs \ - proto/*.proto - -# Declare version ARGs here (not earlier) so the git-hash-bearing values do not -# invalidate the expensive dependency-build layers above on every commit. -ARG OPENSHELL_CARGO_VERSION -ARG OPENSHELL_IMAGE_TAG -RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \ - . cross-build.sh && \ - export_cross_env && \ - export RUSTC_WRAPPER=sccache && \ - export CARGO_BUILD_TARGET="$(rust_target)" && \ - if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ - fi && \ - maturin build --release --target "${CARGO_BUILD_TARGET}" --features bundled-z3 --out /wheels - -FROM scratch AS wheels -COPY --from=builder /wheels/*.whl / diff --git a/deploy/docker/Dockerfile.python-wheels-macos b/deploy/docker/Dockerfile.python-wheels-macos deleted file mode 100644 index 45194b0690..0000000000 --- a/deploy/docker/Dockerfile.python-wheels-macos +++ /dev/null @@ -1,126 +0,0 @@ -# syntax=docker/dockerfile:1.6 - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - - -ARG OSXCROSS_IMAGE=ghcr.io/crazy-max/osxcross:latest -ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python -ARG PYTHON_VERSION=3.12 - -FROM ${OSXCROSS_IMAGE} AS osxcross - -FROM ${PYTHON_IMAGE}:${PYTHON_VERSION}-slim AS builder - -ARG TARGETARCH -ARG CARGO_TARGET_CACHE_SCOPE=default - -ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}" -ENV LD_LIBRARY_PATH="/osxcross/lib" - -COPY --from=osxcross /osxcross /osxcross - -RUN SDKROOT="$(echo /osxcross/SDK/MacOSX*.sdk)" && ln -sfn "${SDKROOT}" /osxcross/SDK/MacOSX.sdk - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - clang \ - cmake \ - curl \ - libclang-dev \ - libssl-dev \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* - -# aws-lc-sys probes with --target=arm64-apple-macosx and clang then looks for -# arm64-apple-macosx-ld. Provide a linker alias to osxcross ld64. -RUN ln -sf /osxcross/bin/arm64-apple-darwin25.1-ld /usr/local/bin/arm64-apple-macosx-ld - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 -RUN rustup target add aarch64-apple-darwin -RUN pip install --no-cache-dir maturin - -WORKDIR /build - -ENV CC_aarch64_apple_darwin=oa64-clang -ENV CXX_aarch64_apple_darwin=oa64-clang++ -ENV AR_aarch64_apple_darwin=aarch64-apple-darwin25.1-ar -ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=oa64-clang -ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_AR=aarch64-apple-darwin25.1-ar -ENV SDKROOT=/osxcross/SDK/MacOSX.sdk -ENV MACOSX_DEPLOYMENT_TARGET=13.3 -ENV CFLAGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 -ENV CXXFLAGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 -ENV BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -isysroot\ ${SDKROOT} - -# Copy dependency manifests first for better caching. -COPY Cargo.toml Cargo.lock ./ -COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml -COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml -COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml -COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml -COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml -COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml -COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml -COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml -COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml -COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml -COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml -COPY crates/openshell-core/build.rs crates/openshell-core/build.rs -COPY proto/ proto/ - -# Create dummy source files to build dependencies. -RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src crates/openshell-bootstrap/src crates/openshell-tui/src && \ - echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ - echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ - touch crates/openshell-core/src/lib.rs && \ - touch crates/openshell-ocsf/src/lib.rs && \ - touch crates/openshell-providers/src/lib.rs && \ - touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-bootstrap/src/lib.rs && \ - touch crates/openshell-policy/src/lib.rs && \ - touch crates/openshell-prover/src/lib.rs && \ - touch crates/openshell-tui/src/lib.rs - -# Build dependencies only (cached unless Cargo.toml/lock changes). -RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-cli --features bundled-z3 2>/dev/null || true - -# Copy actual source code and Python packaging files. -COPY crates/ crates/ -COPY providers/ providers/ -COPY pyproject.toml README.md ./ -COPY python/ python/ - -# Touch source files to ensure they're rebuilt (not the cached dummy). -# Touch build.rs and proto files to force proto code regeneration when the -# cargo target cache mount retains stale OUT_DIR artifacts from prior builds. -RUN touch crates/openshell-cli/src/main.rs \ - crates/openshell-cli/src/lib.rs \ - crates/openshell-bootstrap/src/lib.rs \ - crates/openshell-core/src/lib.rs \ - crates/openshell-providers/src/lib.rs \ - crates/openshell-router/src/lib.rs \ - crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ - crates/openshell-core/build.rs \ - proto/*.proto - -# Declare version ARGs here (not earlier) so the git-hash-bearing values do not -# invalidate the expensive dependency-build layers above on every commit. -ARG OPENSHELL_CARGO_VERSION -ARG OPENSHELL_IMAGE_TAG -RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ - fi && \ - maturin build --release --target aarch64-apple-darwin --features bundled-z3 --out /wheels - -FROM scratch AS wheels -COPY --from=builder /wheels/*.whl / diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 2ac077e7b9..c9de188dec 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -20,6 +20,12 @@ The script detects your operating system and installs the OpenShell CLI and gate You can also download release artifacts directly from the [OpenShell GitHub Releases](https://github.com/NVIDIA/OpenShell/releases) page. +The `openshell` package on PyPI provides the Python SDK only. It does not install the `openshell` CLI. Add the SDK to a Python project with: + +```shell +uv add openshell +``` + Use `openshell status` to confirm the CLI can reach the gateway. ## Supported Compute Drivers diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index ebd7a9880c..8a8b82046e 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -30,12 +30,6 @@ curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | The install script uses Homebrew, RPM, or a Debian package based on your machine. It starts the local gateway server after installation. -If you prefer [uv](https://docs.astral.sh/uv/): - -```shell -uv tool install -U openshell -``` - After installing the CLI, run `openshell --help` in your terminal to view the full CLI reference. diff --git a/docs/get-started/tutorials/docker-compose.mdx b/docs/get-started/tutorials/docker-compose.mdx index 645b5ddc98..59536141f6 100644 --- a/docs/get-started/tutorials/docker-compose.mdx +++ b/docs/get-started/tutorials/docker-compose.mdx @@ -72,13 +72,9 @@ curl -sf http://localhost:8080/healthz curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -**From PyPI (any platform with [uv](https://docs.astral.sh/uv/)):** - -```shell -uv tool install -U openshell -``` - +The `openshell` package on PyPI provides the Python SDK only and does not install the CLI. + On Windows without WSL, install the CLI inside a WSL 2 distribution (for example AlmaLinux or Ubuntu) and run all `openshell` commands from that distribution. diff --git a/flake.nix b/flake.nix index 13c4857bc6..cf7ccebf3b 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,7 @@ llvmPackages.libclang # system dependency for openshell-prover z3 + uv ]; env = { diff --git a/pyproject.toml b/pyproject.toml index fb8aac17a2..247a40b412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [build-system] -requires = ["maturin>=1.5,<2.0"] -build-backend = "maturin" +requires = ["setuptools>=80", "setuptools-scm[simple]>=8"] +build-backend = "setuptools.build_meta" [project] name = "openshell" @@ -29,8 +29,6 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", - "Programming Language :: Rust", - "License :: OSI Approved :: Apache Software License", "Topic :: Security", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -51,33 +49,23 @@ dev = [ "pytest-xdist>=3.0", "ruff>=0.4", "ty>=0.0.1a6", - "maturin>=1.5,<2.0", - "setuptools-scm>=8", "grpcio-tools>=1.60", "pyelftools>=0.30", "pyyaml==6.0.2", "types-pyyaml>=6.0.12.20260518", ] -[tool.uv] -# Don't try to install the root package with uv sync - use uv pip install . instead -# This is needed because maturin bin packages don't install the binary with editable installs -# (both `uv pip install -e .` and `maturin develop` skip binary installation) -# Use `uv pip install .` (without -e) to install the openshell CLI binary -package = false +[tool.setuptools] +package-dir = { "" = "python" } +include-package-data = false -[tool.maturin] -bindings = "bin" -manifest-path = "crates/openshell-cli/Cargo.toml" -python-source = "python" -module-name = "openshell" -# _proto/ stubs are gitignored; force them into the wheel so git-aware -# maturin builds don't drop them (package-relative paths). -include = [ - "openshell/_proto/*_pb2.py", - "openshell/_proto/*_pb2_grpc.py", - "openshell/_proto/*.pyi", -] +[tool.setuptools.packages.find] +where = ["python"] +include = ["openshell*"] +namespaces = false + +[tool.setuptools.package-data] +openshell = ["py.typed", "_proto/*.py", "_proto/*.pyi"] [tool.ruff] target-version = "py311" diff --git a/python/wheel_verifier_test.py b/python/wheel_verifier_test.py new file mode 100644 index 0000000000..777f348f73 --- /dev/null +++ b/python/wheel_verifier_test.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from zipfile import ZipFile + +import pytest + + +def _load_verifier_module(): + path = Path(__file__).resolve().parents[1] / "tasks/scripts/verify-python-wheel.py" + spec = importlib.util.spec_from_file_location("openshell_wheel_verifier", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +verifier = _load_verifier_module() + + +def _wheel_files() -> set[str]: + files = { + "openshell/__init__.py", + "openshell/sandbox.py", + "openshell/py.typed", + "openshell/_proto/__init__.py", + } + for stem in ("datamodel", "inference", "openshell", "options", "sandbox"): + files.add(f"openshell/_proto/{stem}_pb2.py") + files.add(f"openshell/_proto/{stem}_pb2.pyi") + files.add(f"openshell/_proto/{stem}_pb2_grpc.py") + return files + + +def _write_wheel( + directory: Path, + *, + version: str = "1.2.3", + tag: str = "py3-none-any", + files: set[str] | None = None, + entry_points: str | None = None, +) -> Path: + wheel = directory / f"openshell-{version}-{tag}.whl" + dist_info = f"openshell-{version}.dist-info" + with ZipFile(wheel, "w") as archive: + for name in files if files is not None else _wheel_files(): + archive.writestr(name, "") + archive.writestr( + f"{dist_info}/METADATA", + f"Metadata-Version: 2.4\nName: openshell\nVersion: {version}\n", + ) + archive.writestr( + f"{dist_info}/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {tag}\n", + ) + if entry_points is not None: + archive.writestr(f"{dist_info}/entry_points.txt", entry_points) + return wheel + + +def test_accepts_pure_python_sdk_wheel(tmp_path: Path) -> None: + wheel = _write_wheel(tmp_path) + + assert verifier.verify_wheel(wheel, "1.2.3") == wheel + + +def test_requires_exactly_one_wheel_in_directory(tmp_path: Path) -> None: + _write_wheel(tmp_path, version="1.2.3") + _write_wheel(tmp_path, version="1.2.4") + + with pytest.raises(ValueError, match="expected exactly one wheel"): + verifier.verify_wheel(tmp_path) + + +def test_rejects_missing_generated_proto(tmp_path: Path) -> None: + files = _wheel_files() - {"openshell/_proto/sandbox_pb2.py"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match=r"sandbox_pb2\.py"): + verifier.verify_wheel(wheel) + + +def test_rejects_bundled_cli(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell-1.2.3.data/scripts/openshell"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains an openshell executable"): + verifier.verify_wheel(wheel) + + +def test_rejects_bundled_windows_cli(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell-1.2.3.data/scripts/openshell.exe"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains an openshell executable"): + verifier.verify_wheel(wheel) + + +def test_rejects_openshell_console_script(tmp_path: Path) -> None: + wheel = _write_wheel( + tmp_path, + entry_points="[console_scripts]\nopenshell = openshell.cli:main\n", + ) + + with pytest.raises(ValueError, match="defines an openshell console script"): + verifier.verify_wheel(wheel) + + +def test_rejects_native_extension(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell/_native.so"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains native files"): + verifier.verify_wheel(wheel) + + +def test_rejects_bytecode(tmp_path: Path) -> None: + files = _wheel_files() | {"openshell/__pycache__/sandbox.cpython-314.pyc"} + wheel = _write_wheel(tmp_path, files=files) + + with pytest.raises(ValueError, match="contains Python bytecode"): + verifier.verify_wheel(wheel) + + +def test_rejects_platform_wheel(tmp_path: Path) -> None: + wheel = _write_wheel(tmp_path, tag="cp311-cp311-manylinux_2_28_x86_64") + + with pytest.raises(ValueError, match="not platform-independent"): + verifier.verify_wheel(wheel) + + +def test_rejects_unexpected_version(tmp_path: Path) -> None: + wheel = _write_wheel(tmp_path) + + with pytest.raises(ValueError, match=r"expected version 1\.2\.4"): + verifier.verify_wheel(wheel, "1.2.4") diff --git a/tasks/python.toml b/tasks/python.toml index 62b8b4f787..6e4530de9e 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -4,193 +4,35 @@ # Python development, build, lint, and format tasks ["python:dev"] -description = "Install Python package in development mode (builds CLI binary)" +description = "Install the Python SDK in development mode" depends = ["python:proto"] -run = "uv sync --group dev && uv pip install ." +run = "uv sync --group dev" hide = true ["build:python:wheel"] -description = "Build Python wheel with CLI binary (native)" +description = "Build the pure-Python SDK wheel" depends = ["python:proto"] -run = "uv run maturin build --release" -hide = true - -["python:build"] -description = "Alias for build:python:wheel" -depends = ["build:python:wheel"] -hide = true - -["build:python:wheel:linux"] -description = "Build a Linux Python wheel natively on the current host" -depends = ["python:proto"] -run = ''' +run = """ #!/usr/bin/env bash set -euo pipefail -WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:?Set WHEEL_OUTPUT_DIR to a per-platform wheel output directory} -EXPECTED_HOST_ARCH=${EXPECTED_HOST_ARCH:?Set EXPECTED_HOST_ARCH to amd64 or arm64} -ORIGINAL_CARGO_TOML="" - -normalize_arch() { - case "$1" in - x86_64|amd64) echo amd64 ;; - aarch64|arm64) echo arm64 ;; - *) echo "$1" ;; - esac -} - -cleanup() { - if [ -n "$ORIGINAL_CARGO_TOML" ] && [ -f "$ORIGINAL_CARGO_TOML" ]; then - cp "$ORIGINAL_CARGO_TOML" Cargo.toml - rm -f "$ORIGINAL_CARGO_TOML" - fi -} - -trap cleanup EXIT - -if [ "$(uname -s)" != "Linux" ]; then - echo "build:python:wheel:linux requires a Linux host" >&2 - exit 1 -fi - -HOST_ARCH=$(normalize_arch "$(uname -m)") -EXPECTED_HOST_ARCH=$(normalize_arch "$EXPECTED_HOST_ARCH") -if [ "$HOST_ARCH" != "$EXPECTED_HOST_ARCH" ]; then - echo "build:python:wheel:linux expected ${EXPECTED_HOST_ARCH} host, got ${HOST_ARCH}" >&2 - exit 1 -fi - -# Avoid failures when RUSTC_WRAPPER is set to an unresolved `sccache` binary. -if command -v sccache >/dev/null 2>&1; then - export RUSTC_WRAPPER="$(command -v sccache)" -else - unset RUSTC_WRAPPER || true -fi - -if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then - ORIGINAL_CARGO_TOML=$(mktemp) - cp Cargo.toml "$ORIGINAL_CARGO_TOML" - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml -fi - +WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:-target/wheels} rm -rf "$WHEEL_OUTPUT_DIR" mkdir -p "$WHEEL_OUTPUT_DIR" -uv run maturin build --release --features bundled-z3 --out "$WHEEL_OUTPUT_DIR" - -ls -la "$WHEEL_OUTPUT_DIR"/*.whl -''' -hide = true - -["build:python:wheel:linux:amd64"] -description = "Build Python wheel for Linux amd64 natively" -depends = ["EXPECTED_HOST_ARCH=amd64 WHEEL_OUTPUT_DIR=target/wheels/linux-amd64 build:python:wheel:linux"] -hide = true - -["python:build:linux:amd64"] -description = "Alias for build:python:wheel:linux:amd64" -depends = ["build:python:wheel:linux:amd64"] -hide = true - -["build:python:wheel:linux:arm64"] -description = "Build Python wheel for Linux arm64 natively" -depends = ["EXPECTED_HOST_ARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux"] -hide = true - -["python:build:linux:arm64"] -description = "Alias for build:python:wheel:linux:arm64" -depends = ["build:python:wheel:linux:arm64"] -hide = true +uv build --wheel --out-dir "$WHEEL_OUTPUT_DIR" -["build:python:wheel:macos"] -description = "Build Python wheel for macOS arm64" -depends = ["python:proto"] -run = """ -#!/usr/bin/env bash -set -euo pipefail - -if [ "$(uname -s)" = "Darwin" ] && [ -z "${OPENSHELL_CARGO_VERSION:-}" ]; then - # Avoid failures when RUSTC_WRAPPER is set to an unresolved `sccache` binary. - if command -v sccache >/dev/null 2>&1; then - export RUSTC_WRAPPER="$(command -v sccache)" - else - unset RUSTC_WRAPPER || true - fi - - mkdir -p target/wheels - - rustup target add aarch64-apple-darwin - - uv run maturin build --release --target aarch64-apple-darwin --out target/wheels -else - echo "Building macOS wheel via Docker cross-toolchain" - mise run build:python:wheel:macos:docker -fi - -ls -la target/wheels/*.whl -""" -hide = true - -["python:build:macos"] -description = "Alias for build:python:wheel:macos" -depends = ["build:python:wheel:macos"] -hide = true - -["build:python:wheel:macos:docker"] -description = "Build Python wheel for macOS arm64 from Docker" -depends = ["python:proto"] -run = """ -#!/usr/bin/env bash -set -euo pipefail - -source tasks/scripts/container-engine.sh - -sha256_16() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}' - fi -} - -sha256_16_stdin() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 | awk '{print substr($1, 1, 16)}' - fi -} - -CARGO_VERSION=${OPENSHELL_CARGO_VERSION:-} -OSXCROSS_IMAGE_REF=${OSXCROSS_IMAGE:-crazymax/osxcross:latest} -if [ -z "$CARGO_VERSION" ] && [ -n "${CI:-}" ]; then - CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo) +VERIFY_ARGS=() +if [ -n "${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL:-}" ]; then + VERIFY_ARGS+=(--expected-version "$SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL") fi - -LOCK_HASH=$(sha256_16 Cargo.lock) -RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-1.95.0} -CACHE_SCOPE_INPUT="v1|python-wheels-macos|base|${LOCK_HASH}|${RUST_SCOPE}" -CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin) - -mkdir -p target/wheels - -ce build \ - -f deploy/docker/Dockerfile.python-wheels-macos \ - --target wheels \ - --build-arg "OSXCROSS_IMAGE=${OSXCROSS_IMAGE_REF}" \ - --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ - ${CARGO_VERSION:+--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}"} \ - ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} \ - --output type=local,dest=target/wheels \ - . - -ls -la target/wheels/*macosx*arm64.whl +uv run --no-project python tasks/scripts/verify-python-wheel.py "$WHEEL_OUTPUT_DIR" "${VERIFY_ARGS[@]}" """ hide = true -["python:build:macos:docker"] -description = "Alias for build:python:wheel:macos:docker" -depends = ["build:python:wheel:macos:docker"] +["python:build"] +description = "Alias for build:python:wheel" +depends = ["build:python:wheel"] hide = true ["python:lint"] diff --git a/tasks/scripts/verify-python-wheel.py b/tasks/scripts/verify-python-wheel.py new file mode 100644 index 0000000000..6234c0cdb6 --- /dev/null +++ b/tasks/scripts/verify-python-wheel.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import sys +from configparser import ConfigParser +from configparser import Error as ConfigParserError +from email.parser import Parser +from pathlib import Path +from zipfile import BadZipFile, ZipFile + +PROTO_STEMS = ("datamodel", "inference", "openshell", "options", "sandbox") +BYTECODE_SUFFIXES = (".pyc", ".pyo") +NATIVE_SUFFIXES = (".dll", ".dylib", ".exe", ".pyd", ".so") + + +def _find_wheel(path: Path) -> Path: + if path.is_file(): + return path + + wheels = sorted(path.glob("*.whl")) + if len(wheels) != 1: + raise ValueError(f"expected exactly one wheel in {path}, found {len(wheels)}") + return wheels[0] + + +def _required_files() -> set[str]: + files = { + "openshell/__init__.py", + "openshell/sandbox.py", + "openshell/py.typed", + "openshell/_proto/__init__.py", + } + for stem in PROTO_STEMS: + files.add(f"openshell/_proto/{stem}_pb2.py") + files.add(f"openshell/_proto/{stem}_pb2.pyi") + files.add(f"openshell/_proto/{stem}_pb2_grpc.py") + return files + + +def _single_member(names: set[str], suffix: str) -> str: + matches = [name for name in names if name.endswith(suffix)] + if len(matches) != 1: + raise ValueError(f"expected exactly one {suffix} file, found {len(matches)}") + return matches[0] + + +def verify_wheel(path: Path, expected_version: str | None = None) -> Path: + wheel = _find_wheel(path) + if not wheel.name.endswith("-py3-none-any.whl"): + raise ValueError(f"wheel is not platform-independent: {wheel.name}") + + with ZipFile(wheel) as archive: + names = set(archive.namelist()) + missing = sorted(_required_files() - names) + if missing: + raise ValueError(f"wheel is missing required files: {', '.join(missing)}") + + scripts = sorted( + name + for name in names + if not name.endswith("/") + and Path(name).name in {"openshell", "openshell.exe"} + ) + if scripts: + raise ValueError( + f"wheel contains an openshell executable: {', '.join(scripts)}" + ) + + entry_points = [ + name for name in names if name.endswith(".dist-info/entry_points.txt") + ] + if len(entry_points) > 1: + raise ValueError( + f"expected at most one entry_points.txt file, found {len(entry_points)}" + ) + if entry_points: + parser = ConfigParser(interpolation=None) + try: + parser.read_string(archive.read(entry_points[0]).decode()) + except (ConfigParserError, UnicodeDecodeError) as error: + raise ValueError( + "wheel contains invalid entry point metadata" + ) from error + if parser.has_option("console_scripts", "openshell"): + raise ValueError("wheel defines an openshell console script") + + bytecode_files = sorted( + name for name in names if name.endswith(BYTECODE_SUFFIXES) + ) + if bytecode_files: + raise ValueError( + f"wheel contains Python bytecode: {', '.join(bytecode_files)}" + ) + + native_files = sorted(name for name in names if name.endswith(NATIVE_SUFFIXES)) + if native_files: + raise ValueError(f"wheel contains native files: {', '.join(native_files)}") + + metadata_name = _single_member(names, ".dist-info/METADATA") + metadata = Parser().parsestr(archive.read(metadata_name).decode()) + if metadata["Name"] != "openshell": + raise ValueError(f"unexpected distribution name: {metadata['Name']}") + + version = metadata["Version"] + filename_version = wheel.name.removesuffix("-py3-none-any.whl").removeprefix( + "openshell-" + ) + if version != filename_version: + raise ValueError( + f"wheel filename version {filename_version} does not match metadata {version}" + ) + if expected_version is not None and version != expected_version: + raise ValueError(f"expected version {expected_version}, found {version}") + + wheel_name = _single_member(names, ".dist-info/WHEEL") + wheel_metadata = Parser().parsestr(archive.read(wheel_name).decode()) + if wheel_metadata.get_all("Tag", []) != ["py3-none-any"]: + raise ValueError( + f"unexpected wheel tags: {wheel_metadata.get_all('Tag', [])}" + ) + if wheel_metadata["Root-Is-Purelib"] != "true": + raise ValueError("wheel is not marked as pure Python") + + return wheel + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify the OpenShell Python wheel") + parser.add_argument( + "path", type=Path, help="Wheel file or directory containing one wheel" + ) + parser.add_argument("--expected-version") + args = parser.parse_args() + + try: + wheel = verify_wheel(args.path, args.expected_version) + except (BadZipFile, OSError, ValueError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + print(f"Verified {wheel}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tasks/test.toml b/tasks/test.toml index a75b05cce2..e926bd3790 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -9,8 +9,8 @@ depends = ["test:rust", "test:python", "test:sbom", "test:install-sh", "test:pac ["test:docs-website"] description = "Test the docs-website sync script" -# --no-project skips the workspace (maturin) build; --with supplies pytest and -# the script's runtime dep (PyYAML), which live outside the project env. +# --no-project skips installing the OpenShell package; --with supplies pytest +# and the script's runtime dependency (PyYAML), which lives outside the project env. run = "uv run --no-project --with pytest --with pyyaml pytest tasks/scripts/sync_docs_website_test.py" ["test:sbom"] diff --git a/uv.lock b/uv.lock index 1e86bc0df2..b12d0c0fcd 100644 --- a/uv.lock +++ b/uv.lock @@ -302,30 +302,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "maturin" -version = "1.11.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/84/bfed8cc10e2d8b6656cf0f0ca6609218e6fcb45a62929f5094e1063570f7/maturin-1.11.5.tar.gz", hash = "sha256:7579cf47640fb9595a19fe83a742cbf63203f0343055c349c1cab39045a30c29", size = 226885, upload-time = "2026-01-09T11:06:13.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/6c/3443d2f8c6d4eae5fc7479cd4053542aff4c1a8566d0019d0612d241b15a/maturin-1.11.5-py3-none-linux_armv6l.whl", hash = "sha256:edd1d4d35050ea2b9ef42aa01e87fe019a1e822940346b35ccb973e0aa8f6d82", size = 8845897, upload-time = "2026-01-09T11:06:17.327Z" }, - { url = "https://files.pythonhosted.org/packages/c5/03/abf1826d8aebc0d47ef6d21bdd752d98d63ac4372ad2b115db9cd5176229/maturin-1.11.5-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2a596eab137cb3e169b97e89a739515abfa7a8755e2e5f0fc91432ef446f74f4", size = 17233855, upload-time = "2026-01-09T11:06:04.272Z" }, - { url = "https://files.pythonhosted.org/packages/90/a1/5ad62913271724035a7e4bcf796d7c95b4119317ae5f8cb034844aa99bc4/maturin-1.11.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:1c27a2eb47821edf26c75d100b3150b52dca2c1a5f074d7514af06f7a7acb9d5", size = 8881776, upload-time = "2026-01-09T11:06:10.24Z" }, - { url = "https://files.pythonhosted.org/packages/c6/66/997974b44f8d3de641281ec04fbf5b6ca821bdc8291a2fa73305978db74d/maturin-1.11.5-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:f1320dacddcd3aa84a4bdfc77ee6fdb60e4c3835c853d7eb79c09473628b0498", size = 8870347, upload-time = "2026-01-09T11:06:12.178Z" }, - { url = "https://files.pythonhosted.org/packages/58/e0/c8fa042daf0608cc2e9a59b6df3a9e287bfc7f229136f17727f4118bac2d/maturin-1.11.5-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:ffe7418834ff3b4a6c987187b7abb85ba033f4733e089d77d84e2de87057b4e7", size = 9291396, upload-time = "2026-01-09T11:06:02.05Z" }, - { url = "https://files.pythonhosted.org/packages/99/af/9d3edc8375efc8d435d5f24794bc4de234d4e743447592da970d53b31361/maturin-1.11.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:c739b243d012386902f112ea63a54a94848932b70ae3565fa5e121fd1c0200e0", size = 8827831, upload-time = "2026-01-09T11:06:19.523Z" }, - { url = "https://files.pythonhosted.org/packages/8a/12/cc341f6abbf9005f90935a4ee5dc7b30e2df7d1bb90b96d48b756b2c0ee7/maturin-1.11.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:8127d2cd25950bacbcdc8a2ec6daab1d4d27200f7d73964392680ad64d27f7f0", size = 8718895, upload-time = "2026-01-09T11:06:21.617Z" }, - { url = "https://files.pythonhosted.org/packages/76/17/654a59c66287e287373f2a0086e4fc8a23f0545a81c2bd6e324db26a5801/maturin-1.11.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:2a4e872fb78e77748217084ffeb59de565d08a86ccefdace054520aaa7b66db4", size = 11384741, upload-time = "2026-01-09T11:06:15.261Z" }, - { url = "https://files.pythonhosted.org/packages/2e/da/7118de648182971d723ea99d79c55007f96cdafc95f5322cc1ad15f6683e/maturin-1.11.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2079447967819b5cf615e5b5b99a406d662effdc8d6afd493dcd253c6afc3707", size = 9423814, upload-time = "2026-01-09T11:05:57.242Z" }, - { url = "https://files.pythonhosted.org/packages/cf/8f/be14395c6e23b19ddaa0c171e68915bdcd1ef61ad1f411739c6721196903/maturin-1.11.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:50f6c668c1d5d4d4dc1c3ffec7b4270dab493e5b2368f8e4213f4bcde6a50eea", size = 9104378, upload-time = "2026-01-09T11:05:59.835Z" }, - { url = "https://files.pythonhosted.org/packages/77/83/53ea82a2f42a03930ea5545673d11a4ef49bb886827353a701f41a5f11c4/maturin-1.11.5-py3-none-win32.whl", hash = "sha256:49f85ce6cbe478e9743ecddd6da2964afc0ded57013aa4d054256be702d23d40", size = 7738696, upload-time = "2026-01-09T11:06:06.651Z" }, - { url = "https://files.pythonhosted.org/packages/3c/41/353a26d49aa80081c514a6354d429efbecedb90d0153ec598cece3baa607/maturin-1.11.5-py3-none-win_amd64.whl", hash = "sha256:70d3e5beffb9ef9dfae5f3c1a7eeb572091505eb8cb076e9434518df1c42a73b", size = 9029838, upload-time = "2026-01-09T11:05:54.543Z" }, - { url = "https://files.pythonhosted.org/packages/15/67/c94f8f5440bc42d54113a2d99de0d6107f06b5a33f31823e52b2715d856f/maturin-1.11.5-py3-none-win_arm64.whl", hash = "sha256:9348f7f0a346108e0c96e6719be91da4470bd43c15802435e9f4157f5cca43d4", size = 7624029, upload-time = "2026-01-09T11:06:08.728Z" }, -] - [[package]] name = "openshell" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "cloudpickle" }, { name = "grpcio" }, @@ -336,7 +315,6 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "grpcio-tools" }, - { name = "maturin" }, { name = "pyelftools" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -344,7 +322,6 @@ dev = [ { name = "pytest-xdist" }, { name = "pyyaml" }, { name = "ruff" }, - { name = "setuptools-scm" }, { name = "ty" }, { name = "types-pyyaml" }, ] @@ -360,7 +337,6 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "grpcio-tools", specifier = ">=1.60" }, - { name = "maturin", specifier = ">=1.5,<2.0" }, { name = "pyelftools", specifier = ">=0.30" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=0.23" }, @@ -368,7 +344,6 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.0" }, { name = "pyyaml", specifier = "==6.0.2" }, { name = "ruff", specifier = ">=0.4" }, - { name = "setuptools-scm", specifier = ">=8" }, { name = "ty", specifier = ">=0.0.1a6" }, { name = "types-pyyaml", specifier = ">=6.0.12.20260518" }, ] @@ -550,19 +525,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, ] -[[package]] -name = "setuptools-scm" -version = "9.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7b/b1/19587742aad604f1988a8a362e660e8c3ac03adccdb71c96d86526e5eb62/setuptools_scm-9.2.2.tar.gz", hash = "sha256:1c674ab4665686a0887d7e24c03ab25f24201c213e82ea689d2f3e169ef7ef57", size = 203385, upload-time = "2025-10-19T22:08:05.608Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ea/ac2bf868899d0d2e82ef72d350d97a846110c709bacf2d968431576ca915/setuptools_scm-9.2.2-py3-none-any.whl", hash = "sha256:30e8f84d2ab1ba7cb0e653429b179395d0c33775d54807fc5f1dd6671801aef7", size = 62975, upload-time = "2025-10-19T22:08:04.007Z" }, -] - [[package]] name = "tomli" version = "2.4.1"