From 0188f5a0de6cb2312bd683624b02dcb93bccc456 Mon Sep 17 00:00:00 2001 From: Ivan Zatevakhin Date: Sun, 9 Aug 2026 06:19:04 +0100 Subject: [PATCH] feat(sys): add generated vllm.cpp C bindings --- .github/workflows/sys.yml | 41 +++ Justfile | 488 ++++++++++++++++++++++++++++ README.md | 30 +- flake.nix | 4 + vllm-cpp-sys/Cargo.toml | 3 + vllm-cpp-sys/README.md | 30 +- vllm-cpp-sys/build.rs | 190 +++++++++-- vllm-cpp-sys/src/bindings.rs | 174 ++++++++++ vllm-cpp-sys/src/build_support.rs | 67 ++++ vllm-cpp-sys/src/lib.rs | 138 +------- vllm-cpp-sys/tests/build_support.rs | 132 ++++++++ vllm-cpp-sys/tests/layout.c | 63 ++++ vllm-cpp-sys/tests/layout.rs | 182 +++++++++++ vllm-cpp-sys/tests/symbols.rs | 7 +- vllm-cpp-sys/wrapper.h | 1 + vllm-cpp/Cargo.toml | 2 + 16 files changed, 1372 insertions(+), 180 deletions(-) create mode 100644 .github/workflows/sys.yml create mode 100644 Justfile create mode 100644 vllm-cpp-sys/src/bindings.rs create mode 100644 vllm-cpp-sys/src/build_support.rs create mode 100644 vllm-cpp-sys/tests/build_support.rs create mode 100644 vllm-cpp-sys/tests/layout.c create mode 100644 vllm-cpp-sys/tests/layout.rs create mode 100644 vllm-cpp-sys/wrapper.h diff --git a/.github/workflows/sys.yml b/.github/workflows/sys.yml new file mode 100644 index 0000000..ae6f7b6 --- /dev/null +++ b/.github/workflows/sys.yml @@ -0,0 +1,41 @@ +name: sys + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +env: + CMAKE_BUILD_PARALLEL_LEVEL: "2" + CMAKE_GENERATOR: Ninja + +jobs: + conformance: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy,rustfmt + - uses: Swatinem/rust-cache@v2 + - name: install native tools + run: sudo apt-get update && sudo apt-get install --yes binutils build-essential clang cmake jq ninja-build + - name: install pinned maintainer tools + run: | + cargo install bindgen-cli --version 0.72.1 --locked + cargo install just --version 1.40.0 --locked + - name: check formatting + run: just fmt-check + - name: lint + run: just lint + - name: check generated bindings and ffi conformance + run: just sys + - name: test Linux CPU link modes + run: just link-modes + - name: test packaged sys crate and downstream fixture + run: just package-test diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..bf281fd --- /dev/null +++ b/Justfile @@ -0,0 +1,488 @@ +set shell := ["bash", "-euo", "pipefail", "-c"] + +root := justfile_directory() +bindings_file := root + "/vllm-cpp-sys/src/bindings.rs" + +# Maintainer workflows require Just 1.40 or newer. + +export CMAKE_GENERATOR := env_var_or_default("CMAKE_GENERATOR", "Ninja") +export CMAKE_BUILD_PARALLEL_LEVEL := env_var_or_default("CMAKE_BUILD_PARALLEL_LEVEL", "2") + +# List available maintainer workflows. +default: + @just --justfile {{ quote(root + "/Justfile") }} --list + +# Regenerate the checked-in Rust bindings (bindgen 0.72.1 required). +bindings output=bindings_file: + just --justfile '{{ root }}/Justfile' _bindings {{ quote(output) }} + +[private] +_bindings output: + #!/usr/bin/env bash + set -euo pipefail + output={{ quote(output) }} + version=$(bindgen --version) + if [[ $version != 'bindgen 0.72.1' ]]; then + echo "expected bindgen 0.72.1, found: $version" >&2 + exit 1 + fi + mkdir -p "$(dirname "$output")" + output_dir=$(cd "$(dirname "$output")" && pwd -P) + output="$output_dir/$(basename "$output")" + cd {{ quote(root + "/vllm-cpp-sys") }} + bindgen wrapper.h \ + --allowlist-function '^vllm_.*' \ + --allowlist-type '^vllm_.*' \ + --allowlist-var '^VLLM_.*' \ + --no-doc-comments \ + --no-layout-tests \ + --formatter rustfmt \ + --rust-target 1.85 \ + --output "$output" \ + -- \ + -std=c11 \ + -I. + +# Check that the committed bindings match bindgen 0.72.1 output. +bindings-check: + #!/usr/bin/env bash + set -euo pipefail + temp=$(mktemp) + trap 'rm -f "$temp"' EXIT + just --justfile {{ quote(root + "/Justfile") }} _bindings "$temp" + cmp "$temp" {{ quote(bindings_file) }} + +# Check that the wrapper is valid C11 and C++20. +header-check: + #!/usr/bin/env bash + set -euo pipefail + "${CC:-cc}" -std=c11 -fsyntax-only -x c -I'{{ root }}/vllm-cpp-sys' \ + '{{ root }}/vllm-cpp-sys/wrapper.h' + "${CXX:-c++}" -std=c++20 -fsyntax-only -x c++ -I'{{ root }}/vllm-cpp-sys' \ + '{{ root }}/vllm-cpp-sys/wrapper.h' + +# Run focused tests for private build-script artifact selection helpers. +build-support-test: + #!/usr/bin/env bash + set -euo pipefail + temp=$(mktemp -d) + trap 'rm -rf "$temp"' EXIT + rustc --edition=2021 --test -D warnings \ + {{ quote(root + "/vllm-cpp-sys/tests/build_support.rs") }} \ + -o "$temp/build-support-tests" + "$temp/build-support-tests" + +# Run the focused C/Rust layout conformance test. +layout-test: + CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-{{ root }}/target/layout-test}" cargo test --locked -p vllm-cpp-sys --release --test layout + +# Run generated-binding, header, and layout conformance checks. +sys: bindings-check header-check build-support-test layout-test + +# Test all Linux CPU link modes and the exact shared-library exports. +link-modes: + #!/usr/bin/env bash + set -euo pipefail + repo_root={{ quote(root) }} + target_base=${CARGO_TARGET_DIR:-"$repo_root/target/link-modes"} + mkdir -p "$target_base" + target_base=$(cd "$target_base" && pwd -P) + recipe_base="$target_base/vllm-cpp-sys-link-modes" + mkdir -p "$recipe_base" + recipe_base=$(cd "$recipe_base" && pwd -P) + work=$(mktemp -d -- "$recipe_base/run.XXXXXXXXXX") + cleanup() { + case $work in + "$recipe_base"/run.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected link-modes work directory: $work" >&2 ;; + esac + } + trap cleanup EXIT + cd "$repo_root" + export CARGO_TERM_COLOR=never + + find_one() { + local root=$1 + local path_pattern=$2 + local matches=() + mapfile -t matches < <( + find -L "$root" -type f -path "$path_pattern" -print | LC_ALL=C sort + ) + if ((${#matches[@]} != 1)); then + printf 'expected exactly one %s below %s, found %d\n' \ + "$path_pattern" "$root" "${#matches[@]}" >&2 + printf '%s\n' "${matches[@]}" >&2 + exit 1 + fi + printf '%s\n' "${matches[0]}" + } + + find_installed_library() { + local root=$1 + local filename=$2 + local matches=() + mapfile -t matches < <( + find -L "$root" \ + \( -path "*/out/lib/$filename" -o -path "*/out/lib64/$filename" \) \ + -type f -print | LC_ALL=C sort + ) + if ((${#matches[@]} != 1)); then + printf 'expected exactly one installed %s below %s, found %d\n' \ + "$filename" "$root" "${#matches[@]}" >&2 + printf '%s\n' "${matches[@]}" >&2 + exit 1 + fi + printf '%s\n' "${matches[0]}" + } + + check_exports() { + local library=$1 + local actual="$work/exports.actual" + local expected="$work/exports.expected" + nm -D --defined-only "$library" \ + | awk '$2 != "A" { name=$3; sub(/@@.*/, "", name); print name }' \ + | LC_ALL=C sort > "$actual" + cat > "$expected" <<'EOF' + vllm_abi_version + vllm_chat + vllm_chat_stream + vllm_complete + vllm_complete_stream + vllm_completion_free + vllm_engine_free + vllm_engine_load + vllm_last_error + vllm_model_params_default + vllm_request_cancel + vllm_request_done + vllm_request_error + vllm_request_free + vllm_request_submit + vllm_request_wait + vllm_sampling_params_default + vllm_string_free + vllm_version + EOF + diff -u "$expected" "$actual" + } + + bundled_static_target="$work/bundled-static" + echo '==> bundled static' + CARGO_TARGET_DIR="$bundled_static_target" \ + cargo test --locked -p vllm-cpp-sys --release --tests + + bundled_dynamic_target="$work/bundled-dynamic" + echo '==> bundled dynamic build' + CARGO_TARGET_DIR="$bundled_dynamic_target" \ + cargo build --locked -p vllm-cpp-sys --release --features dynamic-link + bundled_dynamic_lib=$(find_installed_library \ + "$bundled_dynamic_target/release/build" libvllm.so) + check_exports "$bundled_dynamic_lib" + bundled_dynamic_lib_dir=$(dirname "$bundled_dynamic_lib") + echo '==> bundled dynamic test' + LD_LIBRARY_PATH="$bundled_dynamic_lib_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + CARGO_TARGET_DIR="$bundled_dynamic_target" \ + cargo test --locked -p vllm-cpp-sys --release --tests --features dynamic-link + + prefix="$work/system-prefix" + mkdir -p "$prefix/include" "$prefix/lib" "$prefix/lib64" "$prefix/blake3-lib" + cp vllm-cpp-sys/vllm.cpp/include/vllm.h "$prefix/include/" + + bundled_static_lib=$(find_installed_library \ + "$bundled_static_target/release/build" libvllm.a) + bundled_static_lib_dir=$(dirname "$bundled_static_lib") + cp "$bundled_static_lib" "$prefix/lib/" + cp -a "$bundled_static_lib_dir"/libvllm.so* "$prefix/lib/" + blake3_lib=$(find_one \ + "$bundled_static_target/release/build" '*/out/build/libblake3_vendored.a') + cp "$blake3_lib" "$prefix/blake3-lib/" + + system_static_target="$work/system-static" + system_static_log="$work/system-static.log" + echo '==> system static' + VLLM_CPP_ROOT="$prefix" \ + VLLM_CPP_BLAKE3_LIB_DIR="$prefix/blake3-lib" \ + CARGO_TARGET_DIR="$system_static_target" \ + cargo test --locked -vv -p vllm-cpp-sys --release --tests \ + --no-default-features --features system 2>&1 | tee "$system_static_log" + grep -Fq "cargo:rerun-if-changed=$prefix/lib/libvllm.a" "$system_static_log" + grep -Fq \ + "cargo:rerun-if-changed=$prefix/blake3-lib/libblake3_vendored.a" \ + "$system_static_log" + blake3_search_line=$(grep -nF -m1 \ + "cargo:rustc-link-search=native=$prefix/blake3-lib" "$system_static_log" \ + | cut -d: -f1 || true) + vllm_search_line=$(grep -nF -m1 \ + "cargo:rustc-link-search=native=$prefix/lib" "$system_static_log" \ + | cut -d: -f1 || true) + if [[ -z $blake3_search_line || -z $vllm_search_line \ + || $blake3_search_line -ge $vllm_search_line ]]; then + echo 'system static build did not select lib over empty lib64 or preserve BLAKE3 link order' >&2 + exit 1 + fi + + echo '==> system static watched-archive rerun' + replacement="$prefix/lib/.libvllm.a.replacement" + cp "$prefix/lib/libvllm.a" "$replacement" + touch "$replacement" + mv "$replacement" "$prefix/lib/libvllm.a" + system_static_rerun_log="$work/system-static-rerun.log" + VLLM_CPP_ROOT="$prefix" \ + VLLM_CPP_BLAKE3_LIB_DIR="$prefix/blake3-lib" \ + CARGO_TARGET_DIR="$system_static_target" \ + cargo test --locked -vv -p vllm-cpp-sys --release --tests \ + --no-default-features --features system 2>&1 | tee "$system_static_rerun_log" + grep -Fq 'Dirty vllm-cpp-sys' "$system_static_rerun_log" + grep -Fq 'libvllm.a' "$system_static_rerun_log" + grep -Fq "$prefix/lib/libvllm.a" "$system_static_rerun_log" + grep -Fq "cargo:rerun-if-changed=$prefix/lib/libvllm.a" \ + "$system_static_rerun_log" + + system_override_static_target="$work/system-override-static" + system_override_static_log="$work/system-override-static.log" + echo '==> system static invalid override' + if VLLM_CPP_ROOT="$prefix" \ + VLLM_CPP_LIB_DIR="$prefix/lib64" \ + VLLM_CPP_BLAKE3_LIB_DIR="$prefix/blake3-lib" \ + CARGO_TARGET_DIR="$system_override_static_target" \ + cargo test --locked -vv -p vllm-cpp-sys --release --tests \ + --no-default-features --features system 2>&1 \ + | tee "$system_override_static_log"; then + echo 'expected system static VLLM_CPP_LIB_DIR override to reject the empty lib64 directory' >&2 + exit 1 + fi + grep -Fq \ + "expected VLLM_CPP_LIB_DIR override artifact libvllm.a at $prefix/lib64/libvllm.a" \ + "$system_override_static_log" + + system_dynamic_target="$work/system-dynamic" + system_dynamic_log="$work/system-dynamic.log" + echo '==> system dynamic' + LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + VLLM_CPP_ROOT="$prefix" \ + CARGO_TARGET_DIR="$system_dynamic_target" \ + cargo test --locked -vv -p vllm-cpp-sys --release --tests \ + --no-default-features --features system,dynamic-link 2>&1 \ + | tee "$system_dynamic_log" + grep -Fq "cargo:rerun-if-changed=$prefix/lib/libvllm.so" \ + "$system_dynamic_log" + grep -Fq "cargo:rustc-link-search=native=$prefix/lib" \ + "$system_dynamic_log" + + system_override_dynamic_target="$work/system-override-dynamic" + system_override_dynamic_log="$work/system-override-dynamic.log" + echo '==> system dynamic invalid override' + if LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + VLLM_CPP_ROOT="$prefix" \ + VLLM_CPP_LIB_DIR="$prefix/lib64" \ + CARGO_TARGET_DIR="$system_override_dynamic_target" \ + cargo test --locked -vv -p vllm-cpp-sys --release --tests \ + --no-default-features --features system,dynamic-link 2>&1 \ + | tee "$system_override_dynamic_log"; then + echo 'expected system dynamic VLLM_CPP_LIB_DIR override to reject the empty lib64 directory' >&2 + exit 1 + fi + grep -Fq \ + "expected VLLM_CPP_LIB_DIR override artifact libvllm.so at $prefix/lib64/libvllm.so" \ + "$system_override_dynamic_log" + + echo 'all four Linux CPU link modes passed' + +# Test the packaged sys crate and an offline downstream fixture. +package-test: + #!/usr/bin/env bash + set -euo pipefail + repo_root={{ quote(root) }} + cd "$repo_root" + package_target=${CARGO_TARGET_DIR:-$repo_root/target/package-gate} + mkdir -p "$package_target" + package_target=$(cd "$package_target" && pwd -P) + export CARGO_TARGET_DIR="$package_target" + + command -v jq >/dev/null || { + echo 'jq is required to parse Cargo metadata' >&2 + exit 1 + } + if [[ ${CARGO_NET_OFFLINE:-false} != true ]]; then + cargo fetch --locked + fi + export CARGO_NET_OFFLINE=true + version=$(cargo metadata --locked --offline --no-deps --format-version 1 \ + | jq -er '[.packages[] | select(.name == "vllm-cpp-sys") | .version] | if length == 1 then .[0] else error("expected exactly one vllm-cpp-sys package") end') + + package_args=() + if [[ -n $(git status --porcelain=v1 --untracked-files=all -- vllm-cpp-sys) ]]; then + package_args+=(--allow-dirty) + fi + cargo package -p vllm-cpp-sys --locked --offline "${package_args[@]}" + package_file="$package_target/package/vllm-cpp-sys-$version.crate" + temp=$(mktemp -d) + trap 'rm -rf "$temp"' EXIT + tar -xzf "$package_file" -C "$temp" + package_root="$temp/vllm-cpp-sys-$version" + temp_target="$temp/target" + + package_size=$(stat -c '%s' "$package_file") + unpacked_size=$(du -sb "$package_root" | cut -f1) + entry_count=$(tar -tzf "$package_file" | wc -l) + max_compressed=$((6 * 1024 * 1024)) + max_unpacked=$((36 * 1024 * 1024)) + max_entries=1300 + ((package_size <= max_compressed)) || { + echo "package exceeds compressed budget: $package_size > $max_compressed" >&2 + exit 1 + } + ((unpacked_size <= max_unpacked)) || { + echo "package exceeds unpacked budget: $unpacked_size > $max_unpacked" >&2 + exit 1 + } + ((entry_count <= max_entries)) || { + echo "package exceeds entry budget: $entry_count > $max_entries" >&2 + exit 1 + } + + required_members=( + Cargo.lock + Cargo.toml + LICENSE-APACHE + LICENSE-MIT + NOTICE + README.md + THIRD_PARTY.md + build.rs + wrapper.h + src/bindings.rs + src/build_support.rs + src/lib.rs + tests/build_support.rs + tests/layout.c + tests/layout.rs + tests/symbols.rs + vllm.cpp/CMakeLists.txt + vllm.cpp/LICENSE + vllm.cpp/NOTICE + vllm.cpp/include/vllm.h + vllm.cpp/src/capi/chat_prompt.cpp + vllm.cpp/src/capi/chat_prompt.h + vllm.cpp/src/capi/engine_handle.h + vllm.cpp/src/capi/vllm_c.cpp + vllm.cpp/src/vllm/version.cpp + vllm.cpp/third_party/README.md + vllm.cpp/third_party/blake3/LICENSE_A2 + vllm.cpp/third_party/blake3/LICENSE_CC0 + vllm.cpp/third_party/minja/LICENSE + vllm.cpp/third_party/nlohmann/json.hpp + ) + for member in "${required_members[@]}"; do + [[ -s $package_root/$member ]] || { + echo "packaged crate is missing required member: $member" >&2 + exit 1 + } + done + + native_inventory() { + local base=$1 + local member + shift + for member in "$@"; do + find "$base/$member" -type f -print + done | sed "s#^$base/##" | LC_ALL=C sort + } + native_members=( + vllm.cpp/CMakeLists.txt + vllm.cpp/LICENSE + vllm.cpp/NOTICE + vllm.cpp/cmake + vllm.cpp/include + vllm.cpp/src + vllm.cpp/third_party/README.md + vllm.cpp/third_party/blake3 + vllm.cpp/third_party/minja + vllm.cpp/third_party/nlohmann + ) + diff -u \ + <(native_inventory "$repo_root/vllm-cpp-sys" "${native_members[@]}") \ + <(native_inventory "$package_root" "${native_members[@]}") + diff -u \ + <(printf '%s\n' CMakeLists.txt LICENSE NOTICE cmake include src third_party | LC_ALL=C sort) \ + <(find "$package_root/vllm.cpp" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort) + denied_members=( + Justfile + layout.c + vllm.cpp/.agents + vllm.cpp/.git + vllm.cpp/.github + vllm.cpp/assets + vllm.cpp/benchmarks + vllm.cpp/docs + vllm.cpp/examples + vllm.cpp/scripts + vllm.cpp/tests + vllm.cpp/tools + vllm.cpp/triton_kernels + vllm.cpp/third_party/doctest + vllm.cpp/third_party/httplib + vllm.cpp/third_party/vulkan + ) + for member in "${denied_members[@]}"; do + [[ ! -e $package_root/$member ]] || { + echo "denied upstream tree or record leaked into package: $member" >&2 + exit 1 + } + done + + ( + cd "$package_root" + CARGO_NET_OFFLINE=true CARGO_TARGET_DIR="$temp_target" \ + cargo test --locked --release --tests --offline + ) + + fixture="$temp/downstream" + mkdir -p "$fixture/src" + cat > "$fixture/Cargo.toml" < "$fixture/src/main.rs" <<'EOF' + use std::ffi::CStr; + + use vllm_cpp_sys as ffi; + + fn main() { + assert_eq!(ffi::VLLM_ABI_VERSION, 10); + assert_eq!(unsafe { ffi::vllm_abi_version() }, 10); + let version = unsafe { CStr::from_ptr(ffi::vllm_version()) }; + assert!(!version.to_bytes().is_empty()); + + let mut params = unsafe { ffi::vllm_model_params_default() }; + params.model_path = c"/nonexistent/vllm-cpp-rs-package-smoke".as_ptr(); + let mut engine = std::ptr::null_mut(); + let status = unsafe { ffi::vllm_engine_load(¶ms, &mut engine) }; + assert_eq!(status, ffi::vllm_status_VLLM_ERR_MODEL_LOAD); + assert!(engine.is_null()); + } + EOF + ( + cd "$fixture" + CARGO_NET_OFFLINE=true cargo generate-lockfile --offline + CARGO_NET_OFFLINE=true CARGO_TARGET_DIR="$temp_target" \ + cargo run --locked --release --offline + ) + +# Check Just and Rust formatting. +fmt-check: + just --unstable --justfile {{ quote(root + "/Justfile") }} --fmt --check + cargo fmt --all -- --check + +# Lint all workspace targets with warnings denied. +lint: + cargo clippy --locked --workspace --all-targets -- -D warnings + +# Run the complete maintainer gate serially; do not pass --jobs. +ci: fmt-check lint sys link-modes package-test diff --git a/README.md b/README.md index ffd26fa..92b359a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Rust bindings for [vllm.cpp](https://github.com/mudler/vllm.cpp), organized as: ## Status -This bootstrap PR establishes the workspace, reproducible bundled CPU build, source packaging, and link smoke tests. Later PRs add generated FFI and the high-level API. +The sys crate provides checked-in generated raw FFI declarations. Conformance checks cover C/Rust layout, all 19 exported C symbols, and C ABI version 10. Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. A high-level API follows in a later PR. vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. @@ -20,7 +20,7 @@ Initial development and testing support Linux CPU builds. They require: - Ninja or another CMake build tool. - A C11 and C++20 compiler. - A system linker and C++ standard library. -- Git for source checkouts. +- Just 1.40 or newer for maintainer workflows, plus Git, `jq`, and GNU tar. This repository provides a Nix development shell with the pinned development tools: @@ -38,31 +38,41 @@ git submodule update --init --recursive ## Build and Test -Inside the Nix shell: +Inside the Nix shell, use the root `Justfile` (requires Just 1.40 or newer) for maintainer workflows: ```console -CMAKE_GENERATOR=Ninja cargo build --release -CMAKE_GENERATOR=Ninja cargo test --workspace --release +CMAKE_GENERATOR=Ninja cargo build --locked --release +CMAKE_GENERATOR=Ninja cargo test --locked --workspace --release +just ci ``` Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The bundled build is deterministic and CPU-only: native tests, examples, the HTTP server, CUDA, Metal, MLX, Vulkan, Triton, and CUTLASS fetching are disabled explicitly. -`build.rs` does not download native dependencies. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. +`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. `just package-test` requires `jq` and GNU tar, fetches locked Cargo dependencies when `CARGO_NET_OFFLINE` is not `true`, and then validates the package offline. + +## Link Modes + +- The default `bundled` feature builds and statically links the pinned CPU-only source. +- `bundled,dynamic-link` builds and dynamically links the pinned shared library. +- `system` requires `--no-default-features` and links a prefix selected by `VLLM_CPP_ROOT`. +- `system,dynamic-link` dynamically links the selected system library. + +System-mode consumer builds validate that `VLLM_CPP_ROOT/include/vllm.h` exists; they do not compare its layout. The maintainer `layout-test` recipe compiles `tests/layout.c` at test runtime against the bundled header and compares its C layouts with the generated Rust declarations. The `link-modes` recipe repeats layout conformance against bundled and system fixture headers and also runs the runtime ABI test. `VLLM_CPP_LIB_DIR` selects a nonstandard vllm library directory. Upstream's normal CMake install does not install `libblake3_vendored.a`, so a stock install works directly with `system,dynamic-link`; system static users must provision that archive separately and set `VLLM_CPP_BLAKE3_LIB_DIR` (or place it in the vllm library directory). Dynamic tests and applications must make `libvllm.so` loader-visible with `LD_LIBRARY_PATH`, rpath, or an installed loader path; Cargo neither deploys the library nor configures rpath. Native Linux CPU is the supported target; cross-compiling the layout integration test is unsupported because it executes the compiled target probe. ## Packaging Inspect and build the sys package with: ```console -cargo package -p vllm-cpp-sys --list -cargo package -p vllm-cpp-sys +cargo package -p vllm-cpp-sys --locked --list +just package-test ``` -The package carries only the native build inputs and required licenses/notices; upstream tests, large fixtures, media, benchmarks, and agent records are excluded. The bootstrap package measures approximately 30 MiB unpacked and 4.2 MiB compressed. +The package carries only the native build inputs and required licenses/notices; upstream tests, large fixtures, media, benchmarks, and agent records are excluded. The package measures approximately 30 MiB unpacked and 4.2 MiB compressed. ## Support -The bootstrap is validated on Linux x86_64 with a bundled static CPU build. Other operating systems, architectures, dynamic/system linking, and accelerator backends are not yet supported by the Rust build. +CI runs Linux x86_64 CPU tests in all four link modes: bundled static/dynamic and system static/dynamic. The system prefix is a test fixture assembled from bundled build outputs; its separately copied BLAKE3 archive demonstrates the explicit system-static contract rather than claiming that a stock upstream install provides one. Other operating systems, architectures, and accelerator backends are not yet supported by the Rust build. ## Licensing and Affiliation diff --git a/flake.nix b/flake.nix index 92243e9..26ca83f 100644 --- a/flake.nix +++ b/flake.nix @@ -23,8 +23,12 @@ packages = [ rustToolchain pkgs.cmake + pkgs.git + pkgs.just + pkgs.jq pkgs.ninja pkgs.pkg-config + pkgs.gnutar pkgs.llvmPackages.clang pkgs.llvmPackages.bintools pkgs.rust-bindgen diff --git a/vllm-cpp-sys/Cargo.toml b/vllm-cpp-sys/Cargo.toml index daf1648..98082c6 100644 --- a/vllm-cpp-sys/Cargo.toml +++ b/vllm-cpp-sys/Cargo.toml @@ -10,6 +10,7 @@ links = "vllm" build = "build.rs" include = [ "/build.rs", + "/wrapper.h", "/src/**", "/tests/**", "/Cargo.toml", @@ -33,6 +34,8 @@ include = [ [features] default = ["bundled"] bundled = [] +system = [] +dynamic-link = [] [build-dependencies] cmake = "0.1.58" diff --git a/vllm-cpp-sys/README.md b/vllm-cpp-sys/README.md index 2ba5838..f679e9c 100644 --- a/vllm-cpp-sys/README.md +++ b/vllm-cpp-sys/README.md @@ -1,8 +1,34 @@ # vllm-cpp-sys -Raw Rust bindings and a bundled native build for [vllm.cpp](https://github.com/mudler/vllm.cpp). +Raw Rust bindings and native linking for [vllm.cpp](https://github.com/mudler/vllm.cpp). -This crate is an implementation detail for `vllm-cpp`. Its public functions are unsafe and mirror the stable C API. The bundled source is pinned to commit `34aedfbe8ed9779697905541a62e2160ccfd9c05` and builds CPU-only by default. +This crate exposes generated unsafe functions that mirror the stable C API. Applications should use the application-facing `vllm-cpp` bindings when implemented. Ordinary consumers do not need Just, bindgen, or libclang. + +## Link Modes + +- `bundled` is enabled by default. It builds the pinned vllm.cpp source as a static CPU library. +- `dynamic-link` makes either source mode link `libvllm` dynamically. +- `system` disables the bundled build and links a caller-provided installation. Disable default features when selecting it. + +`bundled` and `system` are mutually exclusive. Initial support is limited to native Linux CPU builds. + +System mode requires `VLLM_CPP_ROOT`, whose prefix must contain `include/vllm.h` plus a `lib` or `lib64` directory. `VLLM_CPP_LIB_DIR` can override the vllm library directory. Consumer builds validate that the selected system header exists but do not compare its layout. The maintainer integration test compiles its C probe at test runtime against that header, compares its C layouts with the generated Rust declarations, and the runtime test requires ABI version 10. + +Upstream's normal CMake install provides `libvllm` and `vllm.h` but does not install the private `libblake3_vendored.a` target. A stock install therefore works directly with `system,dynamic-link`. System static mode requires callers to provision the matching `libblake3_vendored.a` separately and set `VLLM_CPP_BLAKE3_LIB_DIR`; when the variable is unset, the build script checks the selected vllm library directory for backward compatibility. It does not search arbitrary build trees. + +`dynamic-link` does not copy or package the shared library. Tests and applications must install `libvllm` in a loader-visible location or configure `LD_LIBRARY_PATH`, rpath, or another loader search path. + +## Generated Bindings + +The bundled source is pinned to commit `34aedfbe8ed9779697905541a62e2160ccfd9c05` and exposes C ABI version 10. Bindings are generated with bindgen 0.72.1 from `wrapper.h`, which includes `vllm.cpp/include/vllm.h`, and committed to `src/bindings.rs`. Maintainers use Just 1.40 or newer from the repository root: + +```console +just bindings +just sys +just link-modes +``` + +The conformance gate verifies the generated output, C and C++ header compatibility, C/Rust layout, the exact 19-symbol export set, and compile-time and runtime ABI 10. CI separately runs bundled static/dynamic and fixture-backed system static/dynamic tests; dynamic tests set the required loader path. `build.rs` only performs consumer native build/link integration and does not compile or execute the layout probe. `tests/layout.rs` compiles and executes `tests/layout.c` with the bundled header or `VLLM_CPP_ROOT/include/vllm.h` at test runtime using the Rust standard library. Native Linux CPU is the supported target; cross-compiling that integration test is unsupported. The Rust crate is dual-licensed under MIT or Apache-2.0. The bundled vllm.cpp source retains its upstream Apache-2.0 license and notices. diff --git a/vllm-cpp-sys/build.rs b/vllm-cpp-sys/build.rs index 4838f56..737cdcf 100644 --- a/vllm-cpp-sys/build.rs +++ b/vllm-cpp-sys/build.rs @@ -1,13 +1,21 @@ +#[path = "src/build_support.rs"] +mod build_support; + use std::env; use std::fs; use std::path::{Path, PathBuf}; +use build_support::{find_installed_library_dir, require_library_file, shared_library_name}; + const RERUN_ENV: &[&str] = &[ "CMAKE_BUILD_PARALLEL_LEVEL", "CMAKE_GENERATOR", "CMAKE_GENERATOR_PLATFORM", "CMAKE_GENERATOR_TOOLSET", "CMAKE_PREFIX_PATH", + "VLLM_CPP_ROOT", + "VLLM_CPP_LIB_DIR", + "VLLM_CPP_BLAKE3_LIB_DIR", "CC", "CFLAGS", "CXX", @@ -18,7 +26,7 @@ fn main() { for path in [ "vllm.cpp/CMakeLists.txt", "vllm.cpp/cmake", - "vllm.cpp/include", + "vllm.cpp/include/vllm.h", "vllm.cpp/src", "vllm.cpp/third_party/blake3", "vllm.cpp/third_party/minja", @@ -30,10 +38,32 @@ fn main() { println!("cargo:rerun-if-env-changed={name}"); } - if !cfg!(feature = "bundled") { - panic!("vllm-cpp-sys supports only the default `bundled` feature in this bootstrap"); + if env::var_os("DOCS_RS").is_some() { + return; + } + + let bundled = cfg!(feature = "bundled"); + let system = cfg!(feature = "system"); + match (bundled, system) { + (true, true) => panic!("features `bundled` and `system` are mutually exclusive"), + (false, false) => panic!("enable exactly one of the `bundled` or `system` features"), + _ => {} + } + + let target_os = env::var("CARGO_CFG_TARGET_OS").expect("cargo sets target OS"); + if target_os != "linux" { + panic!("linking is implemented only for Linux, not {target_os}"); + } + + let system_root = system.then(validate_system_root); + if bundled { + build_bundled(); + } else { + link_system(system_root.as_deref().expect("system root was validated")); } +} +fn build_bundled() { let source = Path::new("vllm.cpp"); if !source.join("CMakeLists.txt").is_file() { panic!( @@ -56,62 +86,158 @@ fn main() { .define("VLLM_CPP_CUTLASS_FETCH", "OFF") .define("VLLM_CPP_SANITIZE", "OFF"); + let dynamic_link = cfg!(feature = "dynamic-link"); + let vllm_artifact = if dynamic_link { + shared_library_name("vllm") + } else { + static_library_name("vllm") + }; + let install = config.build(); + let installed_lib_dir = find_installed_library_dir(&install, &vllm_artifact) + .unwrap_or_else(|error| panic!("failed to select bundled library directory: {error}")); + let vllm = require_library_file( + &installed_lib_dir, + &vllm_artifact, + if dynamic_link { + "bundled dynamic vllm" + } else { + "bundled static vllm" + }, + ) + .unwrap_or_else(|error| panic!("{error}")); + println!("cargo:rerun-if-changed={}", vllm.display()); + println!( + "cargo:rustc-link-search=native={}", + installed_lib_dir.display() + ); + + if dynamic_link { + println!("cargo:rustc-link-lib=dylib=vllm"); + return; + } + let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("cargo always sets OUT_DIR")); - let installed_lib_dir = find_installed_library_dir(&install); - let vllm = find_unique_file(&installed_lib_dir, static_library_name("vllm")) - .unwrap_or_else(|error| panic!("failed to locate installed vllm archive: {error}")); let blake3 = find_unique_file( &out_dir.join("build"), static_library_name("blake3_vendored"), ) .unwrap_or_else(|error| panic!("failed to locate blake3_vendored build archive: {error}")); - + println!("cargo:rerun-if-changed={}", blake3.display()); println!( "cargo:rustc-link-search=native={}", - vllm.parent().expect("library has a parent").display() + blake3.parent().expect("library has a parent").display() ); + println!("cargo:rustc-link-lib=static:+whole-archive=vllm"); + println!("cargo:rustc-link-lib=static=blake3_vendored"); + link_platform_dependencies(); +} + +fn validate_system_root() -> PathBuf { + let root = required_path("VLLM_CPP_ROOT"); + let header = root.join("include/vllm.h"); + if !header.is_file() { + panic!( + "VLLM_CPP_ROOT must contain include/vllm.h; missing {}", + header.display() + ); + } + println!("cargo:rerun-if-changed={}", header.display()); + root +} + +fn link_system(root: &Path) { + let dynamic_link = cfg!(feature = "dynamic-link"); + let vllm_artifact = if dynamic_link { + shared_library_name("vllm") + } else { + static_library_name("vllm") + }; + let lib_dir = system_library_dir(root, &vllm_artifact); + let vllm = require_library_file( + &lib_dir, + &vllm_artifact, + if dynamic_link { + "system dynamic vllm" + } else { + "system static vllm" + }, + ) + .unwrap_or_else(|error| panic!("{error}")); + println!("cargo:rerun-if-changed={}", vllm.display()); + + if dynamic_link { + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rustc-link-lib=dylib=vllm"); + return; + } + + let blake3_lib_dir = env::var_os("VLLM_CPP_BLAKE3_LIB_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| lib_dir.clone()); + if !blake3_lib_dir.is_dir() { + panic!( + "VLLM_CPP_BLAKE3_LIB_DIR does not exist: {}", + blake3_lib_dir.display() + ); + } + let blake3 = require_library_file( + &blake3_lib_dir, + &static_library_name("blake3_vendored"), + "system static blake3_vendored; provision it separately and set VLLM_CPP_BLAKE3_LIB_DIR, or use `system,dynamic-link`", + ) + .unwrap_or_else(|error| panic!("{error}")); + println!("cargo:rerun-if-changed={}", blake3.display()); println!( "cargo:rustc-link-search=native={}", - blake3.parent().expect("library has a parent").display() + blake3_lib_dir.display() ); + if blake3_lib_dir != lib_dir { + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + } println!("cargo:rustc-link-lib=static:+whole-archive=vllm"); println!("cargo:rustc-link-lib=static=blake3_vendored"); + link_platform_dependencies(); +} - let target = env::var("CARGO_CFG_TARGET_OS").expect("cargo sets target OS"); - match target.as_str() { - "linux" => { - println!("cargo:rustc-link-lib=dylib=stdc++"); - println!("cargo:rustc-link-lib=dylib=pthread"); - println!("cargo:rustc-link-lib=dylib=dl"); - } - unsupported => { - panic!("bundled linking is not implemented for {unsupported} in this bootstrap") +fn system_library_dir(root: &Path, expected_artifact: &str) -> PathBuf { + if let Some(lib_dir) = env::var_os("VLLM_CPP_LIB_DIR").map(PathBuf::from) { + if !lib_dir.is_dir() { + panic!("VLLM_CPP_LIB_DIR does not exist: {}", lib_dir.display()); } + require_library_file(&lib_dir, expected_artifact, "VLLM_CPP_LIB_DIR override") + .unwrap_or_else(|error| panic!("{error}")); + return lib_dir; } + + find_installed_library_dir(root, expected_artifact).unwrap_or_else(|error| { + panic!( + "failed to locate {expected_artifact} under {}: {error}", + root.display() + ) + }) +} + +fn link_platform_dependencies() { + println!("cargo:rustc-link-lib=dylib=stdc++"); + println!("cargo:rustc-link-lib=dylib=pthread"); + println!("cargo:rustc-link-lib=dylib=dl"); +} + +fn required_path(name: &str) -> PathBuf { + env::var_os(name) + .map(PathBuf::from) + .unwrap_or_else(|| panic!("{name} must be set when the `system` feature is enabled")) } fn static_library_name(stem: &str) -> String { - if cfg!(target_env = "msvc") { + if env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("msvc") { format!("{stem}.lib") } else { format!("lib{stem}.a") } } -fn find_installed_library_dir(install: &Path) -> PathBuf { - for name in ["lib64", "lib"] { - let candidate = install.join(name); - if candidate.is_dir() { - return candidate; - } - } - panic!( - "cmake did not install a lib or lib64 directory below {}", - install.display() - ); -} - fn find_unique_file(root: &Path, name: String) -> Result { let mut matches = Vec::new(); collect_named_files(root, &name, &mut matches).map_err(|error| error.to_string())?; diff --git a/vllm-cpp-sys/src/bindings.rs b/vllm-cpp-sys/src/bindings.rs new file mode 100644 index 0000000..9cd44c6 --- /dev/null +++ b/vllm-cpp-sys/src/bindings.rs @@ -0,0 +1,174 @@ +/* automatically generated by rust-bindgen 0.72.1 */ + +pub const VLLM_ABI_VERSION: u32 = 10; +pub const vllm_status_VLLM_OK: vllm_status = 0; +pub const vllm_status_VLLM_ERR_INVALID_ARGUMENT: vllm_status = 1; +pub const vllm_status_VLLM_ERR_MODEL_LOAD: vllm_status = 2; +pub const vllm_status_VLLM_ERR_RUNTIME: vllm_status = 3; +pub const vllm_status_VLLM_ERR_UNKNOWN: vllm_status = 4; +pub type vllm_status = ::std::os::raw::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_engine { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_request { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_model_params { + pub model_path: *const ::std::os::raw::c_char, + pub tokenizer_config_path: *const ::std::os::raw::c_char, + pub block_size: i32, + pub num_blocks: i32, + pub max_model_len: i32, + pub max_num_seqs: i32, + pub tool_parser: *const ::std::os::raw::c_char, + pub reasoning_parser: *const ::std::os::raw::c_char, + pub speculative_config: *const ::std::os::raw::c_char, + pub enable_prefix_caching: i32, + pub max_num_batched_tokens: i32, + pub scheduling_policy: *const ::std::os::raw::c_char, + pub kv_transfer_config: *const ::std::os::raw::c_char, + pub enable_jump_forward: i32, +} +pub type vllm_logits_processor = ::std::option::Option< + unsafe extern "C" fn( + token_ids: *const i32, + n_token_ids: i32, + logits: *mut f32, + vocab_size: i32, + user_data: *mut ::std::os::raw::c_void, + ), +>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_sampling_params { + pub temperature: f32, + pub top_p: f32, + pub top_k: i32, + pub min_p: f32, + pub max_tokens: i32, + pub seed: u64, + pub has_seed: i32, + pub presence_penalty: f32, + pub frequency_penalty: f32, + pub repetition_penalty: f32, + pub min_tokens: i32, + pub ignore_eos: i32, + pub stop: *const *const ::std::os::raw::c_char, + pub n_stop: i32, + pub structured_json: *const ::std::os::raw::c_char, + pub structured_regex: *const ::std::os::raw::c_char, + pub structured_choice: *const *const ::std::os::raw::c_char, + pub n_structured_choice: i32, + pub structured_grammar: *const ::std::os::raw::c_char, + pub structured_json_object: i32, + pub logits_processor: vllm_logits_processor, + pub logits_processor_user_data: *mut ::std::os::raw::c_void, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_completion { + pub text: *mut ::std::os::raw::c_char, + pub finish_reason: *const ::std::os::raw::c_char, + pub prompt_tokens: i32, + pub completion_tokens: i32, +} +unsafe extern "C" { + pub fn vllm_model_params_default() -> vllm_model_params; +} +unsafe extern "C" { + pub fn vllm_sampling_params_default() -> vllm_sampling_params; +} +unsafe extern "C" { + pub fn vllm_engine_load( + params: *const vllm_model_params, + out: *mut *mut vllm_engine, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_engine_free(engine: *mut vllm_engine); +} +unsafe extern "C" { + pub fn vllm_complete( + engine: *mut vllm_engine, + prompt: *const ::std::os::raw::c_char, + params: *const vllm_sampling_params, + out: *mut vllm_completion, + ) -> vllm_status; +} +pub type vllm_token_callback = ::std::option::Option< + unsafe extern "C" fn( + delta_text: *const ::std::os::raw::c_char, + finished: bool, + user_data: *mut ::std::os::raw::c_void, + ) -> bool, +>; +unsafe extern "C" { + pub fn vllm_complete_stream( + engine: *mut vllm_engine, + prompt: *const ::std::os::raw::c_char, + params: *const vllm_sampling_params, + cb: vllm_token_callback, + user_data: *mut ::std::os::raw::c_void, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_request_submit( + engine: *mut vllm_engine, + prompt: *const ::std::os::raw::c_char, + params: *const vllm_sampling_params, + cb: vllm_token_callback, + user_data: *mut ::std::os::raw::c_void, + out: *mut *mut vllm_request, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_request_cancel(request: *mut vllm_request) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_request_wait(request: *mut vllm_request) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_request_done(request: *const vllm_request) -> bool; +} +unsafe extern "C" { + pub fn vllm_request_error(request: *const vllm_request) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + pub fn vllm_request_free(request: *mut vllm_request); +} +unsafe extern "C" { + pub fn vllm_chat( + engine: *mut vllm_engine, + request_json: *const ::std::os::raw::c_char, + out_response_json: *mut *mut ::std::os::raw::c_char, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_chat_stream( + engine: *mut vllm_engine, + request_json: *const ::std::os::raw::c_char, + cb: vllm_token_callback, + user_data: *mut ::std::os::raw::c_void, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_string_free(s: *mut ::std::os::raw::c_char); +} +unsafe extern "C" { + pub fn vllm_completion_free(out: *mut vllm_completion); +} +unsafe extern "C" { + pub fn vllm_last_error() -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + pub fn vllm_version() -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + pub fn vllm_abi_version() -> i32; +} diff --git a/vllm-cpp-sys/src/build_support.rs b/vllm-cpp-sys/src/build_support.rs new file mode 100644 index 0000000..63e62a9 --- /dev/null +++ b/vllm-cpp-sys/src/build_support.rs @@ -0,0 +1,67 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +const LIB_DIR_CANDIDATES: [&str; 2] = ["lib64", "lib"]; + +pub(crate) fn shared_library_name(stem: &str) -> String { + format!("lib{stem}.so") +} + +pub(crate) fn require_library_file( + directory: &Path, + expected_artifact: &str, + description: &str, +) -> Result { + let path = directory.join(expected_artifact); + if path.is_file() { + Ok(path) + } else { + Err(format!( + "expected {description} artifact {expected_artifact} at {}; install or copy the artifact there", + path.display() + )) + } +} + +pub(crate) fn find_installed_library_dir( + root: &Path, + expected_artifact: &str, +) -> Result { + let searched_dirs: Vec = LIB_DIR_CANDIDATES + .iter() + .map(|name| root.join(name)) + .collect(); + let mut matches: Vec<(PathBuf, PathBuf)> = Vec::new(); + + for directory in &searched_dirs { + let candidate = directory.join(expected_artifact); + if candidate.is_file() { + let canonical = fs::canonicalize(&candidate).unwrap_or_else(|_| candidate.clone()); + if !matches.iter().any(|(_, existing)| existing == &canonical) { + matches.push((directory.clone(), canonical)); + } + } + } + + match matches.as_slice() { + [(directory, _)] => Ok(directory.clone()), + [] => Err(format!( + "expected {expected_artifact} below {} in exactly one standard library directory; searched {}", + root.display(), + searched_dirs + .iter() + .map(|path| path.join(expected_artifact).display().to_string()) + .collect::>() + .join(", ") + )), + _ => Err(format!( + "found ambiguous {expected_artifact} artifacts below {} in {}; set VLLM_CPP_LIB_DIR to the intended directory", + root.display(), + matches + .iter() + .map(|(path, _)| path.display().to_string()) + .collect::>() + .join(", ") + )), + } +} diff --git a/vllm-cpp-sys/src/lib.rs b/vllm-cpp-sys/src/lib.rs index 101cc32..91db72e 100644 --- a/vllm-cpp-sys/src/lib.rs +++ b/vllm-cpp-sys/src/lib.rs @@ -1,138 +1,8 @@ //! Raw bindings to the stable vllm.cpp C API. //! -//! Complete generated bindings will be added in a follow-up. This bootstrap -//! declares the current symbols so the packaged native library can be linked and probed. +//! This crate exposes checked-in generated unsafe FFI declarations. Applications +//! should use the application-facing `vllm-cpp` bindings when implemented. -#![allow(non_camel_case_types)] +#![allow(non_camel_case_types, non_upper_case_globals)] -use std::ffi::{c_char, c_float, c_int, c_void}; - -#[repr(C)] -pub struct vllm_engine { - _private: [u8; 0], -} - -#[repr(C)] -pub struct vllm_request { - _private: [u8; 0], -} - -pub type vllm_status = c_int; -pub type vllm_token_callback = Option< - unsafe extern "C" fn(delta_text: *const c_char, finished: bool, user_data: *mut c_void) -> bool, ->; -pub type vllm_logits_processor = Option< - unsafe extern "C" fn( - token_ids: *const i32, - n_token_ids: i32, - logits: *mut c_float, - vocab_size: i32, - user_data: *mut c_void, - ), ->; - -#[repr(C)] -#[derive(Clone, Copy)] -pub struct vllm_model_params { - pub model_path: *const c_char, - pub tokenizer_config_path: *const c_char, - pub block_size: i32, - pub num_blocks: i32, - pub max_model_len: i32, - pub max_num_seqs: i32, - pub tool_parser: *const c_char, - pub reasoning_parser: *const c_char, - pub speculative_config: *const c_char, - pub enable_prefix_caching: i32, - pub max_num_batched_tokens: i32, - pub scheduling_policy: *const c_char, - pub kv_transfer_config: *const c_char, - pub enable_jump_forward: i32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -pub struct vllm_sampling_params { - pub temperature: c_float, - pub top_p: c_float, - pub top_k: i32, - pub min_p: c_float, - pub max_tokens: i32, - pub seed: u64, - pub has_seed: i32, - pub presence_penalty: c_float, - pub frequency_penalty: c_float, - pub repetition_penalty: c_float, - pub min_tokens: i32, - pub ignore_eos: i32, - pub stop: *const *const c_char, - pub n_stop: i32, - pub structured_json: *const c_char, - pub structured_regex: *const c_char, - pub structured_choice: *const *const c_char, - pub n_structured_choice: i32, - pub structured_grammar: *const c_char, - pub structured_json_object: i32, - pub logits_processor: vllm_logits_processor, - pub logits_processor_user_data: *mut c_void, -} - -#[repr(C)] -pub struct vllm_completion { - pub text: *mut c_char, - pub finish_reason: *const c_char, - pub prompt_tokens: i32, - pub completion_tokens: i32, -} - -unsafe extern "C" { - pub fn vllm_model_params_default() -> vllm_model_params; - pub fn vllm_sampling_params_default() -> vllm_sampling_params; - pub fn vllm_engine_load( - params: *const vllm_model_params, - out: *mut *mut vllm_engine, - ) -> vllm_status; - pub fn vllm_engine_free(engine: *mut vllm_engine); - pub fn vllm_complete( - engine: *mut vllm_engine, - prompt: *const c_char, - params: *const vllm_sampling_params, - out: *mut vllm_completion, - ) -> vllm_status; - pub fn vllm_complete_stream( - engine: *mut vllm_engine, - prompt: *const c_char, - params: *const vllm_sampling_params, - callback: vllm_token_callback, - user_data: *mut c_void, - ) -> vllm_status; - pub fn vllm_request_submit( - engine: *mut vllm_engine, - prompt: *const c_char, - params: *const vllm_sampling_params, - callback: vllm_token_callback, - user_data: *mut c_void, - out: *mut *mut vllm_request, - ) -> vllm_status; - pub fn vllm_request_cancel(request: *mut vllm_request) -> vllm_status; - pub fn vllm_request_wait(request: *mut vllm_request) -> vllm_status; - pub fn vllm_request_done(request: *const vllm_request) -> bool; - pub fn vllm_request_error(request: *const vllm_request) -> *const c_char; - pub fn vllm_request_free(request: *mut vllm_request); - pub fn vllm_chat( - engine: *mut vllm_engine, - request_json: *const c_char, - out_response_json: *mut *mut c_char, - ) -> vllm_status; - pub fn vllm_chat_stream( - engine: *mut vllm_engine, - request_json: *const c_char, - callback: vllm_token_callback, - user_data: *mut c_void, - ) -> vllm_status; - pub fn vllm_string_free(string: *mut c_char); - pub fn vllm_completion_free(completion: *mut vllm_completion); - pub fn vllm_last_error() -> *const c_char; - pub fn vllm_version() -> *const c_char; - pub fn vllm_abi_version() -> i32; -} +include!("bindings.rs"); diff --git a/vllm-cpp-sys/tests/build_support.rs b/vllm-cpp-sys/tests/build_support.rs new file mode 100644 index 0000000..11b6649 --- /dev/null +++ b/vllm-cpp-sys/tests/build_support.rs @@ -0,0 +1,132 @@ +#[path = "../src/build_support.rs"] +mod build_support; + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use build_support::{find_installed_library_dir, require_library_file, shared_library_name}; + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(label: &str) -> Self { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is before the Unix epoch") + .as_nanos(); + let path = env::temp_dir().join(format!( + "vllm-cpp-sys-build-support-{}-{label}-{timestamp}", + std::process::id() + )); + fs::create_dir(&path).expect("failed to create temporary directory"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn write_file(path: &Path) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("failed to create parent directory"); + } + fs::write(path, b"archive").expect("failed to write test artifact"); +} + +#[test] +fn shared_library_name_uses_linux_soname() { + assert_eq!(shared_library_name("vllm"), "libvllm.so"); +} + +#[test] +fn selects_lib_when_lib64_lacks_the_requested_artifact() { + let temp = TempDir::new("select-lib"); + fs::create_dir(temp.path().join("lib64")).expect("failed to create lib64"); + write_file(&temp.path().join("lib/libvllm.a")); + + let selected = find_installed_library_dir(temp.path(), "libvllm.a") + .expect("expected helper to select lib directory"); + + assert_eq!(selected, temp.path().join("lib")); +} + +#[test] +fn selects_lib64_when_it_contains_the_requested_shared_object() { + let temp = TempDir::new("select-lib64"); + fs::create_dir(temp.path().join("lib")).expect("failed to create lib"); + write_file(&temp.path().join("lib64/libvllm.so")); + + let selected = find_installed_library_dir(temp.path(), "libvllm.so") + .expect("expected helper to select lib64 directory"); + + assert_eq!(selected, temp.path().join("lib64")); +} + +#[test] +fn reports_missing_candidate_artifacts() { + let temp = TempDir::new("missing"); + fs::create_dir(temp.path().join("lib")).expect("failed to create lib"); + fs::create_dir(temp.path().join("lib64")).expect("failed to create lib64"); + + let error = find_installed_library_dir(temp.path(), "libvllm.a") + .expect_err("expected helper to report a missing library artifact"); + + assert!(error.contains("expected libvllm.a below")); + assert!(error.contains(&temp.path().display().to_string())); + assert!(error.contains(&temp.path().join("lib").display().to_string())); + assert!(error.contains(&temp.path().join("lib64").display().to_string())); +} + +#[test] +fn reports_ambiguous_installed_library_dirs() { + let temp = TempDir::new("ambiguous"); + write_file(&temp.path().join("lib/libvllm.a")); + write_file(&temp.path().join("lib64/libvllm.a")); + + let error = find_installed_library_dir(temp.path(), "libvllm.a") + .expect_err("expected helper to reject ambiguous library directories"); + + assert!(error.contains("found ambiguous libvllm.a artifacts below")); + assert!(error.contains(&temp.path().join("lib").display().to_string())); + assert!(error.contains(&temp.path().join("lib64").display().to_string())); + assert!(error.contains("VLLM_CPP_LIB_DIR")); +} + +#[test] +fn require_library_file_reports_the_expected_path() { + let temp = TempDir::new("require"); + let expected = temp.path().join("lib/libvllm.a"); + write_file(&expected); + + let actual = require_library_file(&temp.path().join("lib"), "libvllm.a", "system static vllm") + .expect("expected helper to return the exact library path"); + assert_eq!(actual, expected); + + let error = require_library_file( + &temp.path().join("lib"), + "libblake3_vendored.a", + "system static blake3_vendored", + ) + .expect_err("expected helper to report a missing library"); + assert!( + error.contains("expected system static blake3_vendored artifact libblake3_vendored.a at") + ); + assert!(error.contains( + &temp + .path() + .join("lib/libblake3_vendored.a") + .display() + .to_string() + )); +} diff --git a/vllm-cpp-sys/tests/layout.c b/vllm-cpp-sys/tests/layout.c new file mode 100644 index 0000000..923fdcb --- /dev/null +++ b/vllm-cpp-sys/tests/layout.c @@ -0,0 +1,63 @@ +#include + +#include +#include + +#define PRINT_LAYOUT(type) \ + printf(#type " %zu %zu\n", sizeof(type), _Alignof(type)) +#define PRINT_OFFSET(type, field) \ + printf(#type "." #field " %zu\n", offsetof(type, field)) + +int main(void) { + PRINT_LAYOUT(vllm_status); + + PRINT_LAYOUT(vllm_model_params); + PRINT_OFFSET(vllm_model_params, model_path); + PRINT_OFFSET(vllm_model_params, tokenizer_config_path); + PRINT_OFFSET(vllm_model_params, block_size); + PRINT_OFFSET(vllm_model_params, num_blocks); + PRINT_OFFSET(vllm_model_params, max_model_len); + PRINT_OFFSET(vllm_model_params, max_num_seqs); + PRINT_OFFSET(vllm_model_params, tool_parser); + PRINT_OFFSET(vllm_model_params, reasoning_parser); + PRINT_OFFSET(vllm_model_params, speculative_config); + PRINT_OFFSET(vllm_model_params, enable_prefix_caching); + PRINT_OFFSET(vllm_model_params, max_num_batched_tokens); + PRINT_OFFSET(vllm_model_params, scheduling_policy); + PRINT_OFFSET(vllm_model_params, kv_transfer_config); + PRINT_OFFSET(vllm_model_params, enable_jump_forward); + + PRINT_LAYOUT(vllm_sampling_params); + PRINT_OFFSET(vllm_sampling_params, temperature); + PRINT_OFFSET(vllm_sampling_params, top_p); + PRINT_OFFSET(vllm_sampling_params, top_k); + PRINT_OFFSET(vllm_sampling_params, min_p); + PRINT_OFFSET(vllm_sampling_params, max_tokens); + PRINT_OFFSET(vllm_sampling_params, seed); + PRINT_OFFSET(vllm_sampling_params, has_seed); + PRINT_OFFSET(vllm_sampling_params, presence_penalty); + PRINT_OFFSET(vllm_sampling_params, frequency_penalty); + PRINT_OFFSET(vllm_sampling_params, repetition_penalty); + PRINT_OFFSET(vllm_sampling_params, min_tokens); + PRINT_OFFSET(vllm_sampling_params, ignore_eos); + PRINT_OFFSET(vllm_sampling_params, stop); + PRINT_OFFSET(vllm_sampling_params, n_stop); + PRINT_OFFSET(vllm_sampling_params, structured_json); + PRINT_OFFSET(vllm_sampling_params, structured_regex); + PRINT_OFFSET(vllm_sampling_params, structured_choice); + PRINT_OFFSET(vllm_sampling_params, n_structured_choice); + PRINT_OFFSET(vllm_sampling_params, structured_grammar); + PRINT_OFFSET(vllm_sampling_params, structured_json_object); + PRINT_OFFSET(vllm_sampling_params, logits_processor); + PRINT_OFFSET(vllm_sampling_params, logits_processor_user_data); + + PRINT_LAYOUT(vllm_completion); + PRINT_OFFSET(vllm_completion, text); + PRINT_OFFSET(vllm_completion, finish_reason); + PRINT_OFFSET(vllm_completion, prompt_tokens); + PRINT_OFFSET(vllm_completion, completion_tokens); + + PRINT_LAYOUT(vllm_token_callback); + PRINT_LAYOUT(vllm_logits_processor); + return 0; +} diff --git a/vllm-cpp-sys/tests/layout.rs b/vllm-cpp-sys/tests/layout.rs new file mode 100644 index 0000000..8d2ce4c --- /dev/null +++ b/vllm-cpp-sys/tests/layout.rs @@ -0,0 +1,182 @@ +use std::collections::BTreeMap; +use std::env; +use std::fs; +use std::mem::{align_of, offset_of, size_of}; +use std::path::PathBuf; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use vllm_cpp_sys::{ + vllm_completion, vllm_logits_processor, vllm_model_params, vllm_sampling_params, vllm_status, + vllm_token_callback, +}; + +#[test] +fn generated_bindings_match_c_layout() { + let probe = LayoutProbe::compile(); + let output = Command::new(&probe.executable) + .output() + .expect("failed to execute the C layout probe"); + assert!( + output.status.success(), + "C layout probe failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let actual = parse_layouts(&String::from_utf8(output.stdout).expect("probe emits UTF-8")); + + macro_rules! expect_layout { + ($type:ty) => { + assert_eq!( + actual[stringify!($type)], + (size_of::<$type>(), align_of::<$type>()), + "layout mismatch for {}", + stringify!($type) + ); + }; + } + macro_rules! expect_offset { + ($type:ty, $field:ident) => { + assert_eq!( + actual[concat!(stringify!($type), ".", stringify!($field))].0, + offset_of!($type, $field), + "offset mismatch for {}.{}", + stringify!($type), + stringify!($field) + ); + }; + } + + expect_layout!(vllm_status); + + expect_layout!(vllm_model_params); + expect_offset!(vllm_model_params, model_path); + expect_offset!(vllm_model_params, tokenizer_config_path); + expect_offset!(vllm_model_params, block_size); + expect_offset!(vllm_model_params, num_blocks); + expect_offset!(vllm_model_params, max_model_len); + expect_offset!(vllm_model_params, max_num_seqs); + expect_offset!(vllm_model_params, tool_parser); + expect_offset!(vllm_model_params, reasoning_parser); + expect_offset!(vllm_model_params, speculative_config); + expect_offset!(vllm_model_params, enable_prefix_caching); + expect_offset!(vllm_model_params, max_num_batched_tokens); + expect_offset!(vllm_model_params, scheduling_policy); + expect_offset!(vllm_model_params, kv_transfer_config); + expect_offset!(vllm_model_params, enable_jump_forward); + + expect_layout!(vllm_sampling_params); + expect_offset!(vllm_sampling_params, temperature); + expect_offset!(vllm_sampling_params, top_p); + expect_offset!(vllm_sampling_params, top_k); + expect_offset!(vllm_sampling_params, min_p); + expect_offset!(vllm_sampling_params, max_tokens); + expect_offset!(vllm_sampling_params, seed); + expect_offset!(vllm_sampling_params, has_seed); + expect_offset!(vllm_sampling_params, presence_penalty); + expect_offset!(vllm_sampling_params, frequency_penalty); + expect_offset!(vllm_sampling_params, repetition_penalty); + expect_offset!(vllm_sampling_params, min_tokens); + expect_offset!(vllm_sampling_params, ignore_eos); + expect_offset!(vllm_sampling_params, stop); + expect_offset!(vllm_sampling_params, n_stop); + expect_offset!(vllm_sampling_params, structured_json); + expect_offset!(vllm_sampling_params, structured_regex); + expect_offset!(vllm_sampling_params, structured_choice); + expect_offset!(vllm_sampling_params, n_structured_choice); + expect_offset!(vllm_sampling_params, structured_grammar); + expect_offset!(vllm_sampling_params, structured_json_object); + expect_offset!(vllm_sampling_params, logits_processor); + expect_offset!(vllm_sampling_params, logits_processor_user_data); + + expect_layout!(vllm_completion); + expect_offset!(vllm_completion, text); + expect_offset!(vllm_completion, finish_reason); + expect_offset!(vllm_completion, prompt_tokens); + expect_offset!(vllm_completion, completion_tokens); + + expect_layout!(vllm_token_callback); + expect_layout!(vllm_logits_processor); +} + +struct LayoutProbe { + temp_dir: PathBuf, + executable: PathBuf, +} + +impl LayoutProbe { + fn compile() -> Self { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let include_dir = if cfg!(feature = "system") { + PathBuf::from( + env::var_os("VLLM_CPP_ROOT") + .expect("VLLM_CPP_ROOT is required by the system feature"), + ) + .join("include") + } else { + manifest_dir.join("vllm.cpp/include") + }; + let temp_dir = unique_temp_dir(); + fs::create_dir(&temp_dir).expect("failed to create a layout probe directory"); + let probe = Self { + executable: temp_dir.join("vllm-layout"), + temp_dir, + }; + let compiler = env::var_os("CC").unwrap_or_else(|| "cc".into()); + let output = Command::new(compiler) + .arg("-std=c11") + .arg("-Wall") + .arg("-Wextra") + .arg("-I") + .arg(&include_dir) + .arg(manifest_dir.join("tests/layout.c")) + .arg("-o") + .arg(&probe.executable) + .output() + .expect("failed to execute the C compiler for the layout probe"); + assert!( + output.status.success(), + "failed to compile the C layout probe:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + probe + } +} + +impl Drop for LayoutProbe { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.temp_dir); + } +} + +fn unique_temp_dir() -> PathBuf { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is before the Unix epoch") + .as_nanos(); + env::temp_dir().join(format!( + "vllm-cpp-sys-layout-{}-{timestamp}", + std::process::id() + )) +} + +fn parse_layouts(output: &str) -> BTreeMap { + output + .lines() + .map(|line| { + let mut fields = line.split_whitespace(); + let name = fields.next().expect("probe record has a name").to_owned(); + let first = fields + .next() + .expect("probe record has a value") + .parse() + .expect("probe value is an integer"); + let second = fields.next().map_or(0, |value| { + value.parse().expect("probe alignment is an integer") + }); + assert!(fields.next().is_none(), "unexpected probe record: {line}"); + (name, (first, second)) + }) + .collect() +} diff --git a/vllm-cpp-sys/tests/symbols.rs b/vllm-cpp-sys/tests/symbols.rs index b3de44b..34d1d79 100644 --- a/vllm-cpp-sys/tests/symbols.rs +++ b/vllm-cpp-sys/tests/symbols.rs @@ -2,6 +2,8 @@ use std::ffi::{c_char, c_void}; use vllm_cpp_sys as ffi; +const _: [(); 10] = [(); ffi::VLLM_ABI_VERSION as usize]; + unsafe extern "C" fn token_callback( _delta_text: *const c_char, _finished: bool, @@ -35,14 +37,15 @@ fn every_c_api_symbol_links() { #[test] fn reports_expected_abi_and_handles_invalid_model_path() { + assert_eq!(ffi::VLLM_ABI_VERSION, 10); assert_eq!(unsafe { ffi::vllm_abi_version() }, 10); let mut params = unsafe { ffi::vllm_model_params_default() }; - params.model_path = c"/nonexistent/vllm-cpp-rs-bootstrap-model".as_ptr(); + params.model_path = c"/nonexistent/vllm-cpp-rs-sys-model".as_ptr(); let mut engine = std::ptr::null_mut(); let status = unsafe { ffi::vllm_engine_load(¶ms, &mut engine) }; - assert_eq!(status, 2); + assert_eq!(status, ffi::vllm_status_VLLM_ERR_MODEL_LOAD); assert!(engine.is_null()); let error = unsafe { std::ffi::CStr::from_ptr(ffi::vllm_last_error()) }; assert!(!error.to_bytes().is_empty()); diff --git a/vllm-cpp-sys/wrapper.h b/vllm-cpp-sys/wrapper.h new file mode 100644 index 0000000..511fd6a --- /dev/null +++ b/vllm-cpp-sys/wrapper.h @@ -0,0 +1 @@ +#include "vllm.cpp/include/vllm.h" diff --git a/vllm-cpp/Cargo.toml b/vllm-cpp/Cargo.toml index 4dc1782..b6570b9 100644 --- a/vllm-cpp/Cargo.toml +++ b/vllm-cpp/Cargo.toml @@ -10,6 +10,8 @@ description = "Rust bindings for vllm.cpp" [features] default = ["bundled"] bundled = ["vllm-cpp-sys/bundled"] +system = ["vllm-cpp-sys/system"] +dynamic-link = ["vllm-cpp-sys/dynamic-link"] [dependencies] vllm-cpp-sys = { workspace = true, default-features = false }