diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bbe0da..a45d331 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,10 +39,6 @@ add_subdirectory(thirdparty/json EXCLUDE_FROM_ALL) include(cmake/icu.cmake) add_subdirectory(src) -add_executable(clap-model examples/embedding/embedding.cpp) -target_link_libraries(clap-model PRIVATE clap CLI11::CLI11 nlohmann_json::nlohmann_json) -set_target_properties(clap-model PROPERTIES CXX_EXTENSIONS OFF) - # The CLI target differs from the core library target, while its executable # keeps the existing "clap" name and command-line behavior. add_executable(clap-cli src/clap.cpp src/embedding.cpp src/server.cpp src/quantize.cpp) @@ -123,7 +119,7 @@ if(BUILD_TESTING) target_link_libraries(feature-input-test PRIVATE clap nlohmann_json::nlohmann_json) target_include_directories(feature-input-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") add_test(NAME clamp3_bundle_contract - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_bundle_contract.py" "$") + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_bundle_contract.py" "$") add_executable(clap-model-test tests/models/model-test.cpp) target_link_libraries(clap-model-test PRIVATE clap nlohmann_json::nlohmann_json) target_include_directories(clap-model-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") @@ -160,7 +156,7 @@ if(BUILD_TESTING) set_tests_properties(clap_smoke PROPERTIES PASS_REGULAR_EXPRESSION "clap.cpp") add_test(NAME clap_audio_cli - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/audio_cli.py" "$" + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/audio_cli.py" "$" "${CLAP_AUDIO_FIXTURE_DIR}") set_tests_properties(clap_audio_cli PROPERTIES TIMEOUT 30) add_test(NAME clap_embeddings @@ -205,17 +201,17 @@ if(BUILD_TESTING) set_tests_properties(clamp3_model_native_f16 PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120) add_test(NAME clamp3_model_cli COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_cli.py" - "$" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") + "$" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") set_tests_properties(clamp3_model_cli PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120) add_test(NAME clamp3_invalid_metadata COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_invalid.py" - "$" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") + "$" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") set_tests_properties(clamp3_invalid_metadata PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120) endif() option(MERT2_MODEL_PARITY "Run optional official MERT-v2-FullSong oracle tests (torch/torchaudio/transformers/safetensors required)" OFF) add_test(NAME mert_invalid_metadata COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/mert_invalid.py" - "$") + "$") set_tests_properties(mert_invalid_metadata PROPERTIES TIMEOUT 30) option(MERT_MODEL_PARITY "Run optional reviewed official MERT-v1 numerical oracles" OFF) set(MERT_REFERENCE_SOURCE "" CACHE PATH "Reviewed pinned MERT-v1 source (modeling_MERT.py/configuration_MERT.py)") @@ -261,14 +257,14 @@ if(BUILD_TESTING) set_tests_properties(mert2_model_parity PROPERTIES TIMEOUT 600) add_test(NAME mert2_model_cli COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/mert2_cli.py" - "$" "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture") + "$" "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture") set_tests_properties(mert2_model_cli PROPERTIES DEPENDS mert2_model_parity TIMEOUT 180) endif() add_test(NAME muq_conversion_contract COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/muq_conversion_test.py") add_test(NAME muq_quantized_rejection COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/muq_quantized.py" - "$" "$") + "$" "$") set_tests_properties(muq_quantized_rejection PROPERTIES TIMEOUT 120) option(MUQ_MODEL_PARITY "Run optional MuQ official-component waveform oracle tests" OFF) set(MUQ_REFERENCE_SOURCE "" CACHE PATH "Pinned official MuQ conv.py/features.py directory") @@ -287,7 +283,7 @@ if(BUILD_TESTING) add_test(NAME muq_mulan_conversion_contract COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/muq_mulan_conversion_test.py") add_test(NAME muq_mulan_invalid - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/muq_mulan_invalid.py" "$") + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/muq_mulan_invalid.py" "$") option(MUQ_MULAN_MODEL_PARITY "Run tiny pinned official MuLan component oracles" OFF) set(MUQ_MULAN_REFERENCE_SOURCE "" CACHE PATH "Pinned MuLan source directory") if(MUQ_MULAN_MODEL_PARITY) @@ -316,4 +312,4 @@ if(BUILD_TESTING) set_tests_properties(clap_embedding_cli PROPERTIES TIMEOUT 30) endif() -install(TARGETS clap-cli clap-model RUNTIME DESTINATION bin) +install(TARGETS clap-cli RUNTIME DESTINATION bin) diff --git a/README.md b/README.md index 374b355..812aad1 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ C++17/ggml implementations of Hugging Face `laion/larger_clap_music`, CLaMP3, `m-a-p/MERT-v1-95M`, `m-a-p/MERT-v2-FullSong`, OpenMuQ/MuQ and MuQ-MuLan, -with native inference through `clap-model`. The -original `clap` embedding command and HTTP server intentionally retain their dummy behavior. +with native text/audio inference through `clap embedding` and a model-backed +OpenAI-compatible text embeddings endpoint through `clap serve`. Both commands +require a converted GGUF model; inference runs locally without Python or downloads. ## Requirements @@ -88,7 +89,7 @@ sudo apt-get install libvulkan-dev glslc spirv-headers cmake -S . -B build-vulkan -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON cmake --build build-vulkan --config Release --parallel ctest --test-dir build-vulkan -C Release --output-on-failure -./build-vulkan/clap-model --model model.gguf --text 'A jazz quartet.' +./build-vulkan/clap embedding --model model.gguf --text 'A jazz quartet.' ``` The native model's linear layers use the first ggml Vulkan device, with weights @@ -103,8 +104,8 @@ Published releases additionally build `clap-linux-amd64-vulkan` and `clap-linux-arm64-vulkan` on Ubuntu 24.04. They require the system Vulkan loader (`libvulkan1` on Debian/Ubuntu) and a compatible Vulkan ICD/driver; the SDK and `glslc` are build-time dependencies only. Like the CPU release assets, these -`clap` executables still expose the dummy embedding/server interfaces described -below; real model inference currently uses the separately built `clap-model`. +`clap` executables expose the model-backed embedding/server interfaces described +below, using the same native model runtime as local inference. ### CUDA 12.9 release build @@ -134,8 +135,8 @@ All CUDA release presets emit Hopper (`90`) as real and virtual code, retaining PTX while avoiding sccache 0.18.0's PTX-only `compute_90` parsing issue. This adds a CUDA-enabled release build, not new model-backend selection: native -model linear dispatch remains CPU/optional Vulkan as documented above, and the -released `clap` embedding/server interfaces still return dummy embeddings. +model linear dispatch remains CPU/optional Vulkan as documented above. The +embedding command and server use that same native runtime. ### ROCm amd64 release build @@ -170,8 +171,8 @@ are not bundled; retain the SDK's installed directory layout. ggml does not support static ROCm linking. GPU execution also requires a compatible AMD GPU and driver. This adds a release build, not HIP model dispatch: native model linear layers -remain CPU/optional Vulkan, and the released `clap` embedding/server interfaces -still return dummy embeddings. No ROCm inference or GPU tests are run. +remain CPU/optional Vulkan. The embedding command and server use that same +native runtime. No ROCm inference or GPU tests are run. ## Native model layout @@ -179,9 +180,9 @@ The C++ organization follows `llama.cpp/src/models`: shared runtime services are separate from architecture-specific parameter/tensor loading and encoder code. `src/CMakeLists.txt` builds one `clap` core library from `CLAP_CORE_SOURCES` and -`models/*.cpp`. Both executables and the model tests link this same core library; -there is no separate models library. The `clap-cli` target keeps its executable -name `clap`, while native model inference uses `clap-model` as before. +`models/*.cpp`. The CLI and model tests link this same core library; there is no +separate models library. The `clap-cli` target produces the `clap` executable, +which provides the `embedding`, `serve` and `quantize` subcommands. | Path | Responsibility | | --- | --- | @@ -191,7 +192,7 @@ name `clap`, while native model inference uses `clap-model` as before. | `src/model-ops.*` | Shared ggml matrix operations (CPU/optional Vulkan) and CPU normalization | | `src/audio.cpp`, `src/vocab.*` | Native audio frontend, Unicode byte-level BPE and XLM-R Unigram tokenization | | `src/models/models.h`, `src/models/*.cpp` | CLAP, CLaMP3, MERT-v1, MERT2, MuQ and MuQ-MuLan specializations: hyperparameters, tensor contracts, encoders and projections | -| `examples/embedding/embedding.cpp` | Real text/audio inference executable (`clap-model`) | +| `src/clap.cpp`, `src/embedding.*`, `src/server.*` | Model-backed `clap embedding` text/audio CLI and `clap serve` text embeddings endpoint | | `convert_hf_to_gguf.py` | Offline HuggingFace-to-GGUF conversion CLI; no Python at inference | | `conversion/__init__.py` | Architecture-to-module map and lazy model-class lookup | | `conversion/base.py` | Shared model base, architecture detection, safe checkpoint loading and GGUF serialization | @@ -202,8 +203,8 @@ Convert a local `laion/larger_clap_music` checkpoint containing `config.json`, ```sh python convert_hf_to_gguf.py checkpoint/ model.gguf -./build/clap-model --model model.gguf --text 'A jazz quartet playing saxophone.' -./build/clap-model --model model.gguf --audio music.wav +./build/clap embedding --model model.gguf --text 'A jazz quartet playing saxophone.' +./build/clap embedding --model model.gguf --audio music.wav ``` For CLAP, the native executable returns a normalized 512-dimensional JSON array. Audio @@ -223,8 +224,8 @@ python convert_hf_to_gguf.py checkpoint/ model-f32.gguf --dtype f32 ./build/clap quantize model-f32.gguf model-q8_0.gguf # default remains Q8_0 ./build/clap quantize --type Q4_K_M model-f32.gguf model-q4_k_m.gguf # --type Q8_0 is also accepted; preset names are case-insensitive. -./build/clap-model --model model-q8_0.gguf --text 'A jazz quartet.' -./build/clap-model --model model-q8_0.gguf --audio music.wav +./build/clap embedding --model model-q8_0.gguf --text 'A jazz quartet.' +./build/clap embedding --model model-q8_0.gguf --audio music.wav # CLaMP3 also accepts --audio; its bundled MERT tensors remain F32/F16. ``` @@ -365,8 +366,8 @@ test/custom config with `--clamp3-config` and tokenizer with `--tokenizer`: python convert_hf_to_gguf.py \ checkpoint/weights_clamp3_saas_h_size_768_t_model_FacebookAI_xlm-roberta-base_t_length_128_a_size_768_a_layers_12_a_length_128_s_size_768_s_layers_12_p_size_64_p_length_512.pth \ clamp3-f16.gguf --dtype f16 -./build/clap-model --model clamp3-f16.gguf --text 'A piano sonata.' -./build/clap-model --model clamp3-f16.gguf --audio music.flac +./build/clap embedding --model clamp3-f16.gguf --text 'A piano sonata.' +./build/clap embedding --model clamp3-f16.gguf --audio music.flac ``` Every CLaMP3 conversion **automatically bundles** `m-a-p/MERT-v1-95M`. @@ -428,9 +429,9 @@ project's Apache-2.0 license does not relicense those weights. ```sh python convert_hf_to_gguf.py mert-checkpoint/ mert-v1-95m-f32.gguf \ --dtype f32 --mert-revision 12af15fef9d0ac838c3f475bfbbf26d2060dd4f5 -./build/clap-model --model mert-v1-95m-f32.gguf --audio music.wav --frames -./build/clap-model --model mert-v1-95m-f32.gguf --audio music.wav --clamp3-features -./build/clap-model --model clamp3-f16.gguf --audio music.wav +./build/clap embedding --model mert-v1-95m-f32.gguf --audio music.wav --frames +./build/clap embedding --model mert-v1-95m-f32.gguf --audio music.wav --clamp3-features +./build/clap embedding --model clamp3-f16.gguf --audio music.wav ``` `--mert-revision` records the full source SHA in GGUF. Standalone local MERT @@ -534,8 +535,8 @@ Convert a local safetensors snapshot pinned to Hugging Face revision python -m pip install -r requirements.txt python convert_hf_to_gguf.py checkpoint/ muq-f32.gguf \ --muq-revision 0562a57814f6f8bbd9fdea0a25921a2fce1a841a -./build/clap-model --model muq-f32.gguf --audio music.flac -./build/clap-model --model muq-f32.gguf --audio music.flac --frames +./build/clap embedding --model muq-f32.gguf --audio music.flac +./build/clap embedding --model muq-f32.gguf --audio music.flac --frames # F16 saves disk space; MuQ expands it to F32 compute weights on load: python convert_hf_to_gguf.py checkpoint/ muq-f16.gguf --dtype f16 \ --muq-revision 0562a57814f6f8bbd9fdea0a25921a2fce1a841a @@ -614,7 +615,7 @@ quantized-storage rejection without requiring ML packages. ### MuQ-MuLan-large (native joint audio/text embeddings) `OpenMuQ/MuQ-MuLan-large` has native, single-item text and raw-waveform -inference through `clap::Model` and `clap-model`. Published embeddings have 512 +inference through `clap::Model` and `clap embedding`. Published embeddings have 512 components. No Python, model source execution, or downloads occur at inference. The model revision is `2e01c796b71dca71b45251384c04cd7b237c9020`; the reviewed [official source](https://github.com/tencent-ailab/MuQ/tree/28847ea50cd31ac4b8b6a7dacc051ad7d1c7606a) @@ -651,11 +652,11 @@ python convert_hf_to_gguf.py /path/to/muq-mulan artifacts/muq-mulan-f16.gguf \ --text-config /path/to/xlm-roberta-base/config.json \ --tokenizer /path/to/xlm-roberta-base/tokenizer.json -artifacts/build/clap-model --model artifacts/muq-mulan-f16.gguf \ +artifacts/build/clap embedding --model artifacts/muq-mulan-f16.gguf \ --text 'classical genres, hopeful mood, piano.' --threads 2 -artifacts/build/clap-model --model artifacts/muq-mulan-f16.gguf \ +artifacts/build/clap embedding --model artifacts/muq-mulan-f16.gguf \ --text '一首适合海边风景的小提琴曲，节奏欢快' --threads 2 -artifacts/build/clap-model --model artifacts/muq-mulan-f16.gguf \ +artifacts/build/clap embedding --model artifacts/muq-mulan-f16.gguf \ --audio music.flac --threads 2 ``` @@ -722,7 +723,7 @@ ctest --test-dir artifacts/build -R muq_mulan --output-on-failure # child process and exits before the native CLI starts; arrays/metrics are saved. python tests/models/muq_mulan_checkpoint_parity.py \ --checkpoint artifacts/checkpoint --gguf artifacts/muq-mulan-f16.gguf \ - --dtype f16 --cli artifacts/build/clap-model \ + --dtype f16 --cli artifacts/build/clap \ --source artifacts/oracle --muq-source /path/to/pinned/MuQ/source-components \ --out artifacts/published-muq-mulan-f16 --audio music.wav ``` @@ -770,8 +771,8 @@ the full source SHA in GGUF metadata: ```sh python convert_hf_to_gguf.py checkpoint/ mert-v2-fullsong-f16.gguf \ --dtype f16 --mert2-revision 7001f7657276238629e61c7746ce70735e0604c0 -./build/clap-model --model mert-v2-fullsong-f16.gguf --audio complete-song.flac -./build/clap-model --model mert-v2-fullsong-f16.gguf --audio complete-song.flac --frames +./build/clap embedding --model mert-v2-fullsong-f16.gguf --audio complete-song.flac +./build/clap embedding --model mert-v2-fullsong-f16.gguf --audio complete-song.flac --frames ``` Input is mixed to mono and resampled to the checkpoint's 24 kHz rate. The native @@ -866,77 +867,80 @@ python tests/models/clamp3_checkpoint_parity.py build checkpoint/ build/publishe --dtype f16 --gguf clamp3-f16.gguf ``` -For raw music files, use the native `--mert-model` pipeline above. Alternatively, -run `preprocessing/audio/extract_mert.py` in the official `sanderwood/clamp3` -checkout with `--model_path m-a-p/MERT-v1-95M` and `--mean_features`, then pass -its `.npy` to `--audio-features`. Existing dummy interfaces below are unchanged. +For raw music files, use `clap embedding --model clamp3-f16.gguf --audio music.flac`. +The bundled MERT frontend handles waveform feature extraction internally; +external `--mert-model` and `--audio-features` inputs are not supported. ## Embedding command -Generate dummy embeddings directly, without starting a server or loading a model: +Load a converted GGUF and generate real native text or audio embeddings: ```sh -./build/clap embedding -p "Hello World!" -# 1.0000000 0.0000000 0.0000000 - -./build/clap embedding -f prompts.txt --embd-output-format json -./build/clap embedding -p 'hello<#sep#>world' \ - --embd-separator '<#sep#>' --embd-output-format array -# [[1.0,0.0,0.0],[1.0,0.0,0.0]] +./build/clap embedding --model model.gguf --text 'A jazz quartet.' +./build/clap embedding --model model.gguf --audio music.flac ``` -The input and output options follow llama.cpp's `llama-embedding` example: +`--model` (`-m`) is required. Select exactly one input mode: + +- `--text` embeds one string, without separator splitting, and defaults to a + single JSON vector. Empty text is passed to the model, whose tokenizer and + architecture determine whether it is supported. +- `-p/--prompt` or `-f/--file` embed separator-delimited text. Input must not be + empty. `--embd-separator` defaults to an actual newline and must not be empty; + empty segments, including a trailing segment, are passed to the model. +- `-a/--audio` decodes WAV, MP3 or FLAC and defaults to a single JSON vector. + Architecture-specific `--frames`, `--clamp3-features` and `--crop-offset` + behavior is described above. There is no HTTP audio API. + +For prompt/file text input, `--embd-output-format` defaults to `raw` (one +space-separated vector per line). It also accepts `array` (an array of vectors), +`json` (an indexed OpenAI-style embedding list), and `json+` (also includes +`cosineSimilarity` for multiple inputs). These are real model outputs: vector +size, normalization and text/audio support depend on the loaded architecture. +Text inputs are evaluated sequentially, not as a native batched forward pass. +Structured output goes to stdout; errors go to stderr. -- Exactly one of `-p/--prompt` or `-f/--file` is required. Input must not be empty. -- `--embd-separator` splits input into prompts (default: an actual newline). - Empty segments, including a trailing segment, each receive an embedding. - The separator must not be empty. -- `--embd-output-format` accepts `raw` (default, one space-separated vector per - line), `array` (an array of vectors), `json` (`object: "list"` with indexed - embedding objects), or `json+` (also includes `cosineSimilarity` for multiple - prompts). Structured output is written to stdout; errors go to stderr. +```sh +./build/clap embedding --model model.gguf -p "Hello World!" +./build/clap embedding --model model.gguf -f prompts.txt --embd-output-format json +./build/clap embedding --model model.gguf -p 'hello<#sep#>world' \ + --embd-separator '<#sep#>' --embd-output-format array +``` -Every prompt receives the fixed vector `[1.0, 0.0, 0.0]`, just like the server. -All cosine similarities are therefore `1.0`. This is only a subset of the -llama.cpp CLI: model loading, tokenization, inference, pooling, and normalization -options are not implemented. No model argument is needed. +`--threads` (`-t`) selects ggml CPU threads (`0` means automatic). No model is +bundled with the executable, and a missing or invalid model is an error rather +than a fallback to placeholder vectors. ## Embeddings server -Start the server (defaults to `127.0.0.1:8080`): +Start the server with a text-capable GGUF, such as CLAP, CLaMP3 or MuQ-MuLan +(default listen address: `127.0.0.1:8080`): ```sh -./build/clap serve --host 127.0.0.1 --port 8080 +./build/clap serve --model model.gguf --host 127.0.0.1 --port 8080 ``` -Send a request following the llama.cpp `/v1/embeddings` JSON response shape: +The model is loaded once at startup. Its API identifier is the GGUF **filename** +(`model.gguf` in this example), not its full path or a Hugging Face repository ID. +Send an OpenAI-compatible text embeddings request: ```sh curl http://127.0.0.1:8080/v1/embeddings \ -H 'Content-Type: application/json' \ - -d '{"model":"dummy","input":["hello","world"],"encoding_format":"float"}' + -d '{"model":"model.gguf","input":["hello","world"],"encoding_format":"float"}' ``` -```json -{ - "object": "list", - "data": [ - {"object": "embedding", "index": 0, "embedding": [1.0, 0.0, 0.0]}, - {"object": "embedding", "index": 1, "embedding": [1.0, 0.0, 0.0]} - ], - "model": "dummy", - "usage": {"prompt_tokens": 0, "total_tokens": 0} -} -``` +The response is an `object: "list"` with indexed `object: "embedding"` entries +containing real floating-point vectors and the loaded model's filename in +`model`. Embedding dimensions and normalization are those of the native model, +not a fixed server vector. The server does not resize model embeddings. -This is a stub: every input gets the same fixed 3-dimensional vector; no model -is loaded and no tokenization or inference takes place. Usage counts are zero. `input` accepts one non-empty string or an array of 1–1024 non-empty strings. -`model` is optional (defaults to `dummy`) and is echoed in the response. -Only `encoding_format: "float"` (the default) and `dimensions: 3` are supported; -token IDs, multimodal inputs and base64 encoding are not implemented. -Invalid JSON or unsupported parameter values return HTTP 400 with a JSON -`error` object. Request bodies are limited to 1 MiB. +Only floating-point encoding is supported; token-ID inputs, base64 encoding and +multimodal inputs are unsupported. This is a **text-only** endpoint: audio files +must use `clap embedding --audio` or the C++ API, not an HTTP audio request. +Invalid JSON or unsupported parameter values return HTTP 400 with a JSON `error` +object. Request bodies are limited to 1 MiB. Running `clap` without a subcommand still prints `clap.cpp`. The executable is located at `build/clap` for single-configuration generators or diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp deleted file mode 100644 index 52141a3..0000000 --- a/examples/embedding/embedding.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "clap.h" -#include -#include -#include -#include -#include - -int main(int argc, char **argv) { - CLI::App app{"Native CLAP/CLaMP3 embeddings (separate from the legacy dummy clap CLI/server)"}; - std::string path, text, audio; - int threads = 0; - size_t crop = 0; - bool frames = false, clamp3_features = false; - app.add_option("-m,--model", path, "Converted CLAP/CLaMP3/MERT-v1/MERT2/MuQ/MuLan GGUF")->required()->check(CLI::ExistingFile); - auto *text_option = app.add_option("-p,--text", text, "UTF-8 text to embed"); - auto *audio_option = app.add_option("-a,--audio", audio, "WAV, MP3 or FLAC (mixed to mono and resampled for the model)")->check(CLI::ExistingFile); - text_option->excludes(audio_option); - app.add_option("-t,--threads", threads, "ggml CPU threads (0: automatic)")->check(CLI::NonNegativeNumber); - auto *crop_option = app.add_option("--crop-offset", crop, "Sample offset at 48 kHz into CLAP audio longer than ten seconds"); - app.add_flag("--frames", frames, "Return final-layer waveform-encoder frames instead of mean pooling"); - app.add_flag("--clamp3-features", clamp3_features, - "Return standalone MERT-v1 mean-layer/time features (one vector per 5 s window)"); - CLI11_PARSE(app, argc, argv); - try { - if (text_option->count() + audio_option->count() != 1) - throw std::runtime_error("Specify exactly one of --text or --audio"); - if (crop_option->count() && !audio_option->count()) throw std::runtime_error("--crop-offset requires --audio"); - if (frames && (!audio_option->count() || crop_option->count())) - throw std::runtime_error("--frames requires --audio and excludes --crop-offset"); - if (clamp3_features && (!audio_option->count() || frames || crop_option->count())) - throw std::runtime_error("--clamp3-features requires --audio and excludes --frames/--crop-offset"); - std::vector pcm; - int pcm_rate = 0; - if (audio_option->count()) { - auto decoded = clap::decode_audio(audio); - pcm_rate = decoded.sample_rate; - pcm = std::move(decoded.samples); - } - clap::Model model(path, threads); - if (crop_option->count() && model.audio_sample_rate() != 48000) - throw std::runtime_error("--crop-offset requires a CLAP model"); - if (frames || clamp3_features) { - const auto sequence = clamp3_features ? model.clamp3_features(pcm, pcm_rate) : model.audio_frames(pcm, pcm_rate); - std::cout << nlohmann::json{{"frames", sequence.frames}, {"dimension", sequence.dimension}, - {"values", sequence.values}}.dump() << '\n'; - } else { - const auto embedding = text_option->count() ? model.text(text) : model.audio(pcm, pcm_rate, crop); - std::cout << nlohmann::json(embedding).dump() << '\n'; - } - if (!std::cout) throw std::runtime_error("Failed to write representations"); - return 0; - } catch (const std::exception &error) { - std::cerr << error.what() << '\n'; return 1; - } -} diff --git a/src/clap.cpp b/src/clap.cpp index 14a0fd2..9ffc3e2 100644 --- a/src/clap.cpp +++ b/src/clap.cpp @@ -6,30 +6,51 @@ #include #include +#include #include int main(int argc, char **argv) { CLI::App app{"clap.cpp"}; + argv = app.ensure_utf8(argv); std::string host = "127.0.0.1"; int port = 8080; + std::string server_model; + int server_threads = 0; auto *serve = app.add_subcommand("serve", "Start the embeddings HTTP server"); + serve->add_option("-m,--model", server_model, "Converted model GGUF") + ->required()->check(CLI::ExistingFile); + serve->add_option("-t,--threads", server_threads, "ggml CPU threads (0: automatic)") + ->check(CLI::NonNegativeNumber)->capture_default_str(); serve->add_option("--host", host, "Listen address")->capture_default_str(); serve->add_option("--port", port, "Listen port") ->check(CLI::Range(1, 65535))->capture_default_str(); - std::string prompt; + EmbeddingOptions options; std::string file; - std::string separator = "\n"; - std::string format = "raw"; - auto *embedding = app.add_subcommand("embedding", "Generate dummy text embeddings"); - auto *input = embedding->add_option_group("Input", "Choose a prompt or a UTF-8 text file"); + auto *embedding = app.add_subcommand("embedding", "Generate native text or audio embeddings"); + embedding->add_option("-m,--model", options.model_path, "Converted CLAP/CLaMP3/MERT/MERT2/MuQ/MuLan GGUF") + ->required()->check(CLI::ExistingFile); + embedding->add_option("-t,--threads", options.threads, "ggml CPU threads (0: automatic)") + ->check(CLI::NonNegativeNumber)->capture_default_str(); + auto *input = embedding->add_option_group("Input", "Choose one text prompt, text file, or audio file"); input->require_option(1, 1); - input->add_option("-p,--prompt", prompt, "Input text"); + input->add_option("-p,--prompt", options.text, "Separator-delimited text (default output: raw)"); + auto *text_option = input->add_option("--text", options.text, "One UTF-8 text (default output: flat JSON vector)"); auto *file_option = input->add_option("-f,--file", file, "Read input text from a file") ->check(CLI::ExistingFile); - embedding->add_option("--embd-separator", separator, "Input separator (default: newline)"); - embedding->add_option("--embd-output-format", format, "Output format") - ->check(CLI::IsMember({"raw", "array", "json", "json+"}))->capture_default_str(); + auto *audio_option = input->add_option("-a,--audio", options.audio_path, + "WAV, MP3 or FLAC (default output: flat JSON vector)") + ->check(CLI::ExistingFile); + embedding->add_option("--embd-separator", options.separator, "Prompt/file separator (default: newline)"); + embedding->add_option("--embd-output-format", options.format, + "Explicit output: raw rows, array of vectors, JSON list, or JSON list with cosine similarities") + ->check(CLI::IsMember({"raw", "array", "json", "json+"})); + auto *crop_option = embedding->add_option("--crop-offset", options.crop_offset, + "Sample offset at 48 kHz into CLAP audio longer than ten seconds") + ->check(CLI::NonNegativeNumber); + embedding->add_flag("--frames", options.frames, "Return final-layer waveform-encoder frames instead of mean pooling"); + embedding->add_flag("--clamp3-features", options.clamp3_features, + "Return MERT-v1 mean-layer/time features (one vector per 5 s window)"); std::string quantize_input; std::string quantize_output; @@ -47,38 +68,40 @@ int main(int argc, char **argv) { return run_quantize(quantize_input, quantize_output, quantize_type); } - if (*embedding) { - if (separator.empty()) { - std::cerr << "Input separator must not be empty\n"; - return 1; - } - if (*file_option) { - std::ifstream stream(file, std::ios::binary); - if (!stream) { - std::cerr << "Failed to open input file: " << file << '\n'; - return 1; - } - std::string contents; - char buffer[4096]; - while (stream.read(buffer, sizeof(buffer))) { + try { + if (*embedding) { + options.single_text = text_option->count() != 0; + options.crop_requested = crop_option->count() != 0; + if (options.separator.empty()) throw std::runtime_error("Input separator must not be empty"); + if (options.crop_requested && !*audio_option) + throw std::runtime_error("--crop-offset requires --audio"); + if (options.frames && (!*audio_option || options.crop_requested)) + throw std::runtime_error("--frames requires --audio and excludes --crop-offset"); + if (options.clamp3_features && (!*audio_option || options.frames || options.crop_requested)) + throw std::runtime_error("--clamp3-features requires --audio and excludes --frames/--crop-offset"); + if (*file_option) { + std::ifstream stream(file, std::ios::binary); + if (!stream) throw std::runtime_error("Failed to open input file: " + file); + std::string contents; + char buffer[4096]; + while (stream.read(buffer, sizeof(buffer))) { + contents.append(buffer, static_cast(stream.gcount())); + } + if (stream.bad() || !stream.eof()) + throw std::runtime_error("Failed to read input file: " + file); contents.append(buffer, static_cast(stream.gcount())); + options.text = contents; } - if (stream.bad() || !stream.eof()) { - std::cerr << "Failed to read input file: " << file << '\n'; - return 1; - } - contents.append(buffer, static_cast(stream.gcount())); - prompt = contents; + if (!*audio_option && !options.single_text && options.text.empty()) + throw std::runtime_error("Input must not be empty"); + return run_embedding(options); } - if (prompt.empty()) { - std::cerr << "Input must not be empty\n"; - return 1; + if (*serve) { + return run_server(host, port, server_model, server_threads); } - return run_embedding(prompt, separator, format); - } - - if (*serve) { - return run_server(host, port); + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; } std::cout << "clap.cpp\n"; return 0; diff --git a/src/embedding.cpp b/src/embedding.cpp index 9e4c40c..147b8db 100644 --- a/src/embedding.cpp +++ b/src/embedding.cpp @@ -1,55 +1,112 @@ #include "embedding.h" +#include "clap.h" #include +#include +#include +#include #include +#include #include -int run_embedding(const std::string &prompt, const std::string &separator, - const std::string &format) { - // Match llama.cpp's separator semantics, including empty and trailing segments. - std::size_t count = 1; - for (std::size_t pos = 0; (pos = prompt.find(separator, pos)) != std::string::npos; - pos += separator.size()) { - ++count; +namespace { +using Vector = std::vector; +using nlohmann::json; + +// Native projections (notably CLaMP3) are not necessarily unit normalized. +double cosine(const Vector &left, const Vector &right) { + if (left.size() != right.size()) { + throw std::runtime_error("Embedding dimensions do not match"); + } + double dot = 0, left_norm = 0, right_norm = 0; + for (std::size_t i = 0; i < left.size(); ++i) { + dot += static_cast(left[i]) * right[i]; + left_norm += static_cast(left[i]) * left[i]; + right_norm += static_cast(right[i]) * right[i]; } + if (left_norm == 0 || right_norm == 0) return 0; + return std::clamp(dot / (std::sqrt(left_norm) * std::sqrt(right_norm)), -1.0, 1.0); +} - if (format == "raw") { - for (std::size_t i = 0; i < count; ++i) { - std::cout << "1.0000000 0.0000000 0.0000000\n"; +void write_embeddings(const std::vector &vectors, const std::string &format) { + if (format.empty()) { + // Native --text/--audio compatibility: exactly one flat JSON vector. + std::cout << json(vectors.at(0)).dump() << '\n'; + } else if (format == "raw") { + std::cout << std::fixed << std::setprecision(7); + for (const auto &vector : vectors) { + for (std::size_t i = 0; i < vector.size(); ++i) { + if (i) std::cout << ' '; + std::cout << vector[i]; + } + std::cout << '\n'; } - std::cout.flush(); - if (!std::cout) { - std::cerr << "Failed to write embedding output\n"; - return 1; + } else if (format == "array") { + std::cout << json(vectors).dump() << '\n'; + } else { + auto data = json::array(); + for (std::size_t i = 0; i < vectors.size(); ++i) { + data.push_back({{"object", "embedding"}, {"index", i}, {"embedding", vectors[i]}}); } - return 0; + json result = {{"object", "list"}, {"data", data}}; + if (format == "json+" && vectors.size() > 1) { + auto similarities = json::array(); + for (const auto &left : vectors) { + auto row = json::array(); + for (const auto &right : vectors) row.push_back(cosine(left, right)); + similarities.push_back(row); + } + result["cosineSimilarity"] = similarities; + } + std::cout << result.dump() << '\n'; } +} +} // namespace - using nlohmann::json; - auto data = json::array(); - for (std::size_t i = 0; i < count; ++i) { - const auto vector = json::array({1.0f, 0.0f, 0.0f}); - if (format == "array") { - data.push_back(vector); +int run_embedding(const EmbeddingOptions &options) { + try { + // Preserve native CLI error precedence: decode audio before loading weights. + clap::Audio decoded; + if (!options.audio_path.empty()) decoded = clap::decode_audio(options.audio_path); + clap::Model model(options.model_path, options.threads); + std::vector vectors; + if (!options.audio_path.empty()) { + if (options.crop_requested && model.audio_sample_rate() != 48000) { + throw std::runtime_error("--crop-offset requires a CLAP model"); + } + if (options.frames || options.clamp3_features) { + const auto sequence = options.clamp3_features + ? model.clamp3_features(decoded.samples, decoded.sample_rate) + : model.audio_frames(decoded.samples, decoded.sample_rate); + std::cout << json{{"frames", sequence.frames}, {"dimension", sequence.dimension}, + {"values", sequence.values}}.dump() << '\n'; + } else { + vectors.push_back(model.audio(decoded.samples, decoded.sample_rate, options.crop_offset)); + } + } else if (options.single_text) { + vectors.push_back(model.text(options.text)); } else { - data.push_back({{"object", "embedding"}, {"index", i}, {"embedding", vector}}); + // Preserve empty and trailing separator-delimited prompt/file segments. + std::size_t start = 0; + while (true) { + const auto end = options.text.find(options.separator, start); + vectors.push_back(model.text(options.text.substr(start, end == std::string::npos + ? end : end - start))); + if (end == std::string::npos) break; + start = end + options.separator.size(); + } } - } - json result = format == "array" ? data : json{{"object", "list"}, {"data", data}}; - if (format == "json+" && count > 1) { - // All dummy vectors are identical unit vectors, so every cosine is one. - result["cosineSimilarity"] = json::array(); - const json row = std::vector(count, 1.0f); - for (std::size_t i = 0; i < count; ++i) { - result["cosineSimilarity"].push_back(row); + if (!vectors.empty()) { + const std::string format = options.format.empty() && !options.single_text && options.audio_path.empty() + ? "raw" : options.format; + write_embeddings(vectors, format); } - } - std::cout << result.dump() << '\n'; - std::cout.flush(); - if (!std::cout) { - std::cerr << "Failed to write embedding output\n"; + std::cout.flush(); + if (!std::cout) throw std::runtime_error("Failed to write embedding output"); + return 0; + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; return 1; } - return 0; } diff --git a/src/embedding.h b/src/embedding.h index 5430fda..08f4866 100644 --- a/src/embedding.h +++ b/src/embedding.h @@ -1,6 +1,21 @@ #pragma once +#include #include -int run_embedding(const std::string &prompt, const std::string &separator, - const std::string &format); +struct EmbeddingOptions { + std::string model_path; + int threads = 0; + std::string text; + std::string audio_path; + std::string separator = "\n"; + // Empty means native single-vector JSON for --text/--audio, raw for --prompt/--file. + std::string format; + bool single_text = false; + std::size_t crop_offset = 0; + bool crop_requested = false; + bool frames = false; + bool clamp3_features = false; +}; + +int run_embedding(const EmbeddingOptions &options); diff --git a/src/server.cpp b/src/server.cpp index ce4dffe..bfd5a7d 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -1,22 +1,35 @@ #include "server.h" +#include "clap.h" #include #include +#include +#include +#include #include +#include +#include namespace { using nlohmann::json; -void invalid_request(httplib::Response &response, const std::string &message) { - response.status = 400; +void error_response(httplib::Response &response, int status, + const std::string &type, const std::string &message) { + response.status = status; response.set_content(json{{"error", { - {"message", message}, {"type", "invalid_request_error"}, {"code", 400} + {"message", message}, {"type", type}, {"code", status} }}}.dump(), "application/json"); } -void embeddings(const httplib::Request &request, httplib::Response &response) { +void invalid_request(httplib::Response &response, const std::string &message) { + error_response(response, 400, "invalid_request_error", message); +} + +void embeddings(const httplib::Request &request, httplib::Response &response, + clap::Model &model, const std::string &identifier, int dimensions, + std::mutex &inference_mutex) { const auto body = json::parse(request.body, nullptr, false); if (!body.is_object() || !body.contains("input")) { invalid_request(response, "Expected a JSON object with an input field"); @@ -24,9 +37,7 @@ void embeddings(const httplib::Request &request, httplib::Response &response) { } auto inputs = body["input"]; - if (inputs.is_string()) { - inputs = json::array({inputs}); - } + if (inputs.is_string()) inputs = json::array({inputs}); if (!inputs.is_array() || inputs.empty() || inputs.size() > 1024) { invalid_request(response, "input must be a non-empty string or an array of 1 to 1024 strings"); return; @@ -37,8 +48,8 @@ void embeddings(const httplib::Request &request, httplib::Response &response) { return; } } - if (body.contains("model") && !body["model"].is_string()) { - invalid_request(response, "model must be a string"); + if (body.contains("model") && (!body["model"].is_string() || body["model"] != identifier)) { + invalid_request(response, "model must match the loaded model: " + identifier); return; } if (body.contains("encoding_format") && body["encoding_format"] != "float") { @@ -46,34 +57,86 @@ void embeddings(const httplib::Request &request, httplib::Response &response) { return; } if (body.contains("dimensions") && - (!body["dimensions"].is_number_integer() || body["dimensions"] != 3)) { - invalid_request(response, "Only 3 dimensions are supported"); + (!body["dimensions"].is_number_integer() || body["dimensions"] != dimensions)) { + invalid_request(response, "dimensions must match the loaded model: " + std::to_string(dimensions)); return; } - auto data = json::array(); - for (std::size_t index = 0; index < inputs.size(); ++index) { - data.push_back({ - {"object", "embedding"}, {"index", index}, - {"embedding", {1.0f, 0.0f, 0.0f}} - }); + try { + auto data = json::array(); + std::size_t tokens = 0; + bool usage_available = true; + { + // Vocabulary and inference both belong to the same non-thread-safe + // handle. Keep the complete batch atomic with respect to other calls. + const std::lock_guard lock(inference_mutex); + for (std::size_t index = 0; index < inputs.size(); ++index) { + const auto &text = inputs[index].get_ref(); + const auto embedding = model.text(text); + if (embedding.size() != static_cast(dimensions)) + throw std::runtime_error("Model returned an unexpected embedding dimension"); + for (float value : embedding) + if (!std::isfinite(value)) throw std::runtime_error("Model returned a non-finite embedding"); + data.push_back({{"object", "embedding"}, {"index", index}, {"embedding", embedding}}); + // Model::text(string) chooses its architecture's full context. + // The public tokenize() default is 512 and can truncate or be + // unsupported for smaller contexts. Omit usage rather than + // report an inaccurate count in either situation. + try { + const auto ids = model.tokenize(text); + if (ids.size() >= 512) usage_available = false; + else tokens += ids.size(); + } catch (const std::exception &) { + usage_available = false; + } + } + } + json result = {{"object", "list"}, {"data", data}, {"model", identifier}}; + if (usage_available) + result["usage"] = {{"prompt_tokens", tokens}, {"total_tokens", tokens}}; + response.set_content(result.dump(), "application/json"); + } catch (const std::exception &error) { + error_response(response, 500, "server_error", error.what()); + } catch (...) { + error_response(response, 500, "server_error", "Unknown inference error"); } - response.set_content(json{ - {"object", "list"}, {"data", data}, {"model", body.value("model", "dummy")}, - {"usage", {{"prompt_tokens", 0}, {"total_tokens", 0}}} - }.dump(), "application/json"); } } // namespace -int run_server(const std::string &host, int port) { - httplib::Server server; - server.set_payload_max_length(1024 * 1024); - server.Post("/v1/embeddings", embeddings); - if (!server.bind_to_port(host, port)) { - std::cerr << "Failed to bind to " << host << ':' << port << '\n'; +int run_server(const std::string &host, int port, const std::string &model_path, int threads) { + try { + // Load once, before opening the listening socket. A bad model must fail + // startup rather than leave a live server that can never serve requests. + clap::Model model(model_path, threads); + const auto identifier = std::filesystem::path(model_path).filename().string(); + const int dimensions = model.projection_dimension(); + if (dimensions <= 0) throw std::runtime_error("Model projection dimension must be positive"); + std::mutex inference_mutex; + httplib::Server server; + server.set_payload_max_length(1024 * 1024); + server.Post("/v1/embeddings", [&](const httplib::Request &request, httplib::Response &response) { + embeddings(request, response, model, identifier, dimensions, inference_mutex); + }); + server.set_exception_handler([](const httplib::Request &, httplib::Response &response, + std::exception_ptr exception) { + try { + if (exception) std::rethrow_exception(exception); + } catch (const std::exception &error) { + error_response(response, 500, "server_error", error.what()); + return; + } catch (...) { + } + error_response(response, 500, "server_error", "Unknown server error"); + }); + if (!server.bind_to_port(host, port)) { + std::cerr << "Failed to bind to " << host << ':' << port << '\n'; + return 1; + } + std::cout << "Listening on " << host << ':' << port << " with model " << identifier << std::endl; + return server.listen_after_bind() ? 0 : 1; + } catch (const std::exception &error) { + std::cerr << "Server startup failed: " << error.what() << '\n'; return 1; } - std::cout << "Listening on " << host << ':' << port << std::endl; - return server.listen_after_bind() ? 0 : 1; } diff --git a/src/server.h b/src/server.h index a7f5e3f..dd971a5 100644 --- a/src/server.h +++ b/src/server.h @@ -2,4 +2,4 @@ #include -int run_server(const std::string &host, int port); +int run_server(const std::string &host, int port, const std::string &model_path, int threads); diff --git a/tests/audio_cli.py b/tests/audio_cli.py index be8eec3..0f8bac7 100644 --- a/tests/audio_cli.py +++ b/tests/audio_cli.py @@ -10,7 +10,7 @@ def run(*args): - return subprocess.run([cli, "--model", str(model), *args], + return subprocess.run([cli, 'embedding', "--model", str(model), *args], capture_output=True, text=True, timeout=10) diff --git a/tests/embedding_cli.py b/tests/embedding_cli.py index 3f690a9..97545bd 100644 --- a/tests/embedding_cli.py +++ b/tests/embedding_cli.py @@ -1,118 +1,223 @@ -"""Exercise the embedding subcommand without a model or external packages.""" +"""Real embedding CLI contract; optional second argument is a parity fixture root. +Usage: embedding_cli.py CLAP [BUILD_WITH_MODEL_FIXTURES] +The default suite generates a tiny valid GGUF with a scalar text oracle using +only the standard library. Optional existing model-fixture, clamp3-fixture and +mert2-fixture oracles additionally check native text/audio/frame representations. +""" import errno import json +import math from pathlib import Path +import struct import subprocess import sys import tempfile import unittest +import wave + +BINARY = str(Path(sys.argv[1]).resolve()) +FIXTURES = Path(sys.argv[2]) if len(sys.argv) > 2 else None class EmbeddingCLITest(unittest.TestCase): - def run_cli(self, *args): - return subprocess.run( - [sys.argv[1], "embedding", *args], capture_output=True, text=True, timeout=5 - ) + @classmethod + def setUpClass(cls): + from tiny_model import write_model + cls.temporary = tempfile.TemporaryDirectory(prefix="clap embedding ") + cls.addClassCleanup(cls.temporary.cleanup) + cls.tiny_path = Path(cls.temporary.name) / "model.gguf" + cls.tiny_reference = write_model(cls.tiny_path) + + def run_cli(self, *args, command="embedding", **kwargs): + return subprocess.run([BINARY, command, *map(str, args)], + capture_output=True, text=True, timeout=120, **kwargs) def success(self, *args): result = self.run_cli(*args) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stderr, "") return result.stdout + def model_args(self, kind="model"): + if FIXTURES is None: + if kind == "model": + return ["--model", str(self.tiny_path), "--threads", "2"] + self.skipTest("pass a build directory containing parity fixtures for native encoder checks") + assert FIXTURES is not None + path = FIXTURES / (kind + "-fixture") / "model.gguf" + self.assertTrue(path.is_file(), f"missing fixture: {path}") + return ["--model", str(path), "--threads", "2"] + + def vector(self, text, kind="model"): + return json.loads(self.success(*self.model_args(kind), "--text", text)) + + def assert_vector_close(self, actual, expected, tolerance=3e-5): + self.assertEqual(len(actual), len(expected)) + self.assertTrue(all(isinstance(x, (int, float)) and math.isfinite(x) for x in actual)) + self.assertLessEqual(max(abs(a-b) for a, b in zip(actual, expected)), tolerance) + def test_help(self): output = self.success("--help") - for option in ("--prompt", "--file", "--embd-separator", "--embd-output-format"): + for option in ("--model", "--threads", "--prompt", "--text", "--audio", "--file", + "--crop-offset", "--frames", "--clamp3-features", + "--embd-separator", "--embd-output-format"): self.assertIn(option, output) + self.assertNotIn("dummy", output.lower()) + result = self.run_cli("--help", command="serve") + self.assertEqual(result.returncode, 0, result.stderr) + for option in ("--model", "--threads", "--host", "--port"): + self.assertIn(option, result.stdout) + + def test_model_required_for_embedding_and_serve(self): + for command, args in (("embedding", ["--prompt", "hello"]), ("serve", [])): + with self.subTest(command=command): + # Do not hang if the old dummy server starts without a model. + try: + result = subprocess.run([BINARY, command, *args], capture_output=True, + text=True, timeout=1) + except subprocess.TimeoutExpired: + self.fail(f"{command} started without its required --model") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--model", result.stderr) - def test_default_and_raw(self): - for args in (("-p", "hello"), ("--prompt", "hello", "--embd-output-format", "raw")): - with self.subTest(args=args): - self.assertEqual([float(x) for x in self.success(*args).split()], [1, 0, 0]) - - def test_array_and_separator(self): - for prompt, separator in (("hello\nworld", "\n"), ("hello<#sep#>world", "<#sep#>")): - output = self.success("-p", prompt, "--embd-separator", separator, - "--embd-output-format", "array") - self.assertEqual(json.loads(output), [[1, 0, 0], [1, 0, 0]]) - - def test_json(self): - output = self.success("-p", "hello\nworld", "--embd-output-format", "json") - self.assertEqual(json.loads(output), { - "object": "list", "data": [ - {"object": "embedding", "index": i, "embedding": [1, 0, 0]} - for i in range(2) - ] - }) - - def test_json_similarity(self): - for prompt, count in (("hello", 1), ("hello\nworld", 2)): - output = json.loads(self.success("-p", prompt, "--embd-output-format", "json+")) - self.assertEqual(len(output["data"]), count) - if count > 1: - self.assertEqual(output["cosineSimilarity"], [[1, 1], [1, 1]]) - else: - self.assertNotIn("cosineSimilarity", output) + def test_invalid_model_is_not_a_dummy_success(self): + with tempfile.TemporaryDirectory() as directory: + model = Path(directory) / "invalid.gguf" + model.write_bytes(b"not GGUF") + for command, args in (("embedding", ["--prompt", "hello"]), ("serve", [])): + result = self.run_cli("--model", model, "--threads", "1", *args, command=command) + self.assertGreater(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + self.assertTrue(result.stderr) + + def test_text_single_vector_matches_model_oracle(self): + args = self.model_args() + cases = self.tiny_reference["cases"] if FIXTURES is None else [ + {"input": case["text"], "embedding": case["expected"]} + for case in json.loads((FIXTURES / "model-fixture" / "fixture.json").read_text())["text_cases"] + ] + for case in cases: + actual = json.loads(self.success(*args, "--text", case["input"])) + self.assert_vector_close(actual, case["embedding"]) + + def test_prompt_raw_default_and_explicit_formats(self): + expected = self.vector("hello") + args = self.model_args() + for flags in ([], ["--embd-output-format", "raw"]): + self.assert_vector_close([float(x) for x in self.success(*args, "-p", "hello", *flags).split()], expected) + self.assertEqual(json.loads(self.success(*args, "--text", "hello", "--embd-output-format", "array")), [expected]) + for option in ("-p", "--prompt"): + result = json.loads(self.success(*args, option, "hello", "--embd-output-format", "json")) + self.assertEqual(result, {"object": "list", "data": [{"object": "embedding", "index": 0, "embedding": expected}]}) + + def test_separator_and_empty_segments(self): + args = self.model_args() + for text, separator, parts in (("hello\nworld", "\n", ["hello", "world"]), + ("hello<#sep#>world", "<#sep#>", ["hello", "world"]), + ("hello\n\n", "\n", ["hello", "", ""])): + actual = json.loads(self.success(*args, "--prompt", text, "--embd-separator", separator, + "--embd-output-format", "array")) + self.assertEqual(actual, [self.vector(part) for part in parts]) + # --text is one item, never separator-split (including a trailing newline). + direct = self.vector("hello\nworld\n") + self.assertTrue(all(isinstance(x, (int, float)) for x in direct)) def test_file(self): + args = self.model_args() with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "prompts.txt" - path.write_text("你好\nworld", encoding="utf-8") + path.write_bytes("你好\nworld".encode("utf-8")) for flag in ("-f", "--file"): - output = self.success(flag, str(path), "--embd-output-format", "array") - self.assertEqual(json.loads(output), [[1, 0, 0], [1, 0, 0]]) + actual = json.loads(self.success(*args, flag, path, "--embd-output-format", "array")) + self.assertEqual(actual, [self.vector("你好"), self.vector("world")]) + + def test_json_similarity_uses_true_cosine_for_unnormalized_vectors(self): + args = self.model_args("clamp3") + result = json.loads(self.success(*args, "--prompt", "ab c\na b", "--embd-output-format", "json+")) + vectors = [item["embedding"] for item in result["data"]] + norms = [math.sqrt(sum(x*x for x in vector)) for vector in vectors] + self.assertTrue(any(abs(norm - 1) > 1e-3 for norm in norms), "fixture must expose unnormalized outputs") + for i in range(2): + for j in range(2): + expected = sum(a*b for a, b in zip(vectors[i], vectors[j])) / (norms[i]*norms[j]) + self.assertAlmostEqual(result["cosineSimilarity"][i][j], expected, places=6) + single = json.loads(self.success(*args, "-p", "hello", "--embd-output-format", "json+")) + self.assertNotIn("cosineSimilarity", single) + + def test_audio_single_vector_and_native_frames(self): + args = self.model_args("mert2") + fixture = json.loads((FIXTURES / "mert2-fixture" / "fixture.json").read_text()) + case = next(case for case in fixture["waveform_cases"] if case["full_song"]) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "audio.wav" + samples = struct.pack("<" + "f" * len(case["waveform"]), *case["waveform"]) + rate = fixture["sample_rate"] + path.write_bytes(b"RIFF" + struct.pack("0 and message in result.stderr, (key,result.returncode,result.stderr) print(f'{len(cases)} invalid MERT metadata cases rejected') diff --git a/tests/models/mert_parity.py b/tests/models/mert_parity.py index 29d8cb9..0eb374c 100644 --- a/tests/models/mert_parity.py +++ b/tests/models/mert_parity.py @@ -94,7 +94,7 @@ def run(args): normalized = processor(pcm, sampling_rate=24000, return_tensors='pt').input_values with torch.no_grad(): result = network(normalized, output_hidden_states=True) - actual = json.loads(subprocess.check_output([str(args.build/'clap-model'),'-m',str(target), + actual = json.loads(subprocess.check_output([str(args.build/'clap'), 'embedding','-m',str(target), '-a',str(audio),'--frames','-t','2'], text=True)) error(np.asarray(actual['values']).reshape(actual['frames'],actual['dimension']), result.last_hidden_state[0].numpy(), f'{dtype}/{name}/frames', 0.0002) @@ -106,19 +106,19 @@ def run(args): with torch.no_grad(): states=network(chunk,output_hidden_states=True).hidden_states chunks.append(torch.stack(states).mean(-2).mean(0)[0].numpy()) - actual = json.loads(subprocess.check_output([str(args.build/'clap-model'),'-m',str(target), + actual = json.loads(subprocess.check_output([str(args.build/'clap'), 'embedding','-m',str(target), '-a',str(audio),'--clamp3-features','-t','2'], text=True)) error(np.asarray(actual['values']).reshape(actual['frames'],actual['dimension']), np.stack(chunks), f'{dtype}/{name}/clamp3', 0.00002) if name == 'resampled': - resampled = json.loads(subprocess.check_output([str(args.build/'clap-model'), + resampled = json.loads(subprocess.check_output([str(args.build/'clap'), 'embedding', '-m',str(target),'-a',str(args.out/'resampled-input.wav'), '--clamp3-features','-t','2'], text=True)) np.testing.assert_array_equal(resampled['values'], actual['values']) print(f'{dtype}/48kHz CLI -> 24kHz MERT matches native-rate control', flush=True) for n in [0,399,23999]: path=args.out/f'short-{n}.wav';wav(path,np.zeros(n,dtype='float32')) - result=subprocess.run([str(args.build/'clap-model'),'-m',str(target),'-a',str(path), + result=subprocess.run([str(args.build/'clap'), 'embedding','-m',str(target),'-a',str(path), '--clamp3-features'],capture_output=True,text=True) assert result.returncode != 0, (n,result.stdout) print('MERT-v1 numerical parity passed',flush=True) diff --git a/tests/models/mert_pipeline.py b/tests/models/mert_pipeline.py index 3c962f9..57b9906 100644 --- a/tests/models/mert_pipeline.py +++ b/tests/models/mert_pipeline.py @@ -33,15 +33,15 @@ (fixture/'config.json').write_text(json.dumps(c)) subprocess.run([sys.executable,str(root/'convert_hf_to_gguf.py'),str(fixture/'model.pth'),str(fixture/'model.gguf'), '--clamp3-config',str(fixture/'config.json'),'--tokenizer',str(build/'clamp3-fixture/tokenizer.json')],check=True) -cli=str(build/'clap-model') +cli=str(build/'clap') for dtype in ['f32']: mert=str(out/f'model-{dtype}.gguf') for case in ['broadband','two_chunks','discard_tail','resampled-input']: wave=out/f'{case}.wav' - data=json.loads(subprocess.check_output([cli,'-m',mert,'-a',str(wave),'--clamp3-features','-t','2'])) + data=json.loads(subprocess.check_output([cli, 'embedding','-m',mert,'-a',str(wave),'--clamp3-features','-t','2'])) features=np.asarray(data['values'],dtype='float32').reshape(data['frames'],hidden) expected=encode(build,fixture/'model.gguf',features.ravel(),data['frames'],fixture) - actual=json.loads(subprocess.check_output([cli,'-m',str(fixture/'model.gguf'), + actual=json.loads(subprocess.check_output([cli, 'embedding','-m',str(fixture/'model.gguf'), '-a',str(wave),'-t','2'])) np.testing.assert_allclose(actual,expected,rtol=0,atol=1e-6) with torch.no_grad(): @@ -65,6 +65,6 @@ ['-m',clamp,'-a',wave,'--audio-features','obsolete.npy'], ['-m',clamp,'-a',wave,'--mert-model',mert], ]: - result = subprocess.run([cli,*arguments], capture_output=True, text=True) + result = subprocess.run([cli, 'embedding',*arguments], capture_output=True, text=True) assert result.returncode > 0 and result.stderr, (arguments,result.returncode,result.stderr[:1000]) print('Native pipeline and CLI validation passed') diff --git a/tests/models/muq_checkpoint_parity.py b/tests/models/muq_checkpoint_parity.py index e5b7290..c201d74 100644 --- a/tests/models/muq_checkpoint_parity.py +++ b/tests/models/muq_checkpoint_parity.py @@ -78,7 +78,7 @@ wav.write_bytes(b'RIFF' + struct.pack('0 and message in p.stderr,(message,p.returncode,p.stderr) print(message,'rejected:',p.stderr.strip()) @@ -67,7 +67,7 @@ def complete(path,meta,geometry,alter=None,half_name=None): with tempfile.TemporaryDirectory() as temporary: path=Path(temporary)/'complete.gguf' complete(path,base_meta,shapes) - baseline=subprocess.run([str(cli),'--model',str(path),'--text',''],capture_output=True,text=True) + baseline=subprocess.run([str(cli), 'embedding','--model',str(path),'--text',''],capture_output=True,text=True) assert baseline.returncode==0,(baseline.returncode,baseline.stderr) assert json.loads(baseline.stdout)==[0.]*6 cases=[] @@ -84,15 +84,15 @@ def complete(path,meta,geometry,alter=None,half_name=None): cases.append((base_meta,shapes,('conv.conv.0.bn1.running_var',-1.),'variance')) for meta,geometry,alter,message in cases: complete(path,meta,geometry,alter) - proc=subprocess.run([str(cli),'--model',str(path),'--text','a'],capture_output=True,text=True) + proc=subprocess.run([str(cli), 'embedding','--model',str(path),'--text','a'],capture_output=True,text=True) assert proc.returncode>0 and message in proc.stderr,(message,proc.returncode,proc.stderr) overflow=copy.deepcopy(cfg);overflow['text_transformer']['tf_depth']=4294967296 complete(path,dict(base_meta,**{'muq_mulan.config':json.dumps(overflow)}), {k:v for k,v in shapes.items() if not k.startswith('adapter.')}) - proc=subprocess.run([str(cli),'--model',str(path),'--text','a'],capture_output=True,text=True) + proc=subprocess.run([str(cli), 'embedding','--model',str(path),'--text','a'],capture_output=True,text=True) assert proc.returncode>0 and 'dimensions' in proc.stderr, ('depth overflow must fail',proc.returncode,proc.stderr) complete(path,base_meta,shapes,half_name='conformer.layer_norm.weight') - proc=subprocess.run([str(cli),'--model',str(path),'--text','a'],capture_output=True,text=True) + proc=subprocess.run([str(cli), 'embedding','--model',str(path),'--text','a'],capture_output=True,text=True) assert proc.returncode>0 and 'F32' in proc.stderr, ('F16 norm must fail',proc.returncode,proc.stderr) print('Complete tiny MuLan: legacy tokenizer accepted;',len(cases),'malformed contracts rejected') @@ -109,7 +109,7 @@ def complete(path,meta,geometry,alter=None,half_name=None): meta=dict(base_meta, **{'general.architecture':arch,'muq.config':json.dumps(raw)}) geometry = shapes if arch == 'muq_mulan' else {k:v for k,v in shapes.items() if k.startswith(('frontend.','conv.','conformer.'))} complete(path,meta,geometry) - proc=subprocess.run([str(cli),'--model',str(path),'--text','a'],capture_output=True,text=True,timeout=15) + proc=subprocess.run([str(cli), 'embedding','--model',str(path),'--text','a'],capture_output=True,text=True,timeout=15) if name=='valid tiny': ok=proc.returncode==0 if arch=='muq_mulan' else 'no text encoder' in proc.stderr elif name=='valid published': diff --git a/tests/models/muq_mulan_oracle_pipeline.py b/tests/models/muq_mulan_oracle_pipeline.py index 9c50050..d43ac29 100644 --- a/tests/models/muq_mulan_oracle_pipeline.py +++ b/tests/models/muq_mulan_oracle_pipeline.py @@ -18,7 +18,7 @@ out=fixture/('meta-oracle-'+dtype);out.mkdir(exist_ok=True) base=[sys.executable,str(root/'tests/models/muq_mulan_checkpoint_parity.py'), '--checkpoint',str(fixture),'--gguf',str(fixture/f'model-{dtype}.gguf'), - '--dtype',dtype,'--cli',str(build/'clap-model'),'--source',str(source), + '--dtype',dtype,'--cli',str(build/'clap'),'--source',str(source), '--muq-source',str(muq_source),'--out',str(out)] for branch in ('text','audio'): subprocess.run(base+['--branch',branch],check=True) @@ -27,7 +27,7 @@ metrics=[] for index,case in enumerate(cases): args=['--text',case['text']] if branch=='text' else ['--audio',case['wave']] - result=subprocess.run([str(build/'clap-model'),'--model',str(fixture/f'model-{dtype}.gguf'),'--threads','2',*args],check=True,capture_output=True,text=True) + result=subprocess.run([str(build/'clap'), 'embedding','--model',str(fixture/f'model-{dtype}.gguf'),'--threads','2',*args],check=True,capture_output=True,text=True) actual=json.loads(result.stdout);expected=case['expected'] error=max(abs(a-b) for a,b in zip(actual,expected)) (out/f'{branch}-{index}-native.json').write_text(json.dumps(actual)) diff --git a/tests/models/muq_mulan_parity.py b/tests/models/muq_mulan_parity.py index a81a38c..1a0224c 100644 --- a/tests/models/muq_mulan_parity.py +++ b/tests/models/muq_mulan_parity.py @@ -101,7 +101,7 @@ def wave(n): # Native CLI must return joint vectors and reject every unsupported input mode. x=waves[-1][1]; data=x.numpy().astype(' 0 and 'quantized non-dense/embedding tensor' in result.stderr, (name, result) destination = output / (name + '-converted.gguf') diff --git a/tests/models/parity.py b/tests/models/parity.py index 307e4e3..08b366c 100644 --- a/tests/models/parity.py +++ b/tests/models/parity.py @@ -129,6 +129,6 @@ for name, data, error in corruptions: path = out / (name + '.gguf') path.write_bytes(data) - result = subprocess.run([str(build / 'clap-model'), '--model', str(path), '--text', 'hello'], capture_output=True, text=True) + result = subprocess.run([str(build / 'clap'), 'embedding', '--model', str(path), '--text', 'hello'], capture_output=True, text=True) assert result.returncode == 1 and error in result.stderr.lower(), result.stderr print('Transformers F32/F16/Q8_0/Q4_K_M parity, malformed GGUF and unsupported-config rejection passed') diff --git a/tests/models/quantized_checkpoint_smoke.py b/tests/models/quantized_checkpoint_smoke.py index c27ac05..f78b29a 100644 --- a/tests/models/quantized_checkpoint_smoke.py +++ b/tests/models/quantized_checkpoint_smoke.py @@ -24,7 +24,7 @@ for label, path in [('f32', f32), ('quantized', quantized)]: results[label] = {'path': str(path), 'bytes': path.stat().st_size} for mode, value in [('text', prompt), ('audio', str(audio))]: - result = subprocess.run([str(build / 'clap-model'), '--model', str(path), + result = subprocess.run([str(build / 'clap'), 'embedding', '--model', str(path), '--threads', '2', '--' + mode, value], capture_output=True, text=True, timeout=900) assert result.returncode == 0, result.stderr diff --git a/tests/tiny_model.py b/tests/tiny_model.py new file mode 100644 index 0000000..f2979c2 --- /dev/null +++ b/tests/tiny_model.py @@ -0,0 +1,174 @@ +"""Stdlib-only tiny, valid CLAP GGUF and independent scalar text oracle. + +write_model(path, *, broken_projection=False) returns metadata with dimensions, +model (filename), and cases [{input, tokens, embedding}]. No downloads or torch. +The reduced-mel model is for text/tensor tests, NOT waveform frontend coverage. +""" + +import json +import math +from pathlib import Path +import struct + + +def write_model(path, *, broken_projection=False) -> dict: + path = Path(path) + text = dict(hidden_size=8, intermediate_size=16, num_attention_heads=2, + num_hidden_layers=1, vocab_size=261, type_vocab_size=1, + max_position_embeddings=514, bos_token_id=0, pad_token_id=1, + eos_token_id=2, hidden_act="gelu", projection_hidden_act="relu", + projection_dim=5, layer_norm_eps=1e-5) + audio = dict(enable_fusion=False, depths=[1] * 4, num_attention_heads=[1, 1, 2, 4], + patch_size=4, patch_stride=[4, 4], patch_embed_input_channels=1, + qkv_bias=True, enable_patch_layer_norm=True, flatten_patch_embeds=True, + hidden_act="gelu", projection_hidden_act="relu", spec_size=32, + num_mel_bins=32, window_size=1, patch_embeds_hidden_size=2, + hidden_size=16, mlp_ratio=2.0, projection_dim=5) + config = dict(model_type="clap", projection_dim=5, text_config=text, audio_config=audio) + vocab = {"": 0, "": 1, "": 2, "": 3, "": 4} + extra = 256 + for byte in range(256): + regular = 33 <= byte <= 126 or 161 <= byte <= 172 or byte >= 174 + vocab[chr(byte if regular else extra)] = byte + 5 + if not regular: + extra += 1 + tokenizer = dict(model=dict(type="BPE", vocab=vocab, merges=[])) + tensors = {} + + def add(name, shape, norm=False): + serial = len(tensors) + 1 + count = math.prod(shape) + values = [1.0 if norm else 0.18 * math.sin((i + serial * 17) * 0.173) + for i in range(count)] + if broken_projection and name.startswith("text_proj.linear2."): + values = [0.0] * count + # The oracle uses exactly the F32 storage values, not Python doubles. + data = struct.pack("<" + "f" * count, *values) + tensors[name] = (shape, list(struct.unpack("<" + "f" * count, data)), data) + + def norm(name, dim): + add(name + ".weight", [dim], True) + add(name + ".bias", [dim]) + + def linear(name, inputs, outputs, bias=True): + add(name + ".weight", [inputs, outputs]) + if bias: + add(name + ".bias", [outputs]) + + add("audio.patch_embed.proj.weight", [4, 4, 1, 2]) + add("audio.patch_embed.proj.bias", [2]) + norm("audio.patch_embed.norm", 2) + norm("audio.batch_norm", 32) + add("audio.batch_norm.running_mean", [32]) + add("audio.batch_norm.running_var", [32], True) + dim = 2 + for stage in range(4): + p = f"audio.layers.{stage}.blocks.0" + norm(p + ".layernorm_before", dim) + norm(p + ".layernorm_after", dim) + for q in ("query", "key", "value"): + linear(p + ".attention.self." + q, dim, dim) + add(p + ".attention.self.rel_bias", [[1, 1, 2, 4][stage], 1]) + linear(p + ".attention.output.dense", dim, dim) + linear(p + ".intermediate.dense", dim, dim * 2) + linear(p + ".output.dense", dim * 2, dim) + if stage < 3: + down = f"audio.layers.{stage}.downsample" + norm(down + ".norm", dim * 4) + linear(down + ".reduction", dim * 4, dim * 2, False) + dim *= 2 + norm("audio.norm", 16) + add("text.embeddings.word_embeddings.weight", [8, 261]) + add("text.embeddings.position_embeddings.weight", [8, 514]) + add("text.embeddings.token_type_embeddings.weight", [8, 1]) + norm("text.embeddings.LayerNorm", 8) + p = "text.encoder.layer.0" + for q in ("query", "key", "value"): + linear(p + ".attention.self." + q, 8, 8) + linear(p + ".attention.output.dense", 8, 8) + norm(p + ".attention.output.LayerNorm", 8) + linear(p + ".intermediate.dense", 8, 16) + linear(p + ".output.dense", 16, 8) + norm(p + ".output.LayerNorm", 8) + linear("text.pooler.dense", 8, 8) + linear("text_proj.linear1", 8, 5) + linear("text_proj.linear2", 5, 5) + linear("audio_proj.linear1", 16, 5) + linear("audio_proj.linear2", 5, 5) + + def string(value): + raw = value.encode("utf-8") + return struct.pack(" 512 else ids + x = [] + for pos, token in enumerate(ids, 2): + x.append([weight("text.embeddings.word_embeddings.weight")[token * 8 + i] + + weight("text.embeddings.position_embeddings.weight")[pos * 8 + i] + + weight("text.embeddings.token_type_embeddings.weight")[i] for i in range(8)]) + x = normalize(x, "text.embeddings.LayerNorm") + q, k, v = [dense(x, p + ".attention.self." + part) for part in ("query", "key", "value")] + attention = [[0.0] * 8 for _ in ids] + for h in range(2): + for i in range(len(ids)): + scores = [sum(q[i][h * 4 + c] * k[j][h * 4 + c] for c in range(4)) / 2 + for j in range(len(ids))] + maximum = max(scores) + scores = [math.exp(s - maximum) for s in scores] + denominator = sum(scores) + for c in range(4): + attention[i][h * 4 + c] = sum(s / denominator * v[j][h * 4 + c] + for j, s in enumerate(scores)) + x = normalize(residual(dense(attention, p + ".attention.output.dense"), x), + p + ".attention.output.LayerNorm") + y = dense(x, p + ".intermediate.dense") + y = [[0.5 * value * (1 + math.erf(value / math.sqrt(2))) for value in row] for row in y] + x = normalize(residual(dense(y, p + ".output.dense"), x), p + ".output.LayerNorm") + pooled = [[math.tanh(value) for value in dense([x[0]], "text.pooler.dense")[0]]] + projected = dense([[max(value, 0) for value in dense(pooled, "text_proj.linear1")[0]]], + "text_proj.linear2")[0] + length = math.sqrt(sum(value * value for value in projected)) + return dict(input=prompt, tokens=len(ids), + embedding=[value / length for value in projected] if length else []) + + return dict(model=path.name, dimensions=5, + cases=[reference(prompt) for prompt in ("hello", "world", "hello world!", "caf\u00e9", "a", "b")])