feat: support EmbeddingGemma 2 text and audio embeddings

CMakeLists.txt

@@ -73,6 +73,61 @@ if(BUILD_TESTING)
         COMMENT "Generating audio test fixtures"
         VERBATIM)
     add_custom_target(clap-audio-fixtures ALL DEPENDS ${CLAP_AUDIO_FIXTURES})
+    add_executable(clap-gemma-audio-integration-test tests/models/gemma-audio-integration-test.cpp)
+    target_link_libraries(clap-gemma-audio-integration-test PRIVATE clap nlohmann_json::nlohmann_json)
+    add_test(NAME embeddinggemma2_audio_integration
+        COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_audio_integration.py"
+                "$<TARGET_FILE:clap-gemma-audio-integration-test>" "$<TARGET_FILE:clap-cli>")
+    set_tests_properties(embeddinggemma2_audio_integration PROPERTIES TIMEOUT 120)
+    add_executable(clap-gemma-audio-test tests/models/embeddinggemma2_audio-test.cpp)
+    target_include_directories(clap-gemma-audio-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+    target_link_libraries(clap-gemma-audio-test PRIVATE clap nlohmann_json::nlohmann_json)
+    add_test(NAME embeddinggemma2_audio
+        COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_audio.py"
+                --test "$<TARGET_FILE:clap-gemma-audio-test>")
+    set_tests_properties(embeddinggemma2_audio PROPERTIES TIMEOUT 180)
+    add_executable(clap-gemma-soft-test tests/models/gemma-soft-test.cpp)
+    target_include_directories(clap-gemma-soft-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+    target_link_libraries(clap-gemma-soft-test PRIVATE clap nlohmann_json::nlohmann_json)
+    add_test(NAME embeddinggemma2_soft
+        COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_soft.py"
+                "$<TARGET_FILE:clap-gemma-soft-test>")
+    add_executable(clap-embeddinggemma2-test tests/models/embeddinggemma2-test.cpp)
+    target_link_libraries(clap-embeddinggemma2-test PRIVATE clap nlohmann_json::nlohmann_json)
+    add_test(NAME embeddinggemma2_model
+        COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2.py"
+                --test "$<TARGET_FILE:clap-embeddinggemma2-test>")
+    set_tests_properties(embeddinggemma2_model PROPERTIES TIMEOUT 120)
+    add_test(NAME embeddinggemma2_cli
+        COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_cli.py"
+                "$<TARGET_FILE:clap-cli>")
+    set_tests_properties(embeddinggemma2_cli PROPERTIES TIMEOUT 120)
+    add_executable(clap-gemma-tokenizer-test tests/models/gemma-tokenizer-test.cpp)
+    target_include_directories(clap-gemma-tokenizer-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+    target_link_libraries(clap-gemma-tokenizer-test PRIVATE clap nlohmann_json::nlohmann_json ICU::uc ICU::i18n)
+    add_test(NAME gemma_tokenizer COMMAND clap-gemma-tokenizer-test)
+    set(CLAP_GEMMA_TOKENIZER_JSON "" CACHE FILEPATH "Pinned EmbeddingGemma2 tokenizer.json for optional HF parity (Python tokenizers required)")
+    if(CLAP_GEMMA_TOKENIZER_JSON)
+        if(NOT EXISTS "${CLAP_GEMMA_TOKENIZER_JSON}")
+            message(FATAL_ERROR "CLAP_GEMMA_TOKENIZER_JSON does not exist")
+        endif()
+        set(GEMMA_TOKENIZER_FIXTURE "${CMAKE_CURRENT_BINARY_DIR}/gemma-tokenizer-parity")
+        add_test(NAME gemma_tokenizer_hf_fixture
+            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/gemma_tokenizer_parity.py"
+                    "${CLAP_GEMMA_TOKENIZER_JSON}" "${GEMMA_TOKENIZER_FIXTURE}")
+        set_tests_properties(gemma_tokenizer_hf_fixture PROPERTIES FIXTURES_SETUP gemma_tokenizer_hf TIMEOUT 120)
+        add_test(NAME gemma_tokenizer_hf
+            COMMAND clap-gemma-tokenizer-test "${GEMMA_TOKENIZER_FIXTURE}/metadata.json" "${GEMMA_TOKENIZER_FIXTURE}/cases.json")
+        set_tests_properties(gemma_tokenizer_hf PROPERTIES FIXTURES_REQUIRED gemma_tokenizer_hf TIMEOUT 120)
+    endif()
+    add_executable(clap-gemma-storage-test tests/gemma-storage-test.cpp)
+    target_include_directories(clap-gemma-storage-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+    target_link_libraries(clap-gemma-storage-test PRIVATE clap nlohmann_json::nlohmann_json)
+    add_test(NAME gemma_storage COMMAND clap-gemma-storage-test "${CMAKE_CURRENT_BINARY_DIR}/gemma-storage-")
+    add_executable(clap-metadata-test tests/metadata-test.cpp)
+    target_include_directories(clap-metadata-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+    target_link_libraries(clap-metadata-test PRIVATE clap nlohmann_json::nlohmann_json)
+    add_test(NAME clap_metadata COMMAND clap-metadata-test "${CMAKE_CURRENT_BINARY_DIR}/metadata.gguf")
     add_executable(clap-resample-test tests/resample-test.cpp)
     target_link_libraries(clap-resample-test PRIVATE clap)
     add_test(NAME clap_resample COMMAND clap-resample-test)
@@ -87,12 +142,12 @@ if(BUILD_TESTING)
         COMMAND clap-quantize-output-test "${CMAKE_CURRENT_BINARY_DIR}/quantize-publication-fixture")
     add_executable(all-quantization-test tests/all-quantization-test.cpp src/quantize.cpp)
     target_include_directories(all-quantization-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
-    target_link_libraries(all-quantization-test PRIVATE clap)
+    target_link_libraries(all-quantization-test PRIVATE clap nlohmann_json::nlohmann_json)
     add_test(NAME all_architecture_quantization COMMAND all-quantization-test "${CMAKE_CURRENT_BINARY_DIR}/all-quantization-fixture")
     set_tests_properties(all_architecture_quantization PROPERTIES TIMEOUT 120)
     add_executable(clap-q8-test tests/q8-test.cpp)
     target_include_directories(clap-q8-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
-    target_link_libraries(clap-q8-test PRIVATE clap)
+    target_link_libraries(clap-q8-test PRIVATE clap nlohmann_json::nlohmann_json)
     add_test(NAME clap_quantize_cli
         COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/quantize_cli.py" "$<TARGET_FILE:clap-cli>")
     set_tests_properties(clap_quantize_cli PROPERTIES TIMEOUT 30)
@@ -126,7 +181,7 @@ if(BUILD_TESTING)
     add_test(NAME clap_model_unit COMMAND clap-model-test)
     add_executable(tensor-ops-test tests/tensor-ops-test.cpp)
     target_include_directories(tensor-ops-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
-    target_link_libraries(tensor-ops-test PRIVATE clap)
+    target_link_libraries(tensor-ops-test PRIVATE clap nlohmann_json::nlohmann_json)
     add_test(NAME clap_tensor_ops
         COMMAND tensor-ops-test "${CMAKE_CURRENT_BINARY_DIR}/tensor-ops-auto.gguf")
     add_test(NAME clap_tensor_ops_cpu

README.md

@@ -18,6 +18,7 @@ C++17/ggml implementations of audio embedding inference features:
 | [MERT v2](https://huggingface.co/m-a-p/MERT-v2-FullSong) | ❌ | ✅ | 1,024 | 24 kHz |
 | [MuQ](https://huggingface.co/OpenMuQ/MuQ-large-msd-iter) | ❌ | ✅ | 1,024 | 24 kHz |
 | [MuQ-MuLan](https://huggingface.co/OpenMuQ/MuQ-MuLan-large) | ✅ | ✅ | 512 | 24 kHz |
+| [EmbeddingGemma 2](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) | ✅ | ✅ | 768 | 16 kHz |
 
 GGUF models are available at [Audio Embedding GGUF](https://huggingface.co/collections/gorse-io/audio-embedding-gguf) collection.
 
@@ -40,6 +41,58 @@ clap embedding --model model.gguf --text 'A jazz quartet.'
 clap embedding --model model.gguf --audio music.flac
 ```
 
+### EmbeddingGemma 2 (text and audio)
+
+Load the published [Unsloth EmbeddingGemma 2 GGUF](https://huggingface.co/unsloth/embeddinggemma-2-GGUF)
+files directly; no conversion or Python inference is required. Text needs only
+the main model. Audio additionally needs `mmproj-Q8_0.gguf` (or the published
+F16/BF16 mmproj), which contains the audio tower and projector. For example:
+
+```sh
+./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf --text 'Hello world'
+./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \
+  --text 'task: search result | query: How do plants grow?'
+./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \
+  --text 'title: none | text: Plants use sunlight for photosynthesis.'
+./build/clap serve --model embeddinggemma-2-UD-Q4_K_XL.gguf
+```
+
+`encode_text` returns a normalized 768-dimensional embedding. Both BOS and EOS
+are added automatically; mean pooling includes these tokens and any prompt.
+Retrieval prefixes are **not** added automatically: supply
+`task: search result | query: ` for queries and `title: none | text: ` for
+untitled documents, or `title: {title} | text: {content}` for titled documents.
+The tokenizer preserves Unicode, case, and repeated whitespace. Google advertises
+an 8192-token context; use that limit even though the GGUF positional capacity is
+larger. Native computation preserves F32 activations with F16/BF16 storage and
+accepts the published Q4_K/Q5_K/Q6_K/Q8_0 mixed tensor types.
+
+This integration supports text and native waveform embeddings; **`encode_image`
+remains unsupported**, even though the published mmproj also contains vision
+weights. Audio is encoded directly, not transcribed to text:
+
+```sh
+./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \
+  --mmproj mmproj-Q8_0.gguf --audio recording.wav
+```
+
+```cpp
+clap::CLAP model("embeddinggemma-2-UD-Q4_K_XL.gguf", "mmproj-Q8_0.gguf");
+auto audio = clap::decode_audio("recording.wav");
+auto embedding = model.encode_audio(audio.samples, audio.sample_rate);
+```
+
+WAV/MP3/FLAC input is downmixed and resampled to 16 kHz. Audio preprocessing
+follows the published Gemma4 frontend and uses the first 30 seconds, matching
+the default processor's truncation policy. The audio Conformer/projector's
+soft tokens pass through the same text backbone and final mean/L2 pooling,
+producing a normalized 768D embedding in the joint space. Audio requires the
+matching mmproj; text still works without it. Crop offsets and frame APIs are
+not supported for this joint encoder. The HTTP endpoint remains text-only.
+
+Use published GGUFs directly; the checkpoint converter and local quantization
+presets remain scoped to the six existing audio/joint architectures.
+
 ## Embeddings server
 
 Start the server with a text-capable GGUF, such as CLAP, CLaMP3 or MuQ-MuLan
@@ -62,4 +115,4 @@ curl http://127.0.0.1:8080/v1/embeddings \
 ## License
 
 This project is licensed under the [Apache License 2.0](LICENSE).
-Third-party dependencies retain their respective licenses.
+Third-party dependencies retain their respective licenses.

include/clap.h

@@ -42,7 +42,7 @@ Audio resample_audio(const Audio &audio, int sample_rate = 48000);
 std::vector<float> log_mel(const std::vector<float> &pcm, int sample_rate = 48000,
                            std::size_t crop_offset = 0);
 
-// Native CLAP, CLaMP3, MERT-v1, MERT2, MuQ or MuQ-MuLan, selected from GGUF metadata. Weights are
+// Native CLAP, CLaMP3, MERT-v1, MERT2, MuQ, MuQ-MuLan or EmbeddingGemma2, selected from GGUF metadata. Weights are
 // memory-mapped on POSIX platforms; MuQ F16 weights expand to owned F32 buffers.
 // Each call embeds one item; methods on the same instance are not thread-safe.
 // CLAP returns unit-normalized projections. CLaMP3 returns its native unnormalized
@@ -57,9 +57,14 @@ std::vector<float> log_mel(const std::vector<float> &pcm, int sample_rate = 4800
 // an unweighted average without final normalization. No amplitude normalization.
 // MuLan supports neither frame/features APIs nor crop offsets; published width
 // is 512. Its text string API rejects over-context input without truncation.
+// EmbeddingGemma2 text/audio return normalized 768D joint embeddings. Audio
+// requires the separate mmproj constructor, resamples to 16 kHz, and uses the
+// first 30 seconds (published processor policy). No audio frames/crop offsets.
 class CLAP {
 public:
     explicit CLAP(const std::string &gguf_path, int threads = 0);
+    // Optional EmbeddingGemma2 audio tower/projector GGUF. Text works without it.
+    CLAP(const std::string &gguf_path, const std::string &mmproj_path, int threads = 0);
     ~CLAP();
     CLAP(CLAP &&) noexcept;
     CLAP &operator=(CLAP &&) noexcept;

node/README.md

@@ -24,3 +24,16 @@ try {
   model.close(); // optional early release; garbage collection also frees weights
 }
 ```
+
+For EmbeddingGemma 2 audio, pass the matching mmproj in constructor options:
+
+```js
+const model = new Model('embeddinggemma2.gguf', { threads: 4, mmproj: 'mmproj-Q8_0.gguf' });
+const { samples, sampleRate } = decodeAudio('recording.wav');
+const embedding = model.audio(samples, sampleRate);
+model.close();
+```
+
+Omitting `mmproj` preserves text/single-model loading; numeric `threads` is
+still accepted. Missing/incompatible projectors throw native errors. Image
+encoding is not supported.

node/bindings.cpp

@@ -99,8 +99,20 @@ class Model : public Napi::ObjectWrap<Model> {
     explicit Model(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Model>(info) {
         guarded(info.Env(), [&]() -> Napi::Value {
             auto p = path(info[0]);
-            int threads = info[1].IsUndefined() ? 0 : static_cast<int>(integer(info[1], "threads", 0, std::numeric_limits<int>::max()));
-            model_ = std::make_unique<clap::CLAP>(p, threads);
+            auto thread_value = info[1];
+            Napi::Value mmproj_value = info.Env().Undefined();
+            if (info[1].IsObject() && !info[1].IsArray() && !info[1].IsFunction()) {
+                auto options = info[1].As<Napi::Object>();
+                thread_value = options.Get("threads");
+                mmproj_value = options.Get("mmproj");
+            }
+            int threads = thread_value.IsUndefined() ? 0 : static_cast<int>(integer(thread_value, "threads", 0, std::numeric_limits<int>::max()));
+            // Read all option getters before constructing the native model.
+            if (mmproj_value.IsUndefined()) model_ = std::make_unique<clap::CLAP>(p, threads);
+            else {
+                auto mmproj = path(mmproj_value);
+                model_ = std::make_unique<clap::CLAP>(p, mmproj, threads);
+            }
             return info.Env().Undefined();
         });
     }

node/index.d.ts

@@ -8,10 +8,16 @@ export interface Audio {
   samples: Float32Array;
   sampleRate: number;
 }
+export interface ModelOptions {
+  /** Zero selects the native thread default. */
+  threads?: number;
+  /** Separate audio mmproj GGUF for EmbeddingGemma2 audio encoding. */
+  mmproj?: string;
+}
 /** Synchronous CPU inference. Each call blocks the calling JavaScript thread. */
 export class Model {
   /** threads=0 selects the native default; negative/fractional values are rejected. */
-  constructor(path: string, threads?: number);
+  constructor(path: string, threadsOrOptions?: number | ModelOptions);
   readonly projectionDimension: number;
   readonly audioSampleRate: number;
   /** Native tokenizer; maxLength defaults to 512. Returns owned token IDs. */

node/test/addon.test.js

@@ -20,6 +20,23 @@ function near(a, b, tolerance = 2e-5) {
   a.forEach((x, i) => assert.ok(Math.abs(x - b[i]) < tolerance, `${i}: ${x} != ${b[i]}`));
 }
 const pcm = Float32Array.from({ length: 24000 }, (_, i) => 0.2 * Math.sin(i * 0.031));
+test('constructor options preserve text loading and do not ignore mmproj', () => {
+  for (const options of [{}, { threads: 1 }, { threads: 1, mmproj: undefined }]) {
+    const m = new Model(modelPath, options);
+    try { vector(m.text('hello'), 3); } finally { m.close(); }
+  }
+  assert.throws(() => new Model(path.join(fixtures, 'embeddinggemma2-dispatch.gguf'), {
+    threads: 1, mmproj: path.join(fixtures, 'missing-audio-mmproj.gguf')
+  }), /missing-audio-mmproj/);
+  assert.throws(() => new Model(modelPath, { mmproj: 'missing-audio-mmproj.gguf' }),
+    error => error instanceof Error && !(error instanceof TypeError));
+  for (const mmproj of [null, 1, '', 'a\0b']) {
+    assert.throws(() => new Model(modelPath, { mmproj }), TypeError);
+  }
+  for (const threads of [-1, 0.5, NaN, null, '1']) {
+    assert.throws(() => new Model(modelPath, { threads }), TypeError);
+  }
+});
 test('native exports and strict constructor validation', () => {
   assert.equal(typeof Model, 'function');
   for (const p of [undefined, null, 1, '', 'a\0b']) assert.throws(() => new Model(p));

node/test/generate-fixtures.py

@@ -16,6 +16,9 @@
 out.mkdir(parents=True, exist_ok=True)
 fixtures.clap_fixture(out / 'clap.gguf')
 fixtures.mert_fixture(out / 'mert.gguf')
+# Metadata-only probe; missing mmproj must fail before model tensor validation.
+fixtures.write_gguf(out / 'embeddinggemma2-dispatch.gguf',
+                    {'general.architecture': 'gemma-embedding2'}, {})
 with wave.open(str(out / 'stereo.wav'), 'wb') as wav:
     wav.setparams((2, 2, 24000, 0, 'NONE', 'not compressed'))
     wav.writeframes(struct.pack('<hh', 16384, 0) * 24000)

node/test/types.ts

@@ -1,5 +1,10 @@
 import { Model, decodeAudio, resampleAudio, AudioFrames } from '..';
 const model = new Model('model.gguf', 1);
+new Model('model.gguf', {});
+new Model('model.gguf', { threads: 1 });
+new Model('model.gguf', { mmproj: 'audio-mmproj.gguf', threads: 1 });
+// @ts-expect-error mmproj must be a path string
+new Model('model.gguf', { mmproj: 1 });
 const ids: Int32Array = model.tokenize('hello');
 const embedding: Float32Array = model.text(ids);
 model.text('hello'); model.text([0, 2]);

python/README.md

@@ -16,3 +16,14 @@ with CLAP("model.gguf", threads=4) as model:
     audio_embedding = model.encode_audio("dog.wav")
     print(model.projection_dimension, model.audio_sample_rate)
 ```
+
+For EmbeddingGemma 2 audio, also load the matching audio mmproj GGUF:
+
+```python
+with CLAP("embeddinggemma2.gguf", threads=4, mmproj="mmproj-Q8_0.gguf") as model:
+    audio_embedding = model.encode_audio("recording.wav")
+```
+
+`mmproj` accepts a string or `pathlib.Path`. Text still works without it; the
+existing positional `threads` argument is unchanged. Missing/incompatible
+projectors raise native errors. Image encoding is not supported.

python/bindings.cpp

@@ -10,6 +10,7 @@
 #include <filesystem>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <stdexcept>
 #include <vector>
 
@@ -53,10 +54,18 @@ py::array_t<float> matrix(const clap::AudioFrames &frames) {
 // BEFORE reacquiring the GIL. close(), properties and inference share this order.
 class CLAP {
 public:
-    CLAP(const std::filesystem::path &path, int threads) {
+    CLAP(const std::filesystem::path &path, int threads,
+         const std::optional<std::filesystem::path> &mmproj) {
         if (threads < 0) throw std::invalid_argument("threads must be nonnegative");
+        const auto filename = path.u8string();
+        const auto audio_path = mmproj ? std::optional<std::string>(mmproj->u8string()) : std::nullopt;
+        if (filename.find('\0') != std::string::npos ||
+            (audio_path && (audio_path->empty() || audio_path->find('\0') != std::string::npos))) {
+            throw std::invalid_argument("model paths must contain no NUL bytes and mmproj must be nonempty");
+        }
         py::gil_scoped_release release;
-        model_ = std::make_unique<clap::CLAP>(path.u8string(), threads);
+        model_ = audio_path ? std::make_unique<clap::CLAP>(filename, *audio_path, threads)
+                            : std::make_unique<clap::CLAP>(filename, threads);
     }
 
     template <typename F>
@@ -83,7 +92,8 @@ class CLAP {
 PYBIND11_MODULE(_clap_cpp, module) {
     module.doc() = "Native CPU inference for clap.cpp";
     py::class_<CLAP>(module, "CLAP")
-        .def(py::init<const std::filesystem::path &, int>(), py::arg("path"), py::arg("threads") = 0)
+        .def(py::init<const std::filesystem::path &, int, const std::optional<std::filesystem::path> &>(),
+             py::arg("path"), py::arg("threads") = 0, py::kw_only(), py::arg("mmproj") = py::none())
         .def_property_readonly("projection_dimension", [](CLAP &self) {
             return self.call([](clap::CLAP &model) { return model.projection_dimension(); });
         })