diff --git a/CMakeLists.txt b/CMakeLists.txt index a45d331..ac98252 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,61 @@ if(BUILD_TESTING) COMMENT "Generating audio test fixtures" VERBATIM) add_custom_target(clap-audio-fixtures ALL DEPENDS ${CLAP_AUDIO_FIXTURES}) + add_executable(clap-gemma-audio-integration-test tests/models/gemma-audio-integration-test.cpp) + target_link_libraries(clap-gemma-audio-integration-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_audio_integration + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_audio_integration.py" + "$" "$") + set_tests_properties(embeddinggemma2_audio_integration PROPERTIES TIMEOUT 120) + add_executable(clap-gemma-audio-test tests/models/embeddinggemma2_audio-test.cpp) + target_include_directories(clap-gemma-audio-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-audio-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_audio + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_audio.py" + --test "$") + set_tests_properties(embeddinggemma2_audio PROPERTIES TIMEOUT 180) + add_executable(clap-gemma-soft-test tests/models/gemma-soft-test.cpp) + target_include_directories(clap-gemma-soft-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-soft-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_soft + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_soft.py" + "$") + add_executable(clap-embeddinggemma2-test tests/models/embeddinggemma2-test.cpp) + target_link_libraries(clap-embeddinggemma2-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_model + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2.py" + --test "$") + set_tests_properties(embeddinggemma2_model PROPERTIES TIMEOUT 120) + add_test(NAME embeddinggemma2_cli + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_cli.py" + "$") + set_tests_properties(embeddinggemma2_cli PROPERTIES TIMEOUT 120) + add_executable(clap-gemma-tokenizer-test tests/models/gemma-tokenizer-test.cpp) + target_include_directories(clap-gemma-tokenizer-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-tokenizer-test PRIVATE clap nlohmann_json::nlohmann_json ICU::uc ICU::i18n) + add_test(NAME gemma_tokenizer COMMAND clap-gemma-tokenizer-test) + set(CLAP_GEMMA_TOKENIZER_JSON "" CACHE FILEPATH "Pinned EmbeddingGemma2 tokenizer.json for optional HF parity (Python tokenizers required)") + if(CLAP_GEMMA_TOKENIZER_JSON) + if(NOT EXISTS "${CLAP_GEMMA_TOKENIZER_JSON}") + message(FATAL_ERROR "CLAP_GEMMA_TOKENIZER_JSON does not exist") + endif() + set(GEMMA_TOKENIZER_FIXTURE "${CMAKE_CURRENT_BINARY_DIR}/gemma-tokenizer-parity") + add_test(NAME gemma_tokenizer_hf_fixture + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/gemma_tokenizer_parity.py" + "${CLAP_GEMMA_TOKENIZER_JSON}" "${GEMMA_TOKENIZER_FIXTURE}") + set_tests_properties(gemma_tokenizer_hf_fixture PROPERTIES FIXTURES_SETUP gemma_tokenizer_hf TIMEOUT 120) + add_test(NAME gemma_tokenizer_hf + COMMAND clap-gemma-tokenizer-test "${GEMMA_TOKENIZER_FIXTURE}/metadata.json" "${GEMMA_TOKENIZER_FIXTURE}/cases.json") + set_tests_properties(gemma_tokenizer_hf PROPERTIES FIXTURES_REQUIRED gemma_tokenizer_hf TIMEOUT 120) + endif() + add_executable(clap-gemma-storage-test tests/gemma-storage-test.cpp) + target_include_directories(clap-gemma-storage-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-storage-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME gemma_storage COMMAND clap-gemma-storage-test "${CMAKE_CURRENT_BINARY_DIR}/gemma-storage-") + add_executable(clap-metadata-test tests/metadata-test.cpp) + target_include_directories(clap-metadata-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-metadata-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME clap_metadata COMMAND clap-metadata-test "${CMAKE_CURRENT_BINARY_DIR}/metadata.gguf") add_executable(clap-resample-test tests/resample-test.cpp) target_link_libraries(clap-resample-test PRIVATE clap) add_test(NAME clap_resample COMMAND clap-resample-test) @@ -87,12 +142,12 @@ if(BUILD_TESTING) COMMAND clap-quantize-output-test "${CMAKE_CURRENT_BINARY_DIR}/quantize-publication-fixture") add_executable(all-quantization-test tests/all-quantization-test.cpp src/quantize.cpp) target_include_directories(all-quantization-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") - target_link_libraries(all-quantization-test PRIVATE clap) + target_link_libraries(all-quantization-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME all_architecture_quantization COMMAND all-quantization-test "${CMAKE_CURRENT_BINARY_DIR}/all-quantization-fixture") set_tests_properties(all_architecture_quantization PROPERTIES TIMEOUT 120) add_executable(clap-q8-test tests/q8-test.cpp) target_include_directories(clap-q8-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") - target_link_libraries(clap-q8-test PRIVATE clap) + target_link_libraries(clap-q8-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME clap_quantize_cli COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/quantize_cli.py" "$") set_tests_properties(clap_quantize_cli PROPERTIES TIMEOUT 30) @@ -126,7 +181,7 @@ if(BUILD_TESTING) add_test(NAME clap_model_unit COMMAND clap-model-test) add_executable(tensor-ops-test tests/tensor-ops-test.cpp) target_include_directories(tensor-ops-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") - target_link_libraries(tensor-ops-test PRIVATE clap) + target_link_libraries(tensor-ops-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME clap_tensor_ops COMMAND tensor-ops-test "${CMAKE_CURRENT_BINARY_DIR}/tensor-ops-auto.gguf") add_test(NAME clap_tensor_ops_cpu diff --git a/README.md b/README.md index 0724ee0..45e3bbf 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ C++17/ggml implementations of audio embedding inference features: | [MERT v2](https://huggingface.co/m-a-p/MERT-v2-FullSong) | ❌ | ✅ | 1,024 | 24 kHz | | [MuQ](https://huggingface.co/OpenMuQ/MuQ-large-msd-iter) | ❌ | ✅ | 1,024 | 24 kHz | | [MuQ-MuLan](https://huggingface.co/OpenMuQ/MuQ-MuLan-large) | ✅ | ✅ | 512 | 24 kHz | +| [EmbeddingGemma 2](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) | ✅ | ✅ | 768 | 16 kHz | GGUF models are available at [Audio Embedding GGUF](https://huggingface.co/collections/gorse-io/audio-embedding-gguf) collection. @@ -40,6 +41,58 @@ clap embedding --model model.gguf --text 'A jazz quartet.' clap embedding --model model.gguf --audio music.flac ``` +### EmbeddingGemma 2 (text and audio) + +Load the published [Unsloth EmbeddingGemma 2 GGUF](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) +files directly; no conversion or Python inference is required. Text needs only +the main model. Audio additionally needs `mmproj-Q8_0.gguf` (or the published +F16/BF16 mmproj), which contains the audio tower and projector. For example: + +```sh +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf --text 'Hello world' +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \ + --text 'task: search result | query: How do plants grow?' +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \ + --text 'title: none | text: Plants use sunlight for photosynthesis.' +./build/clap serve --model embeddinggemma-2-UD-Q4_K_XL.gguf +``` + +`encode_text` returns a normalized 768-dimensional embedding. Both BOS and EOS +are added automatically; mean pooling includes these tokens and any prompt. +Retrieval prefixes are **not** added automatically: supply +`task: search result | query: ` for queries and `title: none | text: ` for +untitled documents, or `title: {title} | text: {content}` for titled documents. +The tokenizer preserves Unicode, case, and repeated whitespace. Google advertises +an 8192-token context; use that limit even though the GGUF positional capacity is +larger. Native computation preserves F32 activations with F16/BF16 storage and +accepts the published Q4_K/Q5_K/Q6_K/Q8_0 mixed tensor types. + +This integration supports text and native waveform embeddings; **`encode_image` +remains unsupported**, even though the published mmproj also contains vision +weights. Audio is encoded directly, not transcribed to text: + +```sh +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \ + --mmproj mmproj-Q8_0.gguf --audio recording.wav +``` + +```cpp +clap::CLAP model("embeddinggemma-2-UD-Q4_K_XL.gguf", "mmproj-Q8_0.gguf"); +auto audio = clap::decode_audio("recording.wav"); +auto embedding = model.encode_audio(audio.samples, audio.sample_rate); +``` + +WAV/MP3/FLAC input is downmixed and resampled to 16 kHz. Audio preprocessing +follows the published Gemma4 frontend and uses the first 30 seconds, matching +the default processor's truncation policy. The audio Conformer/projector's +soft tokens pass through the same text backbone and final mean/L2 pooling, +producing a normalized 768D embedding in the joint space. Audio requires the +matching mmproj; text still works without it. Crop offsets and frame APIs are +not supported for this joint encoder. The HTTP endpoint remains text-only. + +Use published GGUFs directly; the checkpoint converter and local quantization +presets remain scoped to the six existing audio/joint architectures. + ## Embeddings server Start the server with a text-capable GGUF, such as CLAP, CLaMP3 or MuQ-MuLan @@ -62,4 +115,4 @@ curl http://127.0.0.1:8080/v1/embeddings \ ## License This project is licensed under the [Apache License 2.0](LICENSE). -Third-party dependencies retain their respective licenses. +Third-party dependencies retain their respective licenses. \ No newline at end of file diff --git a/include/clap.h b/include/clap.h index d9c36b3..a5b5b49 100644 --- a/include/clap.h +++ b/include/clap.h @@ -42,7 +42,7 @@ Audio resample_audio(const Audio &audio, int sample_rate = 48000); std::vector log_mel(const std::vector &pcm, int sample_rate = 48000, std::size_t crop_offset = 0); -// Native CLAP, CLaMP3, MERT-v1, MERT2, MuQ or MuQ-MuLan, selected from GGUF metadata. Weights are +// Native CLAP, CLaMP3, MERT-v1, MERT2, MuQ, MuQ-MuLan or EmbeddingGemma2, selected from GGUF metadata. Weights are // memory-mapped on POSIX platforms; MuQ F16 weights expand to owned F32 buffers. // Each call embeds one item; methods on the same instance are not thread-safe. // CLAP returns unit-normalized projections. CLaMP3 returns its native unnormalized @@ -57,9 +57,14 @@ std::vector log_mel(const std::vector &pcm, int sample_rate = 4800 // an unweighted average without final normalization. No amplitude normalization. // MuLan supports neither frame/features APIs nor crop offsets; published width // is 512. Its text string API rejects over-context input without truncation. +// EmbeddingGemma2 text/audio return normalized 768D joint embeddings. Audio +// requires the separate mmproj constructor, resamples to 16 kHz, and uses the +// first 30 seconds (published processor policy). No audio frames/crop offsets. class CLAP { public: explicit CLAP(const std::string &gguf_path, int threads = 0); + // Optional EmbeddingGemma2 audio tower/projector GGUF. Text works without it. + CLAP(const std::string &gguf_path, const std::string &mmproj_path, int threads = 0); ~CLAP(); CLAP(CLAP &&) noexcept; CLAP &operator=(CLAP &&) noexcept; diff --git a/node/README.md b/node/README.md index 0922d19..5a86cbe 100644 --- a/node/README.md +++ b/node/README.md @@ -24,3 +24,16 @@ try { model.close(); // optional early release; garbage collection also frees weights } ``` + +For EmbeddingGemma 2 audio, pass the matching mmproj in constructor options: + +```js +const model = new Model('embeddinggemma2.gguf', { threads: 4, mmproj: 'mmproj-Q8_0.gguf' }); +const { samples, sampleRate } = decodeAudio('recording.wav'); +const embedding = model.audio(samples, sampleRate); +model.close(); +``` + +Omitting `mmproj` preserves text/single-model loading; numeric `threads` is +still accepted. Missing/incompatible projectors throw native errors. Image +encoding is not supported. diff --git a/node/bindings.cpp b/node/bindings.cpp index f896dd5..7c58d74 100644 --- a/node/bindings.cpp +++ b/node/bindings.cpp @@ -99,8 +99,20 @@ class Model : public Napi::ObjectWrap { explicit Model(const Napi::CallbackInfo &info) : Napi::ObjectWrap(info) { guarded(info.Env(), [&]() -> Napi::Value { auto p = path(info[0]); - int threads = info[1].IsUndefined() ? 0 : static_cast(integer(info[1], "threads", 0, std::numeric_limits::max())); - model_ = std::make_unique(p, threads); + auto thread_value = info[1]; + Napi::Value mmproj_value = info.Env().Undefined(); + if (info[1].IsObject() && !info[1].IsArray() && !info[1].IsFunction()) { + auto options = info[1].As(); + thread_value = options.Get("threads"); + mmproj_value = options.Get("mmproj"); + } + int threads = thread_value.IsUndefined() ? 0 : static_cast(integer(thread_value, "threads", 0, std::numeric_limits::max())); + // Read all option getters before constructing the native model. + if (mmproj_value.IsUndefined()) model_ = std::make_unique(p, threads); + else { + auto mmproj = path(mmproj_value); + model_ = std::make_unique(p, mmproj, threads); + } return info.Env().Undefined(); }); } diff --git a/node/index.d.ts b/node/index.d.ts index 5195bba..72ddb0d 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -8,10 +8,16 @@ export interface Audio { samples: Float32Array; sampleRate: number; } +export interface ModelOptions { + /** Zero selects the native thread default. */ + threads?: number; + /** Separate audio mmproj GGUF for EmbeddingGemma2 audio encoding. */ + mmproj?: string; +} /** Synchronous CPU inference. Each call blocks the calling JavaScript thread. */ export class Model { /** threads=0 selects the native default; negative/fractional values are rejected. */ - constructor(path: string, threads?: number); + constructor(path: string, threadsOrOptions?: number | ModelOptions); readonly projectionDimension: number; readonly audioSampleRate: number; /** Native tokenizer; maxLength defaults to 512. Returns owned token IDs. */ diff --git a/node/test/addon.test.js b/node/test/addon.test.js index b248e3c..fb12df5 100644 --- a/node/test/addon.test.js +++ b/node/test/addon.test.js @@ -20,6 +20,23 @@ function near(a, b, tolerance = 2e-5) { a.forEach((x, i) => assert.ok(Math.abs(x - b[i]) < tolerance, `${i}: ${x} != ${b[i]}`)); } const pcm = Float32Array.from({ length: 24000 }, (_, i) => 0.2 * Math.sin(i * 0.031)); +test('constructor options preserve text loading and do not ignore mmproj', () => { + for (const options of [{}, { threads: 1 }, { threads: 1, mmproj: undefined }]) { + const m = new Model(modelPath, options); + try { vector(m.text('hello'), 3); } finally { m.close(); } + } + assert.throws(() => new Model(path.join(fixtures, 'embeddinggemma2-dispatch.gguf'), { + threads: 1, mmproj: path.join(fixtures, 'missing-audio-mmproj.gguf') + }), /missing-audio-mmproj/); + assert.throws(() => new Model(modelPath, { mmproj: 'missing-audio-mmproj.gguf' }), + error => error instanceof Error && !(error instanceof TypeError)); + for (const mmproj of [null, 1, '', 'a\0b']) { + assert.throws(() => new Model(modelPath, { mmproj }), TypeError); + } + for (const threads of [-1, 0.5, NaN, null, '1']) { + assert.throws(() => new Model(modelPath, { threads }), TypeError); + } +}); test('native exports and strict constructor validation', () => { assert.equal(typeof Model, 'function'); for (const p of [undefined, null, 1, '', 'a\0b']) assert.throws(() => new Model(p)); diff --git a/node/test/generate-fixtures.py b/node/test/generate-fixtures.py index 5d620fb..2879c23 100644 --- a/node/test/generate-fixtures.py +++ b/node/test/generate-fixtures.py @@ -16,6 +16,9 @@ out.mkdir(parents=True, exist_ok=True) fixtures.clap_fixture(out / 'clap.gguf') fixtures.mert_fixture(out / 'mert.gguf') +# Metadata-only probe; missing mmproj must fail before model tensor validation. +fixtures.write_gguf(out / 'embeddinggemma2-dispatch.gguf', + {'general.architecture': 'gemma-embedding2'}, {}) with wave.open(str(out / 'stereo.wav'), 'wb') as wav: wav.setparams((2, 2, 24000, 0, 'NONE', 'not compressed')) wav.writeframes(struct.pack(' #include #include +#include #include #include @@ -53,10 +54,18 @@ py::array_t matrix(const clap::AudioFrames &frames) { // BEFORE reacquiring the GIL. close(), properties and inference share this order. class CLAP { public: - CLAP(const std::filesystem::path &path, int threads) { + CLAP(const std::filesystem::path &path, int threads, + const std::optional &mmproj) { if (threads < 0) throw std::invalid_argument("threads must be nonnegative"); + const auto filename = path.u8string(); + const auto audio_path = mmproj ? std::optional(mmproj->u8string()) : std::nullopt; + if (filename.find('\0') != std::string::npos || + (audio_path && (audio_path->empty() || audio_path->find('\0') != std::string::npos))) { + throw std::invalid_argument("model paths must contain no NUL bytes and mmproj must be nonempty"); + } py::gil_scoped_release release; - model_ = std::make_unique(path.u8string(), threads); + model_ = audio_path ? std::make_unique(filename, *audio_path, threads) + : std::make_unique(filename, threads); } template @@ -83,7 +92,8 @@ class CLAP { PYBIND11_MODULE(_clap_cpp, module) { module.doc() = "Native CPU inference for clap.cpp"; py::class_(module, "CLAP") - .def(py::init(), py::arg("path"), py::arg("threads") = 0) + .def(py::init &>(), + py::arg("path"), py::arg("threads") = 0, py::kw_only(), py::arg("mmproj") = py::none()) .def_property_readonly("projection_dimension", [](CLAP &self) { return self.call([](clap::CLAP &model) { return model.projection_dimension(); }); }) diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index af7cd58..69f6309 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -7,7 +7,7 @@ import pytest import clap_cpp -from tiny_models import clap_fixture, mert_fixture, text_reference +from tiny_models import clap_fixture, mert_fixture, text_reference, write_gguf @pytest.fixture(scope="session") @@ -67,6 +67,26 @@ def test_model_loads_from_unicode_path(tmp_path, as_string): rtol=2e-4, atol=2e-5) +def test_optional_mmproj_preserves_existing_loading(model_paths): + with clap_cpp.CLAP(model_paths[0], 1, mmproj=None) as model: + assert_array(model.encode_text("hello"), (3,)) + + +@pytest.mark.parametrize("as_string", [False, True]) +def test_explicit_mmproj_is_not_ignored(model_paths, tmp_path, as_string): + missing = tmp_path / "missing-audio-mmproj.gguf" + # A metadata-only dispatch probe: failure must name the projector, not + # the incomplete model weights. This tests the exact forwarded path. + probe = tmp_path / "embeddinggemma2-dispatch.gguf" + write_gguf(probe, {"general.architecture": "gemma-embedding2"}, {}) + with pytest.raises(RuntimeError, match="missing-audio-mmproj"): + clap_cpp.CLAP(probe, threads=1, + mmproj=str(missing) if as_string else missing) + # Other architectures must not silently ignore a supplied projector either. + with pytest.raises(RuntimeError): + clap_cpp.CLAP(model_paths[0], mmproj=missing) + + def test_properties_are_readonly(model, mert): assert model.projection_dimension == 3 assert model.audio_sample_rate == 48000 diff --git a/rust/README.md b/rust/README.md index 4bea42e..b4c5a27 100644 --- a/rust/README.md +++ b/rust/README.md @@ -18,3 +18,10 @@ fn main() -> Result<(), Box> { Ok(()) } ``` + +For EmbeddingGemma 2 audio, load the main GGUF and matching mmproj with +`CLAP::new_with_mmproj("embeddinggemma2.gguf", "mmproj-Q8_0.gguf", 4)?`, then +call `model.encode_audio("recording.wav", 0)?` as above. Both paths accept +`impl AsRef`; missing/incompatible projectors return errors. The original +`new(path, threads)` remains unchanged, and text does not require an mmproj. +Image encoding is not supported. diff --git a/rust/native/bridge.cpp b/rust/native/bridge.cpp index e029992..9393f45 100644 --- a/rust/native/bridge.cpp +++ b/rust/native/bridge.cpp @@ -19,14 +19,16 @@ extern "C" { // Inline diagnostics keep exception translation allocation-free. // Ops: 0=open, 1=encode_text, 2=encode_audio (file). Result *cr_call(int op, clap::CLAP *model, const char *s, - int threads, size_t crop_offset) noexcept { + int threads, size_t crop_offset, const char *mmproj) noexcept { auto result = std::unique_ptr(new (std::nothrow) Result); if (!result) return nullptr; try { switch (op) { case 0: if (threads < 0) throw std::invalid_argument("threads must be nonnegative"); - result->model = std::make_unique(s, threads); break; + result->model = mmproj ? std::make_unique(s, mmproj, threads) + : std::make_unique(s, threads); + break; case 1: result->floats = model->encode_text(s); break; case 2: { const auto audio = clap::decode_audio(s); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 5e90f82..9537716 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -21,6 +21,7 @@ mod ffi { s: *const c_char, threads: i32, crop_offset: usize, + mmproj: *const c_char, ) -> *mut c_void; pub fn cr_free(p: *mut c_void); pub fn cr_model_free(p: *mut c_void); @@ -56,8 +57,24 @@ fn path(p: &Path) -> Result { Ok(CString::new(p.to_str().ok_or("path must be valid UTF-8")?)?) } // The synchronous bridge borrows the string and owns all returned data. -fn call(op: i32, model: *mut c_void, s: &CStr, threads: i32, crop_offset: usize) -> Result { - let p = unsafe { ffi::cr_call(op, model, s.as_ptr(), threads, crop_offset) }; +fn call( + op: i32, + model: *mut c_void, + s: &CStr, + threads: i32, + crop_offset: usize, + mmproj: Option<&CStr>, +) -> Result { + let p = unsafe { + ffi::cr_call( + op, + model, + s.as_ptr(), + threads, + crop_offset, + mmproj.map_or(std::ptr::null(), CStr::as_ptr), + ) + }; let result = Native(NonNull::new(p).ok_or("native result allocation failed")?); let error = unsafe { CStr::from_ptr(ffi::cr_error(result.ptr())) }; if !error.is_empty() { @@ -91,8 +108,31 @@ impl Drop for CLAP { impl CLAP { /// Load a UTF-8 GGUF path. Zero threads selects the native default. pub fn new(p: impl AsRef, threads: i32) -> Result { - let s = path(p.as_ref())?; - let native = call(0, std::ptr::null_mut(), &s, threads, 0)?; + Self::load(p.as_ref(), None, threads) + } + /// Load a model with a separate audio mmproj GGUF (EmbeddingGemma2). + /// Both paths must be UTF-8 without NUL; zero threads selects the native default. + pub fn new_with_mmproj( + p: impl AsRef, + mmproj: impl AsRef, + threads: i32, + ) -> Result { + Self::load(p.as_ref(), Some(mmproj.as_ref()), threads) + } + fn load(p: &Path, mmproj: Option<&Path>, threads: i32) -> Result { + let s = path(p)?; + let audio_path = mmproj.map(path).transpose()?; + if audio_path.as_ref().is_some_and(|p| p.as_bytes().is_empty()) { + return Err("mmproj path must be nonempty".into()); + } + let native = call( + 0, + std::ptr::null_mut(), + &s, + threads, + 0, + audio_path.as_deref(), + )?; let ptr = NonNull::new(unsafe { ffi::cr_model_take(native.ptr()) }) .ok_or("native model missing")?; Ok(Self { @@ -103,7 +143,7 @@ impl CLAP { /// Encode one UTF-8 string using the model's native context policy. pub fn encode_text(&mut self, text: &str) -> Result> { let s = CString::new(text)?; - Ok(call(1, self.ptr.as_ptr(), &s, 0, 0)?.floats()) + Ok(call(1, self.ptr.as_ptr(), &s, 0, 0, None)?.floats()) } /// Encode a WAV, MP3 or FLAC file. Channels are averaged to mono and the /// source sample rate is converted internally to the model's input rate. @@ -111,6 +151,6 @@ impl CLAP { /// do not support cropping reject nonzero offsets. pub fn encode_audio(&mut self, p: impl AsRef, crop_offset: usize) -> Result> { let s = path(p.as_ref())?; - Ok(call(2, self.ptr.as_ptr(), &s, 0, crop_offset)?.floats()) + Ok(call(2, self.ptr.as_ptr(), &s, 0, crop_offset, None)?.floats()) } } diff --git a/rust/tests/bindings.rs b/rust/tests/bindings.rs index 789186c..2c5e0e8 100644 --- a/rust/tests/bindings.rs +++ b/rust/tests/bindings.rs @@ -43,6 +43,24 @@ fn errors_are_owned_and_inputs_are_checked() { .is_err()); } +#[test] +fn explicit_mmproj_is_not_ignored_and_paths_are_checked() { + let model_path = fixtures().join("模型.gguf"); + let result = CLAP::new_with_mmproj( + fixtures().join("embeddinggemma2-dispatch.gguf"), + fixtures().join("missing-audio-mmproj.gguf"), + 1, + ); + let error = result.err().expect("missing projector must not be ignored"); + assert!(error.to_string().contains("missing-audio-mmproj")); + assert!(CLAP::new_with_mmproj(&model_path, "missing-audio-mmproj.gguf", 1).is_err()); + assert!(CLAP::new_with_mmproj(&model_path, "nul\0.gguf", 1).is_err()); + assert!(CLAP::new_with_mmproj(&model_path, "missing.gguf", -1).is_err()); + // The original constructor remains text-capable without a projector. + let mut model = CLAP::new(&model_path, 1).unwrap(); + assert_eq!(model.encode_text("Hi").unwrap().len(), 3); +} + #[test] fn clap_real_text_and_audio_file() { let mut model = CLAP::new(fixtures().join("模型.gguf"), 1).unwrap(); @@ -91,6 +109,7 @@ fn non_utf8_paths_are_rejected() { use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; let invalid = std::path::Path::new(OsStr::from_bytes(b"invalid\xff.wav")); assert!(CLAP::new(invalid, 1).is_err()); + assert!(CLAP::new_with_mmproj(fixtures().join("模型.gguf"), invalid, 1).is_err()); let mut model = CLAP::new(fixtures().join("模型.gguf"), 1).unwrap(); assert!(model.encode_audio(invalid, 0).is_err()); } diff --git a/rust/tests/generate.py b/rust/tests/generate.py index d834c5e..7ec7629 100644 --- a/rust/tests/generate.py +++ b/rust/tests/generate.py @@ -67,6 +67,9 @@ def add(x, y): (out / 'text-reference.txt').write_text(' '.join(map(str, text_reference([0, 77, 110, 2])))) tiny.clap_fixture(out / '模型.gguf') tiny.mert_fixture(out / 'mert.gguf') +# Metadata-only probe; missing mmproj must fail before model tensor validation. +tiny.write_gguf(out / 'embeddinggemma2-dispatch.gguf', + {'general.architecture': 'gemma-embedding2'}, {}) with wave.open(str(out / '音频.wav'), 'wb') as f: f.setparams((1, 2, 24000, 4, 'NONE', 'not compressed')) f.writeframes(struct.pack('<4h', 0, 8192, -16384, 24576)) diff --git a/src/clap.cpp b/src/clap.cpp index 9ffc3e2..8e3769b 100644 --- a/src/clap.cpp +++ b/src/clap.cpp @@ -28,8 +28,10 @@ int main(int argc, char **argv) { EmbeddingOptions options; std::string file; auto *embedding = app.add_subcommand("embedding", "Generate native text or audio embeddings"); - embedding->add_option("-m,--model", options.model_path, "Converted CLAP/CLaMP3/MERT/MERT2/MuQ/MuLan GGUF") + embedding->add_option("-m,--model", options.model_path, "Model GGUF") ->required()->check(CLI::ExistingFile); + embedding->add_option("--mmproj", options.mmproj_path, "EmbeddingGemma2 audio tower/projector GGUF") + ->check(CLI::ExistingFile); embedding->add_option("-t,--threads", options.threads, "ggml CPU threads (0: automatic)") ->check(CLI::NonNegativeNumber)->capture_default_str(); auto *input = embedding->add_option_group("Input", "Choose one text prompt, text file, or audio file"); diff --git a/src/embedding.cpp b/src/embedding.cpp index da8bed9..6db5e3c 100644 --- a/src/embedding.cpp +++ b/src/embedding.cpp @@ -69,7 +69,7 @@ int run_embedding(const EmbeddingOptions &options) { // Preserve native CLI error precedence: decode audio before loading weights. clap::Audio decoded; if (!options.audio_path.empty()) decoded = clap::decode_audio(options.audio_path); - clap::CLAP model(options.model_path, options.threads); + clap::CLAP model(options.model_path, options.mmproj_path, options.threads); std::vector vectors; if (!options.audio_path.empty()) { if (options.crop_requested && model.audio_sample_rate() != 48000) { diff --git a/src/embedding.h b/src/embedding.h index 08f4866..113fba6 100644 --- a/src/embedding.h +++ b/src/embedding.h @@ -5,6 +5,7 @@ struct EmbeddingOptions { std::string model_path; + std::string mmproj_path; int threads = 0; std::string text; std::string audio_path; diff --git a/src/model-loader.cpp b/src/model-loader.cpp index 750f25d..c96b998 100644 --- a/src/model-loader.cpp +++ b/src/model-loader.cpp @@ -47,10 +47,11 @@ void ModelLoader::norm_shape(const std::string &name, int dim) const { shape(name + ".weight", {dim}); shape(name + ".bias", {dim}); } -void ModelLoader::validate_tensor_coverage() const { +void ModelLoader::validate_tensor_coverage(const std::string &prefix) const { if (!f32_activations && !bundled) return; for (int64_t i=0;ihas_metadata((metadata_prefix + key).c_str()); + return gguf_find_key(gguf, key) >= 0; +} + +nlohmann::json ModelLoader::metadata_value(const char *key) const { + if (source) return source->metadata_value((metadata_prefix + key).c_str()); + const auto id = gguf_find_key(gguf, key); + if (id < 0) throw std::runtime_error(std::string("CLAP: missing metadata: ") + key); + const auto type = gguf_get_kv_type(gguf, id); + if (type == GGUF_TYPE_ARRAY) { + const auto element_type = gguf_get_arr_type(gguf, id); + const size_t count = gguf_get_arr_n(gguf, id); + auto values = nlohmann::json::array(); + if (element_type == GGUF_TYPE_STRING) { + for (size_t i = 0; i < count; ++i) values.push_back(gguf_get_arr_str(gguf, id, i)); + return values; + } + const auto *data = static_cast(gguf_get_arr_data(gguf, id)); + const auto append = [&](auto sample) { + using T = decltype(sample); + for (size_t i = 0; i < count; ++i) { + T value; + std::memcpy(&value, data + i * sizeof(T), sizeof(T)); + values.push_back(value); + } + }; + switch (element_type) { + case GGUF_TYPE_UINT8: append(uint8_t{}); break; + case GGUF_TYPE_INT8: append(int8_t{}); break; + case GGUF_TYPE_UINT16: append(uint16_t{}); break; + case GGUF_TYPE_INT16: append(int16_t{}); break; + case GGUF_TYPE_UINT32: append(uint32_t{}); break; + case GGUF_TYPE_INT32: append(int32_t{}); break; + case GGUF_TYPE_UINT64: append(uint64_t{}); break; + case GGUF_TYPE_INT64: append(int64_t{}); break; + case GGUF_TYPE_FLOAT32: append(float{}); break; + case GGUF_TYPE_FLOAT64: append(double{}); break; + case GGUF_TYPE_BOOL: + for (size_t i = 0; i < count; ++i) values.push_back(data[i] != 0); + break; + default: throw std::runtime_error(std::string("CLAP: unsupported metadata array: ") + key); + } + return values; + } + switch (type) { + case GGUF_TYPE_STRING: return gguf_get_val_str(gguf, id); + case GGUF_TYPE_UINT8: return gguf_get_val_u8(gguf, id); + case GGUF_TYPE_INT8: return gguf_get_val_i8(gguf, id); + case GGUF_TYPE_UINT16: return gguf_get_val_u16(gguf, id); + case GGUF_TYPE_INT16: return gguf_get_val_i16(gguf, id); + case GGUF_TYPE_UINT32: return gguf_get_val_u32(gguf, id); + case GGUF_TYPE_INT32: return gguf_get_val_i32(gguf, id); + case GGUF_TYPE_UINT64: return gguf_get_val_u64(gguf, id); + case GGUF_TYPE_INT64: return gguf_get_val_i64(gguf, id); + case GGUF_TYPE_FLOAT32: return gguf_get_val_f32(gguf, id); + case GGUF_TYPE_FLOAT64: return gguf_get_val_f64(gguf, id); + case GGUF_TYPE_BOOL: return gguf_get_val_bool(gguf, id); + default: throw std::runtime_error(std::string("CLAP: unsupported metadata: ") + key); + } +} + +int64_t ModelLoader::integer(const char *key) const { + const auto value = metadata_value(key); + if (!value.is_number_integer() || (value.is_number_unsigned() && value.get() > INT64_MAX)) + throw std::runtime_error(std::string("CLAP: expected integer metadata: ") + key); + return value.get(); +} +double ModelLoader::number(const char *key) const { + const auto value = metadata_value(key); + if (!value.is_number() || !std::isfinite(value.get())) + throw std::runtime_error(std::string("CLAP: expected finite numeric metadata: ") + key); + return value.get(); +} +bool ModelLoader::boolean(const char *key) const { + const auto value = metadata_value(key); + if (!value.is_boolean()) throw std::runtime_error(std::string("CLAP: expected boolean metadata: ") + key); + return value.get(); +} + bool ModelLoader::has_metadata_prefix(const std::string &prefix) const { if (source) return source->has_metadata_prefix(metadata_prefix + prefix); for (int64_t i = 0; i < gguf_get_n_kv(gguf); ++i) @@ -108,7 +189,9 @@ void ModelLoader::load(const std::string &path, bool inference) { const bool named = architecture >= 0 && gguf_get_kv_type(gguf, architecture) == GGUF_TYPE_STRING; bundled = named && metadata("general.architecture") == "clamp3" && has_metadata_prefix("clamp3.audio_frontend"); const std::string arch = named ? metadata("general.architecture") : ""; - f32_activations = encoder_architecture(arch); + // EmbeddingGemma2 requires F32 activations even with F16/BF16 storage; + // narrow activation packing can overflow its projection/attention stack. + f32_activations = encoder_architecture(arch) || arch == "gemma-embedding2" || arch == "clip"; const auto data_offset = gguf_get_data_offset(gguf); if (!(data_offset <= mapping_size)) throw std::runtime_error("CLAP: truncated GGUF data"); @@ -120,9 +203,11 @@ void ModelLoader::load(const std::string &path, bool inference) { throw std::runtime_error("CLAP: MERT supports F32/F16 weights only"); if (encoder_architecture(arch) && ggml_is_quantized(w->type) && !encoder_quantizable(w, arch)) throw std::runtime_error("CLAP: quantized non-dense/embedding tensor is unsupported: " + tensor_name); + const bool gemma_storage = (arch == "gemma-embedding2" || arch == "clip") && + (w->type == GGML_TYPE_Q5_K || w->type == GGML_TYPE_BF16); if (!(w->type == GGML_TYPE_F32 || w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_Q8_0 || - w->type == GGML_TYPE_Q4_K || w->type == GGML_TYPE_Q6_K)) - throw std::runtime_error("CLAP: only F32/F16/Q8_0/Q4_K/Q6_K weights supported"); + w->type == GGML_TYPE_Q4_K || w->type == GGML_TYPE_Q6_K || gemma_storage)) + throw std::runtime_error("CLAP: unsupported tensor storage type: " + tensor_name); const size_t offset = gguf_get_tensor_offset(gguf, i); if (!(offset <= mapping_size - data_offset && ggml_nbytes(w) <= mapping_size - data_offset - offset)) throw std::runtime_error("CLAP: truncated tensor data"); diff --git a/src/model-loader.h b/src/model-loader.h index e2b8e18..8371bf8 100644 --- a/src/model-loader.h +++ b/src/model-loader.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -28,12 +29,17 @@ class ModelLoader { // Storage tools must not promote F16 bytes before conversion/copying. void load(const std::string &path, bool inference = true); std::string metadata(const char *key) const; + bool has_metadata(const char *key) const; + nlohmann::json metadata_value(const char *key) const; + int64_t integer(const char *key) const; + double number(const char *key) const; + bool boolean(const char *key) const; ggml_tensor *weight(const std::string &name) const; void shape(const std::string &name, std::initializer_list dims) const; void linear_shape(const std::string &name, int in, int out, bool bias = true) const; void norm_shape(const std::string &name, int dim) const; bool f32_activation_compute() const { return source ? source->f32_activation_compute() : f32_activations; } - void validate_tensor_coverage() const; + void validate_tensor_coverage(const std::string &prefix = "") const; private: const ModelLoader *source = nullptr; std::string tensor_prefix, metadata_prefix; diff --git a/src/model-ops.cpp b/src/model-ops.cpp index 710eb5e..77bcc71 100644 --- a/src/model-ops.cpp +++ b/src/model-ops.cpp @@ -160,7 +160,8 @@ void gelu(Matrix &x) { float scalar(const ggml_tensor *t, size_t i) { if (t->type == GGML_TYPE_F32) return static_cast(t->data)[i]; if (t->type == GGML_TYPE_F16) return ggml_fp16_to_fp32(static_cast(t->data)[i]); - if (!(t->type == GGML_TYPE_Q8_0 || t->type == GGML_TYPE_Q4_K || t->type == GGML_TYPE_Q6_K)) + if (t->type == GGML_TYPE_BF16) return ggml_bf16_to_fp32(static_cast(t->data)[i]); + if (!(t->type == GGML_TYPE_Q8_0 || t->type == GGML_TYPE_Q4_K || t->type == GGML_TYPE_Q5_K || t->type == GGML_TYPE_Q6_K)) throw std::runtime_error("CLAP: unsupported scalar tensor type"); // Decode only the containing block (embeddings, relative biases, or norms), // never expand an entire mapped weight tensor/model. diff --git a/src/model.cpp b/src/model.cpp index ce1c81b..9835d9c 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -10,32 +10,53 @@ namespace clap { struct CLAP::Impl { // Declaration order ensures the architecture is destroyed before its mapping. detail::ModelLoader loader; + std::unique_ptr audio_loader; std::unique_ptr architecture; std::unique_ptr vocabulary; - Impl(const std::string &path, int threads) { + Impl(const std::string &path, const std::string &mmproj_path, int threads) { loader.load(path); const auto name = loader.metadata("general.architecture"); + if (!mmproj_path.empty()) { + if (name != "gemma-embedding2") + throw std::runtime_error("CLAP: mmproj is only supported for EmbeddingGemma2"); + audio_loader = std::make_unique(); + audio_loader->load(mmproj_path); + } if (name == "clap") architecture = std::make_unique(loader, threads); else if (name == "clamp3") architecture = std::make_unique(loader, threads); else if (name == "mert2") architecture = std::make_unique(loader, threads); else if (name == "mert") architecture = std::make_unique(loader, threads); else if (name == "muq_mulan") architecture = std::make_unique(loader, threads); else if (name == "muq") architecture = std::make_unique(loader, threads); + else if (name == "gemma-embedding2") architecture = std::make_unique(loader, threads, audio_loader.get()); else throw std::runtime_error("CLAP: unsupported GGUF architecture: " + name); loader.validate_tensor_coverage(); if (architecture->has_text_encoder()) { const auto text = architecture->text_parameters(); - const auto tokenizer = nlohmann::json::parse(loader.metadata((name + ".tokenizer").c_str())); - if (name == "clamp3" || name == "muq_mulan") - vocabulary = std::make_unique(tokenizer, text.bos, text.eos, 3, text.context_length); - else - vocabulary = std::make_unique(tokenizer, text.bos, text.eos, text.context_length); + if (name == "gemma-embedding2") { + nlohmann::json tokenizer = nlohmann::json::object(); + for (const auto *key : {"tokenizer.ggml.model", "tokenizer.ggml.tokens", + "tokenizer.ggml.token_type", "tokenizer.ggml.merges", + "tokenizer.ggml.bos_token_id", "tokenizer.ggml.eos_token_id", + "tokenizer.ggml.unknown_token_id", "tokenizer.ggml.add_bos_token", + "tokenizer.ggml.add_eos_token", "tokenizer.ggml.add_space_prefix"}) + if (loader.has_metadata(key)) tokenizer[key] = loader.metadata_value(key); + vocabulary = std::make_unique(tokenizer, text.context_length); + } else { + const auto tokenizer = nlohmann::json::parse(loader.metadata((name + ".tokenizer").c_str())); + if (name == "clamp3" || name == "muq_mulan") + vocabulary = std::make_unique(tokenizer, text.bos, text.eos, 3, text.context_length); + else + vocabulary = std::make_unique(tokenizer, text.bos, text.eos, text.context_length); + } } } }; -CLAP::CLAP(const std::string &path, int threads) : impl_(std::make_unique(path, threads)) {} +CLAP::CLAP(const std::string &path, int threads) : CLAP(path, std::string(), threads) {} +CLAP::CLAP(const std::string &path, const std::string &mmproj_path, int threads) + : impl_(std::make_unique(path, mmproj_path, threads)) {} CLAP::~CLAP() = default; CLAP::CLAP(CLAP &&) noexcept = default; CLAP &CLAP::operator=(CLAP &&) noexcept = default; diff --git a/src/models/embeddinggemma2-audio.cpp b/src/models/embeddinggemma2-audio.cpp new file mode 100644 index 0000000..5fc0ce0 --- /dev/null +++ b/src/models/embeddinggemma2-audio.cpp @@ -0,0 +1,297 @@ +// Gemma4-audio Conformer contract: Hugging Face modeling_gemma4.py and +// llama.cpp tools/mtmd/models/gemma4a.cpp. Audio files/rate conversion belong +// exclusively to src/audio.cpp; this module accepts mono 16 kHz PCM. +#include "embeddinggemma2-audio.h" +#include "clap.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clap::detail { +namespace { +constexpr double pi = 3.14159265358979323846; +constexpr int mel_bins = 128, max_samples = 480000, left_context = 12; +void require(bool valid, const std::string &message) { + if (!valid) throw std::runtime_error("CLAP: Gemma audio " + message); +} +void silu(Matrix &x) { for (auto &v : x.data) v = v / (1.0f + std::exp(-v)); } +void clip(Matrix &x, float lo, float hi) { for (auto &v : x.data) v = std::clamp(v, lo, hi); } +void finite(const Matrix &x) { + for (float v : x.data) require(std::isfinite(v), "non-finite activation"); +} +// F32 im2col operand and decoded small convolution kernel. Unlike ggml_conv_2d, +// this does not implicitly pack the activation into F16 before convolution. +Matrix convolve(const Matrix &patches, const ggml_tensor *weight, int outputs, int threads) { + Matrix result(patches.rows, outputs); + const size_t count = static_cast(patches.cols) * outputs; + const size_t bytes = (patches.data.size() + result.data.size() + count) * sizeof(float) + 2*1024*1024; + std::unique_ptr ctx(ggml_init({bytes,nullptr,false}),ggml_free); + require(bool(ctx), "convolution allocation failed"); + auto *input=ggml_new_tensor_2d(ctx.get(),GGML_TYPE_F32,patches.cols,patches.rows); + auto *kernel=ggml_new_tensor_2d(ctx.get(),GGML_TYPE_F32,patches.cols,outputs); + std::memcpy(input->data,patches.data.data(),patches.data.size()*sizeof(float)); + auto *decoded=static_cast(kernel->data); + for (size_t i=0;i workspace(plan.work_size); plan.work_data=workspace.data(); + require(ggml_graph_compute(graph,&plan)==GGML_STATUS_SUCCESS,"convolution compute failed"); + std::memcpy(result.data.data(),output->data,result.data.size()*sizeof(float)); + return result; +} +} // namespace + +GemmaAudioEncoder::GemmaAudioEncoder(const ModelLoader &source,int threads) + : loader(source), hidden(0), heads(0), layers(0), feedforward(0), channels{0,0}, output(0), + epsilon(0), ops(source,threads) { + require(loader.metadata("general.architecture")=="clip","requires clip mmproj architecture"); + require(loader.boolean("clip.has_audio_encoder"),"encoder missing"); + require(loader.metadata("clip.audio.projector_type")=="gemma4a","unsupported projector variant"); + auto dimension=[&](const char *key,int max) { + const auto n=loader.integer(key); + require(n>0 && n<=max,std::string("invalid dimension: ")+key); + return static_cast(n); + }; + hidden=dimension("clip.audio.embedding_length",4096); + heads=dimension("clip.audio.attention.head_count",128); + layers=dimension("clip.audio.block_count",128); + feedforward=dimension("clip.audio.feed_forward_length",16384); + require(hidden%heads==0 && hidden%2==0,"incompatible head dimensions"); + require(feedforward==4*hidden,"incompatible feed-forward dimension"); + require(loader.integer("clip.audio.num_mel_bins")==mel_bins,"requires 128 mel bins"); + const int projection=dimension("clip.audio.projection_dim",16384); + epsilon=static_cast(loader.number("clip.audio.attention.layer_norm_epsilon")); + require(std::isfinite(epsilon) && epsilon>0,"invalid norm epsilon"); + for(int i=0;i<2;++i) { + const auto name="a.conv1d."+std::to_string(i); + const auto *w=loader.weight(name+".weight"); + require(w->ne[3]>0 && w->ne[3]<=4096,"invalid convolution channels"); + channels[i]=static_cast(w->ne[3]); + loader.shape(name+".weight",{3,3,i?channels[0]:1,channels[i]}); + loader.shape(name+".norm.weight",{channels[i]}); + require(w->type==GGML_TYPE_F32 || w->type==GGML_TYPE_F16 || w->type==GGML_TYPE_BF16, + "subsampling kernels must be unquantized"); + } + const auto *out=loader.weight("a.pre_encode.out.weight"); + require(out->ne[1]>0 && out->ne[1]<=8192,"invalid output dimension"); + output=static_cast(out->ne[1]); + loader.linear_shape("a.input_projection",32*channels[1],hidden,false); + loader.linear_shape("a.pre_encode.out",hidden,output,true); + loader.linear_shape("mm.a.input_projection",output,projection,false); + if(loader.has_tensor_prefix("mm.a.soft_emb_norm.weight")) loader.shape("mm.a.soft_emb_norm.weight",{output}); + const auto norm=[&](const std::string &name,int n) { + loader.shape(name+".weight",{n}); + auto *w=loader.weight(name+".weight"); + require(!ggml_is_quantized(w->type),"norm vectors must be unquantized: "+name); + for(int i=0;itype),"depthwise kernel must be unquantized"); + loader.shape(p+"per_dim_scale.weight",{hidden/heads}); + for(int d=0;d=0,"invalid converted per-dimension scale"); + } + loader.linear_shape(p+"attn_k_rel",hidden,hidden,false); + for(const auto *name:{"ffn_up","ffn_down","ffn_up_1","ffn_down_1", "attn_q","attn_k", + "attn_v","attn_out","conv_pw1","conv_pw2"}) { + const std::string n=name; + const bool up=n=="ffn_up" || n=="ffn_up_1"; + const bool down=n=="ffn_down" || n=="ffn_down_1"; + loader.linear_shape(p+n,down?feedforward:hidden,up?feedforward:n=="conv_pw1"?2*hidden:hidden,false); + for(const auto *suffix:{"input_min","input_max","output_min","output_max"}) { + loader.shape(p+n+"."+suffix,{1}); + require(std::isfinite(scalar(loader.weight(p+n+"."+suffix),0)),"non-finite clamp: "+p+n); + } + for(const auto *kind:{"input","output"}) { + const auto base=p+n+"."+kind; + require(scalar(loader.weight(base+"_min"),0)<=scalar(loader.weight(base+"_max"),0),"inverted clamp: "+p+n); + } + } + } + // The published file also carries vision weights, which are out of scope. + // Reject unknown audio tensors without requiring coverage of that tower. + loader.validate_tensor_coverage("a."); + loader.validate_tensor_coverage("mm.a."); +} + +Matrix GemmaAudioEncoder::log_mel(const std::vector &pcm) { + require(pcm.size()>160,"input is too short (at least 161 samples required)"); + for(float v:pcm) require(std::isfinite(v),"non-finite PCM input"); + const int samples=static_cast(std::min(pcm.size(),static_cast(max_samples))); + const int frames=(samples-161)/160+1; + Matrix result(frames,mel_bins); + std::array edges; + for(int i=0;i<130;++i) edges[i]=700.0*(std::pow(10.0,std::log10(1.0+8000.0/700.0)*i/129.0)-1.0); + std::array window; + for(int n=0;n<320;++n) window[n]=static_cast(0.5-0.5*std::cos(2*pi*n/320)); + std::array,mel_bins> filters{}; + for(int m=0;m> spectrum(512); + for(int t=0;t{}); + for(int n=0;n<320;++n) { + const int i=t*160+n-160; + const float sample=i<0 || i>=samples?0.0f:pcm[i]*window[n]; + spectrum[n]=sample; + } + clap::fft(spectrum); + std::array magnitude; + for(int k=0;k<257;++k) magnitude[k]=std::abs(spectrum[k]); + for(int m=0;m(std::log(mel+0.001)); + } + } + return result; +} + +Matrix GemmaAudioEncoder::rms(const Matrix &x,const std::string &name) const { + Matrix y(x.rows,x.cols); + const auto *weight=name.empty()?nullptr:loader.weight(name+".weight"); + for(int r=0;r(1.0/std::sqrt(sum/x.cols+epsilon)); + for(int d=0;d=0 && it