From 108fdf5630d622baf6ad9b2340d61bd965601303 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 28 Oct 2026 15:17:56 +0800 Subject: [PATCH 1/2] [verified] feat: support EmbeddingGemma 2 text embeddings --- CMakeLists.txt | 42 +++- README.md | 34 ++- src/clap.cpp | 2 +- src/model-loader.cpp | 90 +++++++- src/model-loader.h | 6 + src/model-ops.cpp | 3 +- src/model.cpp | 22 +- src/models/embeddinggemma2.cpp | 304 +++++++++++++++++++++++++ src/models/models.h | 25 ++ src/vocab.cpp | 148 ++++++++++++ src/vocab.h | 17 ++ tests/gemma-storage-test.cpp | 47 ++++ tests/metadata-test.cpp | 38 ++++ tests/models/EMBEDDINGGEMMA2_SOURCE.md | 77 +++++++ tests/models/embeddinggemma2-test.cpp | 65 ++++++ tests/models/embeddinggemma2.py | 233 +++++++++++++++++++ tests/models/embeddinggemma2_cli.py | 44 ++++ tests/models/gemma-tokenizer-test.cpp | 112 +++++++++ tests/models/gemma_tokenizer_parity.py | 56 +++++ 19 files changed, 1351 insertions(+), 14 deletions(-) create mode 100644 src/models/embeddinggemma2.cpp create mode 100644 tests/gemma-storage-test.cpp create mode 100644 tests/metadata-test.cpp create mode 100644 tests/models/EMBEDDINGGEMMA2_SOURCE.md create mode 100644 tests/models/embeddinggemma2-test.cpp create mode 100644 tests/models/embeddinggemma2.py create mode 100644 tests/models/embeddinggemma2_cli.py create mode 100644 tests/models/gemma-tokenizer-test.cpp create mode 100644 tests/models/gemma_tokenizer_parity.py diff --git a/CMakeLists.txt b/CMakeLists.txt index a45d331..e88ecd0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,42 @@ if(BUILD_TESTING) COMMENT "Generating audio test fixtures" VERBATIM) add_custom_target(clap-audio-fixtures ALL DEPENDS ${CLAP_AUDIO_FIXTURES}) + add_executable(clap-embeddinggemma2-test tests/models/embeddinggemma2-test.cpp) + target_link_libraries(clap-embeddinggemma2-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_model + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2.py" + --test "$") + set_tests_properties(embeddinggemma2_model PROPERTIES TIMEOUT 120) + add_test(NAME embeddinggemma2_cli + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_cli.py" + "$") + set_tests_properties(embeddinggemma2_cli PROPERTIES TIMEOUT 120) + add_executable(clap-gemma-tokenizer-test tests/models/gemma-tokenizer-test.cpp) + target_include_directories(clap-gemma-tokenizer-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-tokenizer-test PRIVATE clap nlohmann_json::nlohmann_json ICU::uc ICU::i18n) + add_test(NAME gemma_tokenizer COMMAND clap-gemma-tokenizer-test) + set(CLAP_GEMMA_TOKENIZER_JSON "" CACHE FILEPATH "Pinned EmbeddingGemma2 tokenizer.json for optional HF parity (Python tokenizers required)") + if(CLAP_GEMMA_TOKENIZER_JSON) + if(NOT EXISTS "${CLAP_GEMMA_TOKENIZER_JSON}") + message(FATAL_ERROR "CLAP_GEMMA_TOKENIZER_JSON does not exist") + endif() + set(GEMMA_TOKENIZER_FIXTURE "${CMAKE_CURRENT_BINARY_DIR}/gemma-tokenizer-parity") + add_test(NAME gemma_tokenizer_hf_fixture + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/gemma_tokenizer_parity.py" + "${CLAP_GEMMA_TOKENIZER_JSON}" "${GEMMA_TOKENIZER_FIXTURE}") + set_tests_properties(gemma_tokenizer_hf_fixture PROPERTIES FIXTURES_SETUP gemma_tokenizer_hf TIMEOUT 120) + add_test(NAME gemma_tokenizer_hf + COMMAND clap-gemma-tokenizer-test "${GEMMA_TOKENIZER_FIXTURE}/metadata.json" "${GEMMA_TOKENIZER_FIXTURE}/cases.json") + set_tests_properties(gemma_tokenizer_hf PROPERTIES FIXTURES_REQUIRED gemma_tokenizer_hf TIMEOUT 120) + endif() + add_executable(clap-gemma-storage-test tests/gemma-storage-test.cpp) + target_include_directories(clap-gemma-storage-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-storage-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME gemma_storage COMMAND clap-gemma-storage-test "${CMAKE_CURRENT_BINARY_DIR}/gemma-storage-") + add_executable(clap-metadata-test tests/metadata-test.cpp) + target_include_directories(clap-metadata-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-metadata-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME clap_metadata COMMAND clap-metadata-test "${CMAKE_CURRENT_BINARY_DIR}/metadata.gguf") add_executable(clap-resample-test tests/resample-test.cpp) target_link_libraries(clap-resample-test PRIVATE clap) add_test(NAME clap_resample COMMAND clap-resample-test) @@ -87,12 +123,12 @@ if(BUILD_TESTING) COMMAND clap-quantize-output-test "${CMAKE_CURRENT_BINARY_DIR}/quantize-publication-fixture") add_executable(all-quantization-test tests/all-quantization-test.cpp src/quantize.cpp) target_include_directories(all-quantization-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") - target_link_libraries(all-quantization-test PRIVATE clap) + target_link_libraries(all-quantization-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME all_architecture_quantization COMMAND all-quantization-test "${CMAKE_CURRENT_BINARY_DIR}/all-quantization-fixture") set_tests_properties(all_architecture_quantization PROPERTIES TIMEOUT 120) add_executable(clap-q8-test tests/q8-test.cpp) target_include_directories(clap-q8-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") - target_link_libraries(clap-q8-test PRIVATE clap) + target_link_libraries(clap-q8-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME clap_quantize_cli COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/quantize_cli.py" "$") set_tests_properties(clap_quantize_cli PROPERTIES TIMEOUT 30) @@ -126,7 +162,7 @@ if(BUILD_TESTING) add_test(NAME clap_model_unit COMMAND clap-model-test) add_executable(tensor-ops-test tests/tensor-ops-test.cpp) target_include_directories(tensor-ops-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") - target_link_libraries(tensor-ops-test PRIVATE clap) + target_link_libraries(tensor-ops-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME clap_tensor_ops COMMAND tensor-ops-test "${CMAKE_CURRENT_BINARY_DIR}/tensor-ops-auto.gguf") add_test(NAME clap_tensor_ops_cpu diff --git a/README.md b/README.md index 0724ee0..d0f5cee 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ C++17/ggml implementations of audio embedding inference features: | [MuQ](https://huggingface.co/OpenMuQ/MuQ-large-msd-iter) | ❌ | ✅ | 1,024 | 24 kHz | | [MuQ-MuLan](https://huggingface.co/OpenMuQ/MuQ-MuLan-large) | ✅ | ✅ | 512 | 24 kHz | +| [EmbeddingGemma 2](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) | ✅ | ❌ | 768 | — | + GGUF models are available at [Audio Embedding GGUF](https://huggingface.co/collections/gorse-io/audio-embedding-gguf) collection. ## Bindings @@ -40,6 +42,36 @@ clap embedding --model model.gguf --text 'A jazz quartet.' clap embedding --model model.gguf --audio music.flac ``` +### EmbeddingGemma 2 (text only) + +Load the published [Unsloth EmbeddingGemma 2 GGUF](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) +files directly; no conversion, Python inference, or `mmproj` is required. For example, +download `embeddinggemma-2-UD-Q4_K_XL.gguf` and run: + +```sh +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf --text 'Hello world' +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \ + --text 'task: search result | query: How do plants grow?' +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \ + --text 'title: none | text: Plants use sunlight for photosynthesis.' +./build/clap serve --model embeddinggemma-2-UD-Q4_K_XL.gguf +``` + +`encode_text` returns a normalized 768-dimensional embedding. Both BOS and EOS +are added automatically; mean pooling includes these tokens and any prompt. +Retrieval prefixes are **not** added automatically: supply +`task: search result | query: ` for queries and `title: none | text: ` for +untitled documents, or `title: {title} | text: {content}` for titled documents. +The tokenizer preserves Unicode, case, and repeated whitespace. Google advertises +an 8192-token context; use that limit even though the GGUF positional capacity is +larger. Native computation preserves F32 activations with F16/BF16 storage and +accepts the published Q4_K/Q5_K/Q6_K/Q8_0 mixed tensor types. + +This integration is **text only**: no `encode_image`, image projector, or audio +encoding is implemented for this architecture. Use the published GGUF directly; +the checkpoint converter and local quantization presets below remain scoped to +the six existing audio/joint architectures. + ## Embeddings server Start the server with a text-capable GGUF, such as CLAP, CLaMP3 or MuQ-MuLan @@ -62,4 +94,4 @@ curl http://127.0.0.1:8080/v1/embeddings \ ## License This project is licensed under the [Apache License 2.0](LICENSE). -Third-party dependencies retain their respective licenses. +Third-party dependencies retain their respective licenses. \ No newline at end of file diff --git a/src/clap.cpp b/src/clap.cpp index 9ffc3e2..544c66b 100644 --- a/src/clap.cpp +++ b/src/clap.cpp @@ -28,7 +28,7 @@ int main(int argc, char **argv) { EmbeddingOptions options; std::string file; auto *embedding = app.add_subcommand("embedding", "Generate native text or audio embeddings"); - embedding->add_option("-m,--model", options.model_path, "Converted CLAP/CLaMP3/MERT/MERT2/MuQ/MuLan GGUF") + embedding->add_option("-m,--model", options.model_path, "Model GGUF") ->required()->check(CLI::ExistingFile); embedding->add_option("-t,--threads", options.threads, "ggml CPU threads (0: automatic)") ->check(CLI::NonNegativeNumber)->capture_default_str(); diff --git a/src/model-loader.cpp b/src/model-loader.cpp index 750f25d..26d37fa 100644 --- a/src/model-loader.cpp +++ b/src/model-loader.cpp @@ -63,6 +63,86 @@ std::string ModelLoader::metadata(const char *key) const { return std::string(gguf_get_val_str(gguf, id)); } +bool ModelLoader::has_metadata(const char *key) const { + if (source) return source->has_metadata((metadata_prefix + key).c_str()); + return gguf_find_key(gguf, key) >= 0; +} + +nlohmann::json ModelLoader::metadata_value(const char *key) const { + if (source) return source->metadata_value((metadata_prefix + key).c_str()); + const auto id = gguf_find_key(gguf, key); + if (id < 0) throw std::runtime_error(std::string("CLAP: missing metadata: ") + key); + const auto type = gguf_get_kv_type(gguf, id); + if (type == GGUF_TYPE_ARRAY) { + const auto element_type = gguf_get_arr_type(gguf, id); + const size_t count = gguf_get_arr_n(gguf, id); + auto values = nlohmann::json::array(); + if (element_type == GGUF_TYPE_STRING) { + for (size_t i = 0; i < count; ++i) values.push_back(gguf_get_arr_str(gguf, id, i)); + return values; + } + const auto *data = static_cast(gguf_get_arr_data(gguf, id)); + const auto append = [&](auto sample) { + using T = decltype(sample); + for (size_t i = 0; i < count; ++i) { + T value; + std::memcpy(&value, data + i * sizeof(T), sizeof(T)); + values.push_back(value); + } + }; + switch (element_type) { + case GGUF_TYPE_UINT8: append(uint8_t{}); break; + case GGUF_TYPE_INT8: append(int8_t{}); break; + case GGUF_TYPE_UINT16: append(uint16_t{}); break; + case GGUF_TYPE_INT16: append(int16_t{}); break; + case GGUF_TYPE_UINT32: append(uint32_t{}); break; + case GGUF_TYPE_INT32: append(int32_t{}); break; + case GGUF_TYPE_UINT64: append(uint64_t{}); break; + case GGUF_TYPE_INT64: append(int64_t{}); break; + case GGUF_TYPE_FLOAT32: append(float{}); break; + case GGUF_TYPE_FLOAT64: append(double{}); break; + case GGUF_TYPE_BOOL: + for (size_t i = 0; i < count; ++i) values.push_back(data[i] != 0); + break; + default: throw std::runtime_error(std::string("CLAP: unsupported metadata array: ") + key); + } + return values; + } + switch (type) { + case GGUF_TYPE_STRING: return gguf_get_val_str(gguf, id); + case GGUF_TYPE_UINT8: return gguf_get_val_u8(gguf, id); + case GGUF_TYPE_INT8: return gguf_get_val_i8(gguf, id); + case GGUF_TYPE_UINT16: return gguf_get_val_u16(gguf, id); + case GGUF_TYPE_INT16: return gguf_get_val_i16(gguf, id); + case GGUF_TYPE_UINT32: return gguf_get_val_u32(gguf, id); + case GGUF_TYPE_INT32: return gguf_get_val_i32(gguf, id); + case GGUF_TYPE_UINT64: return gguf_get_val_u64(gguf, id); + case GGUF_TYPE_INT64: return gguf_get_val_i64(gguf, id); + case GGUF_TYPE_FLOAT32: return gguf_get_val_f32(gguf, id); + case GGUF_TYPE_FLOAT64: return gguf_get_val_f64(gguf, id); + case GGUF_TYPE_BOOL: return gguf_get_val_bool(gguf, id); + default: throw std::runtime_error(std::string("CLAP: unsupported metadata: ") + key); + } +} + +int64_t ModelLoader::integer(const char *key) const { + const auto value = metadata_value(key); + if (!value.is_number_integer() || (value.is_number_unsigned() && value.get() > INT64_MAX)) + throw std::runtime_error(std::string("CLAP: expected integer metadata: ") + key); + return value.get(); +} +double ModelLoader::number(const char *key) const { + const auto value = metadata_value(key); + if (!value.is_number() || !std::isfinite(value.get())) + throw std::runtime_error(std::string("CLAP: expected finite numeric metadata: ") + key); + return value.get(); +} +bool ModelLoader::boolean(const char *key) const { + const auto value = metadata_value(key); + if (!value.is_boolean()) throw std::runtime_error(std::string("CLAP: expected boolean metadata: ") + key); + return value.get(); +} + bool ModelLoader::has_metadata_prefix(const std::string &prefix) const { if (source) return source->has_metadata_prefix(metadata_prefix + prefix); for (int64_t i = 0; i < gguf_get_n_kv(gguf); ++i) @@ -108,7 +188,9 @@ void ModelLoader::load(const std::string &path, bool inference) { const bool named = architecture >= 0 && gguf_get_kv_type(gguf, architecture) == GGUF_TYPE_STRING; bundled = named && metadata("general.architecture") == "clamp3" && has_metadata_prefix("clamp3.audio_frontend"); const std::string arch = named ? metadata("general.architecture") : ""; - f32_activations = encoder_architecture(arch); + // EmbeddingGemma2 requires F32 activations even with F16/BF16 storage; + // narrow activation packing can overflow its projection/attention stack. + f32_activations = encoder_architecture(arch) || arch == "gemma-embedding2"; const auto data_offset = gguf_get_data_offset(gguf); if (!(data_offset <= mapping_size)) throw std::runtime_error("CLAP: truncated GGUF data"); @@ -120,9 +202,11 @@ void ModelLoader::load(const std::string &path, bool inference) { throw std::runtime_error("CLAP: MERT supports F32/F16 weights only"); if (encoder_architecture(arch) && ggml_is_quantized(w->type) && !encoder_quantizable(w, arch)) throw std::runtime_error("CLAP: quantized non-dense/embedding tensor is unsupported: " + tensor_name); + const bool gemma_storage = arch == "gemma-embedding2" && + (w->type == GGML_TYPE_Q5_K || w->type == GGML_TYPE_BF16); if (!(w->type == GGML_TYPE_F32 || w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_Q8_0 || - w->type == GGML_TYPE_Q4_K || w->type == GGML_TYPE_Q6_K)) - throw std::runtime_error("CLAP: only F32/F16/Q8_0/Q4_K/Q6_K weights supported"); + w->type == GGML_TYPE_Q4_K || w->type == GGML_TYPE_Q6_K || gemma_storage)) + throw std::runtime_error("CLAP: unsupported tensor storage type: " + tensor_name); const size_t offset = gguf_get_tensor_offset(gguf, i); if (!(offset <= mapping_size - data_offset && ggml_nbytes(w) <= mapping_size - data_offset - offset)) throw std::runtime_error("CLAP: truncated tensor data"); diff --git a/src/model-loader.h b/src/model-loader.h index e2b8e18..ad1d1b1 100644 --- a/src/model-loader.h +++ b/src/model-loader.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -28,6 +29,11 @@ class ModelLoader { // Storage tools must not promote F16 bytes before conversion/copying. void load(const std::string &path, bool inference = true); std::string metadata(const char *key) const; + bool has_metadata(const char *key) const; + nlohmann::json metadata_value(const char *key) const; + int64_t integer(const char *key) const; + double number(const char *key) const; + bool boolean(const char *key) const; ggml_tensor *weight(const std::string &name) const; void shape(const std::string &name, std::initializer_list dims) const; void linear_shape(const std::string &name, int in, int out, bool bias = true) const; diff --git a/src/model-ops.cpp b/src/model-ops.cpp index 710eb5e..77bcc71 100644 --- a/src/model-ops.cpp +++ b/src/model-ops.cpp @@ -160,7 +160,8 @@ void gelu(Matrix &x) { float scalar(const ggml_tensor *t, size_t i) { if (t->type == GGML_TYPE_F32) return static_cast(t->data)[i]; if (t->type == GGML_TYPE_F16) return ggml_fp16_to_fp32(static_cast(t->data)[i]); - if (!(t->type == GGML_TYPE_Q8_0 || t->type == GGML_TYPE_Q4_K || t->type == GGML_TYPE_Q6_K)) + if (t->type == GGML_TYPE_BF16) return ggml_bf16_to_fp32(static_cast(t->data)[i]); + if (!(t->type == GGML_TYPE_Q8_0 || t->type == GGML_TYPE_Q4_K || t->type == GGML_TYPE_Q5_K || t->type == GGML_TYPE_Q6_K)) throw std::runtime_error("CLAP: unsupported scalar tensor type"); // Decode only the containing block (embeddings, relative biases, or norms), // never expand an entire mapped weight tensor/model. diff --git a/src/model.cpp b/src/model.cpp index ce1c81b..229e0bf 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -22,15 +22,27 @@ struct CLAP::Impl { else if (name == "mert") architecture = std::make_unique(loader, threads); else if (name == "muq_mulan") architecture = std::make_unique(loader, threads); else if (name == "muq") architecture = std::make_unique(loader, threads); + else if (name == "gemma-embedding2") architecture = std::make_unique(loader, threads); else throw std::runtime_error("CLAP: unsupported GGUF architecture: " + name); loader.validate_tensor_coverage(); if (architecture->has_text_encoder()) { const auto text = architecture->text_parameters(); - const auto tokenizer = nlohmann::json::parse(loader.metadata((name + ".tokenizer").c_str())); - if (name == "clamp3" || name == "muq_mulan") - vocabulary = std::make_unique(tokenizer, text.bos, text.eos, 3, text.context_length); - else - vocabulary = std::make_unique(tokenizer, text.bos, text.eos, text.context_length); + if (name == "gemma-embedding2") { + nlohmann::json tokenizer = nlohmann::json::object(); + for (const auto *key : {"tokenizer.ggml.model", "tokenizer.ggml.tokens", + "tokenizer.ggml.token_type", "tokenizer.ggml.merges", + "tokenizer.ggml.bos_token_id", "tokenizer.ggml.eos_token_id", + "tokenizer.ggml.unknown_token_id", "tokenizer.ggml.add_bos_token", + "tokenizer.ggml.add_eos_token", "tokenizer.ggml.add_space_prefix"}) + if (loader.has_metadata(key)) tokenizer[key] = loader.metadata_value(key); + vocabulary = std::make_unique(tokenizer, text.context_length); + } else { + const auto tokenizer = nlohmann::json::parse(loader.metadata((name + ".tokenizer").c_str())); + if (name == "clamp3" || name == "muq_mulan") + vocabulary = std::make_unique(tokenizer, text.bos, text.eos, 3, text.context_length); + else + vocabulary = std::make_unique(tokenizer, text.bos, text.eos, text.context_length); + } } } }; diff --git a/src/models/embeddinggemma2.cpp b/src/models/embeddinggemma2.cpp new file mode 100644 index 0000000..ad2b478 --- /dev/null +++ b/src/models/embeddinggemma2.cpp @@ -0,0 +1,304 @@ +#include "models.h" +#include +#include +#include +#include + +namespace clap::detail { +namespace { +// Names, RMS weights (already +1 in GGUF), NEOX RoPE and PLE order follow +// llama.cpp/src/models/gemma-embedding2.cpp, not the older Gemma embedding model. +constexpr const char *architecture = "gemma-embedding2."; +std::string key(const char *suffix) { return std::string(architecture) + suffix; } +[[noreturn]] void invalid(const std::string &field) { + throw std::runtime_error("CLAP: invalid/unsupported EmbeddingGemma2 " + field); +} +int dimension(const nlohmann::json &value, const std::string &field, int maximum = 65536) { + if (!value.is_number_integer() || value <= 0 || value > maximum) invalid(field); + return value.get(); +} +void fast_gelu(Matrix &x) { + // HF gelu_pytorch_tanh / GGML GELU, rather than erf GELU in older encoders. + for (float &v : x.data) + v = 0.5f * v * (1.0f + std::tanh(0.7978845608028654f * (v + 0.044715f * v*v*v))); +} +} + +EmbeddingGemma2Model::EmbeddingGemma2Model(const ModelLoader &source, int threads) + : loader(source), ops(source, threads) { + load_arch_hparams(); + load_arch_tensors(); +} + +void EmbeddingGemma2Model::load_arch_hparams() { + auto has = [&](const char *suffix) { return loader.has_metadata(key(suffix).c_str()); }; + auto value = [&](const char *suffix) { return loader.metadata_value(key(suffix).c_str()); }; + auto integer = [&](const char *suffix, int maximum = 65536) { + return dimension(value(suffix), suffix, maximum); + }; + auto optional = [&](const char *suffix, int fallback, int maximum = 65536) { + return has(suffix) ? integer(suffix, maximum) : fallback; + }; + auto number = [&](const char *suffix, double fallback) { + if (!has(suffix)) return static_cast(fallback); + const auto v = value(suffix); + if (!v.is_number()) invalid(suffix); + const double n = v.get(); + if (!std::isfinite(n) || n <= 0 || n > std::numeric_limits::max() || + n < std::numeric_limits::min()) invalid(suffix); + return static_cast(n); + }; + hidden = integer("embedding_length"); + layers = integer("block_count", 1024); + output = optional("embedding_length_out", hidden); + per_layer = integer("embedding_length_per_layer_input"); + context = integer("context_length", 1048576); + window = integer("attention.sliding_window", 1048576); + if (!has("attention.layer_norm_rms_epsilon")) invalid("attention.layer_norm_rms_epsilon"); + epsilon = number("attention.layer_norm_rms_epsilon", 1e-6); + rope_base = number("rope.freq_base", 10000); + rope_base_swa = number("rope.freq_base_swa", 10000); + if (has("attention.causal") && (!value("attention.causal").is_boolean() || + value("attention.causal").get())) invalid("causal attention"); + if (has("pooling_type") && (!value("pooling_type").is_number_integer() || + value("pooling_type") != 1)) invalid("pooling_type (mean required)"); + if (has("attention.shared_kv_layers") && (!value("attention.shared_kv_layers").is_number_integer() || + value("attention.shared_kv_layers") != 0)) invalid("shared KV layers"); + if (has("rope.scaling.attn_factor") && number("rope.scaling.attn_factor", 1) != 1) + invalid("rope.scaling.attn_factor"); + if (has("rope.scaling.type")) { + const auto type = value("rope.scaling.type"); + if (type != "none" && type != "linear") invalid("rope.scaling.type"); + if (type == "linear") { + if (!has("rope.scaling.factor")) invalid("rope.scaling.factor"); + rope_scale = 1.0f / number("rope.scaling.factor", 1); + } else if (has("rope.scaling.factor") && number("rope.scaling.factor", 1) != 1) { + invalid("rope.scaling.factor"); + } + } else if (has("rope.scaling.factor")) { + rope_scale = 1.0f / number("rope.scaling.factor", 1); + } + if (!std::isfinite(rope_scale) || rope_scale <= 0) invalid("rope frequency scale"); + const auto tokens = loader.metadata_value("tokenizer.ggml.tokens"); + if (!tokens.is_array() || tokens.empty() || tokens.size() > 1048576) invalid("vocabulary"); + vocab = static_cast(tokens.size()); + auto special = [&](const char *name, int fallback) { + if (!loader.has_metadata(name)) return fallback; + const auto v = loader.metadata_value(name); + if (!v.is_number_integer() || v < 0 || v >= vocab) invalid(name); + return v.get(); + }; + bos = special("tokenizer.ggml.bos_token_id", 2); + eos = special("tokenizer.ggml.eos_token_id", 1); + if (bos >= vocab || eos >= vocab) invalid("special token ids"); + pad = special("tokenizer.ggml.padding_token_id", 0); + (void) special("tokenizer.ggml.unknown_token_id", 3); + auto per_block = [&](const char *suffix, int fallback, bool required) { + std::vector result(layers, fallback); + if (!has(suffix)) { + if (required) invalid(suffix); + return result; + } + const auto v = value(suffix); + if (v.is_array()) { + if (v.size() != static_cast(layers)) invalid(suffix); + for (int i = 0; i < layers; ++i) result[i] = dimension(v[i], suffix); + } else { + std::fill(result.begin(), result.end(), dimension(v, suffix)); + } + return result; + }; + heads = per_block("attention.head_count", 0, true); + kv_heads = has("attention.head_count_kv") ? per_block("attention.head_count_kv", 0, true) : heads; + ffn = per_block("feed_forward_length", 0, true); + local.resize(layers); + if (has("attention.sliding_window_pattern") && value("attention.sliding_window_pattern").is_array()) { + const auto pattern = value("attention.sliding_window_pattern"); + if (pattern.size() != static_cast(layers)) invalid("sliding window pattern length"); + for (int i = 0; i < layers; ++i) { + if (pattern[i].is_boolean()) local[i] = pattern[i].get(); + else if (pattern[i].is_number_integer() && (pattern[i] == 0 || pattern[i] == 1)) + local[i] = pattern[i] == 1; + else invalid("sliding window pattern element"); + } + } else { + const int pattern = optional("attention.sliding_window_pattern", 6, 1048576); + for (int i = 0; i < layers; ++i) local[i] = (i + 1) % pattern != 0; + } + const int full_dimension = optional("attention.key_length", hidden / heads[0]); + const int full_value = optional("attention.value_length", hidden / heads[0]); + const int local_dimension = optional("attention.key_length_swa", full_dimension); + const int local_value = optional("attention.value_length_swa", full_value); + const int full_rotation = optional("rope.dimension_count", full_dimension); + const int local_rotation = optional("rope.dimension_count_swa", full_rotation); + if (full_dimension != full_value || local_dimension != local_value) invalid("key/value head dimension"); + dimensions.resize(layers); rotations.resize(layers); + if (static_cast(per_layer) * layers > std::numeric_limits::max() || + static_cast(hidden) * vocab > std::numeric_limits::max()) invalid("derived tensor products"); + for (int i = 0; i < layers; ++i) { + dimensions[i] = local[i] ? local_dimension : full_dimension; + rotations[i] = local[i] ? local_rotation : full_rotation; + if (dimensions[i] <= 0 || heads[i] % kv_heads[i] || rotations[i] % 2 || + rotations[i] > dimensions[i] || static_cast(heads[i]) * dimensions[i] > 1048576 || + static_cast(hidden) * ffn[i] > std::numeric_limits::max()) invalid("attention/FFN geometry"); + } +} + +void EmbeddingGemma2Model::load_arch_tensors() const { + loader.shape("token_embd.weight", {hidden, vocab}); + loader.linear_shape("per_layer_model_proj", hidden, per_layer * layers, false); + loader.shape("per_layer_proj_norm.weight", {per_layer}); + loader.shape("output_norm.weight", {hidden}); + loader.linear_shape("output", hidden, output, false); + for (int i = 0; i < layers; ++i) { + const auto root = "blk." + std::to_string(i) + "."; + for (const auto *name : {"attn_norm", "post_attention_norm", "ffn_norm", "post_ffw_norm", "post_norm"}) + loader.shape(root + name + ".weight", {hidden}); + loader.shape(root + "attn_q_norm.weight", {dimensions[i]}); + loader.shape(root + "attn_k_norm.weight", {dimensions[i]}); + loader.linear_shape(root + "attn_q", hidden, heads[i] * dimensions[i], false); + loader.linear_shape(root + "attn_k", hidden, kv_heads[i] * dimensions[i], false); + loader.linear_shape(root + "attn_v", hidden, kv_heads[i] * dimensions[i], false); + loader.linear_shape(root + "attn_output", heads[i] * dimensions[i], hidden, false); + loader.linear_shape(root + "ffn_gate", hidden, ffn[i], false); + loader.linear_shape(root + "ffn_up", hidden, ffn[i], false); + loader.linear_shape(root + "ffn_down", ffn[i], hidden, false); + loader.linear_shape(root + "inp_gate", hidden, per_layer, false); + loader.linear_shape(root + "proj", per_layer, hidden, false); + loader.shape(root + "layer_output_scale.weight", {1}); + } +} + +TextParameters EmbeddingGemma2Model::text_parameters() const { return {bos, eos, static_cast(context)}; } +AudioFrames EmbeddingGemma2Model::audio_frames(const std::vector &) const { + throw std::runtime_error("CLAP: EmbeddingGemma2 is text-only; audio is unsupported"); +} + +Matrix EmbeddingGemma2Model::rms(const Matrix &x, const std::string &name, int width) const { + if (!width) width = x.cols; + Matrix out(x.rows, x.cols); + const auto *weight = name.empty() ? nullptr : loader.weight(name + ".weight"); + // GGUF norm weights already contain HF's +1; never add it again. + std::vector scales(width, 1.0f); + if (weight) tensor_row(weight, 0, scales.data()); + for (int row = 0; row < x.rows; ++row) for (int offset = 0; offset < x.cols; offset += width) { + float square = 0; + for (int c = 0; c < width; ++c) square += x[row][offset+c] * x[row][offset+c]; + const float scale = 1.0f / std::sqrt(square / width + epsilon); + for (int c = 0; c < width; ++c) out[row][offset+c] = x[row][offset+c] * scale * scales[c]; + } + return out; +} + +Matrix EmbeddingGemma2Model::attention(const Matrix &x, int layer, const std::vector &mask) const { + const auto root = "blk." + std::to_string(layer) + "."; + const int dim = dimensions[layer], rot = rotations[layer]; + const auto normalized = rms(x, root + "attn_norm"); + Matrix q = rms(ops.linear(normalized, root + "attn_q", false), root + "attn_q_norm", dim); + Matrix k = rms(ops.linear(normalized, root + "attn_k", false), root + "attn_k_norm", dim); + const Matrix v = rms(ops.linear(normalized, root + "attn_v", false), "", dim); + const float base = local[layer] ? rope_base_swa : rope_base; + const float frequency_scale = local[layer] ? 1.0f : rope_scale; + // GGUF NEOX convention rotates the first n_rot channels; partial-RoPE + // channels after n_rot remain untouched. Q/K checkpoint permutation is + // performed by the GGUF converter, not here. + for (Matrix *m : {&q, &k}) { + const int count = m->cols / dim; + for (int c = 0; c < rot / 2; ++c) { + const float frequency = std::pow(base, -2.0f * c / rot) * frequency_scale; + for (int row = 0; row < m->rows; ++row) { + const float cosine = std::cos(row * frequency), sine = std::sin(row * frequency); + for (int head = 0; head < count; ++head) { + float *p = (*m)[row] + head * dim; + const float a = p[c], b = p[c + rot/2]; + p[c] = a*cosine - b*sine; p[c + rot/2] = a*sine + b*cosine; + } + } + } + } + Matrix result(x.rows, q.cols); + // One score row at a time avoids a context^2 attention allocation. + std::vector scores(x.rows); + for (int row = 0; row < x.rows; ++row) { + // GGUF stores twice the HF radius. Symmetric SWA includes both + // endpoints: abs(query_position - key_position) <= window / 2. + const int begin = local[layer] ? std::max(0, row - window / 2) : 0; + const int end = local[layer] ? static_cast(std::min(x.rows, static_cast(row) + window / 2 + 1)) : x.rows; + for (int head = 0; head < heads[layer]; ++head) { + const int kv_head = head / (heads[layer] / kv_heads[layer]); + float maximum = -std::numeric_limits::infinity(); + for (int pos = begin; pos < end; ++pos) { + if (!mask[pos]) continue; + float dot = 0; + for (int c = 0; c < dim; ++c) dot += q[row][head*dim+c] * k[pos][kv_head*dim+c]; + // Unit attention scale: Q and K are already learned RMS-normalized. + scores[pos] = dot; maximum = std::max(maximum, dot); + } + float total = 0; + for (int pos = begin; pos < end; ++pos) + if (mask[pos]) total += (scores[pos] = std::exp(scores[pos] - maximum)); + for (int pos = begin; pos < end; ++pos) { + if (!mask[pos]) continue; + const float probability = scores[pos] / total; + for (int c = 0; c < dim; ++c) result[row][head*dim+c] += probability * v[pos][kv_head*dim+c]; + } + } + } + return rms(ops.linear(result, root + "attn_output", false), root + "post_attention_norm"); +} + +std::vector EmbeddingGemma2Model::text(const std::vector &ids) const { + if (ids.empty() || ids.size() > static_cast(context)) invalid("token sequence length"); + for (int id : ids) if (id < 0 || id >= vocab) invalid("token id"); + std::vector mask(ids.size()); + int real_tokens = 0; + for (size_t i = 0; i < ids.size(); ++i) real_tokens += (mask[i] = ids[i] != pad); + if (!real_tokens) invalid("all-padding token input"); + int64_t width = std::max(std::max(hidden, output), static_cast(per_layer) * layers); + for (int i = 0; i < layers; ++i) width = std::max(width, std::max(heads[i] * dimensions[i], ffn[i])); + if (ids.size() > static_cast(std::numeric_limits::max() / width)) invalid("token matrix size"); + Matrix x(static_cast(ids.size()), hidden); + const auto *embedding = loader.weight("token_embd.weight"); + const float scale = std::sqrt(static_cast(hidden)); + for (int row = 0; row < x.rows; ++row) { + tensor_row(embedding, ids[row], x[row]); + for (int c = 0; c < hidden; ++c) x[row][c] *= scale; + } + Matrix projected = ops.linear(x, "per_layer_model_proj", false); + for (float &v : projected.data) v /= scale; + projected = rms(projected, "per_layer_proj_norm", per_layer); + for (int layer = 0; layer < layers; ++layer) { + const auto root = "blk." + std::to_string(layer) + "."; + add(x, attention(x, layer, mask)); + const auto pre = rms(x, root + "ffn_norm"); + Matrix gate = ops.linear(pre, root + "ffn_gate", false); + fast_gelu(gate); + const Matrix up = ops.linear(pre, root + "ffn_up", false); + for (size_t i = 0; i < gate.data.size(); ++i) gate.data[i] *= up.data[i]; + add(x, rms(ops.linear(gate, root + "ffn_down", false), root + "post_ffw_norm")); + Matrix ple = ops.linear(x, root + "inp_gate", false); + fast_gelu(ple); + for (int row = 0; row < x.rows; ++row) for (int c = 0; c < per_layer; ++c) + ple[row][c] *= projected[row][layer*per_layer+c]; + add(x, rms(ops.linear(ple, root + "proj", false), root + "post_norm")); + const float out_scale = scalar(loader.weight(root + "layer_output_scale.weight"), 0); + for (float &v : x.data) v *= out_scale; + } + // Match the reference graph's operation order even for storage types that + // pack/round the activation operand: projecting a pooled row first is only + // algebraically equivalent, not equivalent under F16/Q8 input packing. + const Matrix final = ops.linear(rms(x, "output_norm"), "output", false); + std::vector result(output, 0.0f); + for (int row = 0; row < final.rows; ++row) if (mask[row]) for (int c = 0; c < output; ++c) + result[c] += final[row][c] / real_tokens; + double square = 0; + for (float v : result) { + if (!std::isfinite(v)) invalid("non-finite output embedding"); + square += static_cast(v) * v; + } + if (!(square > 0)) invalid("zero output embedding"); + const float inverse = static_cast(1.0 / std::sqrt(square)); + for (float &v : result) v *= inverse; + return result; +} +} // namespace clap::detail diff --git a/src/models/models.h b/src/models/models.h index 76d00ae..41fa3b9 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -71,6 +71,31 @@ class Clamp3Model final : public ModelArchitecture { std::unique_ptr frontend; }; +// Gemma-embedding2 GGUF text encoder (EmbeddingGemma 2); no image/audio path. +class EmbeddingGemma2Model final : public ModelArchitecture { +public: + EmbeddingGemma2Model(const ModelLoader &loader, int threads); + std::vector text(const std::vector &ids) const override; + AudioFrames audio_frames(const std::vector &) const override; + TextParameters text_parameters() const override; + int projection_dimension() const override { return output; } + int mel_bins() const override { return 0; } + int sample_rate() const override { return 0; } + bool is_waveform_encoder() const override { return false; } +private: + void load_arch_hparams() override; + void load_arch_tensors() const override; + Matrix rms(const Matrix &x, const std::string &name, int width = 0) const; + Matrix attention(const Matrix &x, int layer, const std::vector &mask) const; + const ModelLoader &loader; + TensorOps ops; + int hidden = 0, output = 0, layers = 0, per_layer = 0, vocab = 0, context = 0, window = 0; + int bos = 2, eos = 1, pad = 0; + float epsilon = 0, rope_base = 0, rope_base_swa = 0, rope_scale = 1; + std::vector heads, kv_heads, dimensions, rotations, ffn; + std::vector local; +}; + // Official m-a-p/MERT-v2-FullSong waveform encoder: checkpoint frontend, // ConvNeXt subsampler and bidirectional rotary Conformer stack. class Mert2Model final : public ModelArchitecture { diff --git a/src/vocab.cpp b/src/vocab.cpp index a4a8e50..62c5cad 100644 --- a/src/vocab.cpp +++ b/src/vocab.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace clap::detail { using json = nlohmann::json; @@ -199,6 +200,153 @@ std::vector ByteBpeVocabulary::tokenize(const std::string &text, size_t max return ids; } +GemmaVocabulary::GemmaVocabulary(const json &metadata, size_t context_length) + : bos(-1), eos(-1), unk(-1), + add_bos(metadata.value("tokenizer.ggml.add_bos_token", false)), + add_eos(metadata.value("tokenizer.ggml.add_eos_token", false)), context_length(context_length) { + if (metadata.at("tokenizer.ggml.model") != "gemma4" || !context_length || + metadata.value("tokenizer.ggml.add_space_prefix", false) || + metadata.value("tokenizer.ggml.remove_extra_whitespaces", false)) + throw std::runtime_error("CLAP: unsupported Gemma tokenizer configuration"); + const auto &tokens = metadata.at("tokenizer.ggml.tokens"); + const auto &types = metadata.at("tokenizer.ggml.token_type"); + const auto &merges = metadata.at("tokenizer.ggml.merges"); + if (!tokens.is_array() || tokens.empty() || tokens.size() > size_t(std::numeric_limits::max()) || + !types.is_array() || types.size() != tokens.size() || !merges.is_array()) + throw std::runtime_error("CLAP: invalid Gemma tokenizer arrays"); + auto bounded_integer = [](const json &value, uint64_t minimum, uint64_t maximum, const char *error) { + // Check the original signed/unsigned JSON value before narrowing to int. + if (!value.is_number_integer() || + (!value.is_number_unsigned() && value.get() < 0) || + value.get() < minimum || value.get() > maximum) + throw std::runtime_error(error); + return value.get(); + }; + auto special_id = [&](const char *key) { + // -1 is an internal sentinel for absent metadata, never an explicit id. + return metadata.contains(key) + ? bounded_integer(metadata.at(key), 0, tokens.size() - 1, "CLAP: invalid Gemma special token id") + : -1; + }; + bos = special_id("tokenizer.ggml.bos_token_id"); + eos = special_id("tokenizer.ggml.eos_token_id"); + unk = special_id("tokenizer.ggml.unknown_token_id"); + if (metadata.contains("tokenizer.ggml.scores")) { + const auto &scores = metadata.at("tokenizer.ggml.scores"); + if (!scores.is_array() || scores.size() != tokens.size()) + throw std::runtime_error("CLAP: invalid Gemma tokenizer scores"); + for (const auto &s : scores) if (!s.is_number() || !std::isfinite(s.get())) + throw std::runtime_error("CLAP: invalid Gemma tokenizer score"); + } + std::fill(std::begin(bytes), std::end(bytes), -1); + for (size_t id = 0; id < tokens.size(); ++id) { + const auto token = tokens.at(id).get(); + const int type = bounded_integer(types.at(id), 1, 6, "CLAP: invalid Gemma token type"); + if (token.empty()) + throw std::runtime_error("CLAP: invalid Gemma token"); + if (!pieces.emplace(token, static_cast(id)).second) + throw std::runtime_error("CLAP: duplicate Gemma token"); + if (type == 2 || type == 3 || type == 4) special.emplace_back(token, static_cast(id)); + if (type == 6) { + if (token.size() != 6 || token.compare(0, 3, "<0x") || token[5] != '>') + throw std::runtime_error("CLAP: invalid Gemma byte token"); + const auto digits = std::string("0123456789ABCDEF"); + const auto a = digits.find(token[3]), b = digits.find(token[4]); + if (a == std::string::npos || b == std::string::npos) + throw std::runtime_error("CLAP: invalid Gemma byte token"); + bytes[a * 16 + b] = static_cast(id); + } + } + for (int id : bytes) if (id < 0) throw std::runtime_error("CLAP: incomplete Gemma byte fallback"); + auto valid = [&](int id) { return id >= 0 && size_t(id) < tokens.size(); }; + if ((add_bos && !valid(bos)) || (add_eos && !valid(eos)) || (unk != -1 && !valid(unk))) + throw std::runtime_error("CLAP: invalid Gemma special token id"); + if (merges.size() > size_t(std::numeric_limits::max())) + throw std::runtime_error("CLAP: too many Gemma merges"); + int rank = 0; + for (const auto &item : merges) { + const auto merge = item.get(); + const auto split = merge.find(' '); + if (split == std::string::npos || split == 0 || split + 1 == merge.size()) + throw std::runtime_error("CLAP: invalid Gemma merge"); + auto pair = std::make_pair(merge.substr(0, split), merge.substr(split + 1)); + if (!pieces.count(pair.first) || !pieces.count(pair.second) || !pieces.count(pair.first + pair.second)) + throw std::runtime_error("CLAP: Gemma merge references missing token"); + ranks.emplace(std::move(pair), rank++); + } + std::sort(special.begin(), special.end(), [](const auto &a, const auto &b) { + return a.first.size() > b.first.size(); + }); +} + +void GemmaVocabulary::encode_piece(const std::string &text, std::vector &ids) const { + // Only ASCII space is normalized. Preserve all other Unicode, whitespace, + // repeated spaces, controls and combining marks exactly (no ICU NFKC). + std::string normalized; + for (char c : text) { if (c == ' ') normalized += "▁"; else normalized += c; } + if (normalized.empty()) return; + struct Symbol { size_t start, size; int previous, next; }; + struct Merge { int rank, left, right; size_t left_size, right_size; }; + struct Compare { bool operator()(const Merge &a, const Merge &b) const { + return a.rank > b.rank || (a.rank == b.rank && a.left > b.left); + }}; + std::vector symbols; + for (size_t pos = 0; pos < normalized.size();) { + const auto c = static_cast(normalized[pos]); + size_t length = c < 0x80 ? 1 : c >= 0xc2 && c <= 0xdf ? 2 : c >= 0xe0 && c <= 0xef ? 3 : c >= 0xf0 && c <= 0xf4 ? 4 : 1; + length = std::min(length, normalized.size() - pos); + for (size_t k = 1; k < length; ++k) if ((static_cast(normalized[pos+k]) & 0xc0) != 0x80) { length = 1; break; } + const int i = static_cast(symbols.size()); + symbols.push_back({pos, length, i-1, i+1}); pos += length; + } + symbols.back().next = -1; + std::priority_queue, Compare> queue; + auto add = [&](int left, int right) { + if (left < 0 || right < 0) return; + const auto &a = symbols[left], &b = symbols[right]; + auto it = ranks.find({normalized.substr(a.start,a.size), normalized.substr(b.start,b.size)}); + if (it != ranks.end()) queue.push({it->second,left,right,a.size,b.size}); + }; + for (size_t i = 1; i < symbols.size(); ++i) add(static_cast(i)-1,static_cast(i)); + while (!queue.empty()) { + const auto merge = queue.top(); queue.pop(); + auto &a = symbols[merge.left], &b = symbols[merge.right]; + if (a.size != merge.left_size || b.size != merge.right_size || a.next != merge.right) continue; + a.size += b.size; b.size = 0; a.next = b.next; + if (a.next >= 0) symbols[a.next].previous = merge.left; + add(a.previous, merge.left); add(merge.left,a.next); + } + for (int i = 0; i >= 0; i = symbols[i].next) { + const auto &s = symbols[i]; + const auto it = pieces.find(normalized.substr(s.start,s.size)); + if (it != pieces.end()) ids.push_back(it->second); + else for (size_t j = s.start; j < s.start+s.size; ++j) ids.push_back(bytes[static_cast(normalized[j])]); + } +} + +std::vector GemmaVocabulary::tokenize(const std::string &text, size_t max_length) const { + const size_t reserved = size_t(add_bos) + size_t(add_eos); + if (!max_length || max_length < reserved || max_length > context_length) + throw std::runtime_error("CLAP: invalid maximum text length"); + std::vector ids; + if (add_bos) ids.push_back(bos); + size_t pos = 0; + while (pos < text.size()) { + size_t next = std::string::npos; + const std::pair *match = nullptr; + for (const auto &s : special) { + const auto found = text.find(s.first,pos); + if (found < next) { next = found; match = &s; } + } + if (!match) { encode_piece(text.substr(pos), ids); break; } + encode_piece(text.substr(pos,next-pos),ids); + ids.push_back(match->second); pos = next + match->first.size(); + } + if (ids.size() > max_length - size_t(add_eos)) ids.resize(max_length-size_t(add_eos)); + if (add_eos) ids.push_back(eos); + return ids; +} + UnigramVocabulary::UnigramVocabulary(const json &tokenizer, int bos, int eos, int unk, size_t context_length) : bos(bos), eos(eos), unk(unk), context_length(context_length) { diff --git a/src/vocab.h b/src/vocab.h index 280700b..3343ff8 100644 --- a/src/vocab.h +++ b/src/vocab.h @@ -35,6 +35,23 @@ class ByteBpeVocabulary final : public Vocabulary { std::unique_ptr pattern; }; +// Gemma 4 / EmbeddingGemma 2 GGUF uses SentencePiece-style raw UTF-8 BPE, +// NOT Unigram or GPT-2 byte encoding. Metadata keys retain tokenizer.ggml.*. +class GemmaVocabulary final : public Vocabulary { +public: + GemmaVocabulary(const nlohmann::json &metadata, size_t context_length); + std::vector tokenize(const std::string &text, size_t max_length) const override; +private: + void encode_piece(const std::string &text, std::vector &ids) const; + int bos, eos, unk; + bool add_bos, add_eos; + size_t context_length; + std::unordered_map pieces; + std::map, int> ranks; + std::vector> special; + int bytes[256]; +}; + // XLM-R's tokenizer.json is a SentencePiece Unigram model represented by the // Hugging Face fast-tokenizer format. Inference stays self-contained: GGUF // carries that JSON and ICU supplies the Unicode normalization/whitespace pass. diff --git a/tests/gemma-storage-test.cpp b/tests/gemma-storage-test.cpp new file mode 100644 index 0000000..e395542 --- /dev/null +++ b/tests/gemma-storage-test.cpp @@ -0,0 +1,47 @@ +#include "model-loader.h" +#include "model-ops.h" +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 2) return 1; + try { + for (const auto type : {GGML_TYPE_Q5_K, GGML_TYPE_BF16}) { + const std::string path = std::string(argv[1]) + std::to_string(type) + ".gguf"; + std::unique_ptr ctx(ggml_init({1024 * 1024, nullptr, false}), ggml_free); + std::unique_ptr file(gguf_init_empty(), gguf_free); + gguf_set_val_str(file.get(), "general.architecture", "gemma-embedding2"); + auto *w = ggml_new_tensor_2d(ctx.get(), type, 256, 2); + ggml_set_name(w, "dense.weight"); + std::vector values(512); + for (size_t i = 0; i < values.size(); ++i) values[i] = 0.1f * std::sin(float(i)); + if (type == GGML_TYPE_BF16) ggml_fp32_to_bf16_row_ref(values.data(), static_cast(w->data), values.size()); + else ggml_quantize_chunk(type, values.data(), w->data, 0, 2, 256, nullptr); + gguf_add_tensor(file.get(), w); + if (!gguf_write_to_file(file.get(), path.c_str(), false)) return 2; + clap::detail::ModelLoader loader; + loader.load(path); + std::vector decoded(512), row(256); + ggml_get_type_traits(type)->to_float(w->data, decoded.data(), decoded.size()); + for (size_t i = 0; i < decoded.size(); ++i) + if (clap::detail::scalar(loader.weight("dense.weight"), i) != decoded[i]) return 3; + clap::detail::tensor_row(loader.weight("dense.weight"), 1, row.data()); + for (size_t i = 0; i < row.size(); ++i) if (row[i] != decoded[256 + i]) return 4; + clap::detail::Matrix x(2, 256); + for (auto &v : x.data) v = 0.25f; + const auto result = clap::detail::TensorOps(loader, 2).linear(x, "dense", false); + for (int c = 0; c < 2; ++c) { + float expected = 0; + for (int i = 0; i < 256; ++i) expected += decoded[c * 256 + i] * 0.25f; + if (std::abs(result[0][c] - expected) > 0.002f) return 5; + } + } + } catch (const std::exception &e) { std::fprintf(stderr, "%s\n", e.what()); return 6; } + return 0; +} diff --git a/tests/metadata-test.cpp b/tests/metadata-test.cpp new file mode 100644 index 0000000..ca87665 --- /dev/null +++ b/tests/metadata-test.cpp @@ -0,0 +1,38 @@ +#include "model-loader.h" +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 2) return 1; + auto *ctx = gguf_init_empty(); + gguf_set_val_str(ctx, "general.architecture", "test"); + gguf_set_val_u32(ctx, "test.count", 17); + gguf_set_val_f32(ctx, "test.epsilon", 0.25f); + gguf_set_val_bool(ctx, "test.flag", true); + const char *tokens[] = {"", "▁hello"}; + gguf_set_arr_str(ctx, "test.tokens", tokens, 2); + const float scores[] = {-1.5f, -2.5f}; + gguf_set_arr_data(ctx, "test.scores", GGUF_TYPE_FLOAT32, scores, 2); + if (!gguf_write_to_file(ctx, argv[1], true)) return 1; + gguf_free(ctx); + try { + clap::detail::ModelLoader loader; + loader.load(argv[1]); + if (!loader.has_metadata("test.count") || loader.has_metadata("test.missing")) return 2; + if (loader.integer("test.count") != 17 || loader.number("test.epsilon") != 0.25 || !loader.boolean("test.flag")) return 3; + if (loader.metadata_value("test.tokens") != nlohmann::json::array({"", "▁hello"})) return 4; + if (loader.metadata_value("test.scores") != nlohmann::json::array({-1.5, -2.5})) return 5; + for (const auto *key : {"test.epsilon", "test.flag", "test.missing"}) { + bool rejected = false; + try { (void)loader.integer(key); } catch (const std::runtime_error &) { rejected = true; } + if (!rejected) return 6; + } + bool rejected = false; + try { (void)loader.boolean("test.count"); } catch (const std::runtime_error &) { rejected = true; } + if (!rejected) return 7; + } catch (const std::exception &e) { std::fprintf(stderr, "%s\n", e.what()); return 8; } + return 0; +} diff --git a/tests/models/EMBEDDINGGEMMA2_SOURCE.md b/tests/models/EMBEDDINGGEMMA2_SOURCE.md new file mode 100644 index 0000000..427e8cd --- /dev/null +++ b/tests/models/EMBEDDINGGEMMA2_SOURCE.md @@ -0,0 +1,77 @@ +# EmbeddingGemma 2 text inference contract + +Primary sources, pinned for reproducibility: + +- llama.cpp `8ca4060b7e30e76baaa5670ee1ec1b760b7f8552`, + `src/models/gemma-embedding2.cpp`, `conversion/gemma.py`, `src/llama-vocab.cpp`. +- Transformers `95bafdcccfffe17ec383999fc39273867b11385e`, + `src/transformers/models/embedding_gemma2/modeling_embedding_gemma2.py`. +- Google `google/embeddinggemma-2`, revision + `26711154b641697c443124e316f3d21587f9ff5d` (config/tokenizer/pooling). +- Unsloth `unsloth/embeddinggemma-2-GGUF`, revision + `031f0d4b35536f69ab3509d4893c923264fcf253`. + +## Scope and numerical details + +Load the standard `gemma-embedding2` GGUF directly. No checkpoint converter, +image/mmproj, image API, or audio encoder is added. Default output is 768D; +retrieval prefixes are explicit input text, not automatically inserted. + +The published model has 24 blocks, width 512, FFN width 2048, four Q heads, +and 512D per-layer projection inputs. Blocks 5/11/17/23 (zero based) use global +attention with head dimension 512 and one KV head. Local blocks use dimension +256 and two KV heads. Attention is bidirectional, scale 1.0. GGUF window 1024 +means inclusive radius 512, not radius 1024. RoPE is half-split NeoX, local +base 10000/global base 1000000. RMS weights multiply directly; do not add 1. +Q/K have learned per-head RMS weights; V RMS has no learned weight. Apply +post-attention/FFN/PLE norms before their residual additions, then learned +layer-output scaling. PLE comes from the initial scaled embeddings, not each +block's hidden states. Final projection precedes masked mean pooling and L2 +normalization. Include BOS/EOS/prompt; exclude pad positions. + +Gemma4 tokenization is raw-UTF-8 BPE using explicit merge ranks. Replace ASCII +spaces with `▁` without Unicode normalization, whitespace collapsing, or dummy +leading space. Honor newline runs, byte fallback, and special tokens. The +published GGUF adds both BOS=2 and EOS=1. Scores are not BPE merge priorities. + +Preserve F32 activations/reductions even when weights use F16/BF16 storage. +The shared storage-only matmul path decodes its current operand; quantized +weights are not a license to round all intermediate activations to F16. + +## Default automated tests + +`embeddinggemma2.py` generates deterministic valid tiny GGUFs and an independent +stdlib scalar oracle. F32/F16 storage, mixed local/global heads, inclusive local +window boundaries, PLE, padding, strings/IDs, pooling, malformed metadata and +shapes are covered. `embeddinggemma2_cli.py` checks CLI and HTTP vectors against +that oracle. All fixtures are generated outside the source tree. Dedicated +native tests cover tokenizer rules and Q5_K/BF16 storage. + +`gemma_tokenizer_parity.py` is an optional published-tokenizer parity generator; +it requires the `tokenizers` package only for testing, never for inference. +It checks the pinned tokenizer's SHA256 before generating cases. Enable its +CTest fixture and native comparison with: + +```sh +cmake -S . -B build -DCLAP_GEMMA_TOKENIZER_JSON=/path/to/tokenizer.json +ctest --test-dir build -R gemma_tokenizer_hf --output-on-failure +``` + +## Published checkpoint validation + +Validated `embeddinggemma-2-UD-Q4_K_XL.gguf` (175673856 bytes), SHA256 +`ea905fd08e8061db77a0031cbb5096d459d7cfd2bdd0ad2ba66e483f16719493`. +This file mixes Q4_K/Q5_K/Q6_K/Q8_0/F32 tensors. Six actual inputs were compared +against unmodified pinned llama.cpp: English, Chinese, query/document prefixes, +and a 602-token input crossing the local attention radius. + +Use `llama-embedding` with `--pooling mean --embd-normalize 2 +--embd-output-format json --no-warmup -c 1024 -b 1024 -ub 1024` and the exact +same literal prompt. Native packed quantized llama.cpp versus this runtime's +F32-activation path had minimum cosine 0.9995059. To separate activation packing +from architecture errors, export the **same quantized weights** losslessly to +F32 storage with the test-only `clap-q8-test decode INPUT OUTPUT`, then run that +F32 model in llama.cpp. Across the same six inputs, minimum cosine was +0.999999926 and maximum absolute component error was 5.332e-5. This is a +same-stored-weight oracle, not a claim of parity against Google's original +unquantized checkpoint or an embedding-quality benchmark. diff --git a/tests/models/embeddinggemma2-test.cpp b/tests/models/embeddinggemma2-test.cpp new file mode 100644 index 0000000..d32083f --- /dev/null +++ b/tests/models/embeddinggemma2-test.cpp @@ -0,0 +1,65 @@ +#include "clap.h" +#include +#include +#include +#include +#include +#include +#include + +static void require(bool ok, const char *message) { + if (!ok) throw std::runtime_error(message); +} +int main(int argc, char **argv) { + try { + require(argc >= 3, "usage: embeddinggemma2-test MODEL FIXTURE [reject-message]"); + if (argc == 4) { + try { clap::CLAP model(argv[1], 2); } + catch (const std::exception &e) { + require(std::string(e.what()).find(argv[3]) != std::string::npos, e.what()); + return 0; + } + throw std::runtime_error("invalid model accepted"); + } + clap::CLAP model(argv[1], 2); + std::ifstream input(argv[2]); + nlohmann::json fixture; input >> fixture; + float maximum = 0; + for (const auto &test : fixture.at("text_cases")) { + const auto text = test.at("text").get(); + require(model.tokenize(text, fixture.at("context").get()) == test.at("ids").get>(), "tiny Gemma4 tokenizer mismatch"); + const auto actual = model.encode_text(text); + const auto expected = test.at("embedding").get>(); + require(actual.size() == expected.size(), "text projection dimension mismatch"); + for (size_t i = 0; i < actual.size(); ++i) { + require(std::isfinite(actual[i]), "non-finite text embedding"); + maximum = std::max(maximum, std::abs(actual[i] - expected[i])); + } + } + for (const auto &test : fixture.at("cases")) { + const auto ids = test.at("ids").get>(); + const auto expected = test.at("embedding").get>(); + const auto actual = model.encode_text(ids); + require(actual.size() == expected.size(), "projection dimension mismatch"); + for (size_t i = 0; i < actual.size(); ++i) { + require(std::isfinite(actual[i]), "non-finite embedding"); + maximum = std::max(maximum, std::abs(actual[i] - expected[i])); + } + require(actual == model.encode_text(ids), "repeated inference changed output"); + } + std::cout << "EmbeddingGemma2 independent scalar oracle max_abs_error=" << maximum << '\n'; + require(maximum <= fixture.value("tolerance", 8e-5f), "independent numerical oracle mismatch"); + for (const auto &ids : {std::vector{}, std::vector{-1}, std::vector{0, 0, 0}, + std::vector{fixture.at("vocab").get()}, + std::vector(fixture.at("context").get() + 1, 3)}) { + bool rejected = false; + try { (void) model.encode_text(ids); } catch (const std::exception &) { rejected = true; } + require(rejected, "invalid token input accepted"); + } + bool rejected = false; + try { (void) model.encode_audio(std::vector(48000, 0.1f)); } + catch (const std::exception &) { rejected = true; } + require(rejected, "text-only model accepted audio"); + return 0; + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; return 1; } +} diff --git a/tests/models/embeddinggemma2.py b/tests/models/embeddinggemma2.py new file mode 100644 index 0000000..1e57c8e --- /dev/null +++ b/tests/models/embeddinggemma2.py @@ -0,0 +1,233 @@ +"""Generated tiny GGUF + independent stdlib scalar EmbeddingGemma2 oracle. + +Contract: llama.cpp src/models/gemma-embedding2.cpp, GGUF architecture +'gemma-embedding2'. No downloads, optional ML packages, or committed binaries. +Run: python embeddinggemma2.py --test /path/to/clap-embeddinggemma2-test +Generate: python embeddinggemma2.py --output /path/to/fixtures +""" +import argparse +import copy +import json +import math +from pathlib import Path +import struct +import subprocess +import tempfile + +ARCH = "gemma-embedding2" + + +def string(s): + raw = s.encode("utf-8") + return struct.pack("", "", "", "", "▁a", "▁b", "c", "d", "e", "f", "g", "h", "▁", "a", "b"] + [f"<0x{i:02X}>" for i in range(256)], + "tokenizer.ggml.scores": [0.0] * vocab, + "tokenizer.ggml.token_type": [3, 3, 3, 2] + [1] * 11 + [6] * 256, + "tokenizer.ggml.merges": ["▁ a", "▁ b"], + "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, + "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.unknown_token_id": 3, + "tokenizer.ggml.add_bos_token": True, "tokenizer.ggml.add_eos_token": True}) + tensors = {} + + def add(name, shape, norm=False, scale=False): + serial = len(tensors) + 1 + values = [(0.9 + 0.17 * math.sin(i + serial) if norm else + 0.83 + 0.03 * serial if scale else + 0.23 * math.sin((i + serial * 11) * 0.311) + 0.04 * math.cos(i * 0.13)) + for i in range(math.prod(shape))] + fmt = "e" if storage == "f16" and len(shape) == 2 else "f" + values = list(struct.unpack("<" + fmt * len(values), struct.pack("<" + fmt * len(values), *values))) + tensors[name + ".weight"] = (shape, values) + + add("token_embd", [hidden, vocab]) + add("per_layer_model_proj", [hidden, ple * layers]) + add("per_layer_proj_norm", [ple], norm=True) + add("output_norm", [hidden], norm=True) + add("output", [hidden, output]) + for layer, dim in enumerate(dims): + root = f"blk.{layer}." + for name, size in [("attn_norm", hidden), ("attn_q_norm", dim), ("attn_k_norm", dim), + ("post_attention_norm", hidden), ("ffn_norm", hidden), + ("post_ffw_norm", hidden), ("post_norm", hidden)]: + add(root + name, [size], norm=True) + for name, ins, outs in [("attn_q", hidden, heads[layer] * dim), ("attn_k", hidden, kv[layer] * dim), + ("attn_v", hidden, kv[layer] * dim), ("attn_output", heads[layer] * dim, hidden), + ("ffn_gate", hidden, ff), ("ffn_up", hidden, ff), ("ffn_down", ff, hidden), + ("inp_gate", hidden, ple), ("proj", ple, hidden)]: + add(root + name, [ins, outs]) + add(root + "layer_output_scale", [1], scale=True) + + def weight(name): + return tensors[name + ".weight"][1] + + def dense(rows, name): + (width, outs), w = tensors[name + ".weight"] + # Weights are storage-rounded above; inference preserves F32 operands + # even for F16 storage (EmbeddingGemma2 can overflow F16 activations). + return [[sum(row[i] * w[j * width + i] for i in range(width)) + for j in range(outs)] for row in rows] + + def norm(rows, name=None): + w = weight(name) if name else None + return [[v / math.sqrt(sum(x*x for x in row) / len(row) + 1e-5) * (w[i] if w else 1) + for i, v in enumerate(row)] for row in rows] + + def plus(a, b): + return [[x+y for x, y in zip(ra, rb)] for ra, rb in zip(a, b)] + + def gelu(x): + return 0.5*x*(1+math.tanh(math.sqrt(2/math.pi)*(x+0.044715*x*x*x))) + + def reference(ids): + x = [[weight("token_embd")[token*hidden+i] * math.sqrt(hidden) for i in range(hidden)] for token in ids] + all_ple = [[v / math.sqrt(hidden) for v in row] for row in dense(x, "per_layer_model_proj")] + inputs = [norm([row[l*ple:(l+1)*ple] for row in all_ple], "per_layer_proj_norm") for l in range(layers)] + for layer, dim in enumerate(dims): + root = f"blk.{layer}." + pre = norm(x, root + "attn_norm") + q, k, v = [dense(pre, root + "attn_" + name) for name in ["q", "k", "v"]] + def head_norm(rows, count, name): + return [sum(norm([row[h*dim:(h+1)*dim] for h in range(count)], name), []) for row in rows] + q = head_norm(q, heads[layer], root + "attn_q_norm") + k = head_norm(k, kv[layer], root + "attn_k_norm") + v = head_norm(v, kv[layer], None) + for rows, count in [(q, heads[layer]), (k, kv[layer])]: + for pos, row in enumerate(rows): + for head in range(count): + # GGUF Gemma uses NEOX pairs in the first n_rot channels. + for i in range(rotary[layer] // 2): + angle = pos * (1000.0 if local[layer] else 10000.0) ** (-2*i/rotary[layer]) + a, b = head*dim+i, head*dim+i+rotary[layer]//2 + row[a], row[b] = row[a]*math.cos(angle)-row[b]*math.sin(angle), row[a]*math.sin(angle)+row[b]*math.cos(angle) + attended = [[0.0]*(heads[layer]*dim) for _ in ids] + for pos in range(len(ids)): + for head in range(heads[layer]): + kh = head // (heads[layer] // kv[layer]) + positions = [j for j in range(len(ids)) if ids[j] != 0 and (not local[layer] or abs(pos-j) <= 4 // 2)] + if not positions: + continue + scores = [sum(q[pos][head*dim+c]*k[j][kh*dim+c] for c in range(dim)) for j in positions] + scores = [math.exp(s-max(scores)) for s in scores] + for c in range(dim): + attended[pos][head*dim+c] = sum(s*v[j][kh*dim+c] for j, s in zip(positions, scores))/sum(scores) + x = plus(x, norm(dense(attended, root + "attn_output"), root + "post_attention_norm")) + pre = norm(x, root + "ffn_norm") + gate, up = dense(pre, root + "ffn_gate"), dense(pre, root + "ffn_up") + gated = [[gelu(a)*b for a, b in zip(ra, rb)] for ra, rb in zip(gate, up)] + x = plus(x, norm(dense(gated, root + "ffn_down"), root + "post_ffw_norm")) + gated = [[gelu(a)*b for a, b in zip(ra, rb)] for ra, rb in zip(dense(x, root + "inp_gate"), inputs[layer])] + x = plus(x, norm(dense(gated, root + "proj"), root + "post_norm")) + x = [[v*weight(root+"layer_output_scale")[0] for v in row] for row in x] + # Project each token before pooling, as in the reference graph. Moving + # mean ahead of a quantized/F16 matmul changes input packing rounding. + x = dense(norm(x, "output_norm"), "output") + pooled = [sum(row[c] for token, row in zip(ids, x) if token != 0)/sum(token != 0 for token in ids) for c in range(output)] + length = math.sqrt(sum(v*v for v in pooled)) + return [v/length for v in pooled] + + cases = [[2, 4, 1], [2, 5, 6, 7, 8, 9, 1], [4], [2, 4, 5, 4, 1], [2] + [4, 5]*7 + [1], + [2, 4, 1, 0, 0, 0, 0], [0, 0, 2, 4, 1], [2, 4, 0, 5, 0, 1]] + # Storage rounding is applied before independent scalar F32-activation math. + oracle: dict = dict(vocab=vocab, context=context, cases=[dict(ids=ids, embedding=reference(ids)) for ids in cases], tolerance=5e-6) + oracle["text_cases"] = [dict(text=text, ids=ids, embedding=reference(ids)) for text, ids in + [("a", [2, 13, 1]), (" a", [2, 4, 1]), (" b", [2, 5, 1]), ("", [2, 1])]] + return metadata, tensors, oracle + + +def run(directory, executable=None): + directory.mkdir(parents=True, exist_ok=True) + for storage in ["f32", "f16"]: + for array_pattern in [False, True]: + metadata, tensors, oracle = fixture(storage, array_pattern) + stem = directory / (storage + ("-pattern-array" if array_pattern else "")) + save(stem.with_suffix(".gguf"), metadata, tensors, storage) + stem.with_suffix(".json").write_text(json.dumps(oracle), encoding="utf-8") + if executable: + subprocess.run([executable, str(stem.with_suffix(".gguf")), str(stem.with_suffix(".json"))], check=True) + if executable: + metadata, tensors, _ = fixture() + cases = [("block_count", 0), ("embedding_length", 1.5), ("attention.head_count", [2, 2]), + ("attention.head_count_kv", [3, 1, 1]), ("attention.key_length_swa", 3), + ("attention.value_length", 4), ("rope.dimension_count", 9), + ("attention.sliding_window_pattern", [True, False]), + ("attention.layer_norm_rms_epsilon", -1.0), ("pooling_type", 2), + ("attention.causal", True), ("rope.scaling.type", "yarn"), + ("attention.shared_kv_layers", 1), ("rope.scaling.attn_factor", 2.0), + ("tokenizer.ggml.bos_token_id", -1), ("tokenizer.ggml.eos_token_id", True)] + for i, (key, value) in enumerate(cases): + invalid = copy.deepcopy(metadata) + invalid[key if key.startswith("tokenizer.") else ARCH + "." + key] = value + path = directory / f"invalid-{i}.gguf" + save(path, invalid, tensors, "f32") + subprocess.run([executable, str(path), "unused", "EmbeddingGemma2"], check=True) + for name in ["blk.0.attn_q.weight", "blk.1.attn_k.weight", "per_layer_proj_norm.weight", "output.weight"]: + invalid = dict(tensors) + shape, values = invalid[name] + invalid[name] = ([shape[0] + 1] + shape[1:], values + [0.0] * math.prod(shape[1:])) + path = directory / "invalid-shape.gguf" + save(path, metadata, invalid, "f32") + subprocess.run([executable, str(path), "unused", name], check=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path) + parser.add_argument("--test") + args = parser.parse_args() + if args.output: + run(args.output, args.test) + else: + with tempfile.TemporaryDirectory(prefix="embeddinggemma2-") as temp: + run(Path(temp), args.test) diff --git a/tests/models/embeddinggemma2_cli.py b/tests/models/embeddinggemma2_cli.py new file mode 100644 index 0000000..a848077 --- /dev/null +++ b/tests/models/embeddinggemma2_cli.py @@ -0,0 +1,44 @@ +"""Text-only EmbeddingGemma2 CLI/HTTP integration, stdlib fixtures.""" +import json +import math +from pathlib import Path +import subprocess +import sys +import tempfile + +from embeddinggemma2 import fixture, save +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from embeddings import running_server, request + + +def check(actual, expected): + assert len(actual) == len(expected) + assert all(math.isfinite(v) for v in actual) + assert abs(sum(v*v for v in actual) - 1) < 1e-5 + assert max(abs(a-b) for a, b in zip(actual, expected)) < 5e-6 + + +def main(binary): + with tempfile.TemporaryDirectory(prefix="embeddinggemma2-cli-") as directory: + for storage in ("f32", "f16"): + metadata, tensors, oracle = fixture(storage) + model = Path(directory) / (storage + ".gguf") + save(model, metadata, tensors, storage) + cases = [case for case in oracle["text_cases"] if case["text"]] + for case in cases: + result = subprocess.run([binary, "embedding", "--model", str(model), + "--threads", "2", "--text", case["text"]], + check=True, capture_output=True, text=True, timeout=30) + check(json.loads(result.stdout), case["embedding"]) + with running_server(binary, model) as url: + status, result = request(url, {"input": [case["text"] for case in cases]}) + assert status == 200, result + assert len(result["data"]) == len(cases) + for i, (entry, case) in enumerate(zip(result["data"], cases)): + assert entry["index"] == i + check(entry["embedding"], case["embedding"]) + print("EmbeddingGemma2 F32/F16 CLI and HTTP match independent scalar oracle") + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/tests/models/gemma-tokenizer-test.cpp b/tests/models/gemma-tokenizer-test.cpp new file mode 100644 index 0000000..d782f24 --- /dev/null +++ b/tests/models/gemma-tokenizer-test.cpp @@ -0,0 +1,112 @@ +#include "vocab.h" +#include +#include +#include +#include +#include +using json = nlohmann::json; +using clap::detail::GemmaVocabulary; + +static void expect(const std::vector &actual, const std::vector &wanted) { + if (actual != wanted) throw std::runtime_error("token mismatch: " + json(actual).dump() + " != " + json(wanted).dump()); +} +static json fixture() { + json m = {{"tokenizer.ggml.model", "gemma4"}, + {"tokenizer.ggml.tokens", {"", "", "", "", "a", "b", "c", "ab", "bc", "▁", "▁a", "", "long", "é", "\n", "\n\n"}}, + {"tokenizer.ggml.token_type", {3,3,3,2,1,1,1,1,1,1,1,4,4,1,1,1}}, + {"tokenizer.ggml.merges", {"b c", "a b", "▁ a", "\n \n"}}, + {"tokenizer.ggml.bos_token_id", 2}, {"tokenizer.ggml.eos_token_id", 1}, + {"tokenizer.ggml.unknown_token_id", 3}, + {"tokenizer.ggml.add_bos_token", true}, {"tokenizer.ggml.add_eos_token", true}}; + const char *hex = "0123456789ABCDEF"; + for (int b = 0; b < 256; ++b) { + std::string token = "<0x"; token += hex[b >> 4]; token += hex[b & 15]; token += '>'; + m["tokenizer.ggml.tokens"].push_back(token); + m["tokenizer.ggml.token_type"].push_back(6); + } + return m; +} +int main(int argc, char **argv) { + try { + auto m = fixture(); GemmaVocabulary v(m, 128); + expect(v.tokenize("abc",128), {2,4,8,1}); // merges, NOT Unigram/longest matching + expect(v.tokenize(" a a ",128), {2,10,9,10,9,1}); + expect(v.tokenize("é",128), {2,13,1}); + expect(v.tokenize("e\xcc\x81",128), {2,16+101,16+204,16+129,1}); // no Unicode normalization + expect(v.tokenize("🙂",128), {2,16+240,16+159,16+153,16+130,1}); + expect(v.tokenize("\t\r\n\n",128), {2,16+9,16+13,15,1}); + expect(v.tokenize("longa",128), {2,2,12,4,1,1}); + expect(v.tokenize("",128), {2,1}); + expect(v.tokenize("aaaa",3), {2,4,1}); + m["tokenizer.ggml.add_bos_token"] = false; m["tokenizer.ggml.add_eos_token"] = false; + GemmaVocabulary bare(m,128); expect(bare.tokenize("",1), {}); expect(bare.tokenize("ab",1), {7}); + bool rejected = false; try { v.tokenize("a",1); } catch (const std::exception &) { rejected = true; } + if (!rejected) throw std::runtime_error("invalid length accepted"); + for (const char *key : {"tokenizer.ggml.tokens", "tokenizer.ggml.token_type", "tokenizer.ggml.merges"}) { + auto bad = fixture(); bad.erase(key); rejected = false; + try { GemmaVocabulary invalid(bad,128); } catch (const std::exception &) { rejected = true; } + if (!rejected) throw std::runtime_error(std::string("missing metadata accepted: ")+key); + } + expect(v.tokenize("\0",128), {2,1}); // C string constructor is empty + expect(v.tokenize(std::string("\0",1),128), {2,16,1}); + auto controls = fixture(); + controls["tokenizer.ggml.add_eos_token"] = false; + expect(GemmaVocabulary(controls,128).tokenize("a",1), {2}); + controls["tokenizer.ggml.add_bos_token"] = false; + controls["tokenizer.ggml.add_eos_token"] = true; + expect(GemmaVocabulary(controls,128).tokenize("a",1), {1}); + auto reject = [&](const json &bad) { + bool caught = false; + try { GemmaVocabulary invalid(bad,128); } catch (const std::exception &) { caught = true; } + if (!caught) throw std::runtime_error("malformed Gemma metadata accepted"); + }; + // Exercise both JSON integer representations before any narrowing to int. + std::vector accepted; + auto reject_integer = [&](const json &bad, const std::string &label) { + try { GemmaVocabulary invalid(bad,128); accepted.push_back(label); } + catch (const std::exception &) {} + }; + for (const json &value : {json(std::int64_t{4294967297LL}), + json(std::int64_t{-4294967295LL}), + json(std::uint64_t{4294967297ULL}), + json(std::numeric_limits::max())}) { + auto bad = fixture(); bad["tokenizer.ggml.token_type"][4] = value; + const auto label = value.type_name() + std::string(":") + value.dump(); + reject_integer(bad, "token_type:" + label); + for (const char *key : {"tokenizer.ggml.bos_token_id", "tokenizer.ggml.eos_token_id", "tokenizer.ggml.unknown_token_id"}) { + bad = fixture(); bad[key] = value; + reject_integer(bad, std::string(key) + ":" + label); + bad["tokenizer.ggml.add_bos_token"] = false; + bad["tokenizer.ggml.add_eos_token"] = false; + reject_integer(bad, std::string(key) + ":disabled:" + label); + } + } + auto optional = fixture(); + optional["tokenizer.ggml.add_bos_token"] = false; + optional["tokenizer.ggml.add_eos_token"] = false; + for (const char *key : {"tokenizer.ggml.bos_token_id", "tokenizer.ggml.eos_token_id", "tokenizer.ggml.unknown_token_id"}) { + auto bad = optional; bad[key] = -1; + reject_integer(bad, std::string(key) + ":explicit -1"); + bad[key] = bad["tokenizer.ggml.tokens"].size(); + reject_integer(bad, std::string(key) + ":out of vocabulary"); + optional.erase(key); + } + expect(GemmaVocabulary(optional,128).tokenize("a",128), {4}); + if (!accepted.empty()) throw std::runtime_error("integer metadata accepted: " + json(accepted).dump()); + auto bad = fixture(); bad["tokenizer.ggml.bos_token_id"] = 2.5; reject(bad); + bad = fixture(); bad["tokenizer.ggml.token_type"][4] = 1.5; reject(bad); + bad = fixture(); bad["tokenizer.ggml.tokens"][4] = ""; reject(bad); + bad = fixture(); bad["tokenizer.ggml.tokens"][4] = "b"; reject(bad); + bad = fixture(); bad["tokenizer.ggml.merges"][0] = "missing c"; reject(bad); + bad = fixture(); bad["tokenizer.ggml.tokens"].erase(16); reject(bad); + bad = fixture(); bad["tokenizer.ggml.add_space_prefix"] = true; reject(bad); + bad = fixture(); bad["tokenizer.ggml.model"] = "llama"; reject(bad); + if (argc == 3) { + json meta, cases; std::ifstream(argv[1]) >> meta; std::ifstream(argv[2]) >> cases; + GemmaVocabulary real(meta,8192); + for (const auto &c : cases) expect(real.tokenize(c.at("text"),8192),c.at("ids").get>()); + std::cout << "HF parity cases: " << cases.size() << '\n'; + } + std::cout << "Gemma tokenizer tests passed\n"; + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; return 1; } +} diff --git a/tests/models/gemma_tokenizer_parity.py b/tests/models/gemma_tokenizer_parity.py new file mode 100644 index 0000000..13141d5 --- /dev/null +++ b/tests/models/gemma_tokenizer_parity.py @@ -0,0 +1,56 @@ +"""Optional HF oracle: python gemma_tokenizer_parity.py TOKENIZER_JSON OUT_DIR. + +Only this test needs Python/tokenizers; native inference does not. Supply +Google embeddinggemma-2/tokenizer.json at revision +26711154b641697c443124e316f3d21587f9ff5d. Then run the native +clap-gemma-tokenizer-test OUT_DIR/metadata.json OUT_DIR/cases.json. +Primary sources: llama.cpp src/llama-vocab.cpp (gemma4), conversion/gemma.py; +HF google/embeddinggemma-2 tokenizer.json (BPE / Replace normalizer). +""" +import hashlib +import json +from pathlib import Path +import random +import sys +from tokenizers import Tokenizer + +source, output = Path(sys.argv[1]), Path(sys.argv[2]) +raw = source.read_bytes() +if hashlib.sha256(raw).hexdigest() != "4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4": + raise ValueError("Expected pinned google/embeddinggemma-2 tokenizer.json") +output.mkdir(parents=True, exist_ok=True) +j = json.loads(raw) +t = Tokenizer.from_file(str(source)) +vocab = j["model"]["vocab"] +tokens = [""] * len(vocab) +for token, idx in vocab.items(): + tokens[idx] = token +types = [1] * len(tokens) +for token in j["added_tokens"]: + types[token["id"]] = 3 if token["special"] else 4 +for idx, token in enumerate(tokens): + if len(token) == 6 and token.startswith("<0x") and token.endswith(">"): + types[idx] = 6 +metadata = { + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.tokens": tokens, + "tokenizer.ggml.token_type": types, + "tokenizer.ggml.merges": [" ".join(m) if isinstance(m, list) else m for m in j["model"]["merges"]], + "tokenizer.ggml.bos_token_id": vocab[""], + "tokenizer.ggml.eos_token_id": vocab[""], + "tokenizer.ggml.unknown_token_id": vocab[""], + "tokenizer.ggml.add_bos_token": True, + "tokenizer.ggml.add_eos_token": True, + "tokenizer.ggml.add_space_prefix": False, +} +texts = ["", "hello world", " hello", " hello world ", "Hello, WORLD! 12345", "e\u0301 é Ａ Straße ffi", "中文 日本語 한국어", "🙂🚀🧑🏽‍💻", "\t\n\r\n", "a\u00a0b\u2003c\u200bd", "a\x00b\x01c", "hello", ""] +texts += ["a" + "\n" * n + "b" for n in range(1, 100)] +texts += ["\n" * n for n in range(1, 100)] +texts += ["\t" * n + " " * n for n in range(1, 65)] +texts += [x["content"] + " hello " + x["content"] for x in j["added_tokens"]] +rng = random.Random(17) +alphabet = "abcABCéé▁\n\t\r 123!?<>中🙂\u0301\u200d\u00a0" +texts += ["".join(rng.choice(alphabet) for _ in range(rng.randrange(1, 120))) for _ in range(500)] +(output / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False), encoding="utf-8") +(output / "cases.json").write_text(json.dumps([{"text": s, "ids": t.encode(s).ids} for s in texts], ensure_ascii=False), encoding="utf-8") +print(f"Generated {len(texts)} HF tokenizer parity cases") From 4309c925bcaecd9d5d5ba6be2c19362ee559b441 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Thu, 29 Oct 2026 00:33:12 +0800 Subject: [PATCH 2/2] [verified] feat: support EmbeddingGemma 2 audio embeddings --- CMakeLists.txt | 19 ++ README.md | 39 ++- include/clap.h | 7 +- node/README.md | 13 + node/bindings.cpp | 16 +- node/index.d.ts | 8 +- node/test/addon.test.js | 17 + node/test/generate-fixtures.py | 3 + node/test/types.ts | 5 + python/README.md | 11 + python/bindings.cpp | 16 +- python/tests/test_bindings.py | 22 +- rust/README.md | 7 + rust/native/bridge.cpp | 6 +- rust/src/lib.rs | 52 ++- rust/tests/bindings.rs | 19 ++ rust/tests/generate.py | 3 + src/clap.cpp | 2 + src/embedding.cpp | 2 +- src/embedding.h | 1 + src/model-loader.cpp | 7 +- src/model-loader.h | 2 +- src/model.cpp | 15 +- src/models/embeddinggemma2-audio.cpp | 297 ++++++++++++++++++ src/models/embeddinggemma2-audio.h | 26 ++ src/models/embeddinggemma2.cpp | 49 ++- src/models/models.h | 14 +- tests/gemma-storage-test.cpp | 58 ++-- tests/models/EMBEDDINGGEMMA2_SOURCE.md | 59 +++- tests/models/embeddinggemma2-test.cpp | 6 + tests/models/embeddinggemma2.py | 19 +- tests/models/embeddinggemma2_audio-test.cpp | 97 ++++++ tests/models/embeddinggemma2_audio.py | 274 ++++++++++++++++ .../embeddinggemma2_audio_integration.py | 43 +++ tests/models/embeddinggemma2_cli.py | 3 + tests/models/embeddinggemma2_soft.py | 16 + tests/models/gemma-audio-integration-test.cpp | 53 ++++ tests/models/gemma-soft-test.cpp | 32 ++ 38 files changed, 1266 insertions(+), 72 deletions(-) create mode 100644 src/models/embeddinggemma2-audio.cpp create mode 100644 src/models/embeddinggemma2-audio.h create mode 100644 tests/models/embeddinggemma2_audio-test.cpp create mode 100644 tests/models/embeddinggemma2_audio.py create mode 100644 tests/models/embeddinggemma2_audio_integration.py create mode 100644 tests/models/embeddinggemma2_soft.py create mode 100644 tests/models/gemma-audio-integration-test.cpp create mode 100644 tests/models/gemma-soft-test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e88ecd0..ac98252 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,25 @@ if(BUILD_TESTING) COMMENT "Generating audio test fixtures" VERBATIM) add_custom_target(clap-audio-fixtures ALL DEPENDS ${CLAP_AUDIO_FIXTURES}) + add_executable(clap-gemma-audio-integration-test tests/models/gemma-audio-integration-test.cpp) + target_link_libraries(clap-gemma-audio-integration-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_audio_integration + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_audio_integration.py" + "$" "$") + set_tests_properties(embeddinggemma2_audio_integration PROPERTIES TIMEOUT 120) + add_executable(clap-gemma-audio-test tests/models/embeddinggemma2_audio-test.cpp) + target_include_directories(clap-gemma-audio-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-audio-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_audio + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_audio.py" + --test "$") + set_tests_properties(embeddinggemma2_audio PROPERTIES TIMEOUT 180) + add_executable(clap-gemma-soft-test tests/models/gemma-soft-test.cpp) + target_include_directories(clap-gemma-soft-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_link_libraries(clap-gemma-soft-test PRIVATE clap nlohmann_json::nlohmann_json) + add_test(NAME embeddinggemma2_soft + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/embeddinggemma2_soft.py" + "$") add_executable(clap-embeddinggemma2-test tests/models/embeddinggemma2-test.cpp) target_link_libraries(clap-embeddinggemma2-test PRIVATE clap nlohmann_json::nlohmann_json) add_test(NAME embeddinggemma2_model diff --git a/README.md b/README.md index d0f5cee..45e3bbf 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,7 @@ C++17/ggml implementations of audio embedding inference features: | [MERT v2](https://huggingface.co/m-a-p/MERT-v2-FullSong) | ❌ | ✅ | 1,024 | 24 kHz | | [MuQ](https://huggingface.co/OpenMuQ/MuQ-large-msd-iter) | ❌ | ✅ | 1,024 | 24 kHz | | [MuQ-MuLan](https://huggingface.co/OpenMuQ/MuQ-MuLan-large) | ✅ | ✅ | 512 | 24 kHz | - -| [EmbeddingGemma 2](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) | ✅ | ❌ | 768 | — | +| [EmbeddingGemma 2](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) | ✅ | ✅ | 768 | 16 kHz | GGUF models are available at [Audio Embedding GGUF](https://huggingface.co/collections/gorse-io/audio-embedding-gguf) collection. @@ -42,11 +41,12 @@ clap embedding --model model.gguf --text 'A jazz quartet.' clap embedding --model model.gguf --audio music.flac ``` -### EmbeddingGemma 2 (text only) +### EmbeddingGemma 2 (text and audio) Load the published [Unsloth EmbeddingGemma 2 GGUF](https://huggingface.co/unsloth/embeddinggemma-2-GGUF) -files directly; no conversion, Python inference, or `mmproj` is required. For example, -download `embeddinggemma-2-UD-Q4_K_XL.gguf` and run: +files directly; no conversion or Python inference is required. Text needs only +the main model. Audio additionally needs `mmproj-Q8_0.gguf` (or the published +F16/BF16 mmproj), which contains the audio tower and projector. For example: ```sh ./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf --text 'Hello world' @@ -67,10 +67,31 @@ an 8192-token context; use that limit even though the GGUF positional capacity i larger. Native computation preserves F32 activations with F16/BF16 storage and accepts the published Q4_K/Q5_K/Q6_K/Q8_0 mixed tensor types. -This integration is **text only**: no `encode_image`, image projector, or audio -encoding is implemented for this architecture. Use the published GGUF directly; -the checkpoint converter and local quantization presets below remain scoped to -the six existing audio/joint architectures. +This integration supports text and native waveform embeddings; **`encode_image` +remains unsupported**, even though the published mmproj also contains vision +weights. Audio is encoded directly, not transcribed to text: + +```sh +./build/clap embedding --model embeddinggemma-2-UD-Q4_K_XL.gguf \ + --mmproj mmproj-Q8_0.gguf --audio recording.wav +``` + +```cpp +clap::CLAP model("embeddinggemma-2-UD-Q4_K_XL.gguf", "mmproj-Q8_0.gguf"); +auto audio = clap::decode_audio("recording.wav"); +auto embedding = model.encode_audio(audio.samples, audio.sample_rate); +``` + +WAV/MP3/FLAC input is downmixed and resampled to 16 kHz. Audio preprocessing +follows the published Gemma4 frontend and uses the first 30 seconds, matching +the default processor's truncation policy. The audio Conformer/projector's +soft tokens pass through the same text backbone and final mean/L2 pooling, +producing a normalized 768D embedding in the joint space. Audio requires the +matching mmproj; text still works without it. Crop offsets and frame APIs are +not supported for this joint encoder. The HTTP endpoint remains text-only. + +Use published GGUFs directly; the checkpoint converter and local quantization +presets remain scoped to the six existing audio/joint architectures. ## Embeddings server diff --git a/include/clap.h b/include/clap.h index d9c36b3..a5b5b49 100644 --- a/include/clap.h +++ b/include/clap.h @@ -42,7 +42,7 @@ Audio resample_audio(const Audio &audio, int sample_rate = 48000); std::vector log_mel(const std::vector &pcm, int sample_rate = 48000, std::size_t crop_offset = 0); -// Native CLAP, CLaMP3, MERT-v1, MERT2, MuQ or MuQ-MuLan, selected from GGUF metadata. Weights are +// Native CLAP, CLaMP3, MERT-v1, MERT2, MuQ, MuQ-MuLan or EmbeddingGemma2, selected from GGUF metadata. Weights are // memory-mapped on POSIX platforms; MuQ F16 weights expand to owned F32 buffers. // Each call embeds one item; methods on the same instance are not thread-safe. // CLAP returns unit-normalized projections. CLaMP3 returns its native unnormalized @@ -57,9 +57,14 @@ std::vector log_mel(const std::vector &pcm, int sample_rate = 4800 // an unweighted average without final normalization. No amplitude normalization. // MuLan supports neither frame/features APIs nor crop offsets; published width // is 512. Its text string API rejects over-context input without truncation. +// EmbeddingGemma2 text/audio return normalized 768D joint embeddings. Audio +// requires the separate mmproj constructor, resamples to 16 kHz, and uses the +// first 30 seconds (published processor policy). No audio frames/crop offsets. class CLAP { public: explicit CLAP(const std::string &gguf_path, int threads = 0); + // Optional EmbeddingGemma2 audio tower/projector GGUF. Text works without it. + CLAP(const std::string &gguf_path, const std::string &mmproj_path, int threads = 0); ~CLAP(); CLAP(CLAP &&) noexcept; CLAP &operator=(CLAP &&) noexcept; diff --git a/node/README.md b/node/README.md index 0922d19..5a86cbe 100644 --- a/node/README.md +++ b/node/README.md @@ -24,3 +24,16 @@ try { model.close(); // optional early release; garbage collection also frees weights } ``` + +For EmbeddingGemma 2 audio, pass the matching mmproj in constructor options: + +```js +const model = new Model('embeddinggemma2.gguf', { threads: 4, mmproj: 'mmproj-Q8_0.gguf' }); +const { samples, sampleRate } = decodeAudio('recording.wav'); +const embedding = model.audio(samples, sampleRate); +model.close(); +``` + +Omitting `mmproj` preserves text/single-model loading; numeric `threads` is +still accepted. Missing/incompatible projectors throw native errors. Image +encoding is not supported. diff --git a/node/bindings.cpp b/node/bindings.cpp index f896dd5..7c58d74 100644 --- a/node/bindings.cpp +++ b/node/bindings.cpp @@ -99,8 +99,20 @@ class Model : public Napi::ObjectWrap { explicit Model(const Napi::CallbackInfo &info) : Napi::ObjectWrap(info) { guarded(info.Env(), [&]() -> Napi::Value { auto p = path(info[0]); - int threads = info[1].IsUndefined() ? 0 : static_cast(integer(info[1], "threads", 0, std::numeric_limits::max())); - model_ = std::make_unique(p, threads); + auto thread_value = info[1]; + Napi::Value mmproj_value = info.Env().Undefined(); + if (info[1].IsObject() && !info[1].IsArray() && !info[1].IsFunction()) { + auto options = info[1].As(); + thread_value = options.Get("threads"); + mmproj_value = options.Get("mmproj"); + } + int threads = thread_value.IsUndefined() ? 0 : static_cast(integer(thread_value, "threads", 0, std::numeric_limits::max())); + // Read all option getters before constructing the native model. + if (mmproj_value.IsUndefined()) model_ = std::make_unique(p, threads); + else { + auto mmproj = path(mmproj_value); + model_ = std::make_unique(p, mmproj, threads); + } return info.Env().Undefined(); }); } diff --git a/node/index.d.ts b/node/index.d.ts index 5195bba..72ddb0d 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -8,10 +8,16 @@ export interface Audio { samples: Float32Array; sampleRate: number; } +export interface ModelOptions { + /** Zero selects the native thread default. */ + threads?: number; + /** Separate audio mmproj GGUF for EmbeddingGemma2 audio encoding. */ + mmproj?: string; +} /** Synchronous CPU inference. Each call blocks the calling JavaScript thread. */ export class Model { /** threads=0 selects the native default; negative/fractional values are rejected. */ - constructor(path: string, threads?: number); + constructor(path: string, threadsOrOptions?: number | ModelOptions); readonly projectionDimension: number; readonly audioSampleRate: number; /** Native tokenizer; maxLength defaults to 512. Returns owned token IDs. */ diff --git a/node/test/addon.test.js b/node/test/addon.test.js index b248e3c..fb12df5 100644 --- a/node/test/addon.test.js +++ b/node/test/addon.test.js @@ -20,6 +20,23 @@ function near(a, b, tolerance = 2e-5) { a.forEach((x, i) => assert.ok(Math.abs(x - b[i]) < tolerance, `${i}: ${x} != ${b[i]}`)); } const pcm = Float32Array.from({ length: 24000 }, (_, i) => 0.2 * Math.sin(i * 0.031)); +test('constructor options preserve text loading and do not ignore mmproj', () => { + for (const options of [{}, { threads: 1 }, { threads: 1, mmproj: undefined }]) { + const m = new Model(modelPath, options); + try { vector(m.text('hello'), 3); } finally { m.close(); } + } + assert.throws(() => new Model(path.join(fixtures, 'embeddinggemma2-dispatch.gguf'), { + threads: 1, mmproj: path.join(fixtures, 'missing-audio-mmproj.gguf') + }), /missing-audio-mmproj/); + assert.throws(() => new Model(modelPath, { mmproj: 'missing-audio-mmproj.gguf' }), + error => error instanceof Error && !(error instanceof TypeError)); + for (const mmproj of [null, 1, '', 'a\0b']) { + assert.throws(() => new Model(modelPath, { mmproj }), TypeError); + } + for (const threads of [-1, 0.5, NaN, null, '1']) { + assert.throws(() => new Model(modelPath, { threads }), TypeError); + } +}); test('native exports and strict constructor validation', () => { assert.equal(typeof Model, 'function'); for (const p of [undefined, null, 1, '', 'a\0b']) assert.throws(() => new Model(p)); diff --git a/node/test/generate-fixtures.py b/node/test/generate-fixtures.py index 5d620fb..2879c23 100644 --- a/node/test/generate-fixtures.py +++ b/node/test/generate-fixtures.py @@ -16,6 +16,9 @@ out.mkdir(parents=True, exist_ok=True) fixtures.clap_fixture(out / 'clap.gguf') fixtures.mert_fixture(out / 'mert.gguf') +# Metadata-only probe; missing mmproj must fail before model tensor validation. +fixtures.write_gguf(out / 'embeddinggemma2-dispatch.gguf', + {'general.architecture': 'gemma-embedding2'}, {}) with wave.open(str(out / 'stereo.wav'), 'wb') as wav: wav.setparams((2, 2, 24000, 0, 'NONE', 'not compressed')) wav.writeframes(struct.pack(' #include #include +#include #include #include @@ -53,10 +54,18 @@ py::array_t matrix(const clap::AudioFrames &frames) { // BEFORE reacquiring the GIL. close(), properties and inference share this order. class CLAP { public: - CLAP(const std::filesystem::path &path, int threads) { + CLAP(const std::filesystem::path &path, int threads, + const std::optional &mmproj) { if (threads < 0) throw std::invalid_argument("threads must be nonnegative"); + const auto filename = path.u8string(); + const auto audio_path = mmproj ? std::optional(mmproj->u8string()) : std::nullopt; + if (filename.find('\0') != std::string::npos || + (audio_path && (audio_path->empty() || audio_path->find('\0') != std::string::npos))) { + throw std::invalid_argument("model paths must contain no NUL bytes and mmproj must be nonempty"); + } py::gil_scoped_release release; - model_ = std::make_unique(path.u8string(), threads); + model_ = audio_path ? std::make_unique(filename, *audio_path, threads) + : std::make_unique(filename, threads); } template @@ -83,7 +92,8 @@ class CLAP { PYBIND11_MODULE(_clap_cpp, module) { module.doc() = "Native CPU inference for clap.cpp"; py::class_(module, "CLAP") - .def(py::init(), py::arg("path"), py::arg("threads") = 0) + .def(py::init &>(), + py::arg("path"), py::arg("threads") = 0, py::kw_only(), py::arg("mmproj") = py::none()) .def_property_readonly("projection_dimension", [](CLAP &self) { return self.call([](clap::CLAP &model) { return model.projection_dimension(); }); }) diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index af7cd58..69f6309 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -7,7 +7,7 @@ import pytest import clap_cpp -from tiny_models import clap_fixture, mert_fixture, text_reference +from tiny_models import clap_fixture, mert_fixture, text_reference, write_gguf @pytest.fixture(scope="session") @@ -67,6 +67,26 @@ def test_model_loads_from_unicode_path(tmp_path, as_string): rtol=2e-4, atol=2e-5) +def test_optional_mmproj_preserves_existing_loading(model_paths): + with clap_cpp.CLAP(model_paths[0], 1, mmproj=None) as model: + assert_array(model.encode_text("hello"), (3,)) + + +@pytest.mark.parametrize("as_string", [False, True]) +def test_explicit_mmproj_is_not_ignored(model_paths, tmp_path, as_string): + missing = tmp_path / "missing-audio-mmproj.gguf" + # A metadata-only dispatch probe: failure must name the projector, not + # the incomplete model weights. This tests the exact forwarded path. + probe = tmp_path / "embeddinggemma2-dispatch.gguf" + write_gguf(probe, {"general.architecture": "gemma-embedding2"}, {}) + with pytest.raises(RuntimeError, match="missing-audio-mmproj"): + clap_cpp.CLAP(probe, threads=1, + mmproj=str(missing) if as_string else missing) + # Other architectures must not silently ignore a supplied projector either. + with pytest.raises(RuntimeError): + clap_cpp.CLAP(model_paths[0], mmproj=missing) + + def test_properties_are_readonly(model, mert): assert model.projection_dimension == 3 assert model.audio_sample_rate == 48000 diff --git a/rust/README.md b/rust/README.md index 4bea42e..b4c5a27 100644 --- a/rust/README.md +++ b/rust/README.md @@ -18,3 +18,10 @@ fn main() -> Result<(), Box> { Ok(()) } ``` + +For EmbeddingGemma 2 audio, load the main GGUF and matching mmproj with +`CLAP::new_with_mmproj("embeddinggemma2.gguf", "mmproj-Q8_0.gguf", 4)?`, then +call `model.encode_audio("recording.wav", 0)?` as above. Both paths accept +`impl AsRef`; missing/incompatible projectors return errors. The original +`new(path, threads)` remains unchanged, and text does not require an mmproj. +Image encoding is not supported. diff --git a/rust/native/bridge.cpp b/rust/native/bridge.cpp index e029992..9393f45 100644 --- a/rust/native/bridge.cpp +++ b/rust/native/bridge.cpp @@ -19,14 +19,16 @@ extern "C" { // Inline diagnostics keep exception translation allocation-free. // Ops: 0=open, 1=encode_text, 2=encode_audio (file). Result *cr_call(int op, clap::CLAP *model, const char *s, - int threads, size_t crop_offset) noexcept { + int threads, size_t crop_offset, const char *mmproj) noexcept { auto result = std::unique_ptr(new (std::nothrow) Result); if (!result) return nullptr; try { switch (op) { case 0: if (threads < 0) throw std::invalid_argument("threads must be nonnegative"); - result->model = std::make_unique(s, threads); break; + result->model = mmproj ? std::make_unique(s, mmproj, threads) + : std::make_unique(s, threads); + break; case 1: result->floats = model->encode_text(s); break; case 2: { const auto audio = clap::decode_audio(s); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 5e90f82..9537716 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -21,6 +21,7 @@ mod ffi { s: *const c_char, threads: i32, crop_offset: usize, + mmproj: *const c_char, ) -> *mut c_void; pub fn cr_free(p: *mut c_void); pub fn cr_model_free(p: *mut c_void); @@ -56,8 +57,24 @@ fn path(p: &Path) -> Result { Ok(CString::new(p.to_str().ok_or("path must be valid UTF-8")?)?) } // The synchronous bridge borrows the string and owns all returned data. -fn call(op: i32, model: *mut c_void, s: &CStr, threads: i32, crop_offset: usize) -> Result { - let p = unsafe { ffi::cr_call(op, model, s.as_ptr(), threads, crop_offset) }; +fn call( + op: i32, + model: *mut c_void, + s: &CStr, + threads: i32, + crop_offset: usize, + mmproj: Option<&CStr>, +) -> Result { + let p = unsafe { + ffi::cr_call( + op, + model, + s.as_ptr(), + threads, + crop_offset, + mmproj.map_or(std::ptr::null(), CStr::as_ptr), + ) + }; let result = Native(NonNull::new(p).ok_or("native result allocation failed")?); let error = unsafe { CStr::from_ptr(ffi::cr_error(result.ptr())) }; if !error.is_empty() { @@ -91,8 +108,31 @@ impl Drop for CLAP { impl CLAP { /// Load a UTF-8 GGUF path. Zero threads selects the native default. pub fn new(p: impl AsRef, threads: i32) -> Result { - let s = path(p.as_ref())?; - let native = call(0, std::ptr::null_mut(), &s, threads, 0)?; + Self::load(p.as_ref(), None, threads) + } + /// Load a model with a separate audio mmproj GGUF (EmbeddingGemma2). + /// Both paths must be UTF-8 without NUL; zero threads selects the native default. + pub fn new_with_mmproj( + p: impl AsRef, + mmproj: impl AsRef, + threads: i32, + ) -> Result { + Self::load(p.as_ref(), Some(mmproj.as_ref()), threads) + } + fn load(p: &Path, mmproj: Option<&Path>, threads: i32) -> Result { + let s = path(p)?; + let audio_path = mmproj.map(path).transpose()?; + if audio_path.as_ref().is_some_and(|p| p.as_bytes().is_empty()) { + return Err("mmproj path must be nonempty".into()); + } + let native = call( + 0, + std::ptr::null_mut(), + &s, + threads, + 0, + audio_path.as_deref(), + )?; let ptr = NonNull::new(unsafe { ffi::cr_model_take(native.ptr()) }) .ok_or("native model missing")?; Ok(Self { @@ -103,7 +143,7 @@ impl CLAP { /// Encode one UTF-8 string using the model's native context policy. pub fn encode_text(&mut self, text: &str) -> Result> { let s = CString::new(text)?; - Ok(call(1, self.ptr.as_ptr(), &s, 0, 0)?.floats()) + Ok(call(1, self.ptr.as_ptr(), &s, 0, 0, None)?.floats()) } /// Encode a WAV, MP3 or FLAC file. Channels are averaged to mono and the /// source sample rate is converted internally to the model's input rate. @@ -111,6 +151,6 @@ impl CLAP { /// do not support cropping reject nonzero offsets. pub fn encode_audio(&mut self, p: impl AsRef, crop_offset: usize) -> Result> { let s = path(p.as_ref())?; - Ok(call(2, self.ptr.as_ptr(), &s, 0, crop_offset)?.floats()) + Ok(call(2, self.ptr.as_ptr(), &s, 0, crop_offset, None)?.floats()) } } diff --git a/rust/tests/bindings.rs b/rust/tests/bindings.rs index 789186c..2c5e0e8 100644 --- a/rust/tests/bindings.rs +++ b/rust/tests/bindings.rs @@ -43,6 +43,24 @@ fn errors_are_owned_and_inputs_are_checked() { .is_err()); } +#[test] +fn explicit_mmproj_is_not_ignored_and_paths_are_checked() { + let model_path = fixtures().join("模型.gguf"); + let result = CLAP::new_with_mmproj( + fixtures().join("embeddinggemma2-dispatch.gguf"), + fixtures().join("missing-audio-mmproj.gguf"), + 1, + ); + let error = result.err().expect("missing projector must not be ignored"); + assert!(error.to_string().contains("missing-audio-mmproj")); + assert!(CLAP::new_with_mmproj(&model_path, "missing-audio-mmproj.gguf", 1).is_err()); + assert!(CLAP::new_with_mmproj(&model_path, "nul\0.gguf", 1).is_err()); + assert!(CLAP::new_with_mmproj(&model_path, "missing.gguf", -1).is_err()); + // The original constructor remains text-capable without a projector. + let mut model = CLAP::new(&model_path, 1).unwrap(); + assert_eq!(model.encode_text("Hi").unwrap().len(), 3); +} + #[test] fn clap_real_text_and_audio_file() { let mut model = CLAP::new(fixtures().join("模型.gguf"), 1).unwrap(); @@ -91,6 +109,7 @@ fn non_utf8_paths_are_rejected() { use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; let invalid = std::path::Path::new(OsStr::from_bytes(b"invalid\xff.wav")); assert!(CLAP::new(invalid, 1).is_err()); + assert!(CLAP::new_with_mmproj(fixtures().join("模型.gguf"), invalid, 1).is_err()); let mut model = CLAP::new(fixtures().join("模型.gguf"), 1).unwrap(); assert!(model.encode_audio(invalid, 0).is_err()); } diff --git a/rust/tests/generate.py b/rust/tests/generate.py index d834c5e..7ec7629 100644 --- a/rust/tests/generate.py +++ b/rust/tests/generate.py @@ -67,6 +67,9 @@ def add(x, y): (out / 'text-reference.txt').write_text(' '.join(map(str, text_reference([0, 77, 110, 2])))) tiny.clap_fixture(out / '模型.gguf') tiny.mert_fixture(out / 'mert.gguf') +# Metadata-only probe; missing mmproj must fail before model tensor validation. +tiny.write_gguf(out / 'embeddinggemma2-dispatch.gguf', + {'general.architecture': 'gemma-embedding2'}, {}) with wave.open(str(out / '音频.wav'), 'wb') as f: f.setparams((1, 2, 24000, 4, 'NONE', 'not compressed')) f.writeframes(struct.pack('<4h', 0, 8192, -16384, 24576)) diff --git a/src/clap.cpp b/src/clap.cpp index 544c66b..8e3769b 100644 --- a/src/clap.cpp +++ b/src/clap.cpp @@ -30,6 +30,8 @@ int main(int argc, char **argv) { auto *embedding = app.add_subcommand("embedding", "Generate native text or audio embeddings"); embedding->add_option("-m,--model", options.model_path, "Model GGUF") ->required()->check(CLI::ExistingFile); + embedding->add_option("--mmproj", options.mmproj_path, "EmbeddingGemma2 audio tower/projector GGUF") + ->check(CLI::ExistingFile); embedding->add_option("-t,--threads", options.threads, "ggml CPU threads (0: automatic)") ->check(CLI::NonNegativeNumber)->capture_default_str(); auto *input = embedding->add_option_group("Input", "Choose one text prompt, text file, or audio file"); diff --git a/src/embedding.cpp b/src/embedding.cpp index da8bed9..6db5e3c 100644 --- a/src/embedding.cpp +++ b/src/embedding.cpp @@ -69,7 +69,7 @@ int run_embedding(const EmbeddingOptions &options) { // Preserve native CLI error precedence: decode audio before loading weights. clap::Audio decoded; if (!options.audio_path.empty()) decoded = clap::decode_audio(options.audio_path); - clap::CLAP model(options.model_path, options.threads); + clap::CLAP model(options.model_path, options.mmproj_path, options.threads); std::vector vectors; if (!options.audio_path.empty()) { if (options.crop_requested && model.audio_sample_rate() != 48000) { diff --git a/src/embedding.h b/src/embedding.h index 08f4866..113fba6 100644 --- a/src/embedding.h +++ b/src/embedding.h @@ -5,6 +5,7 @@ struct EmbeddingOptions { std::string model_path; + std::string mmproj_path; int threads = 0; std::string text; std::string audio_path; diff --git a/src/model-loader.cpp b/src/model-loader.cpp index 26d37fa..c96b998 100644 --- a/src/model-loader.cpp +++ b/src/model-loader.cpp @@ -47,10 +47,11 @@ void ModelLoader::norm_shape(const std::string &name, int dim) const { shape(name + ".weight", {dim}); shape(name + ".bias", {dim}); } -void ModelLoader::validate_tensor_coverage() const { +void ModelLoader::validate_tensor_coverage(const std::string &prefix) const { if (!f32_activations && !bundled) return; for (int64_t i=0;itype) && !encoder_quantizable(w, arch)) throw std::runtime_error("CLAP: quantized non-dense/embedding tensor is unsupported: " + tensor_name); - const bool gemma_storage = arch == "gemma-embedding2" && + const bool gemma_storage = (arch == "gemma-embedding2" || arch == "clip") && (w->type == GGML_TYPE_Q5_K || w->type == GGML_TYPE_BF16); if (!(w->type == GGML_TYPE_F32 || w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_Q8_0 || w->type == GGML_TYPE_Q4_K || w->type == GGML_TYPE_Q6_K || gemma_storage)) diff --git a/src/model-loader.h b/src/model-loader.h index ad1d1b1..8371bf8 100644 --- a/src/model-loader.h +++ b/src/model-loader.h @@ -39,7 +39,7 @@ class ModelLoader { void linear_shape(const std::string &name, int in, int out, bool bias = true) const; void norm_shape(const std::string &name, int dim) const; bool f32_activation_compute() const { return source ? source->f32_activation_compute() : f32_activations; } - void validate_tensor_coverage() const; + void validate_tensor_coverage(const std::string &prefix = "") const; private: const ModelLoader *source = nullptr; std::string tensor_prefix, metadata_prefix; diff --git a/src/model.cpp b/src/model.cpp index 229e0bf..9835d9c 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -10,19 +10,26 @@ namespace clap { struct CLAP::Impl { // Declaration order ensures the architecture is destroyed before its mapping. detail::ModelLoader loader; + std::unique_ptr audio_loader; std::unique_ptr architecture; std::unique_ptr vocabulary; - Impl(const std::string &path, int threads) { + Impl(const std::string &path, const std::string &mmproj_path, int threads) { loader.load(path); const auto name = loader.metadata("general.architecture"); + if (!mmproj_path.empty()) { + if (name != "gemma-embedding2") + throw std::runtime_error("CLAP: mmproj is only supported for EmbeddingGemma2"); + audio_loader = std::make_unique(); + audio_loader->load(mmproj_path); + } if (name == "clap") architecture = std::make_unique(loader, threads); else if (name == "clamp3") architecture = std::make_unique(loader, threads); else if (name == "mert2") architecture = std::make_unique(loader, threads); else if (name == "mert") architecture = std::make_unique(loader, threads); else if (name == "muq_mulan") architecture = std::make_unique(loader, threads); else if (name == "muq") architecture = std::make_unique(loader, threads); - else if (name == "gemma-embedding2") architecture = std::make_unique(loader, threads); + else if (name == "gemma-embedding2") architecture = std::make_unique(loader, threads, audio_loader.get()); else throw std::runtime_error("CLAP: unsupported GGUF architecture: " + name); loader.validate_tensor_coverage(); if (architecture->has_text_encoder()) { @@ -47,7 +54,9 @@ struct CLAP::Impl { } }; -CLAP::CLAP(const std::string &path, int threads) : impl_(std::make_unique(path, threads)) {} +CLAP::CLAP(const std::string &path, int threads) : CLAP(path, std::string(), threads) {} +CLAP::CLAP(const std::string &path, const std::string &mmproj_path, int threads) + : impl_(std::make_unique(path, mmproj_path, threads)) {} CLAP::~CLAP() = default; CLAP::CLAP(CLAP &&) noexcept = default; CLAP &CLAP::operator=(CLAP &&) noexcept = default; diff --git a/src/models/embeddinggemma2-audio.cpp b/src/models/embeddinggemma2-audio.cpp new file mode 100644 index 0000000..5fc0ce0 --- /dev/null +++ b/src/models/embeddinggemma2-audio.cpp @@ -0,0 +1,297 @@ +// Gemma4-audio Conformer contract: Hugging Face modeling_gemma4.py and +// llama.cpp tools/mtmd/models/gemma4a.cpp. Audio files/rate conversion belong +// exclusively to src/audio.cpp; this module accepts mono 16 kHz PCM. +#include "embeddinggemma2-audio.h" +#include "clap.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clap::detail { +namespace { +constexpr double pi = 3.14159265358979323846; +constexpr int mel_bins = 128, max_samples = 480000, left_context = 12; +void require(bool valid, const std::string &message) { + if (!valid) throw std::runtime_error("CLAP: Gemma audio " + message); +} +void silu(Matrix &x) { for (auto &v : x.data) v = v / (1.0f + std::exp(-v)); } +void clip(Matrix &x, float lo, float hi) { for (auto &v : x.data) v = std::clamp(v, lo, hi); } +void finite(const Matrix &x) { + for (float v : x.data) require(std::isfinite(v), "non-finite activation"); +} +// F32 im2col operand and decoded small convolution kernel. Unlike ggml_conv_2d, +// this does not implicitly pack the activation into F16 before convolution. +Matrix convolve(const Matrix &patches, const ggml_tensor *weight, int outputs, int threads) { + Matrix result(patches.rows, outputs); + const size_t count = static_cast(patches.cols) * outputs; + const size_t bytes = (patches.data.size() + result.data.size() + count) * sizeof(float) + 2*1024*1024; + std::unique_ptr ctx(ggml_init({bytes,nullptr,false}),ggml_free); + require(bool(ctx), "convolution allocation failed"); + auto *input=ggml_new_tensor_2d(ctx.get(),GGML_TYPE_F32,patches.cols,patches.rows); + auto *kernel=ggml_new_tensor_2d(ctx.get(),GGML_TYPE_F32,patches.cols,outputs); + std::memcpy(input->data,patches.data.data(),patches.data.size()*sizeof(float)); + auto *decoded=static_cast(kernel->data); + for (size_t i=0;i workspace(plan.work_size); plan.work_data=workspace.data(); + require(ggml_graph_compute(graph,&plan)==GGML_STATUS_SUCCESS,"convolution compute failed"); + std::memcpy(result.data.data(),output->data,result.data.size()*sizeof(float)); + return result; +} +} // namespace + +GemmaAudioEncoder::GemmaAudioEncoder(const ModelLoader &source,int threads) + : loader(source), hidden(0), heads(0), layers(0), feedforward(0), channels{0,0}, output(0), + epsilon(0), ops(source,threads) { + require(loader.metadata("general.architecture")=="clip","requires clip mmproj architecture"); + require(loader.boolean("clip.has_audio_encoder"),"encoder missing"); + require(loader.metadata("clip.audio.projector_type")=="gemma4a","unsupported projector variant"); + auto dimension=[&](const char *key,int max) { + const auto n=loader.integer(key); + require(n>0 && n<=max,std::string("invalid dimension: ")+key); + return static_cast(n); + }; + hidden=dimension("clip.audio.embedding_length",4096); + heads=dimension("clip.audio.attention.head_count",128); + layers=dimension("clip.audio.block_count",128); + feedforward=dimension("clip.audio.feed_forward_length",16384); + require(hidden%heads==0 && hidden%2==0,"incompatible head dimensions"); + require(feedforward==4*hidden,"incompatible feed-forward dimension"); + require(loader.integer("clip.audio.num_mel_bins")==mel_bins,"requires 128 mel bins"); + const int projection=dimension("clip.audio.projection_dim",16384); + epsilon=static_cast(loader.number("clip.audio.attention.layer_norm_epsilon")); + require(std::isfinite(epsilon) && epsilon>0,"invalid norm epsilon"); + for(int i=0;i<2;++i) { + const auto name="a.conv1d."+std::to_string(i); + const auto *w=loader.weight(name+".weight"); + require(w->ne[3]>0 && w->ne[3]<=4096,"invalid convolution channels"); + channels[i]=static_cast(w->ne[3]); + loader.shape(name+".weight",{3,3,i?channels[0]:1,channels[i]}); + loader.shape(name+".norm.weight",{channels[i]}); + require(w->type==GGML_TYPE_F32 || w->type==GGML_TYPE_F16 || w->type==GGML_TYPE_BF16, + "subsampling kernels must be unquantized"); + } + const auto *out=loader.weight("a.pre_encode.out.weight"); + require(out->ne[1]>0 && out->ne[1]<=8192,"invalid output dimension"); + output=static_cast(out->ne[1]); + loader.linear_shape("a.input_projection",32*channels[1],hidden,false); + loader.linear_shape("a.pre_encode.out",hidden,output,true); + loader.linear_shape("mm.a.input_projection",output,projection,false); + if(loader.has_tensor_prefix("mm.a.soft_emb_norm.weight")) loader.shape("mm.a.soft_emb_norm.weight",{output}); + const auto norm=[&](const std::string &name,int n) { + loader.shape(name+".weight",{n}); + auto *w=loader.weight(name+".weight"); + require(!ggml_is_quantized(w->type),"norm vectors must be unquantized: "+name); + for(int i=0;itype),"depthwise kernel must be unquantized"); + loader.shape(p+"per_dim_scale.weight",{hidden/heads}); + for(int d=0;d=0,"invalid converted per-dimension scale"); + } + loader.linear_shape(p+"attn_k_rel",hidden,hidden,false); + for(const auto *name:{"ffn_up","ffn_down","ffn_up_1","ffn_down_1", "attn_q","attn_k", + "attn_v","attn_out","conv_pw1","conv_pw2"}) { + const std::string n=name; + const bool up=n=="ffn_up" || n=="ffn_up_1"; + const bool down=n=="ffn_down" || n=="ffn_down_1"; + loader.linear_shape(p+n,down?feedforward:hidden,up?feedforward:n=="conv_pw1"?2*hidden:hidden,false); + for(const auto *suffix:{"input_min","input_max","output_min","output_max"}) { + loader.shape(p+n+"."+suffix,{1}); + require(std::isfinite(scalar(loader.weight(p+n+"."+suffix),0)),"non-finite clamp: "+p+n); + } + for(const auto *kind:{"input","output"}) { + const auto base=p+n+"."+kind; + require(scalar(loader.weight(base+"_min"),0)<=scalar(loader.weight(base+"_max"),0),"inverted clamp: "+p+n); + } + } + } + // The published file also carries vision weights, which are out of scope. + // Reject unknown audio tensors without requiring coverage of that tower. + loader.validate_tensor_coverage("a."); + loader.validate_tensor_coverage("mm.a."); +} + +Matrix GemmaAudioEncoder::log_mel(const std::vector &pcm) { + require(pcm.size()>160,"input is too short (at least 161 samples required)"); + for(float v:pcm) require(std::isfinite(v),"non-finite PCM input"); + const int samples=static_cast(std::min(pcm.size(),static_cast(max_samples))); + const int frames=(samples-161)/160+1; + Matrix result(frames,mel_bins); + std::array edges; + for(int i=0;i<130;++i) edges[i]=700.0*(std::pow(10.0,std::log10(1.0+8000.0/700.0)*i/129.0)-1.0); + std::array window; + for(int n=0;n<320;++n) window[n]=static_cast(0.5-0.5*std::cos(2*pi*n/320)); + std::array,mel_bins> filters{}; + for(int m=0;m> spectrum(512); + for(int t=0;t{}); + for(int n=0;n<320;++n) { + const int i=t*160+n-160; + const float sample=i<0 || i>=samples?0.0f:pcm[i]*window[n]; + spectrum[n]=sample; + } + clap::fft(spectrum); + std::array magnitude; + for(int k=0;k<257;++k) magnitude[k]=std::abs(spectrum[k]); + for(int m=0;m(std::log(mel+0.001)); + } + } + return result; +} + +Matrix GemmaAudioEncoder::rms(const Matrix &x,const std::string &name) const { + Matrix y(x.rows,x.cols); + const auto *weight=name.empty()?nullptr:loader.weight(name+".weight"); + for(int r=0;r(1.0/std::sqrt(sum/x.cols+epsilon)); + for(int d=0;d=0 && it