diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..e2a1fdf --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,39 @@ +name: Python + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + cpu: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install build and test dependencies + run: python -m pip install build pytest numpy + # build's default path builds the wheel FROM the sdist, exercising both. + - name: Build source distribution and CPU wheel + run: python -m build python --outdir dist + - name: Install wheel + run: python -m pip install --no-deps --no-index --find-links dist clap-cpp + - name: Test installed CPU bindings + run: python -m pytest python/tests -v + - uses: actions/upload-artifact@v4 + with: + name: python-${{ matrix.os }} + path: dist/* diff --git a/README.md b/README.md index c507a6f..374b355 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,14 @@ Audio decoding requires no FFmpeg executable or system codec libraries to build or run the library and executables. Only the test fixture generation step uses FFmpeg. +## Python bindings (CPU) + +Install from a recursive checkout with `python -m pip install ./python`. +The [`python/` package](python/README.md) provides native text/audio embeddings, +NumPy waveform input and output, audio decoding/resampling, and model lifetime +management. It builds a self-contained CPU extension; no PyTorch or GPU runtime +is needed for inference. See the package README for API examples and tests. + ## Build and test ```sh diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000..5b0ec50 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.20) +project(clap_cpp_python VERSION 0.1.0 LANGUAGES C CXX) + +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/native/include/clap.h") + set(CLAP_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/native") +else() + get_filename_component(CLAP_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE) +endif() +foreach(source ggml/CMakeLists.txt thirdparty/json/CMakeLists.txt thirdparty/miniaudio/miniaudio.h) + if(NOT EXISTS "${CLAP_ROOT}/${source}") + message(FATAL_ERROR "Native sources missing: run git submodule update --init --recursive") + endif() +endforeach() + +# Build a self-contained CPU extension, never CLI executables, GPU backends, +# external OpenMP runtimes, or shared libraries requiring a loader search path. +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(GGML_CPU ON CACHE BOOL "" FORCE) +foreach(backend METAL CUDA HIP VULKAN SYCL OPENCL OPENVINO RPC MUSA WEBGPU ZDNN + VIRTGPU VIRTGPU_BACKEND ET HEXAGON ZENDNN BLAS ACCELERATE) + set(GGML_${backend} OFF CACHE BOOL "" FORCE) +endforeach() +foreach(feature BACKEND_DL OPENMP BUILD_TESTS BUILD_EXAMPLES) + set(GGML_${feature} OFF CACHE BOOL "" FORCE) +endforeach() +# Avoid host-specific x86 ISA requirements in redistributed wheels. +foreach(isa SSE42 AVX AVX2 AVX_VNNI BMI2 FMA F16C AVX512 AVX512_VBMI + AVX512_VNNI AVX512_BF16 AMX_TILE AMX_INT8 AMX_BF16) + set(GGML_${isa} OFF CACHE BOOL "" FORCE) +endforeach() +set(CLAP_USE_SYSTEM_ICU OFF CACHE BOOL "" FORCE) +add_subdirectory("${CLAP_ROOT}/ggml" ggml EXCLUDE_FROM_ALL) +add_subdirectory("${CLAP_ROOT}/thirdparty/json" json EXCLUDE_FROM_ALL) +include("${CLAP_ROOT}/cmake/icu.cmake") +add_subdirectory("${CLAP_ROOT}/src" core EXCLUDE_FROM_ALL) + +find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) +find_package(pybind11 CONFIG REQUIRED) +pybind11_add_module(_clap_cpp MODULE bindings.cpp) +target_link_libraries(_clap_cpp PRIVATE clap) +target_compile_features(_clap_cpp PRIVATE cxx_std_17) +set_target_properties(_clap_cpp PROPERTIES CXX_EXTENSIONS OFF) +install(TARGETS _clap_cpp LIBRARY DESTINATION clap_cpp COMPONENT python + RUNTIME DESTINATION clap_cpp COMPONENT python) +install(FILES "${CLAP_ROOT}/ggml/LICENSE" DESTINATION clap_cpp/licenses RENAME ggml.txt COMPONENT python) +install(FILES "${CLAP_ROOT}/thirdparty/miniaudio/LICENSE" DESTINATION clap_cpp/licenses RENAME miniaudio.txt COMPONENT python) +install(FILES "${CLAP_ROOT}/thirdparty/icu/LICENSE" DESTINATION clap_cpp/licenses RENAME icu.txt COMPONENT python) +install(FILES "${CLAP_ROOT}/thirdparty/json/LICENSE.MIT" DESTINATION clap_cpp/licenses RENAME json.txt COMPONENT python) diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..abdb957 --- /dev/null +++ b/python/README.md @@ -0,0 +1,85 @@ +# clap.cpp Python bindings (CPU) + +Build and install from a recursive checkout: + +```sh +git clone --recursive https://github.com/gorse-io/clap.cpp.git +cd clap.cpp +python -m pip install ./python +``` + +Requires Python 3.9+, a C++17 compiler and CMake 3.20+. The build backend installs +CMake/Ninja if necessary. NumPy is the only runtime Python dependency; PyTorch, +FFmpeg, a separate `clap` executable and GPU drivers are not required. +The extension statically links the CPU runtime and bundled ICU. GPU backends and +OpenMP are disabled. x86 wheels use baseline instructions rather than the build +machine's instruction set. No pretrained weights are downloaded automatically. + +## Usage + +Convert a supported checkpoint to GGUF with the repository's conversion tools, +then load it once and reuse it: + +```python +from clap_cpp import Model, decode_audio + +with Model("model.gguf", threads=4) as model: + text_embedding = model.text("A dog barking") + pcm, sample_rate = decode_audio("dog.wav") + audio_embedding = model.audio(pcm, sample_rate=sample_rate) + print(model.projection_dimension, model.audio_sample_rate) +``` + +`Model` uses the same GGUF architecture dispatch as the native library (CLAP, +CLaMP3, MERT-v1, MERT-v2, MuQ and MuQ-MuLan). An operation unsupported by a +particular architecture raises a native error; for example audio-only encoders +cannot embed text. CLaMP3 requires a GGUF with its bundled MERT frontend. + +## API + +- `Model(path, threads=0)`: `str`/`pathlib.Path`; zero selects the native thread + default. `close()` is idempotent and the context manager closes on exit. +- `model.text(text_or_tokens)`: one UTF-8 string or token-ID sequence to a + one-dimensional float32 NumPy embedding. +- `model.tokenize(text, max_length=512)`: native tokenizer returning a list of + integer IDs. Its limit must be compatible with the loaded model; strings may + be truncated by the tokenizer. Prefer `text(str)` for normal use. +- `model.audio(pcm, sample_rate=48000, crop_offset=0)`: mono waveform to a float32 + embedding. Automatic model-rate resampling; crop offset uses the native + model-rate sample units and is not supported by every architecture. +- `model.audio_frames(pcm, sample_rate=48000)`: native frame representations as a + `[frames, dimension]` float32 array, for architectures exposing frames. +- `model.clamp3_features(pcm, sample_rate=24000)`: MERT-v1 CLaMP3 preprocessing + features as a `[windows, dimension]` float32 array (at least one second). +- `model.projection_dimension`, `model.audio_sample_rate`: read-only integers. +- `decode_audio(path)`: `(pcm, sample_rate)`; decodes WAV, MP3 or FLAC, averages + channels to mono, preserves sample rate. +- `resample_audio(pcm, source_rate, target_rate=48000)`: mono float32 array using + the native resampler. + +PCM accepts one-dimensional numeric arrays or sequences. Noncontiguous arrays +and other numeric dtypes are converted to contiguous float32; the binding copies +inputs before releasing the GIL, so inference never retains a Python buffer. +Empty, nonfinite or multidimensional PCM and nonpositive sample rates are +rejected. Returned arrays own their storage and survive model closure. +Native invalid arguments map to `ValueError`, other native runtime errors to +`RuntimeError`. Calls after closing raise `RuntimeError`. + +The binding releases the GIL for inference/loading/decoding/resampling and +serializes all operations on each model (including closing). Different model +instances can execute concurrently. Embedding normalization is architecture +specific and identical to the C++ API; the binding does not normalize outputs. + +## Development and source distributions + +```sh +python -m pip install './python[test]' build +python -m pytest python/tests +python -m build python +``` + +The source distribution includes the pinned native sources and third-party +licenses; a wheel can be rebuilt from that archive without a Git checkout or +CMake-time downloads. Build dependencies must still be available from the Python +package index or a local wheelhouse. Wheel publishing and GPU support are out of +scope for this initial binding. diff --git a/python/bindings.cpp b/python/bindings.cpp new file mode 100644 index 0000000..148b595 --- /dev/null +++ b/python/bindings.cpp @@ -0,0 +1,145 @@ +#include "clap.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using PCM = py::array_t; + +namespace { + +std::vector samples(const PCM &input) { + if (input.ndim() != 1 || input.size() == 0) { + throw std::invalid_argument("PCM must be a nonempty one-dimensional mono array"); + } + std::vector result(input.data(), input.data() + input.size()); + for (float value : result) { + if (!std::isfinite(value)) { + throw std::invalid_argument("PCM samples must be finite"); + } + } + return result; +} + +void validate_rate(int rate) { + if (rate <= 0) throw std::invalid_argument("sample rate must be positive"); +} + +py::array_t array(const std::vector &values) { + py::array_t output(values.size()); + if (!values.empty()) std::memcpy(output.mutable_data(), values.data(), values.size() * sizeof(float)); + return output; +} + +py::array_t matrix(const clap::AudioFrames &frames) { + py::array_t output({frames.frames, frames.dimension}); + if (!frames.values.empty()) { + std::memcpy(output.mutable_data(), frames.values.data(), frames.values.size() * sizeof(float)); + } + return output; +} + +// Always release the GIL BEFORE taking the model mutex; release the mutex +// BEFORE reacquiring the GIL. close(), properties and inference share this order. +class Model { +public: + Model(const std::filesystem::path &path, int threads) { + if (threads < 0) throw std::invalid_argument("threads must be nonnegative"); + py::gil_scoped_release release; + model_ = std::make_unique(path.u8string(), threads); + } + + template + auto call(F &&function) { + py::gil_scoped_release release; + std::lock_guard lock(mutex_); + if (!model_) throw std::runtime_error("Model is closed"); + return function(*model_); + } + + void close() { + py::gil_scoped_release release; + std::lock_guard lock(mutex_); + model_.reset(); + } + +private: + std::unique_ptr model_; + std::mutex mutex_; +}; + +} // namespace + +PYBIND11_MODULE(_clap_cpp, module) { + module.doc() = "Native CPU inference for clap.cpp"; + py::class_(module, "Model") + .def(py::init(), py::arg("path"), py::arg("threads") = 0) + .def_property_readonly("projection_dimension", [](Model &self) { + return self.call([](clap::Model &model) { return model.projection_dimension(); }); + }) + .def_property_readonly("audio_sample_rate", [](Model &self) { + return self.call([](clap::Model &model) { return model.audio_sample_rate(); }); + }) + .def("tokenize", [](Model &self, const std::string &text, py::ssize_t max_length) { + if (max_length <= 0) throw std::invalid_argument("max_length must be positive"); + return self.call([&](clap::Model &model) { return model.tokenize(text, max_length); }); + }, py::arg("text"), py::arg("max_length") = 512) + .def("text", [](Model &self, const std::string &text) { + return array(self.call([&](clap::Model &model) { return model.text(text); })); + }, py::arg("text")) + .def("text", [](Model &self, const std::vector &tokens) { + return array(self.call([&](clap::Model &model) { return model.text(tokens); })); + }, py::arg("tokens")) + .def("audio", [](Model &self, const PCM &pcm, int sample_rate, py::ssize_t crop_offset) { + validate_rate(sample_rate); + if (crop_offset < 0) throw std::invalid_argument("crop_offset must be nonnegative"); + auto input = samples(pcm); + return array(self.call([&](clap::Model &model) { return model.audio(input, sample_rate, crop_offset); })); + }, py::arg("pcm"), py::arg("sample_rate") = 48000, py::arg("crop_offset") = 0) + .def("audio_frames", [](Model &self, const PCM &pcm, int sample_rate) { + validate_rate(sample_rate); + auto input = samples(pcm); + return matrix(self.call([&](clap::Model &model) { return model.audio_frames(input, sample_rate); })); + }, py::arg("pcm"), py::arg("sample_rate") = 48000) + .def("clamp3_features", [](Model &self, const PCM &pcm, int sample_rate) { + validate_rate(sample_rate); + auto input = samples(pcm); + return matrix(self.call([&](clap::Model &model) { return model.clamp3_features(input, sample_rate); })); + }, py::arg("pcm"), py::arg("sample_rate") = 24000) + .def("close", &Model::close) + .def("__enter__", [](Model &self) -> Model & { + self.call([](clap::Model &) {}); + return self; + }, py::return_value_policy::reference_internal) + .def("__exit__", [](Model &self, py::object, py::object, py::object) { self.close(); }); + + module.def("decode_audio", [](const std::filesystem::path &path) { + clap::Audio decoded; + { + py::gil_scoped_release release; + decoded = clap::decode_audio(path.u8string()); + } + return py::make_tuple(array(decoded.samples), decoded.sample_rate); + }, py::arg("path")); + module.def("resample_audio", [](const PCM &pcm, int source_rate, int target_rate) { + validate_rate(source_rate); + validate_rate(target_rate); + clap::Audio input{samples(pcm), source_rate}; + clap::Audio output; + { + py::gil_scoped_release release; + output = clap::resample_audio(input, target_rate); + } + return array(output.samples); + }, py::arg("pcm"), py::arg("source_rate"), py::arg("target_rate") = 48000); +} diff --git a/python/clap_cpp/__init__.py b/python/clap_cpp/__init__.py new file mode 100644 index 0000000..c856d55 --- /dev/null +++ b/python/clap_cpp/__init__.py @@ -0,0 +1,5 @@ +"""CPU audio/text embeddings backed by the native clap.cpp runtime.""" + +from ._clap_cpp import Model, decode_audio, resample_audio + +__all__ = ["Model", "decode_audio", "resample_audio"] diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..ec818b7 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["scikit-build-core>=1.0", "pybind11>=2.13"] +build-backend = "scikit_build_core.build" + +[project] +name = "clap-cpp" +version = "0.1.0" +description = "CPU Python bindings for clap.cpp audio and text embeddings" +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +dependencies = ["numpy>=1.23"] + +[project.optional-dependencies] +test = ["pytest>=7"] + +[tool.scikit-build] +minimum-version = "build-system.requires" +wheel.packages = ["clap_cpp"] +install.components = ["python"] +build.targets = ["_clap_cpp"] +sdist.exclude = ["**/.git", "**/__pycache__/**", "dist/**", "build/**"] + +# The sdist carries the pinned native sources, so it builds without a Git +# checkout, submodule initialization or network fetches during CMake configure. +[tool.scikit-build.sdist.force-include] +"../LICENSE" = "LICENSE" +"../include" = "native/include" +"../src" = "native/src" +"../cmake" = "native/cmake" +"../ggml" = "native/ggml" +"../thirdparty/json" = "native/thirdparty/json" +"../thirdparty/miniaudio" = "native/thirdparty/miniaudio" +"../thirdparty/icu/LICENSE" = "native/thirdparty/icu/LICENSE" +"../thirdparty/icu/icu4c/source/common" = "native/thirdparty/icu/icu4c/source/common" +"../thirdparty/icu/icu4c/source/i18n" = "native/thirdparty/icu/icu4c/source/i18n" +"../thirdparty/icu/icu4c/source/stubdata" = "native/thirdparty/icu/icu4c/source/stubdata" + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py new file mode 100644 index 0000000..fac456d --- /dev/null +++ b/python/tests/test_bindings.py @@ -0,0 +1,288 @@ +"""CPU integration tests: no downloads, mocks, or optional model dependencies.""" +from concurrent.futures import ThreadPoolExecutor +import gc +import math +import wave + +import numpy as np +import pytest + +import clap_cpp +from tiny_models import clap_fixture, mert_fixture, text_reference + + +@pytest.fixture(scope="session") +def model_paths(tmp_path_factory): + root = tmp_path_factory.mktemp("tiny-models") + clap_path, mert_path = root / "clap.gguf", root / "mert.gguf" + clap_fixture(clap_path) + mert_fixture(mert_path) + return clap_path, mert_path + + +@pytest.fixture +def model(model_paths): + with clap_cpp.Model(model_paths[0], threads=2) as result: + yield result + + +@pytest.fixture +def mert(model_paths): + with clap_cpp.Model(str(model_paths[1]), threads=2) as result: + yield result + + +@pytest.fixture +def pcm(): + # A full second also meets the MERT CLaMP3-feature minimum duration. + t = np.arange(24000, dtype=np.float64) / 24000 + return (0.2 * np.sin(2 * np.pi * 440 * t)).astype(np.float32) + + +def assert_array(value, shape): + assert isinstance(value, np.ndarray) + assert value.dtype == np.float32 + assert value.shape == shape + assert np.isfinite(value).all() + + +def test_module_exports(): + assert callable(clap_cpp.Model) + assert callable(clap_cpp.decode_audio) + assert callable(clap_cpp.resample_audio) + + +@pytest.mark.parametrize("as_string", [False, True]) +def test_model_loads_from_unicode_path(tmp_path, as_string): + directory = tmp_path / "模型 café 🎵" + directory.mkdir() + path = directory / "音声 modèle 🎶.gguf" + clap_fixture(path) + with clap_cpp.Model(str(path) if as_string else path, threads=2) as model: + assert model.projection_dimension == 3 + tokens = model.tokenize("hello") + np.testing.assert_allclose(model.text("hello"), text_reference(tokens), + rtol=2e-4, atol=2e-5) + + +def test_properties_are_readonly(model, mert): + assert model.projection_dimension == 3 + assert model.audio_sample_rate == 48000 + assert mert.projection_dimension == 4 + assert mert.audio_sample_rate == 24000 + for name in ("projection_dimension", "audio_sample_rate"): + with pytest.raises(AttributeError): + setattr(model, name, 99) + + +@pytest.mark.parametrize("text", ["hello", "world", "你好 café"]) +def test_text_matches_independent_numpy_oracle(model, text): + tokens = model.tokenize(text) + assert isinstance(tokens, list) + assert tokens and all(isinstance(token, int) for token in tokens) + assert tokens[0] == 0 and tokens[-1] == 2 + actual = model.text(text) + assert_array(actual, (3,)) + np.testing.assert_allclose(actual, text_reference(tokens), rtol=2e-4, atol=2e-5) + np.testing.assert_allclose(model.text(tokens), actual, rtol=1e-5, atol=1e-6) + np.testing.assert_array_equal(model.text(text), actual) + assert np.linalg.norm(actual) == pytest.approx(1, abs=1e-5) + + +def test_text_depends_on_input(model): + assert np.max(np.abs(model.text("hello") - model.text("world"))) > 1e-7 + + +def test_tokenizer_limit(model): + assert len(model.tokenize("hello world" * 10, max_length=8)) <= 8 + + +def test_waveform_inference_and_explicit_resampling(model, pcm): + # Exercise the actual 64-mel waveform frontend, not feature-only tensors. + actual = model.audio(pcm, sample_rate=24000) + assert_array(actual, (3,)) + assert np.linalg.norm(actual) == pytest.approx(1, abs=1e-5) + np.testing.assert_array_equal(model.audio(pcm, sample_rate=24000), actual) + np.testing.assert_allclose(model.audio(pcm.tolist(), sample_rate=24000), actual, + rtol=1e-5, atol=1e-6) + upsampled = clap_cpp.resample_audio(pcm, 24000, 48000) + np.testing.assert_allclose(model.audio(upsampled), actual, rtol=1e-5, atol=1e-6) + # Non-contiguous float64 input must be converted safely, not read as F32. + padded = np.empty((pcm.size, 2), dtype=np.float64) + padded[:, 0] = pcm + padded[:, 1] = 999 + np.testing.assert_allclose(model.audio(padded[:, 0], sample_rate=24000), actual, + rtol=1e-5, atol=1e-6) + + +def test_mert_frames_and_mean_pooling(mert, pcm): + frames = mert.audio_frames(pcm, sample_rate=24000) + assert_array(frames, (75, 4)) + np.testing.assert_allclose(mert.audio(pcm, sample_rate=24000), frames.mean(axis=0), + rtol=1e-5, atol=1e-6) + np.testing.assert_array_equal(mert.audio_frames(pcm, sample_rate=24000), frames) + + +def test_mert_clamp3_features(mert, pcm): + features = mert.clamp3_features(pcm) + assert_array(features, (1, 4)) + np.testing.assert_array_equal(mert.clamp3_features(pcm), features) + # Two retained 5-second windows; discard the final half-second tail. + extended = np.tile(pcm, 11)[:252000] + assert_array(mert.clamp3_features(extended), (2, 4)) + + +@pytest.mark.parametrize("name", ["stereo input.wav", "音声 café 🎶/立体声 modèle 🎵.wav"]) +def test_decoded_stereo_wav_preserves_rate_and_averages_channels(tmp_path, name): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + samples = np.array([[32767, -32768], [16384, 0], [-16384, 8192], [0, 0]], + dtype="': 0, '': 1, '': 2, '': 3, '': 4} + vocab.update({ch: i + 5 for i, ch in enumerate(mapping)}) + tok = dict(model=dict(type='BPE', vocab=vocab, merges=[])) + config = dict(model_type='clap', projection_dim=3, audio_config=a, text_config=t) + write_gguf(path, {'general.architecture': 'clap', 'clap.config': json.dumps(config), + 'clap.tokenizer': json.dumps(tok)}, shapes) + + +def mert_fixture(path): + # One aggressive stride keeps a >=1 second waveform tiny after the frontend. + c = dict(model_type='mert_model', sample_rate=24000, conv_bias=False, + feat_extract_norm='group', hidden_act='gelu', feat_extract_activation='gelu', + hidden_size=4, intermediate_size=8, num_hidden_layers=1, + num_attention_heads=1, num_conv_pos_embeddings=4, + num_conv_pos_embedding_groups=1, conv_dim=[2], conv_kernel=[4], + conv_stride=[320], layer_norm_eps=1e-5) + s = {'frontend.0.conv.weight': (4, 1, 2), 'position.weight': (4, 4, 4), + 'position.bias': (4,)} + def norm(p, d): + s[p + '.weight'] = (d,) + s[p + '.bias'] = (d,) + def linear(p, i, o): + s[p + '.weight'] = (i, o) + s[p + '.bias'] = (o,) + norm('frontend.0.layer_norm', 2) + norm('projection.layer_norm', 2) + linear('projection.projection', 2, 4) + norm('encoder.layer_norm', 4) + for part in ('q_proj', 'k_proj', 'v_proj', 'out_proj'): + linear('layers.0.attention.' + part, 4, 4) + norm('layers.0.layer_norm', 4) + norm('layers.0.final_layer_norm', 4) + linear('layers.0.ffn.in', 4, 8) + linear('layers.0.ffn.out', 8, 4) + write_gguf(path, {'general.architecture': 'mert', 'mert.config': json.dumps(c)}, s) + + +def text_reference(ids): + """Independent NumPy CLAP text forward pass using storage-rounded F32 weights. + + The oracle does not invoke clap_cpp, a CLI, Torch, or ggml. Calculations use + float64 after the exact same F32 storage rounding as the generated GGUF. + """ + import numpy as np + + def weight(name, shape): + count = math.prod(shape) + norm = 'norm' in name.lower() or 'LayerNorm' in name + values = [(1.0 if name.endswith('weight') else 0.0) if norm else + 0.12 * math.sin(i * 0.73 + sum(name.encode()) * 0.17) + for i in range(count)] + return np.array(values, dtype=np.float32).astype(np.float64).reshape(shape) + + def linear(x, prefix, output): + return x @ weight(prefix + '.weight', (output, x.shape[-1])).T + weight( + prefix + '.bias', (output,)) + + def norm(x, prefix): + mean = x.mean(axis=-1, keepdims=True) + var = ((x - mean) ** 2).mean(axis=-1, keepdims=True) + return ((x - mean) / np.sqrt(var + 1e-5) * weight(prefix + '.weight', (4,)) + + weight(prefix + '.bias', (4,))) + + ids = np.asarray(ids) + mask = ids != 1 + positions = np.where(mask, np.cumsum(mask) + 1, 1) + x = (weight('text.embeddings.word_embeddings.weight', (261, 4))[ids] + + weight('text.embeddings.position_embeddings.weight', (514, 4))[positions] + + weight('text.embeddings.token_type_embeddings.weight', (1, 4))[0]) + x = norm(x, 'text.embeddings.LayerNorm') + prefix = 'text.encoder.layer.0' + q, k, v = [linear(x, prefix + '.attention.self.' + part, 4) + for part in ('query', 'key', 'value')] + scores = q @ k.T / 2 + scores[:, ~mask] = -1e30 + probabilities = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probabilities /= probabilities.sum(axis=-1, keepdims=True) + y = linear(probabilities @ v, prefix + '.attention.output.dense', 4) + x = norm(x + y, prefix + '.attention.output.LayerNorm') + y = linear(x, prefix + '.intermediate.dense', 8) + y = 0.5 * y * (1 + np.vectorize(math.erf)(y / math.sqrt(2))) + y = linear(y, prefix + '.output.dense', 4) + x = norm(x + y, prefix + '.output.LayerNorm') + pooled = np.tanh(linear(x[0], 'text.pooler.dense', 4)) + hidden = np.maximum(linear(pooled, 'text_proj.linear1', 3), 0) + result = linear(hidden, 'text_proj.linear2', 3) + return (result / np.linalg.norm(result)).astype(np.float32) diff --git a/src/audio.cpp b/src/audio.cpp index 4539540..3f0cc75 100644 --- a/src/audio.cpp +++ b/src/audio.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -155,7 +156,14 @@ Audio decode_audio(const std::string &path) { // finite-sample check so a bad channel cannot be hidden by downmixing. auto config = ma_decoder_config_init(ma_format_f32, 0, 0); ma_decoder decoder{}; - if (ma_decoder_init_file(path.c_str(), &config, &decoder) == MA_SUCCESS) { +#ifdef _WIN32 + // Native paths are UTF-8; miniaudio's narrow file API uses the ANSI code page. + const auto native_path = std::filesystem::u8path(path); + const auto init_result = ma_decoder_init_file_w(native_path.c_str(), &config, &decoder); +#else + const auto init_result = ma_decoder_init_file(path.c_str(), &config, &decoder); +#endif + if (init_result == MA_SUCCESS) { const auto close = [](ma_decoder *value) { ma_decoder_uninit(value); }; const std::unique_ptr guard(&decoder, close); ma_uint32 channels = 0, sample_rate = 0; diff --git a/src/model-loader.cpp b/src/model-loader.cpp index 43ca9c6..750f25d 100644 --- a/src/model-loader.cpp +++ b/src/model-loader.cpp @@ -3,6 +3,7 @@ #include "model-ops.h" #include #include +#include #include #include #ifndef _WIN32 @@ -91,7 +92,7 @@ void ModelLoader::load(const std::string &path, bool inference) { if (mapping == MAP_FAILED) throw std::runtime_error("CLAP: cannot mmap weights"); #else - std::ifstream stream(path, std::ios::binary | std::ios::ate); + std::ifstream stream(std::filesystem::u8path(path), std::ios::binary | std::ios::ate); if (!(static_cast(stream))) throw std::runtime_error("CLAP: cannot open weights"); mapping_size = static_cast(stream.tellg()); storage.resize(mapping_size);