Skip to content

refactor: separate model loading and execution contexts #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhangzhenghao wants to merge 1 commit into
base: main
Choose a base branch
from
Open

refactor: separate model loading and execution contexts #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions src/model-context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#pragma once

#include "model.h"
#include "model-ops.h"

namespace clap::detail {

// Shared execution state for architecture-specific encoders, corresponding to
// llama.cpp's common graph context. This runtime executes TensorOps eagerly:
// it is not a persistent GGML graph. Keep the context alive across calls so
// backend selection and uploaded-weight caches retain their existing lifetime.
// The model's validated parameters and loader must outlive this context.
class ModelContext {
public:
ModelContext(const ModelLoader &source, int threads) : loader(source), ops(source, threads) {}
virtual ~ModelContext() = default;
ModelContext(const ModelContext &) = delete;
ModelContext &operator=(const ModelContext &) = delete;

virtual std::vector<float> text(const std::vector<int> &ids) const = 0;
virtual AudioFrames audio_frames(const std::vector<float> &pcm) const = 0;
virtual AudioFrames clamp3_features(const std::vector<float> &) const {
throw std::runtime_error("CLaMP3 feature extraction requires a MERT-v1 model");
}
virtual std::vector<float> waveform_embedding(const std::vector<float> &, size_t = 0) const {
throw std::runtime_error("CLAP: this architecture has no waveform embeddings");
}

protected:
const ModelLoader &loader;
TensorOps ops;
};
} // namespace clap::detail
5 changes: 5 additions & 0 deletions src/model.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

#include "clap.h"
#include <cstddef>
#include <memory>
#include <vector>
#include <stdexcept>

namespace clap::detail {
class ModelContext;
struct TextParameters {
int bos;
int eos;
Expand Down Expand Up @@ -37,6 +39,9 @@ class ModelArchitecture {
throw std::runtime_error("CLAP: this architecture has no waveform embeddings");
}
protected:
// Architecture-local execution is constructed only after parameters and
// tensor contracts are validated, like llama.cpp build_arch_graph().
virtual std::unique_ptr<ModelContext> build_arch_context(int threads) const = 0;
virtual void load_arch_hparams() = 0;
virtual void load_arch_tensors() const = 0;
};
Expand Down
72 changes: 62 additions & 10 deletions src/models/clamp3.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "models.h"
#include "model-context.h"
#include <stdexcept>
#include <algorithm>
#include <cmath>
Expand All @@ -7,8 +8,44 @@
namespace clap::detail {
using json = nlohmann::json;

// Execution is kept separate from the validated model/tensor contract below.
struct Clamp3Model::context final : ModelContext {
context(const Clamp3Model &model, int threads)
: ModelContext(model.loader, threads),
model(model),
config(model.config),
audio_config(model.audio_config),
text_config(model.text_config),
frontend(model.frontend) {}

int projection_dimension() const { return model.projection_dimension(); }
std::vector<float> encode_features(const std::vector<float> &features, int frames) const;
Matrix encode(const Matrix &embeddings, const std::vector<int> &mask,
const nlohmann::json &branch, const std::string &prefix) const;
Matrix attention(const Matrix &x, const std::vector<int> &mask,
const std::string &prefix, int heads) const;
std::vector<float> chunked_text(const std::vector<int> &ids) const;
std::vector<float> chunked_audio(const Matrix &features) const;
std::vector<float> pool_project(const Matrix &hidden, const std::vector<int> &mask,
const std::string &projection) const;
std::vector<float> text(const std::vector<int> &ids) const override;
AudioFrames audio_frames(const std::vector<float> &pcm) const override;
std::vector<float> waveform_embedding(const std::vector<float> &pcm, size_t crop_offset = 0) const override;

private:
const Clamp3Model &model;
const nlohmann::json &config;
const nlohmann::json &audio_config;
const nlohmann::json &text_config;
const std::unique_ptr<MertModel> &frontend;
};

std::unique_ptr<ModelContext> Clamp3Model::build_arch_context(int threads) const {
return std::make_unique<context>(*this, threads);
}

Clamp3Model::Clamp3Model(const ModelLoader &source, int threads)
: loader(source), ops(source, threads) {
: loader(source) {
load_arch_hparams();
load_arch_tensors();
if (loader.has_metadata_prefix("clamp3.audio_frontend")) {
Expand All @@ -25,10 +62,11 @@ Clamp3Model::Clamp3Model(const ModelLoader &source, int threads)
} else {
throw std::runtime_error("CLAP: CLaMP3 requires bundled MERT-v1-95M audio frontend metadata; reconvert the checkpoint");
}
inference = build_arch_context(threads);
}
Clamp3Model::~Clamp3Model() = default;
bool Clamp3Model::has_waveform_embeddings() const { return frontend != nullptr; }
std::vector<float> Clamp3Model::waveform_embedding(const std::vector<float> &pcm, size_t crop_offset) const {
std::vector<float> Clamp3Model::context::waveform_embedding(const std::vector<float> &pcm, size_t crop_offset) const {
if (crop_offset != 0) throw std::runtime_error("CLAP: crop offsets are unsupported for this model");
if (!frontend) throw std::runtime_error("CLAP: CLaMP3 requires MERT-v1-95M features");
const auto features = frontend->clamp3_features(pcm);
Expand Down Expand Up @@ -117,11 +155,11 @@ int Clamp3Model::projection_dimension() const { return config.at("projection_dim
int Clamp3Model::mel_bins() const { return 0; }
int Clamp3Model::sample_rate() const { return frontend ? frontend->sample_rate() : 0; }
bool Clamp3Model::is_waveform_encoder() const { return false; }
AudioFrames Clamp3Model::audio_frames(const std::vector<float> &) const {
AudioFrames Clamp3Model::context::audio_frames(const std::vector<float> &) const {
throw std::runtime_error("CLAP: CLaMP3 requires precomputed MERT-v1-95M features");
}

Matrix Clamp3Model::attention(const Matrix &x, const std::vector<int> &mask,
Matrix Clamp3Model::context::attention(const Matrix &x, const std::vector<int> &mask,
const std::string &prefix, int heads) const {
const auto q = ops.linear(x, prefix + ".query");
const auto k = ops.linear(x, prefix + ".key");
Expand Down Expand Up @@ -151,7 +189,7 @@ Matrix Clamp3Model::attention(const Matrix &x, const std::vector<int> &mask,
return result;
}

Matrix Clamp3Model::encode(const Matrix &embeddings, const std::vector<int> &mask,
Matrix Clamp3Model::context::encode(const Matrix &embeddings, const std::vector<int> &mask,
const json &branch, const std::string &prefix) const {
Matrix x = ops.norm(embeddings, prefix + ".embeddings.LayerNorm", branch.at("layer_norm_eps"));
for (int layer = 0; layer < branch.at("num_hidden_layers").get<int>(); ++layer) {
Expand All @@ -169,7 +207,7 @@ Matrix Clamp3Model::encode(const Matrix &embeddings, const std::vector<int> &mas
return x;
}

std::vector<float> Clamp3Model::pool_project(const Matrix &hidden, const std::vector<int> &mask,
std::vector<float> Clamp3Model::context::pool_project(const Matrix &hidden, const std::vector<int> &mask,
const std::string &projection) const {
int count = 0;
Matrix pooled(1, hidden.cols);
Expand All @@ -183,7 +221,7 @@ std::vector<float> Clamp3Model::pool_project(const Matrix &hidden, const std::ve
return ops.linear(pooled, projection).data;
}

std::vector<float> Clamp3Model::chunked_text(const std::vector<int> &ids) const {
std::vector<float> Clamp3Model::context::chunked_text(const std::vector<int> &ids) const {
const int chunk = config.at("chunk_length"), dim = text_config.at("hidden_size");
const int pad = text_config.at("pad_token_id"), vocab = text_config.at("vocab_size");
const auto *words = loader.weight("text.embeddings.word_embeddings.weight");
Expand Down Expand Up @@ -223,7 +261,7 @@ std::vector<float> Clamp3Model::chunked_text(const std::vector<int> &ids) const
return result;
}

std::vector<float> Clamp3Model::chunked_audio(const Matrix &features) const {
std::vector<float> Clamp3Model::context::chunked_audio(const Matrix &features) const {
const int chunk = config.at("chunk_length"), dim = audio_config.at("hidden_size");
const auto *positions = loader.weight("audio.embeddings.position_embeddings.weight");
const auto *types = loader.weight("audio.embeddings.token_type_embeddings.weight");
Expand Down Expand Up @@ -257,13 +295,13 @@ std::vector<float> Clamp3Model::chunked_audio(const Matrix &features) const {
return result;
}

std::vector<float> Clamp3Model::text(const std::vector<int> &ids) const {
std::vector<float> Clamp3Model::context::text(const std::vector<int> &ids) const {
if (ids.empty())
throw std::runtime_error("CLAP: text token sequence is empty");
return chunked_text(ids);
}

std::vector<float> Clamp3Model::encode_features(const std::vector<float> &features, int frames) const {
std::vector<float> Clamp3Model::context::encode_features(const std::vector<float> &features, int frames) const {
const int dim = config.at("audio_feature_dim");
if (!(frames > 0 && frames <= std::numeric_limits<int>::max() - 2 &&
features.size() == static_cast<size_t>(frames) * dim))
Expand All @@ -274,4 +312,18 @@ std::vector<float> Clamp3Model::encode_features(const std::vector<float> &featur
std::copy(features.begin(), features.end(), bordered.data.begin() + dim);
return chunked_audio(bordered);
}

// Model-facing entry points preserve the existing internal/public facade.
std::vector<float> Clamp3Model::encode_features(const std::vector<float> &features, int frames) const {
return static_cast<const context &>(*inference).encode_features(features, frames);
}
std::vector<float> Clamp3Model::text(const std::vector<int> &ids) const {
return inference->text(ids);
}
AudioFrames Clamp3Model::audio_frames(const std::vector<float> &pcm) const {
return inference->audio_frames(pcm);
}
std::vector<float> Clamp3Model::waveform_embedding(const std::vector<float> &pcm, size_t crop_offset) const {
return inference->waveform_embedding(pcm, crop_offset);
}
} // namespace clap::detail
63 changes: 55 additions & 8 deletions src/models/clap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "models.h"
#include "model-context.h"
#include <stdexcept>
#include <ggml-cpu.h>
#include <algorithm>
Expand All @@ -18,10 +19,41 @@ float cubic(float x) {
}
} // namespace

ClapModel::ClapModel(const ModelLoader &source, int threads) : loader(source), ops(source, threads) {
// Execution is kept separate from the validated model/tensor contract below.
struct ClapModel::context final : ModelContext {
context(const ClapModel &model, int threads)
: ModelContext(model.loader, threads),
model(model),
config(model.config),
a(model.a),
t(model.t) {}

int mel_bins() const { return model.mel_bins(); }
std::vector<float> encode_features(const std::vector<float> &features, int frames) const;
Matrix attention(const Matrix &x, const std::string &prefix, int heads, int tokens,
const std::vector<int> *mask = nullptr, int window = 0, int shift = 0, int grid = 0) const;
std::vector<float> project(const Matrix &pooled, const std::string &prefix) const;
std::vector<float> text(const std::vector<int> &ids) const override;
AudioFrames audio_frames(const std::vector<float> &pcm) const override;
std::vector<float> waveform_embedding(const std::vector<float> &pcm, size_t crop_offset = 0) const override;

private:
const ClapModel &model;
const nlohmann::json &config;
const nlohmann::json &a;
const nlohmann::json &t;
};

std::unique_ptr<ModelContext> ClapModel::build_arch_context(int threads) const {
return std::make_unique<context>(*this, threads);
}

ClapModel::ClapModel(const ModelLoader &source, int threads) : loader(source) {
load_arch_hparams();
load_arch_tensors();
inference = build_arch_context(threads);
}
ClapModel::~ClapModel() = default;

void ClapModel::load_arch_hparams() {
config = json::parse(loader.metadata("clap.config"));
Expand Down Expand Up @@ -117,11 +149,11 @@ int ClapModel::projection_dimension() const { return config.at("projection_dim")
int ClapModel::mel_bins() const { return a.at("num_mel_bins"); }
int ClapModel::sample_rate() const { return 48000; }
bool ClapModel::is_waveform_encoder() const { return false; }
AudioFrames ClapModel::audio_frames(const std::vector<float> &) const {
AudioFrames ClapModel::context::audio_frames(const std::vector<float> &) const {
throw std::runtime_error("CLAP: CLAP exposes one pooled audio projection, not frame representations");
}

Matrix ClapModel::attention(const Matrix &x, const std::string &prefix, int heads, int tokens,
Matrix ClapModel::context::attention(const Matrix &x, const std::string &prefix, int heads, int tokens,
const std::vector<int> *mask, int window, int shift, int grid) const {
const auto q = ops.linear(x, prefix + ".query"), k = ops.linear(x, prefix + ".key"), v = ops.linear(x, prefix + ".value");
Matrix result(x.rows, x.cols);
Expand Down Expand Up @@ -163,7 +195,7 @@ Matrix ClapModel::attention(const Matrix &x, const std::string &prefix, int head
return result;
}

std::vector<float> ClapModel::project(const Matrix &pooled, const std::string &prefix) const {
std::vector<float> ClapModel::context::project(const Matrix &pooled, const std::string &prefix) const {
auto h = ops.linear(pooled, prefix + ".linear1");
for (float &v : h.data) v = std::max(v, 0.0f);
auto y = ops.linear(h, prefix + ".linear2");
Expand All @@ -176,7 +208,7 @@ std::vector<float> ClapModel::project(const Matrix &pooled, const std::string &p
return y.data;
}

std::vector<float> ClapModel::text(const std::vector<int> &ids) const {
std::vector<float> ClapModel::context::text(const std::vector<int> &ids) const {
const int dim = t.at("hidden_size"), pad = t.at("pad_token_id"), vocab = t.at("vocab_size");
if (!(!ids.empty() && ids.size() <= static_cast<size_t>(t.at("max_position_embeddings").get<int>() - 2)))
throw std::runtime_error("CLAP: text length outside model context");
Expand Down Expand Up @@ -217,12 +249,12 @@ std::vector<float> ClapModel::text(const std::vector<int> &ids) const {
return project(pooled, "text_proj");
}

std::vector<float> ClapModel::waveform_embedding(const std::vector<float> &pcm, size_t crop_offset) const {
std::vector<float> ClapModel::context::waveform_embedding(const std::vector<float> &pcm, size_t crop_offset) const {
if (mel_bins() != 64) throw std::runtime_error("CLAP: waveform frontend requires 64 mel bins");
return encode_features(clap::log_mel(pcm, 48000, crop_offset), 1001);
}

std::vector<float> ClapModel::encode_features(const std::vector<float> &features, int frames) const {
std::vector<float> ClapModel::context::encode_features(const std::vector<float> &features, int frames) const {
const int bins = a.at("num_mel_bins"), spec = a.at("spec_size"), ratio = spec / bins;
const int target_time = spec * ratio;
if (!(frames > 0 && frames <= target_time && features.size() == static_cast<size_t>(frames) * bins))
Expand Down Expand Up @@ -327,4 +359,19 @@ std::vector<float> ClapModel::encode_features(const std::vector<float> &features
Matrix pooled(1, x.cols);
for (int r = 0; r < x.rows; ++r) for (int c = 0; c < x.cols; ++c) pooled[0][c] += x[r][c] / x.rows;
return project(pooled, "audio_proj");
}} // namespace clap::detail
}

// Model-facing entry points preserve the existing internal/public facade.
std::vector<float> ClapModel::encode_features(const std::vector<float> &features, int frames) const {
return static_cast<const context &>(*inference).encode_features(features, frames);
}
std::vector<float> ClapModel::text(const std::vector<int> &ids) const {
return inference->text(ids);
}
AudioFrames ClapModel::audio_frames(const std::vector<float> &pcm) const {
return inference->audio_frames(pcm);
}
std::vector<float> ClapModel::waveform_embedding(const std::vector<float> &pcm, size_t crop_offset) const {
return inference->waveform_embedding(pcm, crop_offset);
}
} // namespace clap::detail
Loading
Loading