From c8c9f980dcf98fe24f509ce231629cbf86bdeed1 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 21 Oct 2026 10:37:35 +0800 Subject: [PATCH 1/3] feat: add native Hugging Face GGUF downloads --- CMakeLists.txt | 15 +- README.md | 77 ++++++++ src/clap.cpp | 37 +++- src/hf.cpp | 379 ++++++++++++++++++++++++++++++++++++++ src/hf.h | 21 +++ tests/hf_download.py | 372 +++++++++++++++++++++++++++++++++++++ tests/hf_model_fixture.py | 96 ++++++++++ 7 files changed, 993 insertions(+), 4 deletions(-) create mode 100644 src/hf.cpp create mode 100644 src/hf.h create mode 100644 tests/hf_download.py create mode 100644 tests/hf_model_fixture.py diff --git a/CMakeLists.txt b/CMakeLists.txt index a45d331..039aaf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,17 +33,25 @@ endif() add_subdirectory(ggml EXCLUDE_FROM_ALL) add_subdirectory(thirdparty/CLI11 EXCLUDE_FROM_ALL) +set(HTTPLIB_REQUIRE_OPENSSL ON CACHE BOOL "HF downloads require HTTPS" FORCE) add_subdirectory(thirdparty/cpp-httplib EXCLUDE_FROM_ALL) add_subdirectory(thirdparty/json EXCLUDE_FROM_ALL) include(cmake/icu.cmake) add_subdirectory(src) +add_library(clap-hf STATIC src/hf.cpp) +set_target_properties(clap-hf PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_EXTENSIONS OFF) +target_compile_features(clap-hf PUBLIC cxx_std_17) +target_include_directories(clap-hf PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src") +target_link_libraries(clap-hf PRIVATE httplib::httplib nlohmann_json::nlohmann_json) + + # The CLI target differs from the core library target, while its executable # keeps the existing "clap" name and command-line behavior. add_executable(clap-cli src/clap.cpp src/embedding.cpp src/server.cpp src/quantize.cpp) target_compile_features(clap-cli PRIVATE cxx_std_17) -target_link_libraries(clap-cli PRIVATE clap CLI11::CLI11 httplib::httplib nlohmann_json::nlohmann_json) +target_link_libraries(clap-cli PRIVATE clap clap-hf CLI11::CLI11 httplib::httplib nlohmann_json::nlohmann_json) set_target_properties(clap-cli PROPERTIES OUTPUT_NAME clap CXX_EXTENSIONS OFF) if(MSVC) @@ -58,6 +66,11 @@ if(BUILD_TESTING) add_test(NAME clap_conversion_imports COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/conversion.py") set_tests_properties(clap_conversion_imports PROPERTIES TIMEOUT 30) + add_test(NAME clap_hf_download + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/hf_download.py" + "$") + set_tests_properties(clap_hf_download PROPERTIES TIMEOUT 60) + find_program(FFMPEG_EXECUTABLE NAMES ffmpeg REQUIRED) set(CLAP_AUDIO_FIXTURE_DIR "${CMAKE_CURRENT_BINARY_DIR}/tests/audio") set(CLAP_AUDIO_FIXTURES diff --git a/README.md b/README.md index 812aad1..76dc2a2 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ require a converted GGUF model; inference runs locally without Python or downloa - CMake 3.20 or newer - A C++17 compiler +- OpenSSL 3 development headers/libraries and a trusted CA certificate store (HTTPS downloads; e.g. `libssl-dev` and `ca-certificates` on Debian/Ubuntu) - ICU source submodule (included by recursive checkout; no system ICU installation required) - Python 3 (only for model conversion and tests; install Python dependencies with `python -m pip install -r requirements.txt`) - FFmpeg with libmp3lame (only when building tests, to generate WAV/MP3/FLAC fixtures) @@ -174,6 +175,82 @@ This adds a release build, not HIP model dispatch: native model linear layers remain CPU/optional Vulkan. The embedding command and server use that same native runtime. No ROCm inference or GPU tests are run. +## Download GGUF models from Hugging Face + +Download a **converted, unsplit GGUF** with the native C++ HTTP client. No Python, +`huggingface-cli`, Git or git-lfs is needed at runtime. Replace `owner/repo` below +with a repository containing a CLAP/CLaMP3/MERT2 GGUF compatible with this loader: + +```sh +./build/clap download -hf owner/repo +./build/clap download -hf owner/repo:Q8_0 +./build/clap download -hf owner/repo -hff subdirectory/model.gguf +./build/clap embedding -hf owner/repo:Q4_K_M --text 'A jazz quartet.' +# Existing local model inference is unchanged: +./build/clap embedding --model model.gguf --text 'A jazz quartet.' +``` + +`-hf`, `-hff` and `-hft` are aliases for `--hf-repo`, `--hf-file` and +`--hf-token`. The default selection prefers a filename containing `Q4_K_M`, then +`Q8_0`, then the first GGUF in lexical filename order. `:quant` filters filenames +case-insensitively; `-hff` selects an exact repository-relative filename and takes +precedence over the quant selector. Split `-00001-of-00002.gguf` models are rejected +before downloading: the clap loader does not support split GGUFs. Downloading does +**not** convert safetensors/PyTorch weights or make unrelated architectures usable. +For repositories without a GGUF, use the conversion instructions below first. + +The command resolves `main` to an immutable commit and enumerates the recursive +repository tree (including paginated results). Successful downloads print only +the absolute local snapshot path to stdout; download status and errors go to +stderr. `clap embedding` resolves the same cache path before loading the model and +keeps its embedding JSON on stdout. `--model` and `-hf` are mutually exclusive. + +### Cache, authentication and network settings + +Cache root precedence (first nonempty value): + +1. `CLAP_CACHE` +2. `HF_HUB_CACHE` +3. `HUGGINGFACE_HUB_CACHE` (legacy alias) +4. `$HF_HOME/hub` +5. `$XDG_CACHE_HOME/huggingface/hub`, or `$HOME/.cache/huggingface/hub` + (falling back to `%USERPROFILE%/.cache/huggingface/hub` when `HOME` is unset) + +The layout is Hugging Face-compatible: `models--owner--repo/blobs/`, +`snapshots//` (relative symlinks to blobs, or hard links when +symlink creation is unavailable), and `refs/main`. +Completed files are reused. Online calls still refresh metadata; add `--offline` +to use only the locally recorded `main` snapshot without any network requests. +An offline cache miss is an error, not a fallback to the network. + +```sh +CLAP_CACHE=/path/to/models ./build/clap download -hf owner/repo +./build/clap download -hf owner/repo --offline +./build/clap embedding -hf owner/repo --offline --text 'A jazz quartet.' +# Set HF_TOKEN in your environment for private/gated repositories. +# Alternatively: ./build/clap download -hf owner/repo -hft "$HF_TOKEN" +``` + +`-hft` overrides `HF_TOKEN`; prefer the environment to avoid putting credentials +in shell history or process arguments. Tokens are sent as bearer credentials only +to the configured origin, never to a different redirect host. `HF_ENDPOINT` +(default `https://huggingface.co`) accepts an HTTP(S) origin for mirrors or local +testing. Use HTTPS for real credentials. TLS certificate verification remains +enabled; OpenSSL 3 support is required at configure time. + +Downloads stream into `.downloadInProgress` files, resume with validated HTTP +Range responses, and publish completed blobs/snapshots atomically after size +validation (and SHA256 verification for LFS blobs). Network/server failures retry +at most three times per request. Directory traversal and unsafe cache symlinks +are refused, including Windows drive/device/alternate-stream filenames on every +platform. The cache must be writable and support symlinks or hard links; do not +share a writable cache with untrusted users. A per-repository `.clap-download.lock` +rejects concurrent writers; after a crash, remove that stale lock directory only +when no download is running. Partial files remain available for the next resume. +An interrupted snapshot publication is recovered under that lock only when its +`.downloadInProgress` link points to the exact expected blob; unrelated temporary +files or links are refused and left untouched. + ## Native model layout The C++ organization follows `llama.cpp/src/models`: shared runtime services are diff --git a/src/clap.cpp b/src/clap.cpp index 9ffc3e2..0312c33 100644 --- a/src/clap.cpp +++ b/src/clap.cpp @@ -1,4 +1,5 @@ #include "embedding.h" +#include "hf.h" #include "quantize.h" #include "server.h" @@ -8,6 +9,7 @@ #include #include #include +#include int main(int argc, char **argv) { CLI::App app{"clap.cpp"}; @@ -28,8 +30,17 @@ int main(int argc, char **argv) { EmbeddingOptions options; std::string file; auto *embedding = app.add_subcommand("embedding", "Generate native text or audio embeddings"); - embedding->add_option("-m,--model", options.model_path, "Converted CLAP/CLaMP3/MERT/MERT2/MuQ/MuLan GGUF") - ->required()->check(CLI::ExistingFile); + auto *model_option = embedding->add_option("-m,--model", options.model_path, "Converted CLAP/CLaMP3/MERT/MERT2/MuQ/MuLan GGUF") + ->check(CLI::ExistingFile); + clap::HfOptions embedding_hf; + auto *hf_option = embedding->add_option("--hf-repo", embedding_hf.repo, "owner/repo[:quant] (-hf)"); + hf_option->excludes(model_option); + embedding->add_option("--hf-file", embedding_hf.file, "Exact GGUF filename (-hff)")->needs(hf_option); + embedding->add_option("--hf-token", embedding_hf.token, "Bearer token (-hft; defaults to HF_TOKEN)")->needs(hf_option); + embedding->add_flag("--offline", embedding_hf.offline, "Use only cached HF model")->needs(hf_option); + auto *model = embedding->add_option_group("Model"); + model->add_options(model_option, hf_option); + model->require_option(1, 1); embedding->add_option("-t,--threads", options.threads, "ggml CPU threads (0: automatic)") ->check(CLI::NonNegativeNumber)->capture_default_str(); auto *input = embedding->add_option_group("Input", "Choose one text prompt, text file, or audio file"); @@ -62,7 +73,26 @@ int main(int argc, char **argv) { quantize->add_option("INPUT.gguf", quantize_input, "F32/F16 CLAP/CLaMP3/MERT/MERT2/MuQ/MuLan model")->required(); quantize->add_option("OUTPUT.gguf", quantize_output, "Output model (must not exist)")->required(); - CLI11_PARSE(app, argc, argv); + clap::HfOptions hf; + auto *download = app.add_subcommand("download", "Download a Hugging Face GGUF model"); + download->add_option("--hf-repo", hf.repo, "owner/repo[:quant] (-hf)")->required(); + download->add_option("--hf-file", hf.file, "Exact GGUF filename (-hff)"); + download->add_option("--hf-token", hf.token, "Bearer token (-hft; defaults to HF_TOKEN)"); + download->add_flag("--offline", hf.offline, "Use only an existing cached snapshot"); + auto arguments = clap::hf_arguments(argc, argv); + std::vector pointers; + for (auto &arg : arguments) pointers.push_back(arg.data()); + CLI11_PARSE(app, argc, pointers.data()); + if (*download) { + try { + std::cout << clap::hf_download(hf) << '\n'; + if (!std::cout) throw std::runtime_error("Cannot write downloaded path"); + return 0; + } catch (const std::exception &e) { + std::cerr << e.what() << '\n'; + return 1; + } + } if (*quantize) { return run_quantize(quantize_input, quantize_output, quantize_type); @@ -94,6 +124,7 @@ int main(int argc, char **argv) { } if (!*audio_option && !options.single_text && options.text.empty()) throw std::runtime_error("Input must not be empty"); + if (*hf_option) options.model_path = clap::hf_download(embedding_hf); return run_embedding(options); } if (*serve) { diff --git a/src/hf.cpp b/src/hf.cpp new file mode 100644 index 0000000..2f01997 --- /dev/null +++ b/src/hf.cpp @@ -0,0 +1,379 @@ +#include "hf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clap { +namespace { +namespace fs = std::filesystem; +using json = nlohmann::json; +std::string env(const char *name) { + const char *v = std::getenv(name); + return v ? v : ""; +} +void require(bool condition, const std::string &message) { + if (!condition) throw std::runtime_error(message); +} +std::string upper(std::string s) { + for (char &c : s) c = static_cast(std::toupper(static_cast(c))); + return s; +} +bool hex(const std::string &s, size_t n) { + return s.size() == n && s.find_first_not_of("0123456789abcdef") == std::string::npos; +} +void safe_name(const std::string &s) { + const fs::path path(s); + require(!s.empty() && !path.is_absolute() && !path.has_root_name() && !path.has_root_directory() && + s.front() != '/' && s.find_first_of("\\:<>\"|?*") == std::string::npos && + std::none_of(s.begin(), s.end(), [](unsigned char c) { return c < 32; }), + "Unsafe repository filename"); + // Apply Windows rules on every platform: cache names must remain portable and + // must not address drive-relative paths, alternate streams or device names. + for (const auto &part : path) { + const auto name = part.string(); + require(!name.empty() && name != "." && name != ".." && name.back() != '.' && name.back() != ' ', + "Unsafe repository filename"); + const auto device = upper(name.substr(0, name.find('.'))); + require(device != "CON" && device != "PRN" && device != "AUX" && device != "NUL" && + device != "CONIN$" && device != "CONOUT$" && + !std::regex_match(device, std::regex(u8"(COM|LPT)([1-9]|\u00b9|\u00b2|\u00b3)")), + "Unsafe repository filename"); + } +} +fs::path snapshot_path(const fs::path &root, const std::string &name) { + safe_name(name); + const auto base = fs::absolute(root).lexically_normal(); + const auto result = (base / name).lexically_normal(); + const auto prefix = std::mismatch(base.begin(), base.end(), result.begin(), result.end()); + require(prefix.first == base.end() && prefix.second != result.end(), "Unsafe snapshot path outside snapshot root"); + return result; +} +// Refuse symlinks in every directory component, including an explicitly supplied cache root. +void safe_path(const fs::path &p) { + fs::path current; + for (const auto &part : fs::absolute(p)) { + current /= part; + require(!fs::is_symlink(fs::symlink_status(current)), "Unsafe cache symlink: " + current.string()); + } +} +void directories(const fs::path &p) { safe_path(p); fs::create_directories(p); } +std::string read(const fs::path &p) { + safe_path(p); + std::ifstream in(p, std::ios::binary); + return std::string(std::istreambuf_iterator(in), {}); +} +void publish_text(const fs::path &p, const std::string &text) { + directories(p.parent_path()); + safe_path(p); + const auto tmp = fs::path(p.string() + ".downloadInProgress"); + safe_path(tmp); + { std::ofstream out(tmp, std::ios::binary | std::ios::trunc); + out << text; out.close(); require(bool(out), "Cannot write cache metadata"); } + fs::rename(tmp, p); +} +fs::path cache_root() { + for (auto key : {"CLAP_CACHE", "HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"}) + if (!env(key).empty()) return fs::absolute(env(key)); + if (!env("HF_HOME").empty()) return fs::absolute(fs::path(env("HF_HOME")) / "hub"); + fs::path base = env("XDG_CACHE_HOME"); + if (base.empty()) { + const auto home = env("HOME").empty() ? env("USERPROFILE") : env("HOME"); + require(!home.empty(), "Set CLAP_CACHE, HOME or USERPROFILE to locate the model cache"); + base = fs::path(home) / ".cache"; + } + return fs::absolute(base / "huggingface" / "hub"); +} +std::string encode(const std::string &s) { + std::ostringstream out; + for (unsigned char c : s) { + if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') out << c; + else out << '%' << std::uppercase << std::hex << std::setw(2) << std::setfill('0') << unsigned(c); + } + return out.str(); +} +struct Url { std::string origin, target; }; +Url parse_url(const std::string &s) { + std::smatch m; + require(std::regex_match(s, m, std::regex("(https?://[^/?#@]+)(/[^#]*)?")), "Invalid HTTP endpoint or redirect URL"); + return {m[1], m[2].matched ? m[2].str() : "/"}; +} +std::string next_url(const std::string ¤t, const std::string &location) { + const auto u = parse_url(current); + if (location.rfind("//", 0) == 0) return current.substr(0, current.find(':')) + ":" + location; + if (location.rfind("/", 0) == 0) return u.origin + location; + if (location.find("://") != std::string::npos) return location; + return u.origin + u.target.substr(0, u.target.rfind('/') + 1) + location; +} +class Http { + std::string endpoint_, token_; +public: + Http(std::string endpoint, std::string token) : endpoint_(std::move(endpoint)), token_(std::move(token)) {} + // Redirects are deliberately manual: credentials never leave the configured origin. + httplib::Result get(std::string url, const httplib::Headers &extra = {}, + const httplib::ResponseHandler &response = {}, + const httplib::ContentReceiver &receiver = {}) { + for (int redirect = 0; redirect < 10; ++redirect) { + const auto u = parse_url(url); + httplib::Client client(u.origin); + client.set_connection_timeout(10); + client.set_read_timeout(30); + client.set_write_timeout(30); + client.set_follow_location(false); + client.set_decompress(false); + httplib::Headers headers = extra; + headers.emplace("Accept-Encoding", "identity"); + if (u.origin == parse_url(endpoint_).origin && !token_.empty()) + headers.emplace("Authorization", "Bearer " + token_); + bool redirect_response = false; + auto result = receiver ? client.Get(u.target, headers, + [&](const httplib::Response &r) { + redirect_response = r.status >= 300 && r.status < 400; + return redirect_response || response(r); + }, + [&](const char *data, size_t n) { return redirect_response || receiver(data, n); }) : client.Get(u.target, headers); + if (!result || result->status < 300 || result->status >= 400) return result; + const auto target = next_url(url, result->get_header_value("Location")); + require(!(u.origin.rfind("https:", 0) == 0 && target.rfind("http:", 0) == 0), "Refusing HTTPS downgrade"); + url = target; + } + throw std::runtime_error("Too many HTTP redirects"); + } + httplib::Result metadata(const std::string &url) { + for (int attempt = 0; attempt < 3; ++attempt) { + auto r = get(url); + if (r && r->status == 200) return r; + if (r && r->status != 429 && r->status < 500) + throw std::runtime_error("Hugging Face metadata HTTP " + std::to_string(r->status)); + if (attempt == 2) throw std::runtime_error("Hugging Face metadata request failed after 3 attempts"); + std::this_thread::sleep_for(std::chrono::milliseconds(100 * (attempt + 1))); + } + throw std::runtime_error("Unreachable HTTP retry state"); + } +}; +struct File { std::string name, oid; uint64_t size = 0; }; +bool gguf(const std::string &s) { return s.size() > 5 && upper(s.substr(s.size() - 5)) == ".GGUF"; } +File select(std::vector files, const std::string &filename, const std::string &quant) { + std::sort(files.begin(), files.end(), [](const File &a, const File &b) { return a.name < b.name; }); + files.erase(std::remove_if(files.begin(), files.end(), [&](const File &f) { + return !gguf(f.name) || (!filename.empty() && f.name != filename) || + (filename.empty() && !quant.empty() && upper(f.name).find(upper(quant)) == std::string::npos); + }), files.end()); + require(!files.empty(), "No matching GGUF model found (safetensors are not converted)"); + if (filename.empty() && quant.empty()) { + for (auto q : {"Q4_K_M", "Q8_0"}) { + auto it = std::find_if(files.begin(), files.end(), [&](const File &f) { return upper(f.name).find(q) != std::string::npos; }); + if (it != files.end()) { std::rotate(files.begin(), it, it + 1); break; } + } + } + require(!std::regex_search(files.front().name, std::regex("-[0-9]{5}-of-[0-9]{5}\\.gguf$", std::regex::icase)), + "Split GGUF models are unsupported by the clap model loader; select an unsplit GGUF with -hff"); + return files.front(); +} +bool cached(const fs::path &p, const fs::path &blobs, uint64_t size = 0) { + safe_path(p.parent_path()); + if (fs::is_symlink(fs::symlink_status(p))) { + const auto target = fs::weakly_canonical(p.parent_path() / fs::read_symlink(p)); + require(target.parent_path() == fs::weakly_canonical(blobs), "Unsafe snapshot symlink"); + safe_path(target); + } else safe_path(p); + return fs::is_regular_file(p) && (!size || fs::file_size(p) == size); +} +std::string sha256(const fs::path &p) { + std::unique_ptr ctx(EVP_MD_CTX_new(), EVP_MD_CTX_free); + require(ctx && EVP_DigestInit_ex(ctx.get(), EVP_sha256(), nullptr) == 1, "Cannot initialize SHA256"); + std::ifstream in(p, std::ios::binary); + char buffer[65536]; + while (in) { in.read(buffer, sizeof(buffer)); EVP_DigestUpdate(ctx.get(), buffer, static_cast(in.gcount())); } + require(in.eof(), "Cannot hash downloaded model"); + unsigned char digest[EVP_MAX_MD_SIZE]; unsigned int n = 0; + require(EVP_DigestFinal_ex(ctx.get(), digest, &n) == 1, "Cannot finish SHA256"); + std::ostringstream out; + for (unsigned i = 0; i < n; ++i) out << std::hex << std::setw(2) << std::setfill('0') << unsigned(digest[i]); + return out.str(); +} +void download(Http &http, const std::string &url, const File &file, const fs::path &blob) { + safe_path(blob); + if (fs::is_regular_file(blob) && fs::file_size(blob) == file.size) return; + const auto partial = fs::path(blob.string() + ".downloadInProgress"); + for (int attempt = 0; attempt < 3; ++attempt) { + safe_path(partial); + require(!fs::exists(partial) || (fs::is_regular_file(partial) && fs::hard_link_count(partial) == 1), "Unsafe partial download"); + uint64_t offset = fs::exists(partial) ? fs::file_size(partial) : 0; + if (offset >= file.size) { fs::remove(partial); offset = 0; } + std::ofstream out; + uint64_t received = offset; + bool accepted = false; + httplib::Headers headers; + if (offset) headers.emplace("Range", "bytes=" + std::to_string(offset) + "-"); + auto r = http.get(url, headers, [&](const httplib::Response &res) { + if (res.status != 200 && res.status != 206) return false; + if (res.status == 206) { + const auto expected = "bytes " + std::to_string(offset) + "-" + std::to_string(file.size - 1) + "/" + std::to_string(file.size); + if (res.get_header_value("Content-Range") != expected) return false; + } else { offset = 0; received = 0; } + if (res.has_header("Content-Length") && res.get_header_value("Content-Length") != std::to_string(file.size - offset)) return false; + safe_path(partial); + out.open(partial, std::ios::binary | (offset ? std::ios::app : std::ios::trunc)); + accepted = bool(out); + return accepted; + }, [&](const char *data, size_t n) { + if (!accepted || n > file.size - received) return false; + out.write(data, static_cast(n)); received += n; + return bool(out); + }); + if (out.is_open()) out.close(); + if (r && accepted && out && received == file.size) { + if (hex(file.oid, 64) && sha256(partial) != file.oid) { + fs::remove(partial); + throw std::runtime_error("Downloaded model SHA256 mismatch"); + } + safe_path(blob); fs::rename(partial, blob); return; + } + if (r && r->status >= 400 && r->status < 500 && r->status != 429) + throw std::runtime_error("Model download HTTP " + std::to_string(r->status)); + std::this_thread::sleep_for(std::chrono::milliseconds(100 * (attempt + 1))); + } + throw std::runtime_error("Model download failed after 3 attempts (HTTP, range or size validation); partial retained for resume"); +} +} // namespace + +std::string hf_download(const HfOptions &options) { + std::string repo = options.repo, quant; + const auto colon = repo.find(':'); + if (colon != std::string::npos) { quant = repo.substr(colon + 1); repo.resize(colon); require(!quant.empty(), "Empty HF quantization selector"); } + require(std::regex_match(repo, std::regex("[A-Za-z0-9_][A-Za-z0-9_.-]*/[A-Za-z0-9_][A-Za-z0-9_.-]*")) && repo.find("..") == std::string::npos && repo.find("--") == std::string::npos, + "Expected Hugging Face owner/repo[:quant]"); + if (!options.file.empty()) safe_name(options.file); + const auto root = cache_root() / ("models--" + repo.substr(0, repo.find('/')) + "--" + repo.substr(repo.find('/') + 1)); + safe_path(root); + const auto blobs = root / "blobs"; + safe_path(blobs); + if (options.offline) { + auto commit = read(root / "refs/main"); + while (!commit.empty() && std::isspace(static_cast(commit.back()))) commit.pop_back(); + require(hex(commit, 40), "Offline cache miss: refs/main is missing or invalid"); + const auto snapshot = root / "snapshots" / commit; + safe_path(snapshot); + std::vector files; + if (fs::is_directory(snapshot)) { + for (const auto &entry : fs::recursive_directory_iterator(snapshot)) { + if (entry.is_directory()) { require(!entry.is_symlink(), "Unsafe cache directory symlink"); continue; } + if (gguf(entry.path().string()) && cached(entry.path(), blobs)) + files.push_back({entry.path().lexically_relative(snapshot).generic_string(), "", 0}); + } + } + require(!files.empty(), "Offline cache miss: no downloaded GGUF"); + return snapshot_path(snapshot, select(files, options.file, quant).name).string(); + } + directories(root); + const auto lock = root / ".clap-download.lock"; + safe_path(lock); + require(fs::create_directory(lock), "Another download owns this repository cache (remove stale .clap-download.lock after a crash)"); + struct Unlock { fs::path path; ~Unlock() { std::error_code ec; fs::remove(path, ec); } } unlock{lock}; + std::string endpoint = env("HF_ENDPOINT"); + if (endpoint.empty()) endpoint = "https://huggingface.co"; + while (!endpoint.empty() && endpoint.back() == '/') endpoint.pop_back(); + require(parse_url(endpoint).target == "/", "HF_ENDPOINT must be an HTTP(S) origin"); + const auto token = options.token.empty() ? env("HF_TOKEN") : options.token; + require(token.find_first_of("\r\n") == std::string::npos, "Invalid HF token"); + Http http(endpoint, token); + const auto info = json::parse(http.metadata(endpoint + "/api/models/" + repo + "/revision/main")->body); + const std::string commit = info.at("sha"); + require(hex(commit, 40), "Invalid immutable Hugging Face commit"); + std::vector files; + std::string url = endpoint + "/api/models/" + repo + "/tree/" + commit + "?recursive=true&limit=1000"; + std::set visited; + while (!url.empty()) { + require(visited.size() < 1000 && visited.insert(url).second, "Invalid tree pagination"); + require(parse_url(url).origin == parse_url(endpoint).origin, "Cross-origin tree pagination refused"); + auto r = http.metadata(url); + const auto rows = json::parse(r->body); + require(rows.is_array(), "Invalid repository tree"); + for (const auto &row : rows) { + if (row.value("type", "") != "file") continue; + File f; f.name = row.at("path"); safe_name(f.name); + if (!gguf(f.name)) continue; + f.size = row.at("size").get(); + f.oid = row.contains("lfs") && row["lfs"].is_object() ? row["lfs"].at("oid").get() : row.at("oid").get(); + require((hex(f.oid, 40) || hex(f.oid, 64)) && f.size > 0, "Invalid model blob metadata"); + files.push_back(f); + } + url.clear(); + std::smatch m; + const auto link = r->get_header_value("Link"); + if (std::regex_search(link, m, std::regex("<([^>]+)>;\\s*rel=\"next\""))) url = next_url(endpoint + "/", m[1]); + } + const auto file = select(files, options.file, quant); + directories(blobs); + const auto snapshot = snapshot_path(root / "snapshots" / commit, file.name); + if (!cached(snapshot, blobs, file.size)) { + std::cerr << "Downloading " << repo << '/' << file.name << " (" << file.size << " bytes)\n"; + download(http, endpoint + '/' + repo + "/resolve/" + commit + '/' + encode(file.name), file, blobs / file.oid); + directories(snapshot.parent_path()); + require(!fs::exists(snapshot) && !fs::is_symlink(fs::symlink_status(snapshot)), "Invalid existing snapshot file; remove it before retrying"); + const auto tmp = fs::path(snapshot.string() + ".downloadInProgress"); + const auto blob = fs::absolute(blobs / file.oid).lexically_normal(); + // We hold the repository lock. Inspect the link itself, never resolve an + // arbitrary target; only our selected, validated blob may be published. + const auto status = fs::symlink_status(tmp); + if (fs::is_symlink(status)) { + const auto target = fs::read_symlink(tmp); + require(target == blob.lexically_relative(snapshot.parent_path()), + "Unsafe temporary snapshot symlink: " + tmp.string()); + } else if (fs::exists(status)) { + safe_path(tmp); + require(fs::is_regular_file(status) && fs::equivalent(tmp, blob), + "Unsafe temporary snapshot file: " + tmp.string()); + } else { + safe_path(tmp); + std::error_code link_error; + fs::create_symlink(blob.lexically_relative(snapshot.parent_path()), tmp, link_error); + if (link_error) { + // Native Windows commonly denies symlinks without Developer Mode + // or elevation. Hard links preserve the same blob/snapshot layout. + std::error_code hardlink_error; + fs::create_hard_link(blob, tmp, hardlink_error); + require(!hardlink_error, "Cannot publish snapshot (symlink: " + link_error.message() + + "; hard link: " + hardlink_error.message() + "): " + tmp.string()); + } + } + fs::rename(tmp, snapshot); + std::cerr << "Downloaded " << file.size << " bytes\n"; + } + publish_text(root / "refs/main", commit); + return snapshot.string(); +} + +std::vector hf_arguments(int argc, char **argv) { + std::vector args; + bool literal = false; + for (int i = 0; i < argc; ++i) { + std::string arg = argv[i]; + if (!literal) for (const auto &alias : {std::pair{"-hf", "--hf-repo"}, {"-hff", "--hf-file"}, {"-hft", "--hf-token"}}) { + const std::string key = alias.first; + if (arg == key || arg.rfind(key + "=", 0) == 0) { arg.replace(0, key.size(), alias.second); break; } + } + if (arg == "--") literal = true; + args.push_back(std::move(arg)); + } + return args; +} +} // namespace clap diff --git a/src/hf.h b/src/hf.h new file mode 100644 index 0000000..1e59be6 --- /dev/null +++ b/src/hf.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +namespace clap { +struct HfOptions { + std::string repo; + std::string file; + std::string token; + bool offline = false; +}; + +// Resolve a single, unsplit GGUF into the Hugging Face-compatible cache. +// Throws std::runtime_error on invalid input, network or cache errors. +std::string hf_download(const HfOptions &options); + +// CLI11 does not accept multi-character short options. Normalize only the +// documented llama.cpp-style aliases before parsing (including =value forms). +std::vector hf_arguments(int argc, char **argv); +} // namespace clap diff --git a/tests/hf_download.py b/tests/hf_download.py new file mode 100644 index 0000000..2b39a5e --- /dev/null +++ b/tests/hf_download.py @@ -0,0 +1,372 @@ +"""Hermetic CLI integration tests; no Hugging Face access or Python runtime dependency.""" +import hashlib +import http.server +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import threading +import unittest + +CLI = sys.argv[1] +sys.argv = sys.argv[:1] + sys.argv[2:] +COMMIT = "a" * 40 +DATA = b"GGUF" + struct.pack("; rel="next"' + self.send(200, json.dumps(rows).encode(), headers) + elif self.path == "/redirected": + self.send(200, data) + elif self.path.startswith(f"/test/model/resolve/{COMMIT}/"): + if getattr(self.server, "redirect", ""): + self.send(302, b"redirect body", {"Location": self.server.redirect}) + return + if self.server.fail: + self.send(503, b"retry") + return + if getattr(self.server, "drop_once", False): + self.server.drop_once = False + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data[:30]) + return + if getattr(self.server, "corrupt", False): + self.send(200, b"x" * len(data)) + return + offset = int(self.headers.get("Range", "bytes=0-")[6:-1]) + if getattr(self.server, "ignore_range", False): + offset = 0 + if offset and self.server.bad_range: + self.send(206, data[offset:], {"Content-Range": f"bytes 0-{len(data)-1}/{len(data)}"}) + else: + headers = {"Content-Range": f"bytes {offset}-{len(data)-1}/{len(data)}"} if offset else {} + self.send(206 if offset else 200, data[offset:], headers) + else: + self.send(404, b"missing") + + def send(self, status, body, headers=None): + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + for k, v in (headers or {}).items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write(body) + +class Download(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(dir=Path(tempfile.gettempdir()).resolve()) + self.cache = Path(self.tmp.name) / "cache" + self.repo = self.cache / "models--test--model" + self.server = FixtureServer(("127.0.0.1", 0), Handler) + self.server.requests, self.server.files = [], ["nested/model-Q8_0.gguf", "model-Q4_K_M.gguf"] + self.server.fail = self.server.bad_range = False + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.env = dict(os.environ, CLAP_CACHE=str(self.cache), HF_ENDPOINT=f"http://127.0.0.1:{self.server.server_port}") + self.env.pop("HF_TOKEN", None) + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + self.tmp.cleanup() + + def run_cli(self, *args, ok=True, exe=CLI): + p = subprocess.run([exe, *args], env=self.env, text=True, capture_output=True, timeout=15) + if ok: + self.assertEqual(p.returncode, 0, p.stderr) + else: + self.assertNotEqual(p.returncode, 0, p.stdout) + return p + + def download(self, *args, **kw): + return self.run_cli("download", "-hf", "test/model", *args, **kw) + + def test_default_cache_offline_and_token(self): + p = self.download("-hft", "secret") + path = Path(p.stdout.strip()) + self.assertEqual(path, self.repo / "snapshots" / COMMIT / "model-Q4_K_M.gguf") + self.assertEqual(path.read_bytes(), DATA) + self.assertEqual((self.repo / "refs/main").read_text().strip(), COMMIT) + self.assertTrue((self.repo / "blobs" / OID).is_file()) + self.assertTrue(all(h.get("Authorization") == "Bearer secret" for _, h in self.server.requests)) + self.server.requests.clear() + self.assertEqual(self.download("--offline").stdout, p.stdout) + self.assertEqual(self.server.requests, []) + self.download() + self.assertFalse(any("/resolve/" in p for p, _ in self.server.requests)) + + def symlink_or_skip(self, target, link, **kwargs): + try: + link.symlink_to(target, **kwargs) + except OSError as error: + if os.name == "nt" and error.winerror == 1314: + self.skipTest("Windows symlink privilege unavailable") + raise + + def test_windows_special_tree_paths_rejected(self): + for name in ["C:/outside/model.gguf", "C:model.gguf", "nested/model.gguf:stream", + "//server/share/model.gguf", "\\\\?\\C:\\outside\\model.gguf", + "NUL.gguf", "nested/COM1/model.gguf", "nested/LPT9.gguf", + "nested./model.gguf", "nested /model.gguf", "model?.gguf", + "COM¹.gguf", "nested/LPT²/model.gguf", "conout$.gguf"]: + with self.subTest(name=name): + self.server.files = [name] + self.server.requests.clear() + p = self.download(ok=False) + self.assertIn("Unsafe repository filename", p.stderr) + self.assertFalse(any("/resolve/" in p for p, _ in self.server.requests)) + self.assertFalse((self.repo / "refs/main").exists()) + + def test_windows_absolute_explicit_filename_rejected(self): + p = self.download("-hff", "C:/outside/model.gguf", ok=False) + self.assertIn("Unsafe repository filename", p.stderr) + self.assertEqual(self.server.requests, []) + + def test_userprofile_default_cache(self): + for key in ["CLAP_CACHE", "HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE", "HF_HOME", + "XDG_CACHE_HOME", "HOME"]: + self.env.pop(key, None) + home = Path(self.tmp.name) / "windows-home" + self.env["USERPROFILE"] = str(home) + path = Path(self.download().stdout.strip()) + self.assertEqual(path, home / ".cache/huggingface/hub/models--test--model" / + "snapshots" / COMMIT / "model-Q4_K_M.gguf") + self.assertEqual(path.read_bytes(), DATA) + + def test_interrupted_snapshot_publication_recovers(self): + path = Path(self.download("-hff", "nested/model-Q8_0.gguf").stdout.strip()) + blob = self.repo / "blobs" / OID + path.unlink() + temporary = path.with_name(path.name + ".downloadInProgress") + self.symlink_or_skip(os.path.relpath(blob, path.parent), temporary) + self.server.requests.clear() + self.assertEqual(Path(self.download("-hff", "nested/model-Q8_0.gguf").stdout.strip()), path) + self.assertEqual(path.read_bytes(), DATA) + self.assertEqual(blob.read_bytes(), DATA) + self.assertFalse(temporary.is_symlink()) + self.assertFalse(any("/resolve/" in p for p, _ in self.server.requests)) + + def test_interrupted_snapshot_hardlink_recovers(self): + path = Path(self.download().stdout.strip()) + blob = self.repo / "blobs" / OID + path.unlink() + temporary = path.with_name(path.name + ".downloadInProgress") + os.link(blob, temporary) + self.server.requests.clear() + self.assertEqual(Path(self.download().stdout.strip()), path) + self.assertTrue(os.path.samefile(path, blob)) + self.assertFalse(temporary.exists()) + self.assertFalse(any("/resolve/" in p for p, _ in self.server.requests)) + + def test_unrelated_snapshot_temporary_file_refused(self): + path = Path(self.download().stdout.strip()) + path.unlink() + temporary = path.with_name(path.name + ".downloadInProgress") + temporary.write_bytes(DATA) + self.assertIn("Unsafe temporary snapshot", self.download(ok=False).stderr) + self.assertEqual(temporary.read_bytes(), DATA) + self.assertFalse(path.exists()) + + def test_malicious_snapshot_temporary_link_refused(self): + path = Path(self.download().stdout.strip()) + blob = self.repo / "blobs" / OID + outside = Path(self.tmp.name) / "outside.gguf" + outside.write_bytes(DATA) + other_blob = blob.with_name("b" * 64) + other_blob.write_bytes(DATA) + chain = self.repo / "chain" + self.symlink_or_skip(blob, chain) + path.unlink() + temporary = path.with_name(path.name + ".downloadInProgress") + for target in [outside, other_blob, chain]: + with self.subTest(target=target): + self.symlink_or_skip(target, temporary) + self.assertIn("Unsafe", self.download(ok=False).stderr) + self.assertTrue(temporary.is_symlink()) + self.assertFalse(path.exists()) + self.assertEqual(outside.read_bytes(), DATA) + self.assertEqual(other_blob.read_bytes(), DATA) + self.assertEqual(blob.read_bytes(), DATA) + temporary.unlink() + + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux symlink failure injection") + def test_snapshot_hardlink_fallback(self): + library = Path(self.tmp.name) / "deny-symlink.so" + source = '#include \nint symlink(const char *a, const char *b) { errno = EPERM; return -1; }\n' + subprocess.run(["cc", "-shared", "-fPIC", "-x", "c", "-", "-o", str(library)], + input=source, text=True, check=True, capture_output=True) + self.env["LD_PRELOAD"] = str(library) + path = Path(self.download().stdout.strip()) + self.assertFalse(path.is_symlink()) + self.assertTrue(os.path.samefile(path, self.repo / "blobs" / OID)) + self.assertEqual(path.read_bytes(), DATA) + self.assertEqual(Path(self.download("--offline").stdout.strip()), path) + path.rename(path.with_name(path.name + ".downloadInProgress")) + self.server.requests.clear() + self.assertEqual(Path(self.download().stdout.strip()), path) + self.assertTrue(os.path.samefile(path, self.repo / "blobs" / OID)) + self.assertFalse(any("/resolve/" in p for p, _ in self.server.requests)) + + def test_quant_and_explicit_filename(self): + p = self.run_cli("download", "-hf", "test/model:Q8_0") + self.assertTrue(Path(p.stdout.strip()).as_posix().endswith("nested/model-Q8_0.gguf")) + self.assertEqual(self.download("-hff", "nested/model-Q8_0.gguf").stdout, p.stdout) + + def test_resume_and_invalid_range(self): + blob = self.repo / "blobs" / (OID + ".downloadInProgress") + blob.parent.mkdir(parents=True) + blob.write_bytes(DATA[:20]) + self.download() + self.assertTrue(any(h.get("Range") == "bytes=20-" for _, h in self.server.requests)) + self.assertEqual(blob.with_name(OID).read_bytes(), DATA) + + def test_bad_range_not_published(self): + blob = self.repo / "blobs" / (OID + ".downloadInProgress") + blob.parent.mkdir(parents=True) + blob.write_bytes(DATA[:20]) + self.server.bad_range = True + self.download(ok=False) + self.assertFalse(blob.with_name(OID).exists()) + + def test_split_unsupported(self): + self.server.files = ["model-00001-of-00002.gguf", "model-00002-of-00002.gguf"] + p = self.download(ok=False) + self.assertIn("split", p.stderr.lower()) + self.assertFalse(any("/resolve/" in p for p, _ in self.server.requests)) + + def test_cross_origin_redirect_does_not_leak_token(self): + other = FixtureServer(("127.0.0.1", 0), Handler) + other.requests = [] + thread = threading.Thread(target=other.serve_forever, daemon=True) + thread.start() + self.server.redirect = f"http://127.0.0.1:{other.server_port}/redirected" + try: + self.download("-hft", "private-token") + self.assertTrue(other.requests) + self.assertTrue(all("Authorization" not in h for _, h in other.requests)) + finally: + other.shutdown() + other.server_close() + thread.join() + + def test_offline_missing(self): + self.assertIn("offline", self.download("--offline", ok=False).stderr.lower()) + self.assertEqual(self.server.requests, []) + + def test_traversal_and_symlink(self): + self.download("-hff", "../outside.gguf", ok=False) + self.repo.mkdir(parents=True, exist_ok=True) + outside = Path(self.tmp.name) / "outside" + outside.mkdir() + self.symlink_or_skip(outside, self.repo / "blobs", target_is_directory=True) + self.download(ok=False) + self.assertEqual(list(outside.iterdir()), []) + + def test_paginated_tree_prefers_q4(self): + self.server.paginate = True + self.assertTrue(self.download().stdout.strip().endswith("model-Q4_K_M.gguf")) + self.assertTrue(any("cursor=second" in p for p, _ in self.server.requests)) + + def test_interrupted_download_resumes(self): + self.server.drop_once = True + self.assertEqual(Path(self.download().stdout.strip()).read_bytes(), DATA) + self.assertTrue(any(h.get("Range") == "bytes=30-" for _, h in self.server.requests)) + + def test_server_ignoring_range_restarts_safely(self): + partial = self.repo / "blobs" / (OID + ".downloadInProgress") + partial.parent.mkdir(parents=True) + partial.write_bytes(DATA[:20]) + self.server.ignore_range = True + self.assertEqual(Path(self.download().stdout.strip()).read_bytes(), DATA) + + def test_corrupt_download_not_published(self): + self.server.corrupt = True + self.assertIn("SHA256", self.download(ok=False).stderr) + self.assertFalse((self.repo / "blobs" / OID).exists()) + self.assertFalse((self.repo / "refs/main").exists()) + + def test_cache_precedence(self): + self.env["HF_HUB_CACHE"] = str(Path(self.tmp.name) / "hub") + self.assertTrue(self.download().stdout.startswith(str(self.cache))) + del self.env["CLAP_CACHE"] + self.assertTrue(self.download().stdout.startswith(self.env["HF_HUB_CACHE"])) + del self.env["HF_HUB_CACHE"] + self.env["HUGGINGFACE_HUB_CACHE"] = str(Path(self.tmp.name) / "legacy") + self.assertTrue(self.download().stdout.startswith(self.env["HUGGINGFACE_HUB_CACHE"])) + del self.env["HUGGINGFACE_HUB_CACHE"] + self.env["HF_HOME"] = str(Path(self.tmp.name) / "home") + self.assertTrue(self.download().stdout.startswith(str(Path(self.env["HF_HOME"]) / "hub"))) + del self.env["HF_HOME"] + self.env["XDG_CACHE_HOME"] = str(Path(self.tmp.name) / "xdg") + self.assertTrue(self.download().stdout.startswith(str(Path(self.env["XDG_CACHE_HOME"]) / "huggingface/hub"))) + + def test_bounded_retry(self): + self.server.fail = True + self.download(ok=False) + count = sum("/resolve/" in p for p, _ in self.server.requests) + self.assertGreaterEqual(count, 2) + self.assertLessEqual(count, 4) + self.assertFalse((self.repo / "blobs" / OID).exists()) + + def test_model_cli_real_inference(self): + from hf_model_fixture import tiny_clap + self.server.data = tiny_clap() + p = self.run_cli("embedding", "-hf", "test/model", "--text", "hello") + values = json.loads(p.stdout) + self.assertEqual(len(values), 2) + self.assertAlmostEqual(sum(x*x for x in values), 1.0, places=5) + path = self.repo / "snapshots" / COMMIT / "model-Q4_K_M.gguf" + direct = self.run_cli("embedding", "--model", str(path), "--text", "hello") + self.assertEqual(p.stdout, direct.stdout) + self.server.requests.clear() + offline = self.run_cli("embedding", "-hf", "test/model", "--offline", "--text", "hello") + self.assertEqual(p.stdout, offline.stdout) + self.assertEqual(self.server.requests, []) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/hf_model_fixture.py b/tests/hf_model_fixture.py new file mode 100644 index 0000000..0b26a50 --- /dev/null +++ b/tests/hf_model_fixture.py @@ -0,0 +1,96 @@ +"""Small deterministic, valid CLAP GGUF using only the Python standard library. + +All encoder weights are zero; a nonzero output bias makes the normalized +projection deterministic. This exercises real model loading/tokenization/inference, +not numerical parity (covered by the existing optional model oracle tests). +""" +import json +import math +import struct + + +def tiny_clap(): + audio = dict(enable_fusion=False, depths=[1]*4, num_attention_heads=[1]*4, + patch_size=4, patch_stride=[4, 4], patch_embed_input_channels=1, + qkv_bias=True, enable_patch_layer_norm=True, flatten_patch_embeds=True, + hidden_act="gelu", projection_hidden_act="relu", spec_size=32, + num_mel_bins=32, window_size=1, patch_embeds_hidden_size=2, + hidden_size=16, mlp_ratio=2, projection_dim=2) + text = dict(hidden_act="gelu", projection_hidden_act="relu", hidden_size=4, + num_attention_heads=1, num_hidden_layers=1, type_vocab_size=1, + vocab_size=7, max_position_embeddings=32, intermediate_size=8, + projection_dim=2, bos_token_id=0, eos_token_id=2, pad_token_id=1, + layer_norm_eps=1e-5) + config = dict(model_type="clap", audio_config=audio, text_config=text, projection_dim=2) + tokenizer = dict(model=dict(type="BPE", vocab={"": 0, "": 1, "": 2, + "h": 3, "e": 4, "l": 5, "o": 6}, merges=[])) + metadata = {"general.architecture": "clap", "clap.config": json.dumps(config), + "clap.tokenizer": json.dumps(tokenizer)} + tensors = [] + + def tensor(name, shape): + # A positive projection bias avoids a zero-norm output with zero weights. + value = 1.0 if name == "text_proj.linear2.bias" else 0.0 + tensors.append((name, shape, struct.pack(" Date: Sun, 25 Oct 2026 04:02:00 +0800 Subject: [PATCH 2/3] test(hf): cover HTTPS downloads with local TLS fixtures --- CMakeLists.txt | 6 ++- README.md | 4 +- src/hf.cpp | 4 ++ tests/hf_download.py | 87 +++++++++++++++++++++++++++++++++++++++- tests/hf_tls_fixture.cpp | 85 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 tests/hf_tls_fixture.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 039aaf0..59fa89f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,9 +66,13 @@ if(BUILD_TESTING) add_test(NAME clap_conversion_imports COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/conversion.py") set_tests_properties(clap_conversion_imports PROPERTIES TIMEOUT 30) + find_package(OpenSSL REQUIRED COMPONENTS Crypto) + add_executable(clap-hf-tls-fixture tests/hf_tls_fixture.cpp) + target_compile_features(clap-hf-tls-fixture PRIVATE cxx_std_17) + target_link_libraries(clap-hf-tls-fixture PRIVATE OpenSSL::Crypto) add_test(NAME clap_hf_download COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/hf_download.py" - "$") + "$" "$") set_tests_properties(clap_hf_download PROPERTIES TIMEOUT 60) find_program(FFMPEG_EXECUTABLE NAMES ffmpeg REQUIRED) diff --git a/README.md b/README.md index 76dc2a2..fcbb127 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,9 @@ in shell history or process arguments. Tokens are sent as bearer credentials onl to the configured origin, never to a different redirect host. `HF_ENDPOINT` (default `https://huggingface.co`) accepts an HTTP(S) origin for mirrors or local testing. Use HTTPS for real credentials. TLS certificate verification remains -enabled; OpenSSL 3 support is required at configure time. +enabled; OpenSSL 3 support is required at configure time. Set `SSL_CERT_FILE` to a +PEM CA bundle when using a private certificate authority; this overrides the default +trust store without disabling certificate or hostname verification. Downloads stream into `.downloadInProgress` files, resume with validated HTTP Range responses, and publish completed blobs/snapshots atomically after size diff --git a/src/hf.cpp b/src/hf.cpp index 2f01997..aafb84d 100644 --- a/src/hf.cpp +++ b/src/hf.cpp @@ -133,6 +133,10 @@ class Http { for (int redirect = 0; redirect < 10; ++redirect) { const auto u = parse_url(url); httplib::Client client(u.origin); + // Native Windows/macOS trust stores do not use OpenSSL's environment defaults. + // Honor an explicit CA bundle without disabling certificate or hostname checks. + const auto ca_file = env("SSL_CERT_FILE"); + if (!ca_file.empty()) client.set_ca_cert_path(ca_file); client.set_connection_timeout(10); client.set_read_timeout(30); client.set_write_timeout(30); diff --git a/tests/hf_download.py b/tests/hf_download.py index 2b39a5e..2b207f6 100644 --- a/tests/hf_download.py +++ b/tests/hf_download.py @@ -4,6 +4,7 @@ import json import os from pathlib import Path +import ssl import struct import subprocess import sys @@ -12,7 +13,8 @@ import unittest CLI = sys.argv[1] -sys.argv = sys.argv[:1] + sys.argv[2:] +TLS_FIXTURE = sys.argv[2] +sys.argv = sys.argv[:1] + sys.argv[3:] COMMIT = "a" * 40 DATA = b"GGUF" + struct.pack("; rel="next"' + scheme = "https" if isinstance(self.connection, ssl.SSLSocket) else "http" + headers["Link"] = f'<{scheme}://127.0.0.1:{self.server.server_port}/api/models/test/model/tree/{COMMIT}?cursor=second>; rel="next"' self.send(200, json.dumps(rows).encode(), headers) elif self.path == "/redirected": self.send(200, data) @@ -97,6 +107,12 @@ def send(self, status, body, headers=None): self.wfile.write(body) class Download(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tls = tempfile.TemporaryDirectory(dir=Path(tempfile.gettempdir()).resolve()) + cls.addClassCleanup(cls.tls.cleanup) + subprocess.run([TLS_FIXTURE, cls.tls.name], check=True, capture_output=True) + def setUp(self): self.tmp = tempfile.TemporaryDirectory(dir=Path(tempfile.gettempdir()).resolve()) self.cache = Path(self.tmp.name) / "cache" @@ -108,6 +124,73 @@ def setUp(self): self.thread.start() self.env = dict(os.environ, CLAP_CACHE=str(self.cache), HF_ENDPOINT=f"http://127.0.0.1:{self.server.server_port}") self.env.pop("HF_TOKEN", None) + self.env.pop("SSL_CERT_FILE", None) + + def start_server(self, tls=False): + server = FixtureServer(("127.0.0.1", 0), Handler) + if tls: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(Path(self.tls.name) / "server.pem", + Path(self.tls.name) / "server-key.pem") + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server + + def use_https(self): + server = self.start_server(tls=True) + self.env["HF_ENDPOINT"] = f"https://127.0.0.1:{server.server_port}" + self.env["SSL_CERT_FILE"] = str(Path(self.tls.name) / "ca.pem") + return server + + def test_https_download(self): + server = self.use_https() + server.paginate = True + path = Path(self.download("-hft", "private-token").stdout.strip()) + self.assertEqual(path.read_bytes(), DATA) + self.assertEqual(path.name, "model-Q4_K_M.gguf") + self.assertTrue(any("cursor=second" in p for p, _ in server.requests)) + self.assertTrue(any("/resolve/" in p for p, _ in server.requests)) + self.assertTrue(all(h.get("Authorization") == "Bearer private-token" + for _, h in server.requests)) + + def test_https_untrusted_certificate_rejected(self): + server = self.use_https() + del self.env["SSL_CERT_FILE"] + self.download(ok=False) + self.assertEqual(server.requests, []) + self.assertFalse((self.repo / "refs/main").exists()) + + def test_https_hostname_mismatch_rejected(self): + server = self.use_https() + self.env["HF_ENDPOINT"] = f"https://localhost:{server.server_port}" + self.download(ok=False) + self.assertEqual(server.requests, []) + self.assertFalse((self.repo / "refs/main").exists()) + + def test_https_cross_origin_redirect_does_not_leak_token(self): + server = self.use_https() + other = self.start_server(tls=True) + server.redirect = f"https://127.0.0.1:{other.server_port}/redirected" + self.assertEqual(Path(self.download("-hft", "private-token").stdout.strip()).read_bytes(), DATA) + self.assertTrue(server.requests) + self.assertTrue(all(h.get("Authorization") == "Bearer private-token" + for _, h in server.requests)) + self.assertTrue(other.requests) + self.assertTrue(all("Authorization" not in h for _, h in other.requests)) + + def test_https_downgrade_rejected(self): + server = self.use_https() + server.redirect = f"http://127.0.0.1:{self.server.server_port}/redirected" + p = self.download("-hft", "private-token", ok=False) + self.assertIn("Refusing HTTPS downgrade", p.stderr) + self.assertTrue(any("/resolve/" in p for p, _ in server.requests)) + self.assertEqual(self.server.requests, []) + self.assertFalse((self.repo / "blobs" / OID).exists()) + self.assertFalse((self.repo / "refs/main").exists()) def tearDown(self): self.server.shutdown() diff --git a/tests/hf_tls_fixture.cpp b/tests/hf_tls_fixture.cpp new file mode 100644 index 0000000..2b68a2c --- /dev/null +++ b/tests/hf_tls_fixture.cpp @@ -0,0 +1,85 @@ +// Generate short-lived TLS fixtures with the same OpenSSL used by the downloader. +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { +void require(bool ok) { + if (!ok) throw std::runtime_error("Cannot generate HTTPS test certificates"); +} +using Key = std::unique_ptr; +using Cert = std::unique_ptr; +Key key() { + std::unique_ptr ctx( + EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, nullptr), EVP_PKEY_CTX_free); + require(bool(ctx)); + require(EVP_PKEY_keygen_init(ctx.get()) == 1); + require(EVP_PKEY_CTX_set_rsa_keygen_bits(ctx.get(), 2048) == 1); + EVP_PKEY *result = nullptr; + require(EVP_PKEY_keygen(ctx.get(), &result) == 1); + return Key(result, EVP_PKEY_free); +} +void extension(X509 *cert, X509 *issuer, int nid, const char *value) { + X509V3_CTX ctx; + X509V3_set_ctx(&ctx, issuer, cert, nullptr, nullptr, 0); + std::unique_ptr ext( + X509V3_EXT_conf_nid(nullptr, &ctx, nid, value), X509_EXTENSION_free); + require(bool(ext)); + require(X509_add_ext(cert, ext.get(), -1) == 1); +} +Cert certificate(EVP_PKEY *key, X509 *issuer, EVP_PKEY *issuer_key) { + Cert cert(X509_new(), X509_free); + require(bool(cert)); + require(X509_set_version(cert.get(), 2) == 1); + require(ASN1_INTEGER_set(X509_get_serialNumber(cert.get()), issuer ? 2 : 1) == 1); + require(X509_gmtime_adj(X509_getm_notBefore(cert.get()), -3600) != nullptr); + require(X509_gmtime_adj(X509_getm_notAfter(cert.get()), 86400) != nullptr); + require(X509_set_pubkey(cert.get(), key) == 1); + auto *name = X509_get_subject_name(cert.get()); + require(X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, + reinterpret_cast(issuer ? "HF test server" : "HF test CA"), + -1, -1, 0) == 1); + require(X509_set_issuer_name(cert.get(), issuer ? X509_get_subject_name(issuer) : name) == 1); + extension(cert.get(), issuer ? issuer : cert.get(), NID_basic_constraints, + issuer ? "critical,CA:FALSE" : "critical,CA:TRUE"); + extension(cert.get(), issuer ? issuer : cert.get(), NID_key_usage, + issuer ? "critical,digitalSignature,keyEncipherment" : "critical,keyCertSign,cRLSign"); + if (issuer) { + extension(cert.get(), issuer, NID_ext_key_usage, "serverAuth"); + extension(cert.get(), issuer, NID_subject_alt_name, "IP:127.0.0.1"); + } + require(X509_sign(cert.get(), issuer_key ? issuer_key : key, EVP_sha256()) > 0); + return cert; +} +void write(const std::filesystem::path &path, X509 *cert, EVP_PKEY *key = nullptr) { + const auto close = [](FILE *file) { std::fclose(file); }; + std::unique_ptr file(std::fopen(path.string().c_str(), "wb"), close); + require(bool(file)); + require((key ? PEM_write_PrivateKey(file.get(), key, nullptr, nullptr, 0, nullptr, nullptr) + : PEM_write_X509(file.get(), cert)) == 1); +} +} // namespace + +int main(int argc, char **argv) { + try { + if (argc != 2) throw std::runtime_error("Expected fixture directory"); + const std::filesystem::path directory(argv[1]); + auto ca_key = key(); + auto ca = certificate(ca_key.get(), nullptr, nullptr); + auto server_key = key(); + auto server = certificate(server_key.get(), ca.get(), ca_key.get()); + write(directory / "ca.pem", ca.get()); + write(directory / "server.pem", server.get()); + write(directory / "server-key.pem", nullptr, server_key.get()); + return 0; + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From 80b0b4baf38d4153f5c66e53275493b7670362a0 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sun, 25 Oct 2026 05:44:16 +0800 Subject: [PATCH 3/3] test(hf): expose TLS fixture setup diagnostics --- tests/hf_download.py | 3 ++- tests/hf_tls_fixture.cpp | 50 +++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/tests/hf_download.py b/tests/hf_download.py index 2b207f6..0ad4845 100644 --- a/tests/hf_download.py +++ b/tests/hf_download.py @@ -111,7 +111,8 @@ class Download(unittest.TestCase): def setUpClass(cls): cls.tls = tempfile.TemporaryDirectory(dir=Path(tempfile.gettempdir()).resolve()) cls.addClassCleanup(cls.tls.cleanup) - subprocess.run([TLS_FIXTURE, cls.tls.name], check=True, capture_output=True) + # Inherit output so CTest reports the helper's diagnostics on setup failure. + subprocess.run([TLS_FIXTURE, cls.tls.name], check=True) def setUp(self): self.tmp = tempfile.TemporaryDirectory(dir=Path(tempfile.gettempdir()).resolve()) diff --git a/tests/hf_tls_fixture.cpp b/tests/hf_tls_fixture.cpp index 2b68a2c..06495cb 100644 --- a/tests/hf_tls_fixture.cpp +++ b/tests/hf_tls_fixture.cpp @@ -1,51 +1,56 @@ // Generate short-lived TLS fixtures with the same OpenSSL used by the downloader. #include +#include #include #include #include +#include +#include #include #include #include #include +#include namespace { -void require(bool ok) { - if (!ok) throw std::runtime_error("Cannot generate HTTPS test certificates"); +void require(bool ok, const char *operation) { + if (!ok) throw std::runtime_error(std::string("HTTPS certificate fixture failed: ") + operation); } using Key = std::unique_ptr; using Cert = std::unique_ptr; Key key() { std::unique_ptr ctx( EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, nullptr), EVP_PKEY_CTX_free); - require(bool(ctx)); - require(EVP_PKEY_keygen_init(ctx.get()) == 1); - require(EVP_PKEY_CTX_set_rsa_keygen_bits(ctx.get(), 2048) == 1); + require(bool(ctx), "EVP_PKEY_CTX_new_id"); + require(EVP_PKEY_keygen_init(ctx.get()) == 1, "EVP_PKEY_keygen_init"); + require(EVP_PKEY_CTX_set_rsa_keygen_bits(ctx.get(), 2048) == 1, "EVP_PKEY_CTX_set_rsa_keygen_bits"); EVP_PKEY *result = nullptr; - require(EVP_PKEY_keygen(ctx.get(), &result) == 1); + require(EVP_PKEY_keygen(ctx.get(), &result) == 1, "EVP_PKEY_keygen"); return Key(result, EVP_PKEY_free); } void extension(X509 *cert, X509 *issuer, int nid, const char *value) { + std::cerr << "Adding extension " << OBJ_nid2sn(nid) << ": " << value << '\n'; X509V3_CTX ctx; X509V3_set_ctx(&ctx, issuer, cert, nullptr, nullptr, 0); std::unique_ptr ext( X509V3_EXT_conf_nid(nullptr, &ctx, nid, value), X509_EXTENSION_free); - require(bool(ext)); - require(X509_add_ext(cert, ext.get(), -1) == 1); + require(bool(ext), "X509V3_EXT_conf_nid"); + require(X509_add_ext(cert, ext.get(), -1) == 1, "X509_add_ext"); } Cert certificate(EVP_PKEY *key, X509 *issuer, EVP_PKEY *issuer_key) { Cert cert(X509_new(), X509_free); - require(bool(cert)); - require(X509_set_version(cert.get(), 2) == 1); - require(ASN1_INTEGER_set(X509_get_serialNumber(cert.get()), issuer ? 2 : 1) == 1); - require(X509_gmtime_adj(X509_getm_notBefore(cert.get()), -3600) != nullptr); - require(X509_gmtime_adj(X509_getm_notAfter(cert.get()), 86400) != nullptr); - require(X509_set_pubkey(cert.get(), key) == 1); + require(bool(cert), "X509_new"); + require(X509_set_version(cert.get(), 2) == 1, "X509_set_version"); + require(ASN1_INTEGER_set(X509_get_serialNumber(cert.get()), issuer ? 2 : 1) == 1, "ASN1_INTEGER_set"); + require(X509_gmtime_adj(X509_getm_notBefore(cert.get()), -3600) != nullptr, "X509_gmtime_adj(notBefore)"); + require(X509_gmtime_adj(X509_getm_notAfter(cert.get()), 86400) != nullptr, "X509_gmtime_adj(notAfter)"); + require(X509_set_pubkey(cert.get(), key) == 1, "X509_set_pubkey"); auto *name = X509_get_subject_name(cert.get()); require(X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, reinterpret_cast(issuer ? "HF test server" : "HF test CA"), - -1, -1, 0) == 1); - require(X509_set_issuer_name(cert.get(), issuer ? X509_get_subject_name(issuer) : name) == 1); + -1, -1, 0) == 1, "X509_NAME_add_entry_by_txt"); + require(X509_set_issuer_name(cert.get(), issuer ? X509_get_subject_name(issuer) : name) == 1, "X509_set_issuer_name"); extension(cert.get(), issuer ? issuer : cert.get(), NID_basic_constraints, issuer ? "critical,CA:FALSE" : "critical,CA:TRUE"); extension(cert.get(), issuer ? issuer : cert.get(), NID_key_usage, @@ -54,15 +59,16 @@ Cert certificate(EVP_PKEY *key, X509 *issuer, EVP_PKEY *issuer_key) { extension(cert.get(), issuer, NID_ext_key_usage, "serverAuth"); extension(cert.get(), issuer, NID_subject_alt_name, "IP:127.0.0.1"); } - require(X509_sign(cert.get(), issuer_key ? issuer_key : key, EVP_sha256()) > 0); + require(X509_sign(cert.get(), issuer_key ? issuer_key : key, EVP_sha256()) > 0, "X509_sign"); return cert; } void write(const std::filesystem::path &path, X509 *cert, EVP_PKEY *key = nullptr) { + std::cerr << "Writing " << path.string() << '\n'; const auto close = [](FILE *file) { std::fclose(file); }; std::unique_ptr file(std::fopen(path.string().c_str(), "wb"), close); - require(bool(file)); + if (!file) throw std::runtime_error("fopen " + path.string() + ": " + std::strerror(errno)); require((key ? PEM_write_PrivateKey(file.get(), key, nullptr, nullptr, 0, nullptr, nullptr) - : PEM_write_X509(file.get(), cert)) == 1); + : PEM_write_X509(file.get(), cert)) == 1, key ? "PEM_write_PrivateKey" : "PEM_write_X509"); } } // namespace @@ -70,9 +76,14 @@ int main(int argc, char **argv) { try { if (argc != 2) throw std::runtime_error("Expected fixture directory"); const std::filesystem::path directory(argv[1]); + std::cerr << "TLS fixture: " << OpenSSL_version(OPENSSL_VERSION) << '\n'; + std::cerr << "Generating CA key\n"; auto ca_key = key(); + std::cerr << "Generating CA certificate\n"; auto ca = certificate(ca_key.get(), nullptr, nullptr); + std::cerr << "Generating server key\n"; auto server_key = key(); + std::cerr << "Generating server certificate\n"; auto server = certificate(server_key.get(), ca.get(), ca_key.get()); write(directory / "ca.pem", ca.get()); write(directory / "server.pem", server.get()); @@ -80,6 +91,7 @@ int main(int argc, char **argv) { return 0; } catch (const std::exception &error) { std::cerr << error.what() << '\n'; + ERR_print_errors_fp(stderr); return 1; } }