実験: 固定block候補のallocation呼び出しを比較

.github/workflows/native-dev.yml

@@ -87,10 +87,14 @@ jobs:
         run: cmake --build --preset release --target uchardet-fixed-block uchardet-fixed-block-test uchardet-fixed-block-timing --parallel 2
       - name: Fixed-block contract comparisons
         run: timeout 10s build/release/benchmark/uchardet-fixed-block-test
+      - name: Build non-default allocation observers
+        run: cmake --build --preset release --target uchardet-fixed-block-allocations uchardet-fixed-block-allocations-plain --parallel 2
       - name: Tool tests
         env:
           UCHARDET_FIXED_BLOCK: build/release/benchmark/uchardet-fixed-block
           UCHARDET_FIXED_BLOCK_TIMING: build/release/benchmark/uchardet-fixed-block-timing
+          UCHARDET_FIXED_BLOCK_ALLOCATIONS: build/release/benchmark/uchardet-fixed-block-allocations
+          UCHARDET_FIXED_BLOCK_ALLOCATIONS_PLAIN: build/release/benchmark/uchardet-fixed-block-allocations-plain
           UCHARDET_TRACE: build/release/benchmark/uchardet-trace
           UCHARDET_FILTER_PROFILE: build/release/benchmark/uchardet-filter-profile
         run: uv run --no-project --python 3.11 python -m unittest discover -s benchmark -p 'test_*.py'

benchmark/CMakeLists.txt

@@ -17,6 +17,20 @@ target_link_libraries(uchardet-fixed-block ${UCHARDET_LIBRARY})
 add_executable(uchardet-fixed-block-timing EXCLUDE_FROM_ALL fixed-block-timing.cpp)
 target_link_libraries(uchardet-fixed-block-timing ${UCHARDET_LIBRARY})
 
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SIZEOF_VOID_P EQUAL 8
+   AND NOT BUILD_SHARED_LIBS AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+  add_executable(uchardet-fixed-block-allocations-plain EXCLUDE_FROM_ALL fixed-block-allocations.cpp)
+  target_link_libraries(uchardet-fixed-block-allocations-plain ${UCHARDET_LIBRARY})
+  add_executable(uchardet-fixed-block-allocations EXCLUDE_FROM_ALL
+    fixed-block-allocations.cpp ../models/experimental/allocation-hooks.cpp)
+  target_compile_definitions(uchardet-fixed-block-allocations PRIVATE UCHARDET_ALLOCATION_PROBE=1)
+  target_compile_options(uchardet-fixed-block-allocations PRIVATE -fno-lto)
+  target_link_libraries(uchardet-fixed-block-allocations ${UCHARDET_LIBRARY})
+  foreach(symbol malloc calloc realloc free _Znwm _Znam _ZdlPv _ZdaPv)
+    target_link_options(uchardet-fixed-block-allocations PRIVATE "-Wl,--wrap=${symbol}")
+  endforeach()
+endif()
+
 if(TARGET libuchardet_experimental)
   add_executable(uchardet-conformance-experimental EXCLUDE_FROM_ALL uchardet-conformance.cpp)
   target_compile_definitions(uchardet-conformance-experimental PRIVATE UCHARDET_EXPERIMENTAL_INPUT_LIMIT=4096)

benchmark/fixed-block-allocations.cpp

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: MIT
+// Selected static-link call sites, NOT a heap profiler or failure-injection test.
+#include "fixed-block-detector.h"
+#include "../models/experimental/allocation-hooks.hpp"
+#include <cstdint>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+static void begin_count() {
+#ifdef UCHARDET_ALLOCATION_PROBE
+  allocation_calls = {};
+  allocation_tracking = true;
+#endif
+}
+static AllocationCalls end_count() {
+#ifdef UCHARDET_ALLOCATION_PROBE
+  allocation_tracking = false;
+  return allocation_calls;
+#else
+  return {};
+#endif
+}
+static void print_calls(const AllocationCalls& c) {
+  std::cout << "{\"malloc\":" << c.malloc_calls << ",\"calloc\":" << c.calloc_calls
+            << ",\"realloc\":" << c.realloc_calls << ",\"free\":" << c.free_calls
+            << ",\"new\":" << c.new_calls << ",\"new_array\":" << c.new_array_calls
+            << ",\"delete\":" << c.delete_calls << ",\"delete_array\":" << c.delete_array_calls << '}';
+}
+
+// Hex strings make the complete ordered candidate snapshot JSON-safe without
+// allocating anything inside a counted interval.
+static std::string snapshot(uchardet_t handle) {
+  std::ostringstream out;
+  out << uchardet_is_done(handle) << ':' << uchardet_get_n_candidates(handle);
+  for (size_t i = 0; i < uchardet_get_n_candidates(handle); ++i) {
+    for (const char* value : {uchardet_get_encoding(handle, i), uchardet_get_language(handle, i)}) {
+      out << (value ? ":present:" : ":null:");
+      if (value) for (const unsigned char* p = reinterpret_cast<const unsigned char*>(value); *p; ++p)
+        out << std::hex << std::setw(2) << std::setfill('0') << static_cast<unsigned>(*p);
+    }
+    const float confidence = uchardet_get_confidence(handle, i);
+    uint32_t bits;
+    static_assert(sizeof(bits) == sizeof(confidence), "binary32 required");
+    std::memcpy(&bits, &confidence, sizeof(bits));
+    out << ':' << std::hex << std::setw(8) << std::setfill('0') << bits;
+  }
+  return out.str();
+}
+
+class Direct {
+ public:
+  Direct(size_t block, size_t) : detector_(uchardet_new(), &uchardet_delete), block_(block) {
+    if (!detector_) throw std::runtime_error("allocation failed");
+  }
+  void reset() { uchardet_reset(detector_.get()); }
+  void feed(const char* data, size_t length) {
+    for (size_t offset = 0; offset < length && !uchardet_is_done(detector_.get()); offset += block_)
+      if (uchardet_handle_data(detector_.get(), data + offset, std::min(block_, length - offset)))
+        throw std::runtime_error("feed failed");
+  }
+  void finish() { uchardet_data_end(detector_.get()); }
+  uchardet_t handle() const { return detector_.get(); }
+ private:
+  std::unique_ptr<struct uchardet, decltype(&uchardet_delete)> detector_;
+  size_t block_;
+};
+
+template<class Detector>
+static void observe(const char* name, size_t block, size_t chunk, const std::vector<char>& bytes) {
+  AllocationCalls construction, first, warm, destruction;
+  std::string first_snapshot, warm_snapshot;
+  const size_t step = chunk ? chunk : std::max(size_t(1), bytes.size());
+  const auto process = [&](Detector& detector) {
+    detector.reset();
+    for (size_t offset = 0; offset < bytes.size(); offset += step)
+      detector.feed(bytes.data() + offset, std::min(step, bytes.size() - offset));
+    detector.finish();
+  };
+  begin_count();
+  {
+    Detector detector(block, 4096);
+    construction = end_count();
+    begin_count();
+    process(detector);
+    first = end_count();
+    first_snapshot = snapshot(detector.handle());
+    for (size_t i = 0; i < 16; ++i) process(detector);
+    begin_count();
+    process(detector);
+    warm = end_count();
+    warm_snapshot = snapshot(detector.handle());
+    if (warm_snapshot != first_snapshot) throw std::runtime_error("reuse changed candidates");
+    begin_count();
+  }
+  destruction = end_count();
+  std::cout << '"' << name << "\":{\"construction\":";
+  print_calls(construction);
+  std::cout << ",\"first_document\":"; print_calls(first);
+  std::cout << ",\"warm_document\":"; print_calls(warm);
+  std::cout << ",\"destruction\":"; print_calls(destruction);
+  std::cout << ",\"snapshot\":\"" << warm_snapshot << "\"}";
+}
+
+int main(int argc, char** argv) {
+  try {
+    if (argc != 2) throw std::invalid_argument("usage: fixed-block-allocations FILE");
+#ifdef UCHARDET_ALLOCATION_PROBE
+    allocation_self_test();
+#endif
+    std::ifstream input(argv[1], std::ios::binary);
+    if (!input) throw std::runtime_error("cannot open input");
+    std::vector<char> bytes(4097);
+    input.read(bytes.data(), static_cast<std::streamsize>(bytes.size()));
+    bytes.resize(static_cast<size_t>(input.gcount()));
+    if (input.bad() || bytes.size() > 4096) throw std::runtime_error("input exceeds bounded pilot");
+    std::cout << "{\"byte_length\":" << bytes.size() << ",\"block\":1024,\"modes\":{";
+    observe<Direct>("whole", 4096, 0, bytes);
+    std::cout << ',';
+    observe<Direct>("direct_fixed", 1024, 0, bytes);
+    std::cout << ',';
+    observe<experimental::FixedBlockDetector>("adapter_whole", 1024, 0, bytes);
+    std::cout << ',';
+    observe<experimental::FixedBlockDetector>("adapter_byte", 1024, 1, bytes);
+    std::cout << "}}\n";
+    return 0;
+  } catch (const std::exception& error) {
+    end_count();
+    std::cerr << error.what() << '\n';
+    return 1;
+  }
+}

benchmark/fixed-block-allocations.ja.md

@@ -0,0 +1,55 @@
+<!-- SPDX-License-Identifier: MIT -->
+# 固定block試作のallocation API呼び出し
+
+1024-byte候補を、静的リンク時に解決される8経路で観測する。
+既存MITの`models/experimental/allocation-hooks.cpp`を再利用し、自己検証の成功後に
+構築・初回文書・warm文書・破棄を分けて数える。時間計測とは併用しない。
+
+対象: malloc/calloc/realloc/free、throwing・unalignedのscalar/array new/delete。
+対象外: shared library内部、strdup内部、aligned/nothrow/sized/custom allocator、mmap。
+counterはsingle-thread診断用。physical allocation数・要求byte数・live/peak memoryを
+報告するtoolではない。freeが非zeroでmallocがzeroでも、確保がなかったとは解釈しない。
+
+## 条件と結果（2026-09-22）
+
+- 64-bit Linux、GCC16.2.1、C++11/Release、非LTOのstatic library。
+- 固定tuningの16入力（cp1252/UTF-8各8）、各4096 bytes以下、process timeout10秒。
+- 旧whole、直接1024-byte feed、adapter外部whole、adapter外部1-byteを比較。
+- 初回後に16回処理し、次の1回のreset/feed/finalizeをwarm区間として観測。
+- 入力I/Oとsnapshotの文字列構築は計測区間外。
+
+全16入力で:
+
+- adapterの構築は直接fixedに対し`new`が1回増加（1→2）。buffer確保と対応する。
+- 初回文書とwarm文書の対象8 counterは直接fixedとadapterで一致。
+- adapterの外部wholeと1-byteは、全counterと候補snapshotが一致。
+- adapter破棄の`delete`は直接fixedより1回多い。
+- 計測あり/なしで全方式の候補数・順序・encoding・language null区別・confidence bits・doneが一致。
+- 直接fixedとadapterのsnapshotは、保存済み1024-byte canonical観測とも一致。
+- 同じreportへの再実行は全byte一致。
+
+例: 1708-byte cp1252文書ではwarm中malloc/newが0、freeが56だった。
+候補のstrdupによる確保をこのhookは数えないため、「warm中allocationなし」とは主張しない。
+初期構築+1回だけからmemory gateの10%条件を判断することもできない。
+
+## 再現
+
+```sh
+cmake --build /tmp/uchardet-fixed-block-build --target \
+  uchardet-fixed-block-allocations uchardet-fixed-block-allocations-plain
+uv run --no-project python benchmark/fixed_block_allocations.py \
+  /workspace/archives/v3-corpus/paris-training-tuning-generated-v1/manifest.json \
+  /workspace/archives/v3-corpus/fixed-block-tuning-v3.json \
+  /tmp/uchardet-fixed-block-build/benchmark/uchardet-fixed-block-allocations-plain \
+  /tmp/uchardet-fixed-block-build/benchmark/uchardet-fixed-block-allocations \
+  /workspace/archives/v3-corpus/fixed-block-allocations-v2.json
+```
+
+targetはLinux/64-bit/GCCまたはClang/static構成でのみ作成し、EXCLUDE_FROM_ALL、installなし。
+`/workspace`は実際の保存先に置き換える。独立holdoutは未開封。
+report content hash: `f1e073ce9e72d360435be5986af752c18e4f01d48b0edd29d1fc90ccceb22dad`。
+binary・driver・入力hashと全counterを保存している。
+追加3 tests成功、ローカルbenchmark suiteは41成功・5 skip。
+
+初回/peak/live memory、エラー/OOM、追加fuzzの検証ではない。P01を再開しない。
+既定API/model/engineは変更せず、標準採用は引き続き保留する。

benchmark/fixed_block_allocations.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: MIT
+"""Bounded call-site counts, with uninstrumented and frozen-output comparisons."""
+
+import argparse
+import json
+import subprocess
+from pathlib import Path
+
+from fixed_block_compare import canonical, content_hash, manifest_hash, sha, write_idempotent
+from fixed_block_timing import FROZEN
+
+
+def signature(record):
+    value = f"{int(record['final_done'])}:{record['candidate_count']}"
+    for candidate in record["candidates"]:
+        for field in ("encoding", "language"):
+            text = candidate[field]
+            value += ":null:" if text is None else ":present:" + text.encode().hex()
+        value += ":" + candidate["confidence_bits"]
+    return value
+
+
+def run(manifest_path, previous_path, plain, counted):
+    manifest = json.loads(manifest_path.read_text())
+    previous = json.loads(previous_path.read_text())
+    if content_hash(previous) != FROZEN or previous.get("content_hash") != FROZEN:
+        raise ValueError("requires frozen tuning pilot")
+    if manifest_hash(manifest) != previous["manifest_hash"]:
+        raise ValueError("manifest changed")
+    samples = {sample["id"]: sample for sample in manifest["samples"]}
+    binaries = {"plain": sha(plain), "counted": sha(counted)}
+    documents = []
+    root = manifest_path.parent.resolve()
+    stages = ("construction", "first_document", "warm_document", "destruction")
+    for old in previous["documents"]:
+        sample = samples[old["sample_id"]]
+        path = (root / sample["path"]).resolve()
+        if sample["split"] != "tuning" or not path.is_relative_to(root):
+            raise ValueError("requires tuning input inside corpus root")
+        if path.stat().st_size > 4096 or sha(path) != old["sha256"]:
+            raise ValueError("input changed or exceeds pilot")
+        observations = {}
+        for name, binary in (("plain", plain), ("counted", counted)):
+            result = subprocess.run(
+                [str(binary), str(path)], check=True, capture_output=True, text=True, timeout=10
+            )
+            observations[name] = json.loads(result.stdout)
+        reference, observed = observations["plain"], observations["counted"]
+        if (
+            observed["byte_length"] != old["byte_length"]
+            or reference["byte_length"] != old["byte_length"]
+        ):
+            raise ValueError("input length mismatch")
+        for mode in observed["modes"]:
+            if observed["modes"][mode]["snapshot"] != reference["modes"][mode]["snapshot"]:
+                raise ValueError("instrumentation changed candidates")
+            if any(value for stage in stages for value in reference["modes"][mode][stage].values()):
+                raise ValueError("plain build unexpectedly counts allocations")
+        expected = signature(old["blocks"]["1024"]["observations"]["0"])
+        for mode in ("direct_fixed", "adapter_whole", "adapter_byte"):
+            if observed["modes"][mode]["snapshot"] != expected:
+                raise ValueError("result differs from frozen canonical feed")
+        documents.append(dict(sample_id=sample["id"], sha256=old["sha256"], observation=observed))
+    if binaries != {"plain": sha(plain), "counted": sha(counted)}:
+        raise ValueError("binary changed during observation")
+    result = dict(
+        schema="fixed-block-allocation-calls-v1",
+        pilot_hash=FROZEN,
+        binaries=binaries,
+        driver_hash=sha(Path(__file__)),
+        documents=documents,
+        metric="selected static-link API call counts, not physical allocations or memory",
+        excluded=(
+            "shared-library internals (including strdup), aligned/nothrow/sized allocation, mmap"
+        ),
+        self_test="eight paths checked by counted executable before input processing",
+    )
+    result["content_hash"] = content_hash(result)
+    return result
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ("manifest", "previous", "plain", "counted", "output"):
+        parser.add_argument(name, type=Path)
+    args = parser.parse_args()
+    result = run(args.manifest, args.previous, args.plain.resolve(), args.counted.resolve())
+    write_idempotent(args.output, canonical(result))
+    print(result["content_hash"])

benchmark/test_fixed_block_allocations.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: MIT
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from fixed_block_allocations import signature
+
+
+class AllocationSignatureTests(unittest.TestCase):
+    def test_language_presence_and_confidence_bits_are_preserved(self):
+        record = dict(
+            final_done=False,
+            candidate_count=1,
+            candidates=[dict(encoding="ASCII", language=None, confidence_bits="00000000")],
+        )
+        original = signature(record)
+        record["candidates"][0]["language"] = ""
+        self.assertNotEqual(signature(record), original)
+        record["candidates"][0]["language"] = None
+        record["candidates"][0]["confidence_bits"] = "80000000"
+        self.assertNotEqual(signature(record), original)
+
+
+@unittest.skipUnless(
+    os.environ.get("UCHARDET_FIXED_BLOCK_ALLOCATIONS")
+    and os.environ.get("UCHARDET_FIXED_BLOCK_ALLOCATIONS_PLAIN"),
+    "set allocation binaries",
+)
+class AllocationNativeTests(unittest.TestCase):
+    def invoke(self, binary, data):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "input"
+            path.write_bytes(data)
+            return subprocess.run([binary, str(path)], capture_output=True, text=True, timeout=10)
+
+    def test_plain_and_instrumented_results_match(self):
+        for data in (b"", b"ASCII", "café et thé".encode("cp1252"), "日本語".encode()):
+            outputs = []
+            for name in (
+                "UCHARDET_FIXED_BLOCK_ALLOCATIONS_PLAIN",
+                "UCHARDET_FIXED_BLOCK_ALLOCATIONS",
+            ):
+                result = self.invoke(os.environ[name], data)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                outputs.append(json.loads(result.stdout))
+            plain, counted = outputs
+            for mode in counted["modes"]:
+                self.assertEqual(
+                    plain["modes"][mode]["snapshot"], counted["modes"][mode]["snapshot"]
+                )
+            for mode in ("adapter_whole", "adapter_byte"):
+                self.assertEqual(
+                    counted["modes"][mode]["snapshot"], counted["modes"]["direct_fixed"]["snapshot"]
+                )
+                self.assertEqual(
+                    counted["modes"][mode]["construction"]["new"],
+                    counted["modes"]["direct_fixed"]["construction"]["new"] + 1,
+                )
+
+    def test_input_cap(self):
+        result = self.invoke(os.environ["UCHARDET_FIXED_BLOCK_ALLOCATIONS"], b"a" * 4097)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(result.stdout, "")
+
+
+if __name__ == "__main__":
+    unittest.main()