lib: cmetrics: upgrade to v2.3.1

lib/cmetrics/.github/workflows/build.yaml

@@ -33,7 +33,7 @@ jobs:
           .\scripts\win_build.bat
       - name: Run unit tests.
         run: |
-          ctest --rerun-failed --output-on-failure -C Debug --test-dir .\tests\
+          ctest --test-dir build --output-on-failure --no-tests=error
 
   build-centos:
     name: CentOS 7 build to confirm no issues once used downstream
@@ -116,7 +116,7 @@ jobs:
           submodules: true
 
       - name: Build on ${{ matrix.os }} with ${{ matrix.compiler }}
-        uses: uraimo/run-on-arch-action@v3.2.0
+        uses: uraimo/run-on-arch-action@v3.2.1
         with:
           arch: aarch64
           distro: ubuntu_latest

lib/cmetrics/.github/workflows/packages.yaml

@@ -22,7 +22,7 @@ jobs:
         with:
           submodules: true
 
-      - uses: uraimo/run-on-arch-action@v3.2.0
+      - uses: uraimo/run-on-arch-action@v3.2.1
         name: Build the ${{matrix.format}} packages
         with:
           arch: aarch64

lib/cmetrics/CMakeLists.txt

@@ -5,8 +5,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # CMetrics Version
 set(CMT_VERSION_MAJOR  2)
-set(CMT_VERSION_MINOR  2)
-set(CMT_VERSION_PATCH  5)
+set(CMT_VERSION_MINOR  3)
+set(CMT_VERSION_PATCH  1)
 set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}")
 
 # Include helpers

lib/cmetrics/benchmarks/README.md

@@ -22,7 +22,7 @@ REPETITIONS=5 benchmarks/run-perf.sh \
 The executable also accepts individual workloads:
 
 ```text
-cmt-benchmark lookup|update|prometheus|opentelemetry|opentelemetry-mixed CARDINALITY OPERATIONS
+cmt-benchmark cat|lookup|update|prometheus|opentelemetry|opentelemetry-mixed CARDINALITY OPERATIONS
 ```
 
 The `opentelemetry` workload repeatedly encodes a labeled counter with the
@@ -36,3 +36,11 @@ Use the reported in-process `elapsed_ns` for the operation itself and `perf
 stat` for whole-process hardware counters. Whole-process counters include
 series construction and teardown by design, exposing setup complexity as well
 as steady-state behavior.
+
+The `cat` workload builds OPERATIONS contexts with CARDINALITY distinct series
+per context, then times concatenating them into one context. Source construction,
+verification of every merged value, and teardown are outside `elapsed_ns`.
+For the textfile workload, run `cmt-benchmark cat 2000 50` (100,000 series).
+On Windows, build with `cmake --build build-perf --config Release` and use
+`build-perf/benchmarks/Release/cmt-benchmark.exe`; timing uses
+`QueryPerformanceCounter`. Linux `perf stat` counters are not available there.

lib/cmetrics/benchmarks/benchmark.c

@@ -7,18 +7,32 @@
 
 #include <cmetrics/cmetrics.h>
 #include <cmetrics/cmt_counter.h>
+#include <cmetrics/cmt_cat.h>
+#include <cmetrics/cmt_map.h>
+#include <cmetrics/cmt_compat.h>
 #include <cmetrics/cmt_encode_opentelemetry.h>
 #include <cmetrics/cmt_encode_prometheus.h>
 #include <cmetrics/cmt_gauge.h>
 #include <cmetrics/cmt_histogram.h>
 
 static uint64_t monotonic_ns(void)
 {
+#ifdef _WIN32
+    LARGE_INTEGER counter;
+    LARGE_INTEGER frequency;
+
+    QueryPerformanceCounter(&counter);
+    QueryPerformanceFrequency(&frequency);
+    return (uint64_t) (counter.QuadPart / frequency.QuadPart) * 1000000000ULL +
+           (uint64_t) (counter.QuadPart % frequency.QuadPart) * 1000000000ULL /
+           frequency.QuadPart;
+#else
     struct timespec timestamp;
 
     clock_gettime(CLOCK_MONOTONIC, &timestamp);
 
     return (uint64_t) timestamp.tv_sec * 1000000000ULL + timestamp.tv_nsec;
+#endif
 }
 
 static size_t parse_size(const char *text, const char *name)
@@ -303,13 +317,98 @@ static int benchmark_opentelemetry_mixed(size_t cardinality, size_t operations)
     return 0;
 }
 
+static int benchmark_cat(size_t cardinality, size_t operations)
+{
+    size_t source_index;
+    size_t series_index;
+    size_t total;
+    int result = -1;
+    uint64_t start;
+    uint64_t elapsed;
+    double value;
+    char source_label[32];
+    char series_label[32];
+    char *keys[] = {"source", "series"};
+    char *values[] = {source_label, series_label};
+    struct cmt **sources;
+    struct cmt *destination;
+    struct cmt_counter *counter;
+
+    if (operations > SIZE_MAX / cardinality) {
+        return -1;
+    }
+    total = operations * cardinality;
+    sources = calloc(operations, sizeof(*sources));
+    destination = cmt_create();
+    if (sources == NULL || destination == NULL) {
+        free(sources);
+        if (destination != NULL) {
+            cmt_destroy(destination);
+        }
+        return -1;
+    }
+    for (source_index = 0; source_index < operations; source_index++) {
+        sources[source_index] = cmt_create();
+        if (sources[source_index] == NULL) {
+            goto cleanup;
+        }
+        counter = cmt_counter_create(sources[source_index], "bench", "", "counter",
+                                     "benchmark", 2, keys);
+        if (counter == NULL) {
+            goto cleanup;
+        }
+        snprintf(source_label, sizeof(source_label), "%zu", source_index);
+        for (series_index = 0; series_index < cardinality; series_index++) {
+            snprintf(series_label, sizeof(series_label), "%zu", series_index);
+            if (cmt_counter_set(counter, 1, (double) series_index, 2, values) != 0) {
+                goto cleanup;
+            }
+        }
+    }
+    start = monotonic_ns();
+    for (source_index = 0; source_index < operations; source_index++) {
+        if (cmt_cat(destination, sources[source_index]) != 0) {
+            goto cleanup;
+        }
+    }
+    elapsed = monotonic_ns() - start;
+    counter = cfl_list_entry(destination->counters.next, struct cmt_counter, _head);
+    if ((size_t) cfl_list_size(&counter->map->metrics) != total) {
+        goto cleanup;
+    }
+    for (source_index = 0; source_index < operations; source_index++) {
+        snprintf(source_label, sizeof(source_label), "%zu", source_index);
+        for (series_index = 0; series_index < cardinality; series_index++) {
+            snprintf(series_label, sizeof(series_label), "%zu", series_index);
+            if (cmt_counter_get_val(counter, 2, values, &value) != 0 ||
+                value != (double) series_index) {
+                goto cleanup;
+            }
+        }
+    }
+    printf("benchmark=cat cardinality=%zu operations=%zu series=%zu elapsed_ns=%"
+           PRIu64 " ns_per_series=%.2f\n", cardinality, operations, total,
+           elapsed, (double) elapsed / total);
+    result = 0;
+
+cleanup:
+    for (source_index = 0; source_index < operations; source_index++) {
+        if (sources[source_index] != NULL) {
+            cmt_destroy(sources[source_index]);
+        }
+    }
+    free(sources);
+    cmt_destroy(destination);
+    return result;
+}
+
 int main(int argc, char **argv)
 {
     size_t cardinality;
     size_t operations;
 
     if (argc != 4) {
-        fprintf(stderr, "usage: %s lookup|update|prometheus|opentelemetry|"
+        fprintf(stderr, "usage: %s cat|lookup|update|prometheus|opentelemetry|"
                         "opentelemetry-mixed "
                         "CARDINALITY OPERATIONS\n",
                 argv[0]);
@@ -320,6 +419,9 @@ int main(int argc, char **argv)
     operations = parse_size(argv[3], "operations");
     cmt_initialize();
 
+    if (strcmp(argv[1], "cat") == 0) {
+        return benchmark_cat(cardinality, operations) == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+    }
     if (strcmp(argv[1], "lookup") == 0) {
         return benchmark_lookup(cardinality, operations) == 0 ?
                EXIT_SUCCESS : EXIT_FAILURE;

lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h

@@ -57,6 +57,8 @@ struct cmt_msgpack_decode_context {
     int                aggregation_type;
     int                metric_value_type_set;
     int                metric_section_set;
+    /* Appended to keep the established field layout. */
+    int                allow_reset;
 };
 
 int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size, 

lib/cmetrics/include/cmetrics/cmt_map.h

@@ -49,8 +49,11 @@ struct cmt_map {
     struct cfl_list *metric_buckets;
     size_t metric_bucket_count;
     size_t indexed_metric_count;
-    /* Most recently created metric; only changed with the metric list. */
+    /* End of the indexed prefix. Direct list insertions must append; removal
+     * must use cmt_map_metric_destroy so this cursor cannot dangle. */
     struct cmt_metric *last_metric;
+    /* Owned family metadata; independent of metric names and sibling scopes. */
+    struct cfl_kvlist *otlp_metadata;
 };
 
 struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts,

lib/cmetrics/include/cmetrics/cmt_metric.h

@@ -75,6 +75,9 @@ struct cmt_metric {
     int hash_indexed;
     struct cmt_map *map;
     struct cfl_list _hash_head;
+
+    /* Owned OTLP datapoint metadata; independent of string label identity. */
+    struct cfl_kvlist *otlp_metadata;
 };
 
 struct cmt_exp_histogram_snapshot {
@@ -136,6 +139,18 @@ void cmt_metric_hist_set(struct cmt_metric *metric, uint64_t timestamp,
 
 uint64_t cmt_metric_hist_get_value(struct cmt_metric *metric, int bucket_id);
 
+/* Convert cumulative native buckets without unsigned underflow. */
+static inline uint64_t cmt_metric_hist_get_bucket_count(struct cmt_metric *metric, int bucket_id)
+{
+    uint64_t current;
+    uint64_t previous;
+
+    current = cmt_metric_hist_get_value(metric, bucket_id);
+    previous = bucket_id == 0 ? 0 : cmt_metric_hist_get_value(metric, bucket_id - 1);
+    return current >= previous ? current - previous : 0;
+}
+
+
 double cmt_metric_hist_get_sum_value(struct cmt_metric *metric);
 
 uint64_t cmt_metric_hist_get_count_value(struct cmt_metric *metric);

lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h

@@ -0,0 +1,35 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  CMetrics
+ *  ========
+ *  Copyright 2021-2022 The CMetrics Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef CMT_OTLP_METADATA_H
+#define CMT_OTLP_METADATA_H
+
+#include <cmetrics/cmetrics.h>
+#include <cmetrics/cmt_map.h>
+#include <mpack/mpack.h>
+
+struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source);
+int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context);
+int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context);
+int cmt_otlp_metadata_restore(struct cmt *context);
+int cmt_otlp_histogram_restore(struct cmt *context);
+int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample);
+int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample);
+
+#endif

lib/cmetrics/include/cmetrics/cmt_variant_utils.h

@@ -398,10 +398,11 @@ static inline int unpack_cfl_kvlist_depth(mpack_reader_t *reader,
         key_value = NULL;
     }
 
-    mpack_done_map(reader);
-
-    if (mpack_reader_error(reader) != mpack_ok) {
-        result = -9;
+    if (result == 0) {
+        mpack_done_map(reader);
+        if (mpack_reader_error(reader) != mpack_ok) {
+            result = -9;
+        }
     }
 
     if (result != 0) {
@@ -477,6 +478,7 @@ static inline int unpack_cfl_variant_string(mpack_reader_t *reader,
     }
 
     (*value)->type = CFL_VARIANT_STRING;
+    (*value)->size = value_length;
 
     return 0;
 }
@@ -528,6 +530,7 @@ static inline int unpack_cfl_variant_binary(mpack_reader_t *reader,
     }
 
     (*value)->type = CFL_VARIANT_BYTES;
+    (*value)->size = value_length;
 
     return 0;
 }

lib/cmetrics/scripts/win_build.bat

@@ -1,6 +1,8 @@
+@echo off
 setlocal
 git submodule update --init --recursive
-cd build
-cmake -G "NMake Makefiles"  -DCMT_TESTS=On ..\
-cmake --build .
-endlocal
+if errorlevel 1 exit /b %errorlevel%
+cmake -S . -B build -G "NMake Makefiles" -DCMT_TESTS=On -DCMAKE_BUILD_TYPE=Debug
+if errorlevel 1 exit /b %errorlevel%
+cmake --build build
+endlocal & exit /b %errorlevel%

lib/cmetrics/src/CMakeLists.txt

@@ -15,6 +15,7 @@ set(src
   cmt_summary.c
   cmt_histogram.c
   cmt_exp_histogram.c
+  cmt_otlp_metadata.c
   cmt_metric.c
   cmt_metric_histogram.c
   cmt_map.c