diff --git a/lib/cmetrics/.github/workflows/build.yaml b/lib/cmetrics/.github/workflows/build.yaml index 222925961d3..10da0499a95 100644 --- a/lib/cmetrics/.github/workflows/build.yaml +++ b/lib/cmetrics/.github/workflows/build.yaml @@ -33,7 +33,7 @@ jobs: .\scripts\win_build.bat - name: Run unit tests. run: | - ctest --rerun-failed --output-on-failure -C Debug --test-dir .\tests\ + ctest --test-dir build --output-on-failure --no-tests=error build-centos: name: CentOS 7 build to confirm no issues once used downstream @@ -116,7 +116,7 @@ jobs: submodules: true - name: Build on ${{ matrix.os }} with ${{ matrix.compiler }} - uses: uraimo/run-on-arch-action@v3.2.0 + uses: uraimo/run-on-arch-action@v3.2.1 with: arch: aarch64 distro: ubuntu_latest diff --git a/lib/cmetrics/.github/workflows/packages.yaml b/lib/cmetrics/.github/workflows/packages.yaml index 462ed10b101..2dc146d8f3f 100644 --- a/lib/cmetrics/.github/workflows/packages.yaml +++ b/lib/cmetrics/.github/workflows/packages.yaml @@ -22,7 +22,7 @@ jobs: with: submodules: true - - uses: uraimo/run-on-arch-action@v3.2.0 + - uses: uraimo/run-on-arch-action@v3.2.1 name: Build the ${{matrix.format}} packages with: arch: aarch64 diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index 50f92c591c6..75a6236ed35 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -5,8 +5,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 2) -set(CMT_VERSION_MINOR 2) -set(CMT_VERSION_PATCH 5) +set(CMT_VERSION_MINOR 3) +set(CMT_VERSION_PATCH 1) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers diff --git a/lib/cmetrics/benchmarks/README.md b/lib/cmetrics/benchmarks/README.md index 1aa43c558e8..aee55326452 100644 --- a/lib/cmetrics/benchmarks/README.md +++ b/lib/cmetrics/benchmarks/README.md @@ -22,7 +22,7 @@ REPETITIONS=5 benchmarks/run-perf.sh \ The executable also accepts individual workloads: ```text -cmt-benchmark lookup|update|prometheus|opentelemetry|opentelemetry-mixed CARDINALITY OPERATIONS +cmt-benchmark cat|lookup|update|prometheus|opentelemetry|opentelemetry-mixed CARDINALITY OPERATIONS ``` The `opentelemetry` workload repeatedly encodes a labeled counter with the @@ -36,3 +36,11 @@ Use the reported in-process `elapsed_ns` for the operation itself and `perf stat` for whole-process hardware counters. Whole-process counters include series construction and teardown by design, exposing setup complexity as well as steady-state behavior. + +The `cat` workload builds OPERATIONS contexts with CARDINALITY distinct series +per context, then times concatenating them into one context. Source construction, +verification of every merged value, and teardown are outside `elapsed_ns`. +For the textfile workload, run `cmt-benchmark cat 2000 50` (100,000 series). +On Windows, build with `cmake --build build-perf --config Release` and use +`build-perf/benchmarks/Release/cmt-benchmark.exe`; timing uses +`QueryPerformanceCounter`. Linux `perf stat` counters are not available there. diff --git a/lib/cmetrics/benchmarks/benchmark.c b/lib/cmetrics/benchmarks/benchmark.c index 13a8661d18c..79a73eabf2c 100644 --- a/lib/cmetrics/benchmarks/benchmark.c +++ b/lib/cmetrics/benchmarks/benchmark.c @@ -7,6 +7,9 @@ #include #include +#include +#include +#include #include #include #include @@ -14,11 +17,22 @@ static uint64_t monotonic_ns(void) { +#ifdef _WIN32 + LARGE_INTEGER counter; + LARGE_INTEGER frequency; + + QueryPerformanceCounter(&counter); + QueryPerformanceFrequency(&frequency); + return (uint64_t) (counter.QuadPart / frequency.QuadPart) * 1000000000ULL + + (uint64_t) (counter.QuadPart % frequency.QuadPart) * 1000000000ULL / + frequency.QuadPart; +#else struct timespec timestamp; clock_gettime(CLOCK_MONOTONIC, ×tamp); return (uint64_t) timestamp.tv_sec * 1000000000ULL + timestamp.tv_nsec; +#endif } static size_t parse_size(const char *text, const char *name) @@ -303,13 +317,98 @@ static int benchmark_opentelemetry_mixed(size_t cardinality, size_t operations) return 0; } +static int benchmark_cat(size_t cardinality, size_t operations) +{ + size_t source_index; + size_t series_index; + size_t total; + int result = -1; + uint64_t start; + uint64_t elapsed; + double value; + char source_label[32]; + char series_label[32]; + char *keys[] = {"source", "series"}; + char *values[] = {source_label, series_label}; + struct cmt **sources; + struct cmt *destination; + struct cmt_counter *counter; + + if (operations > SIZE_MAX / cardinality) { + return -1; + } + total = operations * cardinality; + sources = calloc(operations, sizeof(*sources)); + destination = cmt_create(); + if (sources == NULL || destination == NULL) { + free(sources); + if (destination != NULL) { + cmt_destroy(destination); + } + return -1; + } + for (source_index = 0; source_index < operations; source_index++) { + sources[source_index] = cmt_create(); + if (sources[source_index] == NULL) { + goto cleanup; + } + counter = cmt_counter_create(sources[source_index], "bench", "", "counter", + "benchmark", 2, keys); + if (counter == NULL) { + goto cleanup; + } + snprintf(source_label, sizeof(source_label), "%zu", source_index); + for (series_index = 0; series_index < cardinality; series_index++) { + snprintf(series_label, sizeof(series_label), "%zu", series_index); + if (cmt_counter_set(counter, 1, (double) series_index, 2, values) != 0) { + goto cleanup; + } + } + } + start = monotonic_ns(); + for (source_index = 0; source_index < operations; source_index++) { + if (cmt_cat(destination, sources[source_index]) != 0) { + goto cleanup; + } + } + elapsed = monotonic_ns() - start; + counter = cfl_list_entry(destination->counters.next, struct cmt_counter, _head); + if ((size_t) cfl_list_size(&counter->map->metrics) != total) { + goto cleanup; + } + for (source_index = 0; source_index < operations; source_index++) { + snprintf(source_label, sizeof(source_label), "%zu", source_index); + for (series_index = 0; series_index < cardinality; series_index++) { + snprintf(series_label, sizeof(series_label), "%zu", series_index); + if (cmt_counter_get_val(counter, 2, values, &value) != 0 || + value != (double) series_index) { + goto cleanup; + } + } + } + printf("benchmark=cat cardinality=%zu operations=%zu series=%zu elapsed_ns=%" + PRIu64 " ns_per_series=%.2f\n", cardinality, operations, total, + elapsed, (double) elapsed / total); + result = 0; + +cleanup: + for (source_index = 0; source_index < operations; source_index++) { + if (sources[source_index] != NULL) { + cmt_destroy(sources[source_index]); + } + } + free(sources); + cmt_destroy(destination); + return result; +} + int main(int argc, char **argv) { size_t cardinality; size_t operations; if (argc != 4) { - fprintf(stderr, "usage: %s lookup|update|prometheus|opentelemetry|" + fprintf(stderr, "usage: %s cat|lookup|update|prometheus|opentelemetry|" "opentelemetry-mixed " "CARDINALITY OPERATIONS\n", argv[0]); @@ -320,6 +419,9 @@ int main(int argc, char **argv) operations = parse_size(argv[3], "operations"); cmt_initialize(); + if (strcmp(argv[1], "cat") == 0) { + return benchmark_cat(cardinality, operations) == 0 ? EXIT_SUCCESS : EXIT_FAILURE; + } if (strcmp(argv[1], "lookup") == 0) { return benchmark_lookup(cardinality, operations) == 0 ? EXIT_SUCCESS : EXIT_FAILURE; diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h index fd632247c6d..4bffc827c3f 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h @@ -57,6 +57,8 @@ struct cmt_msgpack_decode_context { int aggregation_type; int metric_value_type_set; int metric_section_set; + /* Appended to keep the established field layout. */ + int allow_reset; }; int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size, diff --git a/lib/cmetrics/include/cmetrics/cmt_map.h b/lib/cmetrics/include/cmetrics/cmt_map.h index 6a3c7e79620..e85b83b17bc 100644 --- a/lib/cmetrics/include/cmetrics/cmt_map.h +++ b/lib/cmetrics/include/cmetrics/cmt_map.h @@ -49,8 +49,11 @@ struct cmt_map { struct cfl_list *metric_buckets; size_t metric_bucket_count; size_t indexed_metric_count; - /* Most recently created metric; only changed with the metric list. */ + /* End of the indexed prefix. Direct list insertions must append; removal + * must use cmt_map_metric_destroy so this cursor cannot dangle. */ struct cmt_metric *last_metric; + /* Owned family metadata; independent of metric names and sibling scopes. */ + struct cfl_kvlist *otlp_metadata; }; struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, diff --git a/lib/cmetrics/include/cmetrics/cmt_metric.h b/lib/cmetrics/include/cmetrics/cmt_metric.h index a45eaba5b50..8499851b4eb 100644 --- a/lib/cmetrics/include/cmetrics/cmt_metric.h +++ b/lib/cmetrics/include/cmetrics/cmt_metric.h @@ -75,6 +75,9 @@ struct cmt_metric { int hash_indexed; struct cmt_map *map; struct cfl_list _hash_head; + + /* Owned OTLP datapoint metadata; independent of string label identity. */ + struct cfl_kvlist *otlp_metadata; }; struct cmt_exp_histogram_snapshot { @@ -136,6 +139,18 @@ void cmt_metric_hist_set(struct cmt_metric *metric, uint64_t timestamp, uint64_t cmt_metric_hist_get_value(struct cmt_metric *metric, int bucket_id); +/* Convert cumulative native buckets without unsigned underflow. */ +static inline uint64_t cmt_metric_hist_get_bucket_count(struct cmt_metric *metric, int bucket_id) +{ + uint64_t current; + uint64_t previous; + + current = cmt_metric_hist_get_value(metric, bucket_id); + previous = bucket_id == 0 ? 0 : cmt_metric_hist_get_value(metric, bucket_id - 1); + return current >= previous ? current - previous : 0; +} + + double cmt_metric_hist_get_sum_value(struct cmt_metric *metric); uint64_t cmt_metric_hist_get_count_value(struct cmt_metric *metric); diff --git a/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h new file mode 100644 index 00000000000..fef8eb3c57a --- /dev/null +++ b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h @@ -0,0 +1,35 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CMT_OTLP_METADATA_H +#define CMT_OTLP_METADATA_H + +#include +#include +#include + +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source); +int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context); +int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context); +int cmt_otlp_metadata_restore(struct cmt *context); +int cmt_otlp_histogram_restore(struct cmt *context); +int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample); +int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample); + +#endif diff --git a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h index 922f870fcea..819f929cf1b 100644 --- a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h +++ b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h @@ -398,10 +398,11 @@ static inline int unpack_cfl_kvlist_depth(mpack_reader_t *reader, key_value = NULL; } - mpack_done_map(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - result = -9; + if (result == 0) { + mpack_done_map(reader); + if (mpack_reader_error(reader) != mpack_ok) { + result = -9; + } } if (result != 0) { @@ -477,6 +478,7 @@ static inline int unpack_cfl_variant_string(mpack_reader_t *reader, } (*value)->type = CFL_VARIANT_STRING; + (*value)->size = value_length; return 0; } @@ -528,6 +530,7 @@ static inline int unpack_cfl_variant_binary(mpack_reader_t *reader, } (*value)->type = CFL_VARIANT_BYTES; + (*value)->size = value_length; return 0; } diff --git a/lib/cmetrics/scripts/win_build.bat b/lib/cmetrics/scripts/win_build.bat index 907baf75176..f178cb4d1fb 100755 --- a/lib/cmetrics/scripts/win_build.bat +++ b/lib/cmetrics/scripts/win_build.bat @@ -1,6 +1,8 @@ +@echo off setlocal git submodule update --init --recursive -cd build -cmake -G "NMake Makefiles" -DCMT_TESTS=On ..\ -cmake --build . -endlocal +if errorlevel 1 exit /b %errorlevel% +cmake -S . -B build -G "NMake Makefiles" -DCMT_TESTS=On -DCMAKE_BUILD_TYPE=Debug +if errorlevel 1 exit /b %errorlevel% +cmake --build build +endlocal & exit /b %errorlevel% diff --git a/lib/cmetrics/src/CMakeLists.txt b/lib/cmetrics/src/CMakeLists.txt index a4bed296713..f73ed74dcb2 100644 --- a/lib/cmetrics/src/CMakeLists.txt +++ b/lib/cmetrics/src/CMakeLists.txt @@ -15,6 +15,7 @@ set(src cmt_summary.c cmt_histogram.c cmt_exp_histogram.c + cmt_otlp_metadata.c cmt_metric.c cmt_metric_histogram.c cmt_map.c diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index 2ab52f01274..aba564add61 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -500,6 +501,248 @@ static inline void cat_scalar_value(struct cmt_metric *metric_dst, } } +static int copy_point_metadata(struct cmt_metric *dst, struct cmt_metric *src) +{ + struct cfl_kvlist *copy; + + if (src->otlp_metadata == NULL) { + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = NULL; + return 0; + } + + copy = cmt_otlp_metadata_clone(src->otlp_metadata); + if (copy == NULL) { + return -1; + } + + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = copy; + + return 0; +} + +static int sample_labels_match(struct cmt_metric *metric, int count, char **labels) +{ + int index; + struct cfl_list *head; + struct cmt_map_label *label; + + index = 0; + cfl_list_foreach(head, &metric->labels) { + if (index >= count) { + return CMT_FALSE; + } + + label = cfl_list_entry(head, struct cmt_map_label, _head); + if ((label->name == NULL) != (labels[index] == NULL)) { + return CMT_FALSE; + } + if (label->name != NULL && strcmp(label->name, labels[index]) != 0) { + return CMT_FALSE; + } + index++; + } + + return index == count; +} + +/* OTLP permits several data points of one series in a single request when + * their time windows differ; those are distinct points, not duplicates. */ +static int same_data_point(struct cmt_metric *a, struct cmt_metric *b) +{ + if (cmt_metric_get_timestamp(a) != cmt_metric_get_timestamp(b)) { + return CMT_FALSE; + } + + if (cmt_metric_has_start_timestamp(a) != cmt_metric_has_start_timestamp(b)) { + return CMT_FALSE; + } + + if (cmt_metric_has_start_timestamp(a) && + cmt_metric_get_start_timestamp(a) != cmt_metric_get_start_timestamp(b)) { + return CMT_FALSE; + } + + return CMT_TRUE; +} + +static struct cmt_metric *append_point_sample(struct cmt_map *dst, + struct cmt_metric *src, + int count, char **labels) +{ + int index; + struct cmt_metric *metric; + struct cmt_map_label *label; + + metric = calloc(1, sizeof(struct cmt_metric)); + if (metric == NULL) { + cmt_errno(); + return NULL; + } + + cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); + metric->map = dst; + metric->hash = src->hash; + cfl_list_add(&metric->_head, &dst->metrics); + + for (index = 0; index < count; index++) { + label = calloc(1, sizeof(struct cmt_map_label)); + if (label == NULL) { + cmt_errno(); + cmt_map_metric_destroy(metric); + return NULL; + } + + if (labels[index] != NULL) { + label->name = cfl_sds_create(labels[index]); + if (label->name == NULL) { + free(label); + cmt_map_metric_destroy(metric); + return NULL; + } + } + cfl_list_add(&label->_head, &metric->labels); + } + + return metric; +} + +/* + * Resolve the destination for a sample carrying OTLP metadata. Like any other + * sample it merges into an existing sample with the same labels, so contexts + * concatenated more than once never duplicate a series. A new sample is only + * appended for a distinct data point (different time window) of a series. + */ +static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map *dst, + struct cmt_metric *src, + int count, char **labels) +{ + struct cfl_list *head; + struct cmt_metric *metric; + + if (count > 0 && count != dst->label_count) { + /* OTLP points of one metric may carry fewer attributes than the + * family has label keys. The index only holds complete label sets, + * so such samples are matched by value and appended. */ + cfl_list_foreach(head, &dst->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (sample_labels_match(metric, count, labels) && + (src->otlp_metadata == NULL || same_data_point(metric, src))) { + return metric; + } + } + return append_point_sample(dst, src, count, labels); + } + + if (count == 0) { + /* Label-less points always merge into the static metric. */ + return cmt_map_metric_get(opts, dst, 0, NULL, CMT_TRUE); + } + + metric = cmt_map_metric_get(opts, dst, count, labels, CMT_FALSE); + if (metric == NULL) { + /* No sample has these labels yet: create an indexed one. */ + return cmt_map_metric_get(opts, dst, count, labels, CMT_TRUE); + } + + if (same_data_point(metric, src)) { + return metric; + } + + /* Rare path: a series with several data points. */ + cfl_list_foreach(head, &dst->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (sample_labels_match(metric, count, labels) && same_data_point(metric, src)) { + return metric; + } + } + + return append_point_sample(dst, src, count, labels); +} + +/* + * Unit and static sample state describe a family. They are set when cmt_cat() + * creates it, and again for each contributor while it holds no samples, then + * never changed: later contributors cannot reinterpret samples already + * merged. Label-less families are created with their static sample set: a + * new destination must not report a sample its source never had. + */ +static int init_family(struct cmt_map *dst, struct cmt_map *family, + struct cmt_map *points) +{ + if (dst->unit != NULL) { + cfl_sds_destroy(dst->unit); + dst->unit = NULL; + } + if (family->unit != NULL) { + dst->unit = cfl_sds_create_len(family->unit, cfl_sds_len(family->unit)); + if (dst->unit == NULL) { + return -1; + } + } + if (!points->metric_static_set) { + dst->metric_static_set = CMT_FALSE; + } + return 0; +} + +static int map_has_samples(struct cmt_map *map) +{ + return map->metric_static_set || cfl_list_size(&map->metrics) > 0; +} + +/* Unspecified temporality (written by releases that did not record it) is + * compatible with either. */ +static int aggregation_compatible(int dst, int src) +{ + return dst == src || + dst == CMT_AGGREGATION_TYPE_UNSPECIFIED || + src == CMT_AGGREGATION_TYPE_UNSPECIFIED; +} + +/* An empty unit is unspecified and compatible with any. Values are never + * converted, so two explicit units must be equal. */ +static int units_compatible(struct cmt_map *dst, struct cmt_map *src) +{ + if (dst->unit == NULL || src->unit == NULL || + cfl_sds_len(dst->unit) == 0 || cfl_sds_len(src->unit) == 0) { + return CMT_TRUE; + } + return cfl_sds_len(dst->unit) == cfl_sds_len(src->unit) && + memcmp(dst->unit, src->unit, cfl_sds_len(src->unit)) == 0; +} + +static int label_keys_match(struct cmt_map *left, + struct cmt_map *right) +{ + struct cfl_list *left_head; + struct cfl_list *right_head; + struct cmt_map_label *left_label; + struct cmt_map_label *right_label; + + left_head = left->label_keys.next; + right_head = right->label_keys.next; + + while (left_head != &left->label_keys && + right_head != &right->label_keys) { + left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); + right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); + + if (left_label->name == NULL || right_label->name == NULL || + strcmp(left_label->name, right_label->name) != 0) { + return CMT_FALSE; + } + + left_head = left_head->next; + right_head = right_head->next; + } + + return left_head == &left->label_keys && + right_head == &right->label_keys; +} + int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) { int c; @@ -508,9 +751,33 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cfl_list *head; struct cmt_metric *metric_dst; struct cmt_metric *metric_src; + struct cfl_kvlist *metadata; struct cmt_summary *summary; struct cmt_histogram *histogram_src; struct cmt_histogram *histogram_dst; + int same_keys; + + /* Only OTLP points of the same family carry fewer values than its label + * keys; any other mismatch has no meaning here. Check every sample before + * changing the destination. */ + same_keys = label_keys_match(dst, src); + cfl_list_foreach(head, &src->metrics) { + metric_src = cfl_list_entry(head, struct cmt_metric, _head); + c = cfl_list_size(&metric_src->labels); + if (c > 0 && c != dst->label_count && + (c > dst->label_count || !same_keys)) { + return -1; + } + } + + if (src->otlp_metadata) { + metadata = cmt_otlp_metadata_clone(src->otlp_metadata); + if (!metadata) { + return -1; + } + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = metadata; + } /* Handle static metric (no labels case) */ if (src->metric_static_set) { @@ -542,6 +809,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } } + if (copy_point_metadata(metric_dst, metric_src) != 0) { + return -1; + } cat_scalar_value(metric_dst, metric_src); } @@ -555,7 +825,12 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } c = cfl_list_size(&metric_src->labels); - metric_dst = cmt_map_metric_get(opts, dst, c, labels, CMT_TRUE); + if (metric_src->otlp_metadata != NULL || (c > 0 && c != dst->label_count)) { + metric_dst = get_point_sample(opts, dst, metric_src, c, labels); + } + else { + metric_dst = cmt_map_metric_get(opts, dst, c, labels, CMT_TRUE); + } free(labels); if (!metric_dst) { @@ -584,6 +859,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } } + if (copy_point_metadata(metric_dst, metric_src) != 0) { + return -1; + } cat_scalar_value(metric_dst, metric_src); } @@ -703,40 +981,12 @@ static struct cmt_summary *summary_lookup(struct cmt *cmt, struct cmt_opts *opts return NULL; } -static int summary_label_keys_match(struct cmt_map *left, - struct cmt_map *right) -{ - struct cfl_list *left_head; - struct cfl_list *right_head; - struct cmt_map_label *left_label; - struct cmt_map_label *right_label; - - left_head = left->label_keys.next; - right_head = right->label_keys.next; - - while (left_head != &left->label_keys && - right_head != &right->label_keys) { - left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); - right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); - - if (left_label->name == NULL || right_label->name == NULL || - strcmp(left_label->name, right_label->name) != 0) { - return CMT_FALSE; - } - - left_head = left_head->next; - right_head = right_head->next; - } - - return left_head == &left->label_keys && - right_head == &right->label_keys; -} - int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_counter *c; @@ -756,6 +1006,7 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -763,6 +1014,22 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, return -1; } + /* Temporality and monotonicity are family semantics: a contributor + * with samples must not change how existing samples are read. */ + if (created || !map_has_samples(c->map)) { + if (init_family(c->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + c->aggregation_type = counter->aggregation_type; + c->allow_reset = counter->allow_reset; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(c->aggregation_type, counter->aggregation_type) || + c->allow_reset != counter->allow_reset || + !units_compatible(c->map, map))) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&c->opts, c->map, filtered_map); if (ret == -1) { @@ -784,6 +1051,7 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_gauge *g; @@ -803,6 +1071,7 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -810,6 +1079,16 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, return -1; } + if (created || !map_has_samples(g->map)) { + if (init_family(g->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(g->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&g->opts, g->map, filtered_map); if (ret == -1) { @@ -831,6 +1110,7 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_untyped *u; @@ -850,6 +1130,7 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -857,6 +1138,16 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, return -1; } + if (created || !map_has_samples(u->map)) { + if (init_family(u->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(u->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&u->opts, u->map, filtered_map); if (ret == -1) { @@ -878,6 +1169,7 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_histogram *hist; @@ -909,6 +1201,7 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, opts->name, opts->description, buckets, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -916,6 +1209,19 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, return -1; } + if (created || !map_has_samples(hist->map)) { + if (init_family(hist->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + hist->aggregation_type = histogram->aggregation_type; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(hist->aggregation_type, + histogram->aggregation_type) || + !units_compatible(hist->map, map))) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&hist->opts, hist->map, filtered_map); if (ret == -1) { @@ -938,6 +1244,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, size_t i; int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_summary *sum; @@ -952,7 +1259,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, sum = summary_lookup(cmt, opts); if (sum != NULL) { - if (!summary_label_keys_match(sum->map, map)) { + if (!label_keys_match(sum->map, map)) { free(labels); return -1; } @@ -987,6 +1294,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, summary->quantiles_count, quantiles, map->label_count, labels); + created = CMT_TRUE; free(quantiles); } @@ -995,6 +1303,16 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, return -1; } + if (created || !map_has_samples(sum->map)) { + if (init_family(sum->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(sum->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&sum->opts, sum->map, filtered_map); if (ret == -1) { @@ -1016,6 +1334,7 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_exp_histogram *eh; @@ -1034,6 +1353,7 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1041,7 +1361,18 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr return -1; } - eh->aggregation_type = exp_histogram->aggregation_type; + if (created || !map_has_samples(eh->map)) { + if (init_family(eh->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + eh->aggregation_type = exp_histogram->aggregation_type; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(eh->aggregation_type, + exp_histogram->aggregation_type) || + !units_compatible(eh->map, map))) { + return -1; + } if (filtered_map != NULL) { ret = cmt_cat_copy_map(&eh->opts, eh->map, filtered_map); diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index 59f12da6218..d1a46905ac9 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -292,10 +293,7 @@ static int unpack_opts(mpack_reader_t *reader, struct cmt_map *map) } } if (NULL == opts->name) { - opts->name = cfl_sds_create(""); - if (NULL == opts->name) { - return CMT_DECODE_MSGPACK_ALLOCATION_ERROR; - } + return CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; } /* Allocate enough space for the three components, the separators @@ -1220,6 +1218,16 @@ static int unpack_metric_hash(mpack_reader_t *reader, size_t index, void *contex return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->hash); } +static int unpack_metric_otlp(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context = context; + + if (decode_context->metric->otlp_metadata != NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return unpack_cfl_kvlist(reader, &decode_context->metric->otlp_metadata); +} + static int unpack_metric(mpack_reader_t *reader, struct cmt_msgpack_decode_context *decode_context, struct cmt_metric **out_metric) @@ -1241,6 +1249,7 @@ static int unpack_metric(mpack_reader_t *reader, {"histogram", unpack_metric_histogram}, {"exp_histogram", unpack_metric_exp_histogram}, {"hash", unpack_metric_hash}, + {"otlp", unpack_metric_otlp}, {NULL, NULL} }; @@ -1328,6 +1337,8 @@ static int unpack_metric(mpack_reader_t *reader, free(metric->exp_hist_negative_buckets); } + cfl_kvlist_destroy(metric->otlp_metadata); + free(metric); } else { @@ -1362,8 +1373,8 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void old_negative_buckets = NULL; if (0 == cfl_list_size(&metric->labels)) { - /* Should we care about finding more than one "implicitly static metric" in - * the array? + /* Every label-less entry is the static metric; when several are + * present the last one wins, exactly as released readers do. */ decode_context->map->metric_static_set = 1; @@ -1440,6 +1451,9 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void cmt_atomic_load(&metric->val_int64)); cmt_atomic_store(&decode_context->map->metric.val_uint64, cmt_atomic_load(&metric->val_uint64)); + cfl_kvlist_destroy(decode_context->map->metric.otlp_metadata); + decode_context->map->metric.otlp_metadata = metric->otlp_metadata; + metric->otlp_metadata = NULL; decode_context->map->metric.hash = metric->hash; cmt_atomic_store(&decode_context->map->metric.timestamp, cmt_atomic_load(&metric->timestamp)); @@ -1545,6 +1559,24 @@ static int unpack_meta_aggregation_type(mpack_reader_t *reader, size_t index, vo return result; } +static int unpack_meta_allow_reset(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + uint64_t value; + int result; + + decode_context = context; + result = cmt_mpack_consume_uint_tag(reader, &value); + if (result != CMT_DECODE_MSGPACK_SUCCESS) { + return result; + } + if (value > 1) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + decode_context->allow_reset = value; + return CMT_DECODE_MSGPACK_SUCCESS; +} + static int unpack_meta_opts(mpack_reader_t *reader, size_t index, void *context) { struct cmt_msgpack_decode_context *decode_context; @@ -1705,6 +1737,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co {"buckets", unpack_meta_buckets}, {"quantiles", unpack_meta_quantiles}, {"aggregation_type", unpack_meta_aggregation_type}, + {"allow_reset", unpack_meta_allow_reset}, {NULL, NULL} }; @@ -1761,6 +1794,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co else if(decode_context->map->type == CMT_COUNTER) { counter = (struct cmt_counter *) decode_context->map->parent; counter->aggregation_type = decode_context->aggregation_type; + counter->allow_reset = decode_context->allow_reset; } } @@ -2245,8 +2279,22 @@ int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size mpack_reader_init_data(&reader, &in_buf[*offset], in_size); result = unpack_context(&reader, cmt); + if (result == CMT_DECODE_MSGPACK_SUCCESS && cmt_otlp_histogram_restore(cmt) != 0) { + result = CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } + if (result == CMT_DECODE_MSGPACK_SUCCESS && cmt_otlp_metadata_restore(cmt) != 0) { + result = CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } - remainder = mpack_reader_remaining(&reader, NULL); + /* Semantic rejection can leave containers open without an MPack error. + * Preserve the consumed offset, then cancel tracking before destruction. */ + if (CMT_DECODE_MSGPACK_SUCCESS != result && mpack_reader_error(&reader) == mpack_ok) { + remainder = (size_t) (reader.end - reader.data); + mpack_reader_flag_error(&reader, mpack_error_invalid); + } + else { + remainder = mpack_reader_remaining(&reader, NULL); + } *offset += in_size - remainder; diff --git a/lib/cmetrics/src/cmt_decode_opentelemetry.c b/lib/cmetrics/src/cmt_decode_opentelemetry.c index 6aafa4abbbf..04210a15e7a 100644 --- a/lib/cmetrics/src/cmt_decode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_decode_opentelemetry.c @@ -20,6 +20,7 @@ #include "cmt_protobuf.h" #include #include +#include #include #include #include @@ -35,8 +36,13 @@ #include #include #include +#include -static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source); +/* Distinguish absent optional values from allocation failures. */ +#define CLONE_UNSUPPORTED_VALUE -1 + +static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source, + int *status); static int clone_array(struct cfl_array *target, Opentelemetry__Proto__Common__V1__ArrayValue *source); @@ -61,14 +67,18 @@ static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars, size_t exemplar_count); -static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source) +static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source, + int *status) { struct cfl_kvlist *new_child_kvlist; struct cfl_array *new_child_array; struct cfl_variant *result_instance = NULL; int result; - if (source == NULL) { + *status = CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + if (source == NULL || source->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET) { + *status = CLONE_UNSUPPORTED_VALUE; return NULL; } if (source->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { @@ -103,6 +113,7 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result = clone_kvlist(new_child_kvlist, source->kvlist_value); if (result) { cfl_variant_destroy(result_instance); + *status = result; return NULL; } @@ -124,6 +135,7 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result = clone_array(new_child_array, source->array_value); if (result) { cfl_variant_destroy(result_instance); + *status = result; return NULL; } @@ -132,6 +144,13 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result_instance = cfl_variant_create_from_bytes((char *) source->bytes_value.data, source->bytes_value.len, CFL_FALSE); } + else { + *status = CLONE_UNSUPPORTED_VALUE; + } + + if (result_instance != NULL) { + *status = CMT_DECODE_OPENTELEMETRY_SUCCESS; + } return result_instance; } @@ -165,9 +184,9 @@ static int clone_array_entry(struct cfl_array *target, return CMT_DECODE_OPENTELEMETRY_SUCCESS; } - new_child_instance = clone_variant(source); + new_child_instance = clone_variant(source, &result); if (new_child_instance == NULL) { - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + return result; } result = cfl_array_append(target, new_child_instance); @@ -212,10 +231,10 @@ static int clone_kvlist_entry(struct cfl_kvlist *target, return CMT_DECODE_OPENTELEMETRY_SUCCESS; } - new_child_instance = clone_variant(source->value); + new_child_instance = clone_variant(source->value, &result); if (new_child_instance == NULL) { - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + return result; } result = cfl_kvlist_insert(target, source->key, new_child_instance); @@ -341,81 +360,71 @@ static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *samp return cfl_hash_64bits_digest(&state); } -static char *map_type_to_key(int map_type) +static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) { - switch (map_type) { - case CMT_COUNTER: - return "counter"; - case CMT_GAUGE: - return "gauge"; - case CMT_UNTYPED: - return "untyped"; - case CMT_SUMMARY: - return "summary"; - case CMT_HISTOGRAM: - return "histogram"; - case CMT_EXP_HISTOGRAM: - return "exp_histogram"; - default: - return "unknown"; + if (!map->otlp_metadata) { + map->otlp_metadata = cfl_kvlist_create(); } + return map->otlp_metadata; } -static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) +static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + uint64_t timestamp) { - struct cfl_kvlist *otlp_root; - struct cfl_kvlist *metrics_root; - struct cfl_kvlist *type_root; - - if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { - return NULL; - } - - otlp_root = get_or_create_external_metadata_kvlist(cmt->external_metadata, "otlp"); - if (otlp_root == NULL) { + if (sample == NULL) { return NULL; } - - metrics_root = get_or_create_external_metadata_kvlist(otlp_root, "metrics"); - if (metrics_root == NULL) { - return NULL; + if (sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + sample->hash = compute_metric_hash(map, sample); } - - type_root = get_or_create_external_metadata_kvlist(metrics_root, map_type_to_key(map->type)); - if (type_root == NULL) { - return NULL; + if (sample->otlp_metadata == NULL) { + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata && cmt_otlp_capture_labels(map, sample) != 0) { + return NULL; + } } - - return get_or_create_external_metadata_kvlist(type_root, map->opts->fqname); + return sample->otlp_metadata; } -static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, - struct cmt_map *map, - struct cmt_metric *sample, - uint64_t timestamp) +/* Keep telemetry attributes typed; string labels remain available to native exporters. */ +static int clone_point_attributes(struct cfl_kvlist *metadata, + Opentelemetry__Proto__Common__V1__KeyValue **attributes, + size_t count) { - char key[128]; - struct cfl_kvlist *metric_context; - struct cfl_kvlist *datapoints_context; + struct cfl_kvlist *copy; + size_t index; + int result; - if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { - sample->hash = compute_metric_hash(map, sample); + copy = cfl_kvlist_create(); + if (copy == NULL) { + return -1; } - - metric_context = get_or_create_metric_metadata_context(cmt, map); - if (metric_context == NULL) { - return NULL; + for (index = 0; index < count; index++) { + if (attributes[index] != NULL && + (attributes[index]->value == NULL || + attributes[index]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET)) { + continue; + } + result = clone_kvlist_entry(copy, attributes[index]); + if (result == CLONE_UNSUPPORTED_VALUE) { + /* A partial typed tree would change the attributes. Retain the + * label-derived representation when any nested value is absent. */ + cfl_kvlist_destroy(copy); + return 0; + } + if (attributes[index] == NULL || result != 0) { + cfl_kvlist_destroy(copy); + return -1; + } } - - datapoints_context = get_or_create_external_metadata_kvlist(metric_context, "datapoints"); - if (datapoints_context == NULL) { - return NULL; + if (cfl_kvlist_insert_kvlist(metadata, "attributes", copy) != 0) { + cfl_kvlist_destroy(copy); + return -1; } - - snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, - sample != NULL ? sample->hash : 0, timestamp); - - return get_or_create_external_metadata_kvlist(datapoints_context, key); + return 0; } static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, @@ -724,6 +733,30 @@ static int decode_data_point_labels(struct cmt *cmt, return result; } +/* A repeated label-less point replaces the static sample's previous point. */ +static void reset_static_data_point(struct cmt_map *map) +{ + struct cmt_metric *metric; + struct cmt_histogram *histogram; + + metric = &map->metric; + + if (metric->otlp_metadata != NULL) { + cfl_kvlist_destroy(metric->otlp_metadata); + metric->otlp_metadata = NULL; + } + + if (map->type == CMT_HISTOGRAM && metric->hist_buckets != NULL) { + histogram = (struct cmt_histogram *) map->parent; + memset(metric->hist_buckets, 0, + sizeof(uint64_t) * (histogram->buckets->count + 1)); + } + else if (map->type == CMT_SUMMARY && metric->sum_quantiles != NULL) { + memset(metric->sum_quantiles, 0, + sizeof(uint64_t) * metric->sum_quantiles_count); + } +} + static int decode_numerical_data_point(struct cmt *cmt, struct cmt_map *map, Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point) @@ -737,9 +770,9 @@ static int decode_numerical_data_point(struct cmt *cmt, result = CMT_DECODE_OPENTELEMETRY_SUCCESS; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -768,7 +801,9 @@ static int decode_numerical_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -799,6 +834,10 @@ static int decode_numerical_data_point(struct cmt *cmt, } point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -909,9 +948,9 @@ static int decode_summary_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -942,7 +981,9 @@ static int decode_summary_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -973,6 +1014,10 @@ static int decode_summary_data_point(struct cmt *cmt, data_point->count); point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -1022,6 +1067,7 @@ static int decode_histogram_data_point(struct cmt *cmt, { int static_metric_detected; struct cmt_histogram *histogram; + uint64_t cumulative = 0; struct cmt_metric *sample; int result; size_t index; @@ -1079,9 +1125,9 @@ static int decode_histogram_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -1112,7 +1158,9 @@ static int decode_histogram_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -1135,8 +1183,11 @@ static int decode_histogram_data_point(struct cmt *cmt, for (index = 0 ; index < data_point->n_bucket_counts; index++) { - cmt_metric_hist_set(sample, data_point->time_unix_nano, - index, data_point->bucket_counts[index]); + if (UINT64_MAX - cumulative < data_point->bucket_counts[index]) { + return CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + } + cumulative += data_point->bucket_counts[index]; + cmt_metric_hist_set(sample, data_point->time_unix_nano, index, cumulative); } cmt_metric_hist_sum_set(sample, data_point->time_unix_nano, @@ -1145,6 +1196,10 @@ static int decode_histogram_data_point(struct cmt *cmt, data_point->count); point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -1345,9 +1400,9 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -1376,6 +1431,9 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, } else { sample = &map->metric; + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -1438,6 +1496,10 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, struct cfl_kvlist *point_metadata; point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index 4341b9c651f..a9b2555f80f 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -264,6 +265,7 @@ static int pack_metric(mpack_writer_t *writer, struct cmt_map *map, struct cmt_m for (index = 0 ; index <= histogram->buckets->count ; index++) { + /* Wire buckets stay cumulative, as every released reader expects. */ mpack_write_uint(writer, cmt_metric_hist_get_value(metric, index)); } @@ -477,14 +479,14 @@ static int pack_context_header(mpack_writer_t *writer, struct cmt *cmt) mpack_start_map(writer, 3); mpack_write_cstr(writer, "cmetrics"); - result = pack_cfl_variant_kvlist(writer, cmt->internal_metadata); + result = cmt_otlp_metadata_pack(writer, cmt); if (result != 0) { return -1; } mpack_write_cstr(writer, "external"); - result = pack_cfl_variant_kvlist(writer, cmt->external_metadata); + result = cmt_otlp_metadata_pack_external(writer, cmt); if (result != 0) { return -2; diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 65b6399ee40..d3b8fcbc02b 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -18,6 +18,7 @@ */ #include #include +#include #include #include #include @@ -82,7 +83,7 @@ struct cfl_kvlist *fetch_metadata_kvlist_key(struct cfl_kvlist *kvlist, char *ke entry_variant = cfl_kvlist_fetch(kvlist, key); - if (entry_variant != NULL) { + if (entry_variant != NULL && entry_variant->type == CFL_VARIANT_KVLIST) { entry_kvlist = entry_variant->data.as_kvlist; } else { @@ -144,7 +145,8 @@ static char *map_type_to_key(int map_type) } } -static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +static struct cfl_kvlist *get_legacy_metric_otlp_metadata_context(struct cmt *cmt, + struct cmt_map *map) { struct cfl_kvlist *otlp_root; struct cfl_kvlist *metrics_root; @@ -172,6 +174,15 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return fetch_metadata_kvlist_key(type_root, map->opts->fqname); } +static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +{ + if (map != NULL && map->otlp_metadata) { + return map->otlp_metadata; + } + + return get_legacy_metric_otlp_metadata_context(cmt, map); +} + static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map, struct cmt_metric *sample) @@ -180,7 +191,13 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cfl_kvlist *metric_context; struct cfl_kvlist *datapoints_context; - metric_context = get_metric_otlp_metadata_context(cmt, map); + if (sample != NULL && sample->otlp_metadata != NULL) { + return sample->otlp_metadata; + } + + /* Native family metadata never carries points. A point an intermediary + * left unbound keeps its legacy entry, as released readers do. */ + metric_context = get_legacy_metric_otlp_metadata_context(cmt, map); if (metric_context == NULL) { return NULL; } @@ -541,7 +558,10 @@ static size_t resolve_target_scope_index(struct cmt_opentelemetry_context *conte return 0; } - metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + metadata = fetch_metadata_kvlist_key(metric_context, "routing"); + if (metadata == NULL) { + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + } if (metadata == NULL) { return 0; } @@ -1294,6 +1314,174 @@ static inline Opentelemetry__Proto__Common__V1__KeyValue **cfl_kvlist_to_otlp_kv } +static void *attribute_arena_alloc(void *data, size_t size) +{ + return cfl_arena_malloc(data, size); +} + +static void attribute_arena_free(void *data, void *pointer) +{ + /* The encoder releases all datapoint objects with their arena. */ +} + +/* + * TRUE when every value can be represented as an OTLP AnyValue: NULL values, + * unsigned integers above INT64_MAX and unknown types have no OTLP form. + */ +static int otlp_variant_is_representable(struct cfl_variant *value, int depth) +{ + size_t index; + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_array *array; + + if (value == NULL || depth > 64) { + return CMT_FALSE; + } + + switch (value->type) { + case CFL_VARIANT_STRING: + case CFL_VARIANT_BOOL: + case CFL_VARIANT_INT: + case CFL_VARIANT_DOUBLE: + case CFL_VARIANT_BYTES: + case CFL_VARIANT_REFERENCE: + return CMT_TRUE; + case CFL_VARIANT_UINT: + return value->data.as_uint64 <= INT64_MAX; + case CFL_VARIANT_ARRAY: + array = value->data.as_array; + for (index = 0; index < array->entry_count; index++) { + if (!otlp_variant_is_representable(array->entries[index], depth + 1)) { + return CMT_FALSE; + } + } + return CMT_TRUE; + case CFL_VARIANT_KVLIST: + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!otlp_variant_is_representable(pair->val, depth + 1)) { + return CMT_FALSE; + } + } + return CMT_TRUE; + default: + return CMT_FALSE; + } +} + +static int otlp_kvlist_is_representable(struct cfl_kvlist *kvlist) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!otlp_variant_is_representable(pair->val, 0)) { + return CMT_FALSE; + } + } + + return CMT_TRUE; +} + +static int apply_typed_point_attributes(struct cmt_opentelemetry_context *context, + struct cmt_map *map, struct cmt_metric *sample, + void *data_point) +{ + struct cfl_kvlist *metadata; + struct cfl_kvlist *attributes; + struct cfl_arena *arena; + struct cfl_list *head; + struct cmt_label *label; + ProtobufCAllocator allocator; + Opentelemetry__Proto__Common__V1__KeyValue **heap; + Opentelemetry__Proto__Common__V1__KeyValue **replacement; + Opentelemetry__Proto__Common__V1__KeyValue ***target; + size_t *target_count; + uint8_t *buffer; + size_t count; + size_t size; + size_t index; + size_t static_count; + + metadata = get_data_point_otlp_metadata_context(context->cmt, map, sample); + attributes = fetch_metadata_kvlist_key(metadata, "attributes"); + if (attributes == NULL || !cmt_otlp_attributes_match_labels(map, sample)) { + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } + + /* typed values without an OTLP form keep the label derived attributes */ + if (!otlp_kvlist_is_representable(attributes)) { + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } + count = cfl_kvlist_count(attributes); + static_count = cfl_list_size(&context->cmt->static_labels->list); + heap = cfl_kvlist_to_otlp_kvpair_list(attributes); + if (heap == NULL) { + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + arena = get_context_arena(context); + allocator.alloc = attribute_arena_alloc; + allocator.free = attribute_arena_free; + allocator.allocator_data = arena; + replacement = cfl_arena_calloc(arena, count + static_count + 1, sizeof(*replacement)); + if (replacement == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + for (index = 0; index < count; index++) { + size = opentelemetry__proto__common__v1__key_value__get_packed_size(heap[index]); + buffer = malloc(size ? size : 1); + if (buffer == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + opentelemetry__proto__common__v1__key_value__pack(heap[index], buffer); + replacement[index] = opentelemetry__proto__common__v1__key_value__unpack( + &allocator, size, buffer); + free(buffer); + if (replacement[index] == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + otlp_kvpair_list_destroy(heap, count); + index = count; + cfl_list_foreach(head, &context->cmt->static_labels->list) { + label = cfl_list_entry(head, struct cmt_label, _head); + replacement[index] = initialize_string_attribute(arena, label->key, label->val); + if (replacement[index++] == NULL) { + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + switch (map->type) { + case CMT_COUNTER: + case CMT_GAUGE: + case CMT_UNTYPED: + target = &((Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point)->n_attributes; + break; + case CMT_HISTOGRAM: + target = &((Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point)->n_attributes; + break; + case CMT_EXP_HISTOGRAM: + target = &((Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point)->n_attributes; + break; + case CMT_SUMMARY: + target = &((Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point)->n_attributes; + break; + default: + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + *target = replacement; + *target_count = count + static_count; + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; +} + static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_kvlist_to_otlp_any_value(struct cfl_variant *value) { size_t entry_count; @@ -1453,6 +1641,7 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_binary_to_ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_to_otlp_any_value(struct cfl_variant *value) { Opentelemetry__Proto__Common__V1__AnyValue *result; + struct cfl_variant signed_value; if (value->type == CFL_VARIANT_STRING) { result = cfl_variant_string_to_otlp_any_value(value); @@ -1463,6 +1652,17 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_to_otlp_an else if (value->type == CFL_VARIANT_INT) { result = cfl_variant_int64_to_otlp_any_value(value); } + else if (value->type == CFL_VARIANT_UINT) { + if (value->data.as_uint64 > INT64_MAX) { + result = NULL; + } + else { + signed_value = *value; + signed_value.type = CFL_VARIANT_INT; + signed_value.data.as_int64 = (int64_t) value->data.as_uint64; + result = cfl_variant_int64_to_otlp_any_value(&signed_value); + } + } else if (value->type == CFL_VARIANT_DOUBLE) { result = cfl_variant_double_to_otlp_any_value(value); } @@ -1518,6 +1718,10 @@ static int64_t fetch_metadata_int64_key(struct cfl_kvlist *metadata, char *key_n return 0; } + if (value->type == CFL_VARIANT_UINT && value->data.as_uint64 <= INT64_MAX) { + return (int64_t) value->data.as_uint64; + } + if (value->type != CFL_VARIANT_INT) { *error_flag = CMT_TRUE; @@ -1751,11 +1955,11 @@ static Opentelemetry__Proto__Metrics__V1__ResourceMetrics ** void destroy_instrumentation_scope(Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) { - if (scope->name != NULL) { + if (is_string_releaseable(scope->name)) { cfl_sds_destroy(scope->name); } - if (scope->version != NULL) { + if (is_string_releaseable(scope->version)) { cfl_sds_destroy(scope->version); } @@ -2367,7 +2571,9 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * if (bucket_list != NULL) { for (index = 0 ; index < bucket_count ; index++) { - data_point->bucket_counts[index] = bucket_list[index]; + data_point->bucket_counts[index] = index == 0 ? bucket_list[index] : + (bucket_list[index] >= bucket_list[index - 1] ? + bucket_list[index] - bucket_list[index - 1] : 0); } } } @@ -3417,6 +3623,11 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } } + result = apply_typed_point_attributes(context, map, sample, data_point); + if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { + destroy_data_point(data_point, map->type); + return result; + } apply_data_point_metadata_from_otlp_context(context->cmt, map, sample, data_point); result = append_data_point_to_metric(metric, (void *) data_point, sample_index); diff --git a/lib/cmetrics/src/cmt_encode_text.c b/lib/cmetrics/src/cmt_encode_text.c index 7d499d72cee..1835c11461e 100644 --- a/lib/cmetrics/src/cmt_encode_text.c +++ b/lib/cmetrics/src/cmt_encode_text.c @@ -188,6 +188,10 @@ static struct cfl_kvlist *get_data_point_metadata_context(struct cmt *cmt, const char *type_key; char key[128]; + if (metric != NULL && metric->otlp_metadata != NULL) { + return metric->otlp_metadata; + } + type_key = map_type_to_otlp_key(map->type); if (type_key == NULL) { return NULL; diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index fb4fe9a150e..91822021317 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -46,6 +46,8 @@ static void metric_release_storage(struct cmt_metric *metric) free(metric->exp_hist_positive_buckets); free(metric->exp_hist_negative_buckets); free(metric->sum_quantiles); + cfl_kvlist_destroy(metric->otlp_metadata); + metric->otlp_metadata = NULL; metric->hist_buckets = NULL; metric->exp_hist_positive_buckets = NULL; @@ -236,6 +238,8 @@ static struct cmt_metric *metric_prepare_storage(struct cmt_map *map, static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash, int labels_count, char **labels_val) { + int indexed_prefix = CMT_TRUE; + struct cmt_metric *match = NULL; struct cfl_list *head; struct cmt_metric *metric; @@ -260,18 +264,27 @@ static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash, } } - /* Decoders can populate the public metric list directly. Search only - * entries that have not yet been indexed, then index a successful match. */ - cfl_list_foreach(head, &map->metrics) { + /* Decoders append directly to the public metric list. Index only the + * suffix after the known indexed prefix, including nonmatching entries, + * so misses do not repeatedly walk every previously indexed metric. */ + head = map->last_metric != NULL ? map->last_metric->_head.next : map->metrics.next; + while (head != &map->metrics) { metric = cfl_list_entry(head, struct cmt_metric, _head); - if (!metric->hash_indexed && metric->hash == hash && + head = head->next; + metric_index_add(map, metric); + if (!metric->hash_indexed) { + indexed_prefix = CMT_FALSE; + } + if (indexed_prefix) { + map->last_metric = metric; + } + if (match == NULL && metric->hash == hash && metric_labels_match(metric, labels_count, labels_val)) { - metric_index_add(map, metric); - return metric; + match = metric; } } - return NULL; + return match; } static struct cmt_metric *map_metric_create(struct cmt_map *map, uint64_t hash, @@ -451,7 +464,12 @@ static struct cmt_metric *map_metric_get_unlocked(struct cmt_opts *opts, } cfl_list_add(&metric->_head, &map->metrics); metric_index_add(map, metric); - map->last_metric = metric; + /* Do not skip an unindexed entry if allocating the index failed. */ + if (metric->hash_indexed && + (metric->_head.prev == &map->metrics || + (map->last_metric != NULL && metric->_head.prev == &map->last_metric->_head))) { + map->last_metric = metric; + } return metric_prepare_storage(map, metric, write_op); } @@ -506,16 +524,13 @@ void cmt_map_destroy(struct cmt_map *map) } /* histogram and quantile allocation for static metric */ - if (map->metric_static_set) { - metric = &map->metric; - - metric_release_storage(metric); - } + metric_release_storage(&map->metric); if (map->unit != NULL) { cfl_sds_destroy(map->unit); } + cfl_kvlist_destroy(map->otlp_metadata); free(map->metric_buckets); free(map); diff --git a/lib/cmetrics/src/cmt_mpack_utils.c b/lib/cmetrics/src/cmt_mpack_utils.c index ea67d5efe86..ddbc4f9a7e0 100644 --- a/lib/cmetrics/src/cmt_mpack_utils.c +++ b/lib/cmetrics/src/cmt_mpack_utils.c @@ -332,5 +332,11 @@ int cmt_mpack_peek_array_length(mpack_reader_t *reader) return 0; } + /* Callers size allocations from this value before the array is read, so + * a declared count that cmt_mpack_unpack_array() rejects reports zero. */ + if (CMT_MPACK_MAX_ARRAY_ENTRY_COUNT < mpack_tag_array_count(&tag)) { + return 0; + } + return mpack_tag_array_count(&tag); } diff --git a/lib/cmetrics/src/cmt_otlp_metadata.c b/lib/cmetrics/src/cmt_otlp_metadata.c new file mode 100644 index 00000000000..72487a1cff0 --- /dev/null +++ b/lib/cmetrics/src/cmt_otlp_metadata.c @@ -0,0 +1,1684 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define METADATA_KEY "__cmetrics_native_metadata_v1" +/* Written only by unreleased builds; ignored on input, never emitted. */ +#define HISTOGRAM_MARKER_KEY "__cmetrics_histogram_buckets" + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *source); + +static struct cfl_variant *clone_variant(struct cfl_variant *source) +{ + size_t index; + struct cfl_array *array; + struct cfl_kvlist *kvlist; + struct cfl_variant *entry; + struct cfl_variant *copy; + + switch (source->type) { + case CFL_VARIANT_STRING: + return cfl_variant_create_from_string_s(source->data.as_string, + cfl_variant_size_get(source), CFL_FALSE); + case CFL_VARIANT_REFERENCE: + return cfl_variant_create_from_string(source->data.as_reference); + case CFL_VARIANT_BYTES: + return cfl_variant_create_from_bytes(source->data.as_bytes, + cfl_variant_size_get(source), CFL_FALSE); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool(source->data.as_bool); + case CFL_VARIANT_INT: + return cfl_variant_create_from_int64(source->data.as_int64); + case CFL_VARIANT_UINT: + return cfl_variant_create_from_uint64(source->data.as_uint64); + case CFL_VARIANT_DOUBLE: + return cfl_variant_create_from_double(source->data.as_double); + case CFL_VARIANT_NULL: + return cfl_variant_create_from_null(); + case CFL_VARIANT_ARRAY: + array = cfl_array_create(source->data.as_array->entry_count); + if (array == NULL) { + return NULL; + } + for (index = 0; index < source->data.as_array->entry_count; index++) { + entry = clone_variant(source->data.as_array->entries[index]); + if (entry == NULL || cfl_array_append(array, entry) != 0) { + cfl_variant_destroy(entry); + cfl_array_destroy(array); + return NULL; + } + } + copy = cfl_variant_create_from_array(array); + if (copy == NULL) { + cfl_array_destroy(array); + } + return copy; + case CFL_VARIANT_KVLIST: + kvlist = clone_kvlist(source->data.as_kvlist); + if (kvlist == NULL) { + return NULL; + } + copy = cfl_variant_create_from_kvlist(kvlist); + if (copy == NULL) { + cfl_kvlist_destroy(kvlist); + } + return copy; + default: + return NULL; + } +} + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *source) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_kvlist *copy; + struct cfl_variant *value; + + copy = cfl_kvlist_create(); + if (copy == NULL) { + return NULL; + } + cfl_list_foreach(head, &source->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + value = clone_variant(pair->val); + if (value == NULL || + cfl_kvlist_insert_s(copy, pair->key, cfl_sds_len(pair->key), value) != 0) { + cfl_variant_destroy(value); + cfl_kvlist_destroy(copy); + return NULL; + } + } + return copy; +} + +/* + * Deep copy, preserving key order and duplicate keys. Unlike a msgpack round + * trip it has no nesting limit, so metadata accepted by a decoder can always + * be copied. + */ +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source) +{ + if (source == NULL) { + return NULL; + } + return clone_kvlist(source); +} + +static int visit_maps(struct cmt *context, + int (*callback)(struct cmt_map *, void *), void *data) +{ + struct cfl_list *head; + int result; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + + + cfl_list_foreach(head, &context->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + result = callback(counter->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + result = callback(gauge->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + result = callback(untyped->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + result = callback(summary->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + result = callback(histogram->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + result = callback(exp_histogram->map, data); + if (result != 0) { + return result; + } + } + return 0; +} + +static int count_maps(struct cmt_map *map, void *data) +{ + (*(size_t *) data)++; + return 0; +} + +static int has_metadata(struct cmt_map *map, void *data) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if (map->otlp_metadata || map->metric.otlp_metadata || + (map->type == CMT_COUNTER && ((struct cmt_counter *) map->parent)->allow_reset)) { + return 1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (sample->otlp_metadata) { + return 1; + } + } + return 0; +} + +static int pack_optional_map(mpack_writer_t *writer, struct cfl_kvlist *map) +{ + mpack_writer_t metadata_writer; + char *buffer = NULL; + size_t size = 0; + int result; + + if (map == NULL) { + mpack_write_nil(writer); + return 0; + } + /* A binary sidecar does not add nesting to the established wire metadata + * tree. Older readers can retain/ignore it even for deeply nested values. */ + mpack_writer_init_growable(&metadata_writer, &buffer, &size); + result = pack_cfl_variant_kvlist(&metadata_writer, map); + if (mpack_writer_destroy(&metadata_writer) != mpack_ok || result || size > UINT32_MAX) { + free(buffer); + return -1; + } + mpack_write_bin(writer, buffer, size); + free(buffer); + return 0; +} + +static void fingerprint_u64(cfl_hash_state_t *state, uint64_t value) +{ + unsigned char bytes[8]; + int index; + + /* byte order independent: writer and reader may differ */ + for (index = 0; index < 8; index++) { + bytes[index] = (unsigned char) (value >> (index * 8)); + } + cfl_hash_64bits_update(state, bytes, sizeof(bytes)); +} + +static uint64_t double_bits(double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* + * Fingerprint of the complete value a point carries through the MessagePack + * codec, read as pack_metric() reads it, distributions included. Points of + * one series and time window that differ only in what the labels cannot + * hold, such as attribute types, usually differ in value too. + */ +static int point_fingerprint(struct cmt_map *map, struct cmt_metric *sample, + uint64_t *fingerprint) +{ + struct cmt_exp_histogram_snapshot snapshot; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + cfl_hash_state_t state; + int64_t int64_value; + uint64_t uint64_value; + size_t index; + int type; + + cfl_hash_64bits_reset(&state); + fingerprint_u64(&state, map->type); + if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + for (index = 0; index <= histogram->buckets->count; index++) { + fingerprint_u64(&state, cmt_metric_hist_get_value(sample, index)); + } + fingerprint_u64(&state, cmt_metric_hist_get_count_value(sample)); + fingerprint_u64(&state, double_bits(cmt_metric_hist_get_sum_value(sample))); + } + else if (map->type == CMT_SUMMARY) { + summary = (struct cmt_summary *) map->parent; + fingerprint_u64(&state, cmt_atomic_load(&sample->sum_quantiles_set)); + for (index = 0; sample->sum_quantiles != NULL && + index < summary->quantiles_count; index++) { + fingerprint_u64(&state, cmt_atomic_load(&sample->sum_quantiles[index])); + } + fingerprint_u64(&state, cmt_summary_get_count_value(sample)); + fingerprint_u64(&state, double_bits(cmt_summary_get_sum_value(sample))); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (cmt_metric_exp_hist_get_snapshot(sample, &snapshot) != 0) { + return -1; + } + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.scale); + fingerprint_u64(&state, snapshot.zero_count); + fingerprint_u64(&state, double_bits(snapshot.zero_threshold)); + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.positive_offset); + fingerprint_u64(&state, snapshot.positive_count); + for (index = 0; index < snapshot.positive_count; index++) { + fingerprint_u64(&state, snapshot.positive_buckets[index]); + } + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.negative_offset); + fingerprint_u64(&state, snapshot.negative_count); + for (index = 0; index < snapshot.negative_count; index++) { + fingerprint_u64(&state, snapshot.negative_buckets[index]); + } + fingerprint_u64(&state, snapshot.count); + fingerprint_u64(&state, snapshot.sum_set); + fingerprint_u64(&state, snapshot.sum); + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + } + else { + cmt_metric_get_value_snapshot(sample, &type, &int64_value, &uint64_value); + fingerprint_u64(&state, type); + if (type == CMT_METRIC_VALUE_INT64) { + fingerprint_u64(&state, (uint64_t) int64_value); + } + else if (type == CMT_METRIC_VALUE_UINT64) { + fingerprint_u64(&state, uint64_value); + } + else { + fingerprint_u64(&state, double_bits(cmt_metric_get_value(sample))); + } + } + *fingerprint = cfl_hash_64bits_digest(&state); + return 0; +} + +/* + * A point is identified as in cmt_cat() by label values, time and start + * time, plus a fingerprint of its values. The series hash concatenates label + * values without boundaries, so distinct label sets can share it: it only + * accelerates the comparison. Values are written as pack_metric() writes + * labels, so they compare against the labels a reader decodes. The start + * time is appended only when set, so its absence stays distinct. A point + * whose fingerprint cannot be computed records zero and is unlikely to bind. + */ +static void pack_point_id(mpack_writer_t *writer, struct cmt_map *map, + struct cmt_metric *sample) +{ + int has_start; + uint64_t fingerprint; + struct cfl_list *head; + struct cmt_map_label *label; + + if (point_fingerprint(map, sample, &fingerprint) != 0) { + fingerprint = 0; + } + has_start = cmt_metric_has_start_timestamp(sample); + mpack_start_array(writer, has_start ? 5 : 4); + mpack_write_uint(writer, sample->hash); + mpack_write_uint(writer, cmt_metric_get_timestamp(sample)); + mpack_start_array(writer, cfl_list_size(&sample->labels)); + cfl_list_foreach(head, &sample->labels) { + label = cfl_list_entry(head, struct cmt_map_label, _head); + if (label->name == NULL) { + mpack_write_nil(writer); + } + else { + mpack_write_cstr(writer, label->name); + } + } + mpack_finish_array(writer); + mpack_write_uint(writer, fingerprint); + if (has_start) { + mpack_write_uint(writer, cmt_metric_get_start_timestamp(sample)); + } + mpack_finish_array(writer); +} + +static int pack_map_metadata(struct cmt_map *map, void *data) +{ + mpack_writer_t *writer = data; + struct cfl_list *head; + struct cmt_metric *sample; + + mpack_start_map(writer, 6); + mpack_write_cstr(writer, "type"); + mpack_write_int(writer, map->type); + mpack_write_cstr(writer, "name"); + mpack_write_cstr(writer, map->opts->fqname); + mpack_write_cstr(writer, "family"); + if (pack_optional_map(writer, map->otlp_metadata) != 0) { + return -1; + } + mpack_write_cstr(writer, "allow_reset"); + mpack_write_bool(writer, map->type == CMT_COUNTER && + ((struct cmt_counter *) map->parent)->allow_reset); + mpack_write_cstr(writer, "points"); + mpack_start_array(writer, cfl_list_size(&map->metrics) + !!map->metric_static_set); + if (map->metric_static_set && pack_optional_map(writer, map->metric.otlp_metadata) != 0) { + return -1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (pack_optional_map(writer, sample->otlp_metadata) != 0) { + return -1; + } + } + mpack_finish_array(writer); + + /* Identity of each entry in "points". An intermediary + * that edits points without updating the sidecar (any released reader) + * changes them, so readers bind by identity instead of by position. + * Readers of v2.3.0 ignore this key. */ + mpack_write_cstr(writer, "point_ids"); + mpack_start_array(writer, cfl_list_size(&map->metrics) + !!map->metric_static_set); + if (map->metric_static_set) { + pack_point_id(writer, map, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + pack_point_id(writer, map, sample); + } + mpack_finish_array(writer); + mpack_finish_map(writer); + return 0; +} + +/* The established internal metadata map accepts arbitrary typed keys. No new + * metric/map wire fields are emitted, so old decoders still consume the chunk. + * Histogram buckets are never re-shaped here: they stay cumulative on the wire, + * exactly as released readers and writers expect. */ +int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t count = 0; + size_t entries; + int has_sidecar; + int has_marker; + + has_sidecar = visit_maps(context, has_metadata, NULL) != 0; + has_marker = cfl_kvlist_fetch(context->internal_metadata, + HISTOGRAM_MARKER_KEY) != NULL; + if (!has_sidecar && !has_marker) { + return pack_cfl_variant_kvlist(writer, context->internal_metadata); + } + if (has_sidecar && cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY)) { + return -1; + } + + /* A stale bucket marker is never forwarded. */ + entries = has_sidecar; + cfl_list_foreach(head, &context->internal_metadata->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, HISTOGRAM_MARKER_KEY) != 0) { + entries++; + } + } + + mpack_start_map(writer, entries); + cfl_list_foreach(head, &context->internal_metadata->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, HISTOGRAM_MARKER_KEY) == 0) { + continue; + } + mpack_write_str(writer, pair->key, cfl_sds_len(pair->key)); + if (pack_cfl_variant(writer, pair->val) != 0) { + return -1; + } + } + + if (!has_sidecar) { + mpack_finish_map(writer); + return 0; + } + + mpack_write_cstr(writer, METADATA_KEY); + visit_maps(context, count_maps, &count); + mpack_start_array(writer, count); + if (visit_maps(context, pack_map_metadata, writer) != 0) { + return -1; + } + mpack_finish_array(writer); + mpack_finish_map(writer); + return 0; +} + +/* + * Fluent Bit 5.1 and older keep OTLP point metadata in the external metadata + * tree: otlp -> metrics -> -> -> {metadata, datapoints}, where a + * point is keyed by ":". They do not read the binary + * sidecar, so the encoder also writes that tree for families with metadata, + * with the keys those readers use. The tree is built on a copy: the context + * itself is not modified. + */ +static const char *legacy_point_keys[] = { + "start_time_unix_nano", "flags", "number_value_case", "has_sum", "has_min", + "min", "has_max", "max", "exemplars", NULL +}; + +static const char *legacy_type_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *legacy_child(struct cfl_kvlist *parent, const char *key) +{ + struct cfl_variant *value; + struct cfl_kvlist *child; + + value = cfl_kvlist_fetch(parent, (char *) key); + if (value != NULL && value->type == CFL_VARIANT_KVLIST) { + return value->data.as_kvlist; + } + if (value != NULL) { + cfl_kvlist_remove(parent, (char *) key); + } + child = cfl_kvlist_create(); + if (child == NULL) { + return NULL; + } + if (cfl_kvlist_insert_kvlist(parent, (char *) key, child) != 0) { + cfl_kvlist_destroy(child); + return NULL; + } + return child; +} + +static int legacy_add_point(struct cfl_kvlist *datapoints, struct cmt_metric *sample) +{ + char key[64]; + size_t index; + struct cfl_variant *value; + struct cfl_variant *copy; + struct cfl_kvlist *point; + + if (sample->otlp_metadata == NULL) { + return 0; + } + + snprintf(key, sizeof(key), "%" PRIx64 ":%" PRIu64, + sample->hash, cmt_metric_get_timestamp(sample)); + cfl_kvlist_remove(datapoints, key); + point = legacy_child(datapoints, key); + if (point == NULL) { + return -1; + } + + for (index = 0; legacy_point_keys[index] != NULL; index++) { + value = cfl_kvlist_fetch(sample->otlp_metadata, (char *) legacy_point_keys[index]); + if (value == NULL) { + continue; + } + copy = clone_variant(value); + if (copy == NULL || + cfl_kvlist_insert(point, (char *) legacy_point_keys[index], copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + return 0; +} + +static int legacy_add_map(struct cmt_map *map, void *data) +{ + struct cfl_kvlist *metrics = data; + struct cfl_kvlist *type_root; + struct cfl_kvlist *family; + struct cfl_kvlist *datapoints; + struct cfl_kvlist *copy; + struct cfl_variant *metadata; + struct cfl_list *head; + struct cmt_metric *sample; + int points; + + metadata = map->otlp_metadata ? cfl_kvlist_fetch(map->otlp_metadata, "metadata") : NULL; + points = map->metric_static_set && map->metric.otlp_metadata != NULL; + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + points |= sample->otlp_metadata != NULL; + } + if (!points && (metadata == NULL || metadata->type != CFL_VARIANT_KVLIST)) { + return 0; + } + + type_root = legacy_child(metrics, legacy_type_key(map->type)); + if (type_root == NULL) { + return -1; + } + family = legacy_child(type_root, map->opts->fqname); + if (family == NULL) { + return -1; + } + + if (metadata != NULL && metadata->type == CFL_VARIANT_KVLIST) { + copy = clone_kvlist(metadata->data.as_kvlist); + cfl_kvlist_remove(family, "metadata"); + if (copy == NULL || cfl_kvlist_insert_kvlist(family, "metadata", copy) != 0) { + cfl_kvlist_destroy(copy); + return -1; + } + } + + if (!points) { + return 0; + } + datapoints = legacy_child(family, "datapoints"); + if (datapoints == NULL) { + return -1; + } + if (map->metric_static_set && legacy_add_point(datapoints, &map->metric) != 0) { + return -1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (legacy_add_point(datapoints, sample) != 0) { + return -1; + } + } + return 0; +} + +static int has_legacy_metadata(struct cmt_map *map, void *data) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if ((map->otlp_metadata && cfl_kvlist_fetch(map->otlp_metadata, "metadata")) || + (map->metric_static_set && map->metric.otlp_metadata)) { + return 1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (sample->otlp_metadata) { + return 1; + } + } + return 0; +} + +int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context) +{ + int result; + struct cfl_kvlist *external; + struct cfl_kvlist *otlp; + struct cfl_kvlist *metrics; + + if (visit_maps(context, has_legacy_metadata, NULL) == 0) { + return pack_cfl_variant_kvlist(writer, context->external_metadata); + } + + external = clone_kvlist(context->external_metadata); + if (external == NULL) { + return -1; + } + otlp = legacy_child(external, "otlp"); + metrics = otlp ? legacy_child(otlp, "metrics") : NULL; + if (metrics == NULL || visit_maps(context, legacy_add_map, metrics) != 0) { + cfl_kvlist_destroy(external); + return -1; + } + + result = pack_cfl_variant_kvlist(writer, external); + cfl_kvlist_destroy(external); + return result; +} + +struct staged_family_metadata { + struct cmt_map *map; + struct cfl_kvlist *family; + struct cfl_kvlist **points; + struct cfl_array *point_ids; + size_t *order; + unsigned char *point_flags; + size_t point_count; + int allow_reset; + int restore_points; +}; + +/* point_flags bits, per staged point: identity recorded more than once, + * native metadata read back, bound to a decoded sample */ +#define POINT_AMBIGUOUS 1 +#define POINT_HAS_METADATA 2 +#define POINT_MATCHED 4 + +struct restore_state { + struct cfl_array *families; + size_t index; + struct staged_family_metadata *staged; +}; + +static int restore_optional_map(struct cfl_kvlist **target, struct cfl_variant *value) +{ + struct cfl_kvlist *copy = NULL; + mpack_reader_t reader; + size_t remaining; + int result; + + if (value == NULL || value->type == CFL_VARIANT_NULL) { + return 0; + } + if (value->type == CFL_VARIANT_BYTES) { + mpack_reader_init_data(&reader, value->data.as_bytes, cfl_sds_len(value->data.as_bytes)); + result = unpack_cfl_kvlist(&reader, ©); + if (result != 0) { + /* A rejected value can leave containers open without setting an + * MPack error (for example, at the nesting limit). Cancel tracking + * before destroying this reader for optional metadata. */ + mpack_reader_flag_error(&reader, mpack_error_data); + } + remaining = result == 0 ? mpack_reader_remaining(&reader, NULL) : 0; + if (mpack_reader_destroy(&reader) != mpack_ok || result || remaining) { + cfl_kvlist_destroy(copy); + return -1; + } + } + else { + return -1; + } + if (!copy) { + return -1; + } + cfl_kvlist_destroy(*target); + *target = copy; + return 0; +} + +/* Unsigned values are read back as CFL_VARIANT_INT with the same bits. */ +static int is_integer(struct cfl_variant *value) +{ + return value->type == CFL_VARIANT_INT || value->type == CFL_VARIANT_UINT; +} + +static uint64_t integer_bits(struct cfl_variant *value) +{ + if (value->type == CFL_VARIANT_INT) { + return (uint64_t) value->data.as_int64; + } + return value->data.as_uint64; +} + +static int label_values_valid(struct cfl_variant *values) +{ + struct cfl_array *array; + size_t index; + + if (values->type != CFL_VARIANT_ARRAY) { + return CMT_FALSE; + } + array = values->data.as_array; + for (index = 0; index < array->entry_count; index++) { + if (array->entries[index]->type != CFL_VARIANT_STRING && + array->entries[index]->type != CFL_VARIANT_NULL) { + return CMT_FALSE; + } + } + return CMT_TRUE; +} + +/* Each identifier is [series hash, timestamp, label values, fingerprint] + * with the start timestamp appended when set, one per "points" entry. */ +static int point_ids_valid(struct cfl_variant *ids, size_t count) +{ + struct cfl_array *id; + size_t index; + + if (ids->type != CFL_VARIANT_ARRAY || ids->data.as_array->entry_count != count) { + return CMT_FALSE; + } + for (index = 0; index < count; index++) { + if (ids->data.as_array->entries[index]->type != CFL_VARIANT_ARRAY) { + return CMT_FALSE; + } + id = ids->data.as_array->entries[index]->data.as_array; + if (id->entry_count < 4 || id->entry_count > 5 || + !is_integer(id->entries[0]) || !is_integer(id->entries[1]) || + !label_values_valid(id->entries[2]) || !is_integer(id->entries[3]) || + (id->entry_count == 5 && !is_integer(id->entries[4]))) { + return CMT_FALSE; + } + } + return CMT_TRUE; +} + +static int compare_u64(uint64_t left, uint64_t right) +{ + return left < right ? -1 : left > right; +} + +/* Order a label value against one read back from an identifier: nil first, + * then by length and bytes, so values keep their boundaries. */ +static int compare_label_value(struct cfl_variant *value, cfl_sds_t label) +{ + size_t length; + int result; + + if (value->type == CFL_VARIANT_NULL || label == NULL) { + return (value->type != CFL_VARIANT_NULL) - (label != NULL); + } + length = cfl_sds_len(value->data.as_string); + if (length != cfl_sds_len(label)) { + return length < cfl_sds_len(label) ? -1 : 1; + } + result = memcmp(value->data.as_string, label, length); + return (result > 0) - (result < 0); +} + +static int compare_id_header(struct cfl_array *id, uint64_t hash, uint64_t timestamp, + uint64_t fingerprint, int has_start, uint64_t start, + size_t label_count) +{ + int result; + + result = compare_u64(integer_bits(id->entries[0]), hash); + if (result == 0) { + result = compare_u64(integer_bits(id->entries[1]), timestamp); + } + if (result == 0) { + result = compare_u64(integer_bits(id->entries[3]), fingerprint); + } + if (result == 0) { + result = (id->entry_count == 5) - (has_start != 0); + } + if (result == 0 && has_start) { + result = compare_u64(integer_bits(id->entries[4]), start); + } + if (result == 0) { + result = compare_u64(id->entries[2]->data.as_array->entry_count, label_count); + } + return result; +} + +/* Order an identifier against a decoded sample with its fingerprint. */ +static int compare_id_sample(struct cfl_array *id, struct cmt_metric *sample, + uint64_t fingerprint) +{ + struct cfl_array *values; + struct cfl_list *head; + struct cmt_map_label *label; + size_t index = 0; + int has_start; + int result; + + has_start = cmt_metric_has_start_timestamp(sample); + result = compare_id_header(id, sample->hash, cmt_metric_get_timestamp(sample), + fingerprint, has_start, + has_start ? cmt_metric_get_start_timestamp(sample) : 0, + cfl_list_size(&sample->labels)); + values = id->entries[2]->data.as_array; + cfl_list_foreach(head, &sample->labels) { + if (result != 0) { + break; + } + label = cfl_list_entry(head, struct cmt_map_label, _head); + result = compare_label_value(values->entries[index++], label->name); + } + return result; +} + +/* Order two identifiers. */ +static int compare_ids(struct cfl_array *left, struct cfl_array *right) +{ + struct cfl_array *left_values; + struct cfl_array *right_values; + struct cfl_variant *value; + size_t index; + int has_start; + int result; + + has_start = right->entry_count == 5; + right_values = right->entries[2]->data.as_array; + result = compare_id_header(left, integer_bits(right->entries[0]), + integer_bits(right->entries[1]), + integer_bits(right->entries[3]), has_start, + has_start ? integer_bits(right->entries[4]) : 0, + right_values->entry_count); + left_values = left->entries[2]->data.as_array; + for (index = 0; result == 0 && index < left_values->entry_count; index++) { + value = right_values->entries[index]; + result = compare_label_value(left_values->entries[index], + value->type == CFL_VARIANT_NULL ? + NULL : value->data.as_string); + } + return result; +} + +static int compare_staged(struct cfl_array *ids, size_t left, size_t right) +{ + return compare_ids(ids->entries[left]->data.as_array, + ids->entries[right]->data.as_array); +} + +static void sift_down(struct cfl_array *ids, size_t *order, size_t root, size_t count) +{ + size_t child; + size_t swap; + + while ((child = root * 2 + 1) < count) { + if (child + 1 < count && compare_staged(ids, order[child], order[child + 1]) < 0) { + child++; + } + if (compare_staged(ids, order[root], order[child]) >= 0) { + return; + } + swap = order[root]; + order[root] = order[child]; + order[child] = swap; + root = child; + } +} + +/* Sort identifier positions in place. The comparison needs the identifiers, + * which qsort() cannot receive without shared state. */ +static void sort_point_ids(struct cfl_array *ids, size_t *order, size_t count) +{ + size_t index; + size_t swap; + + for (index = count / 2; index-- > 0;) { + sift_down(ids, order, index, count); + } + for (index = count; index-- > 1;) { + swap = order[0]; + order[0] = order[index]; + order[index] = swap; + sift_down(ids, order, 0, index); + } +} + +/* Find the staged entry recorded for this sample, or point_count. */ +static size_t find_point_id(struct staged_family_metadata *staged, struct cmt_metric *sample, + uint64_t fingerprint) +{ + size_t low = 0; + size_t high = staged->point_count; + size_t middle; + + while (low < high) { + middle = low + (high - low) / 2; + if (compare_id_sample(staged->point_ids->entries[staged->order[middle]] + ->data.as_array, sample, fingerprint) < 0) { + low = middle + 1; + } + else { + high = middle; + } + } + if (low >= staged->point_count || + compare_id_sample(staged->point_ids->entries[staged->order[low]]->data.as_array, + sample, fingerprint) != 0) { + return staged->point_count; + } + return staged->order[low]; +} + +/* + * Take the staged metadata recorded for this sample. Without identifiers + * (v2.3.0) points bind by position. Otherwise each sample is looked up by + * identity through the sorted identifiers, wherever it now sits. A sample + * whose identity is not recorded gets no metadata, and neither does one + * whose identity was recorded more than once: nothing tells those points + * apart, so whichever binding was chosen could be another point's. + */ +static struct cfl_kvlist *take_point_metadata(struct staged_family_metadata *staged, + struct cmt_metric *sample, size_t position) +{ + struct cfl_kvlist *metadata; + uint64_t fingerprint; + size_t index; + + if (staged->point_ids == NULL) { + index = position; + } + else { + if (point_fingerprint(staged->map, sample, &fingerprint) != 0) { + return NULL; + } + index = find_point_id(staged, sample, fingerprint); + if (index >= staged->point_count || + (staged->point_flags[index] & POINT_AMBIGUOUS)) { + return NULL; + } + staged->point_flags[index] |= POINT_MATCHED; + } + metadata = staged->points[index]; + staged->points[index] = NULL; + return metadata; +} + +static int stage_map_metadata(struct cmt_map *map, void *data) +{ + struct restore_state *state = data; + struct staged_family_metadata *staged; + struct cfl_variant *family; + struct cfl_variant *points; + struct cfl_variant *reset; + struct cfl_variant *ids; + size_t index; + + staged = &state->staged[state->index]; + family = state->families->entries[state->index++]; + staged->map = map; + points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); + reset = cfl_kvlist_fetch(family->data.as_kvlist, "allow_reset"); + if (!reset || reset->type != CFL_VARIANT_BOOL) { + return -1; + } + staged->allow_reset = reset->data.as_bool; + + /* metadata that cannot be read back is skipped for this family only */ + if (restore_optional_map(&staged->family, + cfl_kvlist_fetch(family->data.as_kvlist, "family")) != 0) { + staged->family = NULL; + } + + ids = cfl_kvlist_fetch(family->data.as_kvlist, "point_ids"); + if (ids != NULL) { + /* Points are bound by labels, time and start time. Identifiers that + * cannot be read back leave the family without point metadata. */ + staged->restore_points = point_ids_valid(ids, points->data.as_array->entry_count); + staged->point_ids = staged->restore_points ? ids->data.as_array : NULL; + } + else { + /* v2.3.0 sidecars carry no identifiers and bind by position. Label-less + * entries fold into the static metric on decode, so the point list of + * that family may no longer line up: keep the family metadata but skip + * its point metadata rather than misattributing it. */ + staged->restore_points = points->data.as_array->entry_count == + cfl_list_size(&map->metrics) + !!map->metric_static_set; + } + if (!staged->restore_points) { + return 0; + } + + staged->point_count = points->data.as_array->entry_count; + if (staged->point_count > 0) { + staged->points = calloc(staged->point_count, sizeof(*staged->points)); + if (!staged->points) { + return -1; + } + } + if (staged->point_ids != NULL && staged->point_count > 0) { + staged->point_flags = calloc(staged->point_count, sizeof(*staged->point_flags)); + if (!staged->point_flags) { + return -1; + } + } + for (index = 0; index < staged->point_count; index++) { + /* a point whose metadata cannot be read back keeps none here, and + * keeps its legacy entry: only metadata read back supersedes it */ + if (restore_optional_map(&staged->points[index], + points->data.as_array->entries[index]) != 0) { + staged->points[index] = NULL; + } + if (staged->point_flags != NULL && staged->points[index] != NULL) { + staged->point_flags[index] |= POINT_HAS_METADATA; + } + } + if (staged->point_ids != NULL && staged->point_count > 0) { + staged->order = malloc(staged->point_count * sizeof(*staged->order)); + if (!staged->order) { + return -1; + } + for (index = 0; index < staged->point_count; index++) { + staged->order[index] = index; + } + sort_point_ids(staged->point_ids, staged->order, staged->point_count); + + for (index = 1; index < staged->point_count; index++) { + if (compare_staged(staged->point_ids, staged->order[index - 1], + staged->order[index]) == 0) { + staged->point_flags[staged->order[index - 1]] |= POINT_AMBIGUOUS; + staged->point_flags[staged->order[index]] |= POINT_AMBIGUOUS; + } + } + } + return 0; +} + +static struct cfl_kvlist *legacy_type_root(struct cmt *context, struct cmt_map *map) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(context->external_metadata, "otlp"); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + value = cfl_kvlist_fetch(value->data.as_kvlist, "metrics"); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + value = cfl_kvlist_fetch(value->data.as_kvlist, (char *) legacy_type_key(map->type)); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return value->data.as_kvlist; +} + +static struct cfl_kvlist *legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_kvlist *type_root; + struct cfl_variant *value; + + type_root = legacy_type_root(context, map); + if (type_root == NULL) { + return NULL; + } + value = cfl_kvlist_fetch(type_root, map->opts->fqname); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return value->data.as_kvlist; +} + +/* Remove legacy containers left empty once native metadata replaced them. */ +static void prune_legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_kvlist *family; + struct cfl_variant *datapoints; + + family = legacy_family(context, map); + if (family == NULL) { + return; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints != NULL && datapoints->type == CFL_VARIANT_KVLIST && + cfl_kvlist_count(datapoints->data.as_kvlist) == 0) { + cfl_kvlist_remove(family, "datapoints"); + } + if (cfl_kvlist_count(family) == 0) { + cfl_kvlist_remove(legacy_type_root(context, map), map->opts->fqname); + } +} + +struct legacy_point_key { + uint64_t hash; + uint64_t timestamp; +}; + +static int compare_legacy_point_keys(const void *left, const void *right) +{ + const struct legacy_point_key *a = left; + const struct legacy_point_key *b = right; + + if (a->hash != b->hash) { + return a->hash < b->hash ? -1 : 1; + } + return (a->timestamp > b->timestamp) - (a->timestamp < b->timestamp); +} + +static void add_sample_key(struct legacy_point_key *keys, size_t *count, + struct cmt_metric *sample) +{ + keys[*count].hash = sample->hash; + keys[*count].timestamp = cmt_metric_get_timestamp(sample); + (*count)++; +} + +/* + * Append the keys two or more decoded samples share. Distinct series can share + * a series hash, and windows of one series a time, so such a legacy entry may + * be any of their points': it is no sample's fallback. Returns -1 when the + * samples cannot be gathered. + */ +static int add_shared_sample_keys(struct legacy_point_key *keys, size_t *count, + struct cmt_map *map) +{ + struct legacy_point_key *samples; + struct cfl_list *head; + size_t total = 0; + size_t index; + + samples = malloc((cfl_list_size(&map->metrics) + 1) * sizeof(*samples)); + if (samples == NULL) { + return -1; + } + if (map->metric_static_set) { + add_sample_key(samples, &total, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + add_sample_key(samples, &total, cfl_list_entry(head, struct cmt_metric, _head)); + } + qsort(samples, total, sizeof(*samples), compare_legacy_point_keys); + for (index = 1; index < total; index++) { + if (compare_legacy_point_keys(&samples[index - 1], &samples[index]) == 0 && + (*count == 0 || + compare_legacy_point_keys(&keys[*count - 1], &samples[index]) != 0)) { + keys[(*count)++] = samples[index]; + } + } + free(samples); + return 0; +} + +static void add_restored_key(struct legacy_point_key *keys, size_t *count, + struct cmt_metric *sample) +{ + if (sample->otlp_metadata != NULL) { + keys[*count].hash = sample->hash; + keys[*count].timestamp = cmt_metric_get_timestamp(sample); + (*count)++; + } +} + +/* A legacy "hash:timestamp" key names a restored point when it reads back + * as one of the restored keys in its canonical form, ignoring case as + * cfl_kvlist_remove() did. */ +static int legacy_key_listed(cfl_sds_t name, struct legacy_point_key *keys, size_t count) +{ + struct legacy_point_key key; + char canonical[64]; + char *end; + size_t length; + size_t index; + + key.hash = strtoull(name, &end, 16); + if (*end != ':') { + return CMT_FALSE; + } + key.timestamp = strtoull(end + 1, NULL, 10); + snprintf(canonical, sizeof(canonical), "%" PRIx64 ":%" PRIu64, + key.hash, key.timestamp); + length = strlen(canonical); + if (cfl_sds_len(name) != length) { + return CMT_FALSE; + } + for (index = 0; index < length; index++) { + if (tolower((unsigned char) name[index]) != canonical[index]) { + return CMT_FALSE; + } + } + return bsearch(&key, keys, count, sizeof(*keys), compare_legacy_point_keys) != NULL; +} + +/* + * Drop legacy copies the native metadata supersedes, in one pass over the + * legacy entries: removing them one by one would rescan the list for every + * point. + * + * With identifiers, a legacy entry, keyed by series hash and time only, stays + * only when every point recorded under its key was bound to a decoded sample + * by a unique identity and has no native metadata to read back: the entry is + * then that sample's own, as in released readers. It is dropped when native + * metadata replaced it, and when a recorded point was not bound (removed, + * changed by an intermediary, or ambiguous): the entry describes another or + * an older point. Legacy entries of points never recorded stay. + * Without identifiers (v2.3.0), only the copies of restored points are + * dropped, so points an intermediary left unbound keep what released readers + * keep. + */ +static void drop_legacy_points(struct cfl_kvlist *family, + struct staged_family_metadata *staged) +{ + struct cmt_map *map = staged->map; + struct cfl_array *id; + size_t index; + struct legacy_point_key *keys; + struct cfl_variant *datapoints; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *tmp; + struct cmt_metric *sample; + size_t count = 0; + + if (family == NULL) { + return; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints == NULL || datapoints->type != CFL_VARIANT_KVLIST) { + return; + } + keys = malloc(((staged->point_ids != NULL ? staged->point_count : 0) + + cfl_list_size(&map->metrics) + 1) * sizeof(*keys)); + if (keys == NULL) { + /* no entry can be shown to be a safe fallback */ + cfl_kvlist_remove(family, "datapoints"); + return; + } + if (staged->point_ids != NULL) { + for (index = 0; index < staged->point_count; index++) { + if ((staged->point_flags[index] & POINT_MATCHED) && + !(staged->point_flags[index] & POINT_HAS_METADATA)) { + continue; + } + id = staged->point_ids->entries[index]->data.as_array; + keys[count].hash = integer_bits(id->entries[0]); + keys[count].timestamp = integer_bits(id->entries[1]); + count++; + } + } + else { + if (map->metric_static_set) { + add_restored_key(keys, &count, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + add_restored_key(keys, &count, sample); + } + } + qsort(keys, count, sizeof(*keys), compare_legacy_point_keys); + cfl_list_foreach_safe(head, tmp, &datapoints->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (count > 0 && legacy_key_listed(pair->key, keys, count)) { + cfl_kvpair_destroy(pair); + } + } + free(keys); +} + +static void commit_map_metadata(struct cmt *context, + struct staged_family_metadata *staged) +{ + struct cmt_map *map = staged->map; + struct cfl_kvlist *legacy; + struct cfl_list *head; + struct cmt_metric *sample; + size_t index = 0; + + legacy = legacy_family(context, map); + if (legacy != NULL && staged->family != NULL) { + cfl_kvlist_remove(legacy, "metadata"); + } + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = staged->family; + staged->family = NULL; + if (map->type == CMT_COUNTER) { + ((struct cmt_counter *) map->parent)->allow_reset = staged->allow_reset; + } + if (!staged->restore_points) { + prune_legacy_family(context, map); + return; + } + if (map->metric_static_set) { + cfl_kvlist_destroy(map->metric.otlp_metadata); + map->metric.otlp_metadata = take_point_metadata(staged, &map->metric, index++); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = take_point_metadata(staged, sample, index++); + } + drop_legacy_points(legacy, staged); + prune_legacy_family(context, map); +} + +static int check_map_metadata_shape(struct cmt_map *map, void *data) +{ + struct restore_state *state = data; + struct cfl_variant *family; + struct cfl_variant *points; + struct cfl_variant *name; + struct cfl_variant *type; + + if (state->index >= state->families->entry_count) { + return -1; + } + family = state->families->entries[state->index++]; + if (family->type != CFL_VARIANT_KVLIST) { + return -1; + } + points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); + name = cfl_kvlist_fetch(family->data.as_kvlist, "name"); + type = cfl_kvlist_fetch(family->data.as_kvlist, "type"); + if (points == NULL || points->type != CFL_VARIANT_ARRAY) { + return -1; + } + if (name != NULL && (name->type != CFL_VARIANT_STRING || + cfl_sds_len(name->data.as_string) != cfl_sds_len(map->opts->fqname) || + memcmp(name->data.as_string, map->opts->fqname, + cfl_sds_len(map->opts->fqname)) != 0)) { + return -1; + } + if (type != NULL && !((type->type == CFL_VARIANT_INT && type->data.as_int64 == map->type) || + (type->type == CFL_VARIANT_UINT && type->data.as_uint64 == map->type))) { + return -1; + } + return 0; +} + +/* + * Drop legacy entries whose key two or more decoded samples share. Distinct + * series can share a series hash, and windows of one series a time, so such + * an entry may be any of their points': it is no sample's fallback. This + * holds whether native metadata was restored, skipped or absent, so it runs + * for every family. Entries one sample owns stay. + */ +static int drop_shared_legacy_keys(struct cmt_map *map, void *data) +{ + struct cmt *context = data; + struct legacy_point_key *keys; + struct cfl_kvlist *family; + struct cfl_variant *datapoints; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *tmp; + size_t count = 0; + + family = legacy_family(context, map); + if (family == NULL) { + return 0; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints == NULL || datapoints->type != CFL_VARIANT_KVLIST) { + return 0; + } + keys = malloc((cfl_list_size(&map->metrics) + 1) * sizeof(*keys)); + if (keys == NULL || add_shared_sample_keys(keys, &count, map) != 0) { + /* no entry can be shown to be a safe fallback */ + cfl_kvlist_remove(family, "datapoints"); + } + else if (count > 0) { + cfl_list_foreach_safe(head, tmp, &datapoints->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (legacy_key_listed(pair->key, keys, count)) { + cfl_kvpair_destroy(pair); + } + } + } + free(keys); + prune_legacy_family(context, map); + return 0; +} + +static void restore_native_metadata(struct cmt *context); + +int cmt_otlp_metadata_restore(struct cmt *context) +{ + restore_native_metadata(context); + visit_maps(context, drop_shared_legacy_keys, context); + /* Malformed optional metadata never invalidates the native metric payload. */ + return 0; +} + +static void restore_native_metadata(struct cmt *context) +{ + struct cfl_variant *value; + struct restore_state state; + int result; + size_t family_index; + size_t point_index; + + value = cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY); + if (value == NULL) { + return; + } + if (value->type != CFL_VARIANT_ARRAY) { + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return; + } + state.families = value->data.as_array; + state.index = 0; + result = visit_maps(context, check_map_metadata_shape, &state); + if (result != 0 || state.index != state.families->entry_count) { + /* Optional metadata must not invalidate an otherwise usable chunk. */ + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return; + } + state.staged = calloc(state.families->entry_count, sizeof(*state.staged)); + if (!state.staged && state.families->entry_count > 0) { + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return; + } + state.index = 0; + result = visit_maps(context, stage_map_metadata, &state); + if (result == 0) { + for (family_index = 0; family_index < state.families->entry_count; family_index++) { + commit_map_metadata(context, &state.staged[family_index]); + } + } + for (family_index = 0; family_index < state.families->entry_count; family_index++) { + cfl_kvlist_destroy(state.staged[family_index].family); + if (state.staged[family_index].points != NULL) { + for (point_index = 0; point_index < state.staged[family_index].point_count; point_index++) { + cfl_kvlist_destroy(state.staged[family_index].points[point_index]); + } + } + free(state.staged[family_index].points); + free(state.staged[family_index].order); + free(state.staged[family_index].point_flags); + } + free(state.staged); + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); +} + +int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_list *key_head; + struct cfl_list *value_head; + struct cmt_map_label *key; + struct cmt_map_label *value; + struct cfl_kvlist *projection; + + if (!sample->otlp_metadata) { + return -1; + } + projection = cfl_kvlist_create(); + if (!projection) { + return -1; + } + key_head = map->label_keys.next; + value_head = sample->labels.next; + while (key_head != &map->label_keys && value_head != &sample->labels) { + key = cfl_list_entry(key_head, struct cmt_map_label, _head); + value = cfl_list_entry(value_head, struct cmt_map_label, _head); + if (key->name && value->name && cfl_kvlist_insert_string(projection, key->name, value->name)) { + cfl_kvlist_destroy(projection); + return -1; + } + key_head = key_head->next; + value_head = value_head->next; + } + cfl_kvlist_remove(sample->otlp_metadata, "label_projection"); + if (cfl_kvlist_insert_kvlist(sample->otlp_metadata, "label_projection", projection)) { + cfl_kvlist_destroy(projection); + return -1; + } + return 0; +} + +int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_variant *projection; + struct cfl_list *key_head; + struct cfl_list *value_head; + struct cfl_list *original_head; + struct cfl_kvpair *original; + struct cmt_map_label *key; + struct cmt_map_label *value; + + if (!sample->otlp_metadata) { + return CMT_FALSE; + } + projection = cfl_kvlist_fetch(sample->otlp_metadata, "label_projection"); + if (!projection || projection->type != CFL_VARIANT_KVLIST) { + return CMT_FALSE; + } + key_head = map->label_keys.next; + value_head = sample->labels.next; + original_head = projection->data.as_kvlist->list.next; + while (key_head != &map->label_keys && value_head != &sample->labels) { + key = cfl_list_entry(key_head, struct cmt_map_label, _head); + value = cfl_list_entry(value_head, struct cmt_map_label, _head); + if (key->name && value->name) { + if (original_head == &projection->data.as_kvlist->list) { + return CMT_FALSE; + } + original = cfl_list_entry(original_head, struct cfl_kvpair, _head); + if (original->val->type != CFL_VARIANT_STRING || + strcmp(original->key, key->name) || strcmp(original->val->data.as_string, value->name)) { + return CMT_FALSE; + } + original_head = original_head->next; + } + key_head = key_head->next; + value_head = value_head->next; + } + return original_head == &projection->data.as_kvlist->list && value_head == &sample->labels; +} + +/* + * CMetrics stores classic histogram buckets cumulatively (including +Inf) and + * the msgpack wire format carries them exactly like that. Fluent Bit v5.1.0 + * and older, however, stored OTLP-decoded histograms with independent bucket + * counts in the same fields, so chunks written by those releases (forward + * peers, filesystem backlog) can contain either shape without any marker. + * + * Detect the shape from the sample itself instead of trusting a marker: + * + * - monotonic buckets whose last (+Inf) value equals count: cumulative. + * - otherwise, buckets whose sum equals count: independent counts. + * - anything else is left untouched (cumulative by default). + * + * Both rules can only match at once when every bucket but +Inf is zero, in + * which case both interpretations are identical, so the check is unambiguous + * for consistent data. + */ +static int histogram_sample_is_independent(struct cmt_metric *sample, size_t count) +{ + uint64_t total; + uint64_t value; + uint64_t previous; + uint64_t sample_count; + size_t index; + int monotonic; + + if (sample->hist_buckets == NULL || count == 0) { + return CMT_FALSE; + } + + sample_count = cmt_metric_hist_get_count_value(sample); + total = 0; + previous = 0; + monotonic = CMT_TRUE; + + for (index = 0; index < count; index++) { + value = sample->hist_buckets[index]; + if (value < previous) { + monotonic = CMT_FALSE; + } + previous = value; + + if (value > UINT64_MAX - total) { + /* An overflowing sum cannot match any count. */ + return CMT_FALSE; + } + total += value; + } + + if (monotonic && sample->hist_buckets[count - 1] == sample_count) { + return CMT_FALSE; + } + + return total == sample_count; +} + +static void normalize_histogram_sample(struct cmt_metric *sample, size_t count) +{ + uint64_t total; + size_t index; + + if (!histogram_sample_is_independent(sample, count)) { + return; + } + + total = 0; + for (index = 0; index < count; index++) { + /* No overflow: the sum was validated against count. */ + total += sample->hist_buckets[index]; + sample->hist_buckets[index] = total; + } +} + +int cmt_otlp_histogram_restore(struct cmt *context) +{ + struct cfl_list *head; + struct cfl_list *sample_head; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + + /* A bucket marker is advisory at most: drop it whatever its value. */ + if (cfl_kvlist_fetch(context->internal_metadata, HISTOGRAM_MARKER_KEY) != NULL) { + cfl_kvlist_remove(context->internal_metadata, HISTOGRAM_MARKER_KEY); + } + + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + if (histogram->buckets == NULL) { + continue; + } + + if (histogram->map->metric_static_set) { + normalize_histogram_sample(&histogram->map->metric, + histogram->buckets->count + 1); + } + + cfl_list_foreach(sample_head, &histogram->map->metrics) { + sample = cfl_list_entry(sample_head, struct cmt_metric, _head); + normalize_histogram_sample(sample, histogram->buckets->count + 1); + } + } + + return 0; +} diff --git a/lib/cmetrics/tests/CMakeLists.txt b/lib/cmetrics/tests/CMakeLists.txt index 5b9c9bc3a15..06555709152 100644 --- a/lib/cmetrics/tests/CMakeLists.txt +++ b/lib/cmetrics/tests/CMakeLists.txt @@ -20,6 +20,8 @@ set(UNIT_TESTS_FILES msgpack_temporality.c format_conversion.c expire.c + otlp_pipeline.c + msgpack_compat.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index 63852e623a0..80d483221b8 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -28,6 +28,12 @@ #include #include #include +#include + +#include +#include +#include +#include #include "cmt_tests.h" @@ -1083,7 +1089,587 @@ void test_cat_static_labels_empty() cmt_destroy(dst); } +/* Decode into a fresh map, whose series have not entered the lookup index. */ +static struct cmt *decode_copy(struct cmt *source) +{ + char *buffer; + size_t size; + size_t offset = 0; + struct cmt *decoded = NULL; + + TEST_ASSERT(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + cmt_encode_msgpack_destroy(buffer); + return decoded; +} + +void test_cat_decoded_index_lifecycle() +{ + struct cmt *source; + struct cmt *destination; + struct cmt *appended; + struct cmt_gauge *gauge; + struct cmt_gauge *merged; + struct cmt_gauge *extra; + struct cmt_metric *metric; + struct cfl_list *head; + struct cfl_list *tmp; + double value; + char *keys[] = {"first", "second"}; + char *left[] = {"ab", "c"}; + char *right[] = {"a", "bc"}; + char *late[] = {"late", NULL}; + char *missing[] = {"missing", ""}; + + source = cmt_create(); + TEST_ASSERT(source != NULL); + gauge = cmt_gauge_create(source, "test", "cat", "index", "Index lifecycle", 2, keys); + TEST_ASSERT(gauge != NULL); + /* These label tuples deliberately produce the same concatenated hash input. */ + TEST_ASSERT(cmt_gauge_set(gauge, 10, 1, 2, left) == 0); + TEST_ASSERT(cmt_gauge_set(gauge, 10, 2, 2, right) == 0); + destination = decode_copy(source); + merged = cfl_list_entry(destination->gauges.next, struct cmt_gauge, _head); + TEST_CHECK(merged->map->indexed_metric_count == 0); + TEST_ASSERT(cmt_gauge_get_val(merged, 2, left, &value) == 0); + TEST_CHECK(value == 1); + TEST_ASSERT(cmt_gauge_get_val(merged, 2, right, &value) == 0); + TEST_CHECK(value == 2); + TEST_CHECK(merged->map->indexed_metric_count == 2); + TEST_CHECK(cmt_gauge_get_val(merged, 2, missing, &value) != 0); + TEST_ASSERT(cmt_cat(destination, source) == 0); + TEST_CHECK(cfl_list_size(&merged->map->metrics) == 2); + cmt_destroy(source); + + source = cmt_create(); + TEST_ASSERT(source != NULL); + gauge = cmt_gauge_create(source, "test", "cat", "index", "Index lifecycle", 2, keys); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 20, 3, 2, late) == 0); + appended = decode_copy(source); + extra = cfl_list_entry(appended->gauges.next, struct cmt_gauge, _head); + /* Exercise a decoder-style append after lookups have already indexed a prefix. */ + cfl_list_foreach_safe(head, tmp, &extra->map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + TEST_CHECK(!metric->hash_indexed); + cfl_list_del(head); + cfl_list_add(head, &merged->map->metrics); + } + cmt_destroy(appended); + cmt_destroy(source); + TEST_ASSERT(cmt_gauge_get_val(merged, 2, late, &value) == 0); + TEST_CHECK(value == 3); + TEST_CHECK(merged->map->indexed_metric_count == 3); + metric = cmt_map_metric_get(&merged->opts, merged->map, 2, late, CMT_FALSE); + TEST_ASSERT(metric != NULL); + cmt_map_metric_destroy(metric); + TEST_CHECK(cmt_gauge_get_val(merged, 2, late, &value) != 0); + TEST_ASSERT(cmt_gauge_get_val(merged, 2, right, &value) == 0); + TEST_CHECK(value == 2); + TEST_ASSERT(cmt_gauge_set(merged, 30, 4, 2, missing) == 0); + cmt_map_metrics_expire(merged->map, 20); + TEST_CHECK(cfl_list_size(&merged->map->metrics) == 1); + TEST_CHECK(cmt_gauge_get_val(merged, 2, left, &value) != 0); + TEST_ASSERT(cmt_gauge_get_val(merged, 2, missing, &value) == 0); + TEST_CHECK(value == 4); + cmt_map_metrics_expire(merged->map, 40); + TEST_CHECK(cfl_list_size(&merged->map->metrics) == 0); + TEST_ASSERT(cmt_gauge_set(merged, 50, 5, 2, late) == 0); + TEST_ASSERT(cmt_gauge_get_val(merged, 2, late, &value) == 0); + TEST_CHECK(value == 5); + cmt_destroy(destination); +} + +void test_cat_many_contexts() +{ + int batch; + int index; + double value; + char label[32]; + char *keys[] = {"series"}; + char *values[] = {label}; + struct cmt *source; + struct cmt *destination; + struct cmt_gauge *gauge; + + destination = cmt_create(); + TEST_ASSERT(destination != NULL); + for (batch = 0; batch < 10; batch++) { + source = cmt_create(); + TEST_ASSERT(source != NULL); + gauge = cmt_gauge_create(source, "test", "cat", "many", "Many series", 1, keys); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < 1000; index++) { + snprintf(label, sizeof(label), "%d", batch * 1000 + index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, batch * 1000 + index, 1, values) == 0); + } + TEST_ASSERT(cmt_cat(destination, source) == 0); + /* Repeated concatenation must update existing identities, not duplicate them. */ + TEST_ASSERT(cmt_cat(destination, source) == 0); + cmt_destroy(source); + } + TEST_CHECK(cfl_list_size(&destination->gauges) == 1); + gauge = cfl_list_entry(destination->gauges.next, struct cmt_gauge, _head); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 10000); + for (index = 0; index < 10000; index++) { + snprintf(label, sizeof(label), "%d", index); + TEST_ASSERT(cmt_gauge_get_val(gauge, 1, values, &value) == 0); + TEST_CHECK(value == index); + } + cmt_destroy(destination); +} + +static struct cmt *create_otlp_gauge_context(char *label, double value, uint64_t flags) +{ + struct cmt *cmt; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + + cmt = cmt_create(); + if (cmt == NULL) { + return NULL; + } + + gauge = cmt_gauge_create(cmt, "", "", "otlp_gauge", "otlp gauge", + 1, (char *[]) {"key"}); + if (gauge == NULL || + cmt_gauge_set(gauge, 100, value, 1, (char *[]) {label}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata == NULL || + cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags) != 0) { + cmt_destroy(cmt); + return NULL; + } + + return cmt; +} + +static uint64_t sample_flags(struct cmt_metric *sample) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(sample->otlp_metadata, "flags"); + if (value == NULL) { + return UINT64_MAX; + } + if (value->type == CFL_VARIANT_UINT) { + return value->data.as_uint64; + } + if (value->type == CFL_VARIANT_INT) { + return (uint64_t) value->data.as_int64; + } + return UINT64_MAX; +} + +void test_cat_clears_stale_otlp_point_metadata() +{ + int label_count; + char *keys[] = {"key"}; + char *values[] = {"v"}; + struct cmt *dst; + struct cmt *src; + struct cmt_gauge *dst_gauge; + struct cmt_gauge *src_gauge; + struct cmt_metric *dst_sample; + struct cmt_metric *src_sample; + + /* Exercise both the static and label-indexed sample replacement paths. */ + for (label_count = 0; label_count <= 1; label_count++) { + dst = cmt_create(); + src = cmt_create(); + TEST_ASSERT(dst != NULL && src != NULL); + + dst_gauge = cmt_gauge_create(dst, "", "", "otlp_gauge", "otlp gauge", + label_count, keys); + src_gauge = cmt_gauge_create(src, "", "", "otlp_gauge", "otlp gauge", + label_count, keys); + TEST_ASSERT(dst_gauge != NULL && src_gauge != NULL); + TEST_ASSERT(cmt_gauge_set(dst_gauge, 100, 1.0, label_count, values) == 0); + TEST_ASSERT(cmt_gauge_set(src_gauge, 200, 2.5, label_count, values) == 0); + + dst_sample = cmt_map_metric_get(&dst_gauge->opts, dst_gauge->map, + label_count, values, CMT_FALSE); + src_sample = cmt_map_metric_get(&src_gauge->opts, src_gauge->map, + label_count, values, CMT_FALSE); + TEST_ASSERT(dst_sample != NULL && src_sample != NULL); + dst_sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(dst_sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(dst_sample->otlp_metadata, + "flags", 1) == 0); + cmt_metric_set_start_timestamp(dst_sample, 50); + TEST_ASSERT(src_sample->otlp_metadata == NULL); + + TEST_ASSERT(cmt_cat(dst, src) == 0); + TEST_CHECK(dst_sample->otlp_metadata == NULL); + TEST_CHECK(cmt_metric_get_value(dst_sample) == 2.5); + TEST_CHECK(cmt_metric_get_timestamp(dst_sample) == 200); + TEST_CHECK(!cmt_metric_has_start_timestamp(dst_sample)); + TEST_CHECK(cfl_list_size(&dst_gauge->map->metrics) == label_count); + + /* Repeated copies must also tolerate an already empty destination. */ + TEST_ASSERT(cmt_cat(dst, src) == 0); + TEST_CHECK(dst_sample->otlp_metadata == NULL); + + cmt_destroy(src); + cmt_destroy(dst); + } +} + +/* Samples carrying OTLP metadata must merge by labels, never duplicate. */ +void test_cat_otlp_metadata_samples_merge_by_labels() +{ + int ret; + struct cmt *first; + struct cmt *second; + struct cmt *other; + struct cmt *final; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cmt_map_label *label; + struct cfl_list *head; + int found_v; + int found_w; + + first = create_otlp_gauge_context("v", 1.0, 1); + second = create_otlp_gauge_context("v", 2.0, 2); + other = create_otlp_gauge_context("w", 3.0, 3); + final = cmt_create(); + TEST_ASSERT(first != NULL && second != NULL && other != NULL && final != NULL); + + ret = cmt_cat(final, first); + TEST_CHECK(ret == 0); + ret = cmt_cat(final, second); + TEST_CHECK(ret == 0); + ret = cmt_cat(final, second); + TEST_CHECK(ret == 0); + + TEST_CHECK(cfl_list_size(&final->gauges) == 1); + gauge = cfl_list_entry_first(&final->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 1); + + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_CHECK(cmt_metric_get_value(sample) == 2.0); + TEST_CHECK(sample_flags(sample) == 2); + + /* The merged sample remains reachable through the label index. */ + ret = cmt_gauge_set(gauge, 200, 5.0, 1, (char *[]) {"v"}); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 1); + TEST_CHECK(cmt_metric_get_value(sample) == 5.0); + + /* A distinct label set is still appended. */ + ret = cmt_cat(final, other); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 2); + + found_v = 0; + found_w = 0; + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + if (strcmp(label->name, "v") == 0) { + found_v++; + TEST_CHECK(sample_flags(sample) == 2); + } + else if (strcmp(label->name, "w") == 0) { + found_w++; + TEST_CHECK(sample_flags(sample) == 3); + TEST_CHECK(cmt_metric_get_value(sample) == 3.0); + } + } + TEST_CHECK(found_v == 1); + TEST_CHECK(found_w == 1); + + cmt_destroy(first); + cmt_destroy(second); + cmt_destroy(other); + cmt_destroy(final); +} + +/* Distinct OTLP data points of one series (different time windows) are kept, + * but concatenating them again must not duplicate either of them. */ +void test_cat_otlp_metadata_distinct_points_kept() +{ + int ret; + int pass; + struct cmt *source; + struct cmt *final; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cmt_metric *second; + struct cmt_map_label *label; + struct cfl_list *head; + uint64_t timestamps; + + source = create_otlp_gauge_context("v", 1.0, 1); + TEST_ASSERT(source != NULL); + + /* Add a second data point of the same series at a later time, the way + * the OTLP decoder appends independent points. */ + gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); + first = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + second = calloc(1, sizeof(struct cmt_metric)); + TEST_ASSERT(second != NULL); + cfl_list_init(&second->labels); + cfl_list_init(&second->_hash_head); + second->map = gauge->map; + second->hash = first->hash; + cfl_list_add(&second->_head, &gauge->map->metrics); + TEST_ASSERT(cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"v"}, + CMT_FALSE) == first); + label = calloc(1, sizeof(struct cmt_map_label)); + TEST_ASSERT(label != NULL); + label->name = cfl_sds_create("v"); + TEST_ASSERT(label->name != NULL); + cfl_list_add(&label->_head, &second->labels); + cmt_metric_set_double(second, 200, 2.0); + second->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(second->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(second->otlp_metadata, "flags", 2) == 0); + + final = cmt_create(); + TEST_ASSERT(final != NULL); + + for (pass = 0; pass < 2; pass++) { + ret = cmt_cat(final, source); + TEST_CHECK(ret == 0); + + gauge = cfl_list_entry_first(&final->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 2); + + timestamps = 0; + cfl_list_foreach(head, &gauge->map->metrics) { + first = cfl_list_entry(head, struct cmt_metric, _head); + timestamps += cmt_metric_get_timestamp(first); + if (cmt_metric_get_timestamp(first) == 100) { + TEST_CHECK(sample_flags(first) == 1); + TEST_CHECK(cmt_metric_get_value(first) == 1.0); + } + else { + TEST_CHECK(sample_flags(first) == 2); + TEST_CHECK(cmt_metric_get_value(first) == 2.0); + } + } + TEST_CHECK(timestamps == 300); + } + + cmt_destroy(source); + cmt_destroy(final); +} + +/* A family with more label keys than the destination cannot be merged: the + * destination must stay encodable and keep its samples. */ +void test_cat_rejects_larger_label_schema() +{ + struct cmt *a; + struct cmt *b; + struct cmt *dst; + struct cmt_gauge *gauge_a; + struct cmt_gauge *gauge_b; + struct cmt_gauge *out; + cfl_sds_t wire; + + a = cmt_create(); + b = cmt_create(); + dst = cmt_create(); + TEST_ASSERT(a != NULL && b != NULL && dst != NULL); + gauge_a = cmt_gauge_create(a, "", "", "g", "g", 1, (char *[]) {"a"}); + gauge_b = cmt_gauge_create(b, "", "", "g", "g", 2, (char *[]) {"a", "b"}); + TEST_ASSERT(gauge_a != NULL && gauge_b != NULL); + TEST_ASSERT(cmt_gauge_set(gauge_a, 100, 1, 1, (char *[]) {"x"}) == 0); + TEST_ASSERT(cmt_gauge_set(gauge_b, 200, 2, 2, (char *[]) {"x", "y"}) == 0); + + TEST_CHECK(cmt_cat(dst, a) == 0); + TEST_CHECK(cmt_cat(dst, b) == -1); + + out = cfl_list_entry_first(&dst->gauges, struct cmt_gauge, _head); + TEST_CHECK(out->map->label_count == 1); + TEST_CHECK(cfl_list_size(&out->map->metrics) == 1); + wire = cmt_encode_opentelemetry_create(dst); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + cmt_encode_opentelemetry_destroy(wire); + } + + cmt_destroy(a); + cmt_destroy(b); + cmt_destroy(dst); +} + +static struct cmt *semantics_counter(int aggregation_type, int allow_reset, int with_sample, + char *unit) +{ + struct cmt *ctx; + struct cmt_counter *counter; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + counter = cmt_counter_create(ctx, "", "", "requests", "requests", 1, (char *[]) {"site"}); + TEST_ASSERT(counter != NULL); + counter->aggregation_type = aggregation_type; + counter->allow_reset = allow_reset; + if (unit != NULL) { + counter->map->unit = cfl_sds_create(unit); + TEST_ASSERT(counter->map->unit != NULL); + } + if (with_sample) { + TEST_ASSERT(cmt_counter_set(counter, 100, 5, 1, (char *[]) {"b"}) == 0); + } + return ctx; +} + +static struct cmt *semantics_histogram(int aggregation_type) +{ + struct cmt *ctx; + struct cmt_histogram *histogram; + uint64_t buckets[] = {1, 2}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "latency", "latency", + cmt_histogram_buckets_create(1, 1.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + histogram->aggregation_type = aggregation_type; + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, buckets, 1.5, 2, 0, NULL) == 0); + return ctx; +} + +/* Later contributors must not change how samples already in an existing + * family are read: an empty one is a no-op and a conflicting one fails + * without changes. */ +void test_cat_keeps_existing_family_semantics() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + + /* empty contributors: delta, non-monotonic, with a unit */ + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_TRUE, CMT_FALSE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + TEST_CHECK(counter->allow_reset == CMT_FALSE); + TEST_CHECK(counter->map->unit == NULL); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + + /* contributors with samples and conflicting semantics */ + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_FALSE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_TRUE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + TEST_CHECK(counter->allow_reset == CMT_FALSE); + TEST_CHECK(cmt_metric_get_value(cfl_list_entry_first(&counter->map->metrics, + struct cmt_metric, _head)) == 5); + cmt_destroy(dst); + + /* unspecified temporality (Fluent Bit 5.0.x histograms) merges with + * either and keeps the destination's */ + dst = semantics_histogram(CMT_AGGREGATION_TYPE_CUMULATIVE); + src = semantics_histogram(CMT_AGGREGATION_TYPE_UNSPECIFIED); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + histogram = cfl_list_entry_first(&dst->histograms, struct cmt_histogram, _head); + TEST_CHECK(histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + src = semantics_histogram(CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + cmt_destroy(dst); +} + +/* A destination family that holds no samples has nothing to reinterpret: + * it takes the semantics of the first contributor that brings samples. */ +void test_cat_empty_family_adopts_semantics() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_FALSE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 0); + + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_TRUE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(counter->allow_reset == CMT_TRUE); + TEST_CHECK(counter->map->unit != NULL && strcmp(counter->map->unit, "ms") == 0); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + + /* once it holds samples, its semantics stay */ + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + cmt_destroy(dst); +} + +/* Values are never converted, so families with different explicit units + * cannot be merged. An empty unit is unspecified. */ +void test_cat_rejects_conflicting_units() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_FALSE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, ""); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + + TEST_CHECK(strcmp(counter->map->unit, "s") == 0); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + cmt_destroy(dst); +} + TEST_LIST = { + {"cat_rejects_larger_label_schema", test_cat_rejects_larger_label_schema}, + {"cat_keeps_existing_family_semantics", test_cat_keeps_existing_family_semantics}, + {"cat_rejects_conflicting_units", test_cat_rejects_conflicting_units}, + {"cat_empty_family_adopts_semantics", test_cat_empty_family_adopts_semantics}, + {"cat_clears_stale_otlp_point_metadata", test_cat_clears_stale_otlp_point_metadata}, + {"cat_otlp_metadata_distinct_points_kept", test_cat_otlp_metadata_distinct_points_kept}, + {"cat_otlp_metadata_samples_merge_by_labels", test_cat_otlp_metadata_samples_merge_by_labels}, + {"cat_many_contexts", test_cat_many_contexts}, + {"cat_decoded_index_lifecycle", test_cat_decoded_index_lifecycle}, {"cat_static_labels_source_dynamic_conflict", test_cat_static_labels_source_dynamic_conflict}, {"cat_static_labels_dynamic_conflict", test_cat_static_labels_dynamic_conflict}, {"cat_static_labels_chain", test_cat_static_labels_chain}, diff --git a/lib/cmetrics/tests/data/all-types-release.c b/lib/cmetrics/tests/data/all-types-release.c new file mode 100644 index 00000000000..fbf53d38edf --- /dev/null +++ b/lib/cmetrics/tests/data/all-types-release.c @@ -0,0 +1,103 @@ +/* + * Generator for all-types-v5.0.0.msgpack, all-types-v5.0.11.msgpack and + * all-types-v5.1.5.msgpack: one context with every metric type, written by the + * CMetrics library bundled with that Fluent Bit release (2.0.5, 2.1.5 and + * 2.2.5). These are the payloads those agents send over forward and keep in + * filesystem chunks, so current readers must keep decoding them. + * + * Build it against the headers and static libraries of lib/cmetrics, lib/cfl + * and lib/fluent-otel-proto from the Fluent Bit release tag and redirect stdout + * to the fixture. The fixtures are versioned on purpose: tests must never + * regenerate them with the current encoder. tests/msgpack_compat.c asserts + * the values below. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_counter *counter; + struct cmt_counter *delta; + struct cmt_gauge *gauge; + struct cmt_gauge *signed_gauge; + struct cmt_gauge *unsigned_gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *metric; + double quantiles[] = {0.5, 0.9}; + double quantile_values[] = {0.2, 0.8}; + uint64_t buckets[] = {1, 3, 4, 5}; + uint64_t positive[] = {1, 2}; + uint64_t negative[] = {3}; + char *buffer; + size_t size; + + cmt_label_add(ctx, "cluster", "prod"); + + counter = cmt_counter_create(ctx, "fixture", "http", "requests_total", "requests", + 1, (char *[]) {"method"}); + cmt_counter_set(counter, 1000, 10, 1, (char *[]) {"GET"}); + cmt_counter_set(counter, 1000, 4, 1, (char *[]) {"POST"}); + metric = cmt_map_metric_get(&counter->opts, counter->map, 1, (char *[]) {"GET"}, 0); + cmt_metric_set_start_timestamp(metric, 500); + + delta = cmt_counter_create(ctx, "", "", "fixture_delta_total", "delta", 0, NULL); + delta->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_counter_allow_reset(delta); + cmt_counter_set(delta, 1000, 3, 0, NULL); + + gauge = cmt_gauge_create(ctx, "", "", "fixture_temperature", "temperature", + 1, (char *[]) {"room"}); + cmt_gauge_set(gauge, 1000, -12.5, 1, (char *[]) {"lab"}); + cmt_gauge_set(gauge, 1000, 21, 1, (char *[]) {""}); + + signed_gauge = cmt_gauge_create(ctx, "", "", "fixture_signed", "signed", 0, NULL); + cmt_gauge_set(signed_gauge, 1000, 0, 0, NULL); + cmt_metric_set_int64(&signed_gauge->map->metric, 1000, -9007199254740993LL); + + unsigned_gauge = cmt_gauge_create(ctx, "", "", "fixture_unsigned", "unsigned", 0, NULL); + cmt_gauge_set(unsigned_gauge, 1000, 0, 0, NULL); + cmt_metric_set_uint64(&unsigned_gauge->map->metric, 1000, UINT64_MAX); + + untyped = cmt_untyped_create(ctx, "", "", "fixture_untyped", "untyped", + 1, (char *[]) {"k"}); + cmt_untyped_set(untyped, 1000, 1.5, 1, (char *[]) {"v"}); + + histogram = cmt_histogram_create(ctx, "", "", "fixture_latency_seconds", "latency", + cmt_histogram_buckets_create(3, 0.1, 1.0, 10.0), + 1, (char *[]) {"route"}); + histogram->aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + histogram->map->unit = cfl_sds_create("s"); + cmt_histogram_set_default(histogram, 1000, buckets, 6.5, 5, 1, (char *[]) {"/"}); + + summary = cmt_summary_create(ctx, "", "", "fixture_rpc_seconds", "rpc", 2, quantiles, + 1, (char *[]) {"svc"}); + cmt_summary_set_default(summary, 1000, quantile_values, 10, 20, 1, (char *[]) {"api"}); + + exp_histogram = cmt_exp_histogram_create(ctx, "", "", "fixture_exp", "exp", + 1, (char *[]) {"op"}); + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_exp_histogram_set_default(exp_histogram, 1000, 2, 1, 0.001, -1, 2, positive, + 0, 1, negative, 1, 4.5, 7, 1, (char *[]) {"read"}); + + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { + return 1; + } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/all-types-v5.0.0.msgpack b/lib/cmetrics/tests/data/all-types-v5.0.0.msgpack new file mode 100644 index 00000000000..52f1f32233c Binary files /dev/null and b/lib/cmetrics/tests/data/all-types-v5.0.0.msgpack differ diff --git a/lib/cmetrics/tests/data/all-types-v5.0.11.msgpack b/lib/cmetrics/tests/data/all-types-v5.0.11.msgpack new file mode 100644 index 00000000000..20fe5c0e8f9 Binary files /dev/null and b/lib/cmetrics/tests/data/all-types-v5.0.11.msgpack differ diff --git a/lib/cmetrics/tests/data/all-types-v5.1.5.msgpack b/lib/cmetrics/tests/data/all-types-v5.1.5.msgpack new file mode 100644 index 00000000000..b850b0d40fc Binary files /dev/null and b/lib/cmetrics/tests/data/all-types-v5.1.5.msgpack differ diff --git a/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c new file mode 100644 index 00000000000..e598dc275c6 --- /dev/null +++ b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c @@ -0,0 +1,39 @@ +/* + * Generator for histogram-cumulative-v5.1.0.msgpack, a fixture written by the + * bundled CMetrics library of Fluent Bit v5.1.0. It holds the native layout + * that release used for histograms built with cmt_histogram_observe() or + * decoded from Prometheus: cumulative buckets [1, 3, 5, 6], bounds + * [0.1, 0.5, 1] and count 6, once as the static sample and once labeled + * path="/a". The chunk is unmarked; readers infer the layout per sample. + * + * Build it against the headers and static libraries of a v5.1.0 checkout and + * redirect stdout to the fixture. The fixture is versioned on purpose: tests + * must never regenerate it with the current encoder. + */ +#include +#include +#include +#include +#include +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_histogram *h; + char *keys[] = {"path"}; + char *vals[] = {"/a"}; + double samples[] = {0.05, 0.2, 0.3, 0.7, 0.8, 2.0}; + char *buffer; + size_t size; + int i; + h = cmt_histogram_create(ctx, "", "", "req_seconds", "request latency", + cmt_histogram_buckets_create(3, 0.1, 0.5, 1.0), 1, keys); + for (i = 0; i < 6; i++) { + cmt_histogram_observe(h, 100, samples[i], 1, vals); + cmt_histogram_observe(h, 100, samples[i], 0, NULL); + } + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { return 1; } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack new file mode 100644 index 00000000000..7384021aed0 Binary files /dev/null and b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack differ diff --git a/lib/cmetrics/tests/data/histogram-v5.1.0.c b/lib/cmetrics/tests/data/histogram-v5.1.0.c new file mode 100644 index 00000000000..0743581e549 --- /dev/null +++ b/lib/cmetrics/tests/data/histogram-v5.1.0.c @@ -0,0 +1,33 @@ +/* + * Generator for histogram-v5.1.0.msgpack, a fixture written by the bundled + * CMetrics library of Fluent Bit v5.1.0. It holds an OTLP-style histogram with + * independent bucket counts [5, 1, 3], count 9, sum 12 and bounds [1, 2]; that + * release stored OTLP-decoded histograms in this representation and wrote no + * bucket-layout marker. + * + * Build it against the headers and static libraries of a v5.1.0 checkout and + * redirect stdout to the fixture. The fixture is versioned on purpose: tests + * must never regenerate it with the current encoder. + */ +#include +#include +#include +#include +#include +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_histogram *h; + uint64_t buckets[] = {5, 1, 3}; + char *buffer; + size_t size; + h = cmt_histogram_create(ctx, "", "", "legacy_histogram", "legacy", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + h->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_histogram_set_default(h, 100, buckets, 12.0, 9, 0, NULL); + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { return 1; } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack b/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack new file mode 100644 index 00000000000..014606de3a1 Binary files /dev/null and b/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack differ diff --git a/lib/cmetrics/tests/encoding.c b/lib/cmetrics/tests/encoding.c index e75b65595e7..12e4ac98cdc 100644 --- a/lib/cmetrics/tests/encoding.c +++ b/lib/cmetrics/tests/encoding.c @@ -417,7 +417,7 @@ static void pack_msgpack_basic_meta(mpack_writer_t *writer, mpack_write_cstr(writer, "meta"); if (payload_type == MSGPACK_MALFORMED_DUPLICATE_TYPE) { - mpack_start_map(writer, 5); + mpack_start_map(writer, 6); mpack_write_cstr(writer, "ver"); mpack_write_uint(writer, MSGPACK_ENCODER_VERSION); @@ -427,7 +427,7 @@ static void pack_msgpack_basic_meta(mpack_writer_t *writer, mpack_write_uint(writer, CMT_GAUGE); } else { - mpack_start_map(writer, 4); + mpack_start_map(writer, payload_type == MSGPACK_MALFORMED_EXP_HIST_SCALE_OVERFLOW ? 4 : 5); mpack_write_cstr(writer, "ver"); mpack_write_uint(writer, MSGPACK_ENCODER_VERSION); @@ -583,6 +583,7 @@ static void assert_malformed_msgpack_rejected(enum malformed_msgpack_payload pay ret = cmt_decode_msgpack_create(&decoded, payload, payload_size, &offset); TEST_CHECK(ret != CMT_DECODE_MSGPACK_SUCCESS); + TEST_MSG("malformed payload type: %d", payload_type); TEST_CHECK(decoded == NULL); cmt_encode_msgpack_destroy(payload); diff --git a/lib/cmetrics/tests/issues.c b/lib/cmetrics/tests/issues.c index c88974d271c..a41cd6494ee 100644 --- a/lib/cmetrics/tests/issues.c +++ b/lib/cmetrics/tests/issues.c @@ -222,6 +222,11 @@ static void check_variant_nesting_limit(int use_maps, size_t nesting_depth, mpack_reader_init_data(&reader, buffer, size); result = unpack_cfl_variant(&reader, &variant); TEST_CHECK((result == 0) == (expected_result == 0)); + if (result != 0) { + TEST_CHECK(variant == NULL); + /* A semantic rejection need not consume the remaining containers. */ + mpack_reader_flag_error(&reader, mpack_error_invalid); + } if (variant != NULL) { cfl_variant_destroy(variant); diff --git a/lib/cmetrics/tests/msgpack_abi.c b/lib/cmetrics/tests/msgpack_abi.c index c9d5d13e318..44a903dd160 100644 --- a/lib/cmetrics/tests/msgpack_abi.c +++ b/lib/cmetrics/tests/msgpack_abi.c @@ -347,7 +347,352 @@ void test_msgpack_abi_histogram_buckets_reject_mismatch() cmt_destroy(cmt); } +static struct cmt *create_static_gauge_with_metadata(void) +{ + struct cmt *source; + struct cmt_gauge *gauge; + struct cfl_kvlist *metadata; + + source = cmt_create(); + if (source == NULL) { + return NULL; + } + + gauge = cmt_gauge_create(source, "", "", "windows", "windows", 0, NULL); + if (gauge == NULL || cmt_gauge_set(gauge, 1, 1.0, 0, NULL) != 0) { + cmt_destroy(source); + return NULL; + } + + metadata = cfl_kvlist_create(); + gauge->map->metric.otlp_metadata = metadata; + if (metadata == NULL || + cfl_kvlist_insert_uint64(metadata, "flags", 7) != 0 || + cfl_kvlist_insert_string(metadata, "text", "value") != 0 || + cfl_kvlist_insert_bytes(metadata, "binary", "\0\xff", 2, CFL_FALSE) != 0) { + cmt_destroy(source); + return NULL; + } + + return source; +} + +static void test_point_metadata_and_unlabeled_windows(void) +{ + struct cmt *source; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *result; + struct cmt_metric *point; + struct cfl_variant *value; + char *buffer; + size_t size; + size_t offset; + int ret; + + /* Typed static point metadata survives a round trip. */ + source = create_static_gauge_with_metadata(); + TEST_ASSERT(source != NULL); + TEST_CHECK(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + offset = 0; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_ASSERT(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(result != NULL); + TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 1); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(value != NULL); + TEST_CHECK((value->type == CFL_VARIANT_UINT && value->data.as_uint64 == 7) || + (value->type == CFL_VARIANT_INT && value->data.as_int64 == 7)); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "text"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == CFL_VARIANT_STRING && cfl_variant_size_get(value) == 5); + TEST_CHECK(memcmp(value->data.as_string, "value", 5) == 0); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "binary"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == CFL_VARIANT_BYTES && cfl_variant_size_get(value) == 2); + TEST_CHECK(memcmp(value->data.as_bytes, "\0\xff", 2) == 0); + TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(source); + + /* A further label-less point is the same series: like released readers, + * it replaces the static metric (last one wins) instead of creating a + * duplicate series. The sidecar binds point metadata by point identity, + * so the surviving point keeps its own metadata. */ + source = create_static_gauge_with_metadata(); + TEST_ASSERT(source != NULL); + gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); + point = calloc(1, sizeof(*point)); + TEST_ASSERT(point != NULL); + cfl_list_init(&point->labels); + point->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(point->otlp_metadata != NULL); + TEST_CHECK(cfl_kvlist_insert_uint64(point->otlp_metadata, "flags", 3) == 0); + cmt_metric_set_double(point, 2, 2.0); + cfl_list_add(&point->_head, &gauge->map->metrics); + TEST_CHECK(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + offset = 0; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_ASSERT(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(result != NULL); + TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); + TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 2); + TEST_CHECK(cmt_metric_get_value(&result->map->metric) == 2.0); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(value != NULL); + TEST_CHECK((value->type == CFL_VARIANT_UINT && value->data.as_uint64 == 3) || + (value->type == CFL_VARIANT_INT && value->data.as_int64 == 3)); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(source); +} + +static void test_index_resize_preserves_family_metadata(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_variant *value; + char label[32]; + char *values[] = {label}; + char *keys[] = {"series"}; + size_t index; + char *buffer; + size_t size; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "resize", "resize", 1, keys); + TEST_ASSERT(gauge != NULL); + gauge->map->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(gauge->map->otlp_metadata, "family", "retained") == 0); + for (index = 0; index < 300; index++) { + snprintf(label, sizeof(label), "%zu", index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, index, 1, values) == 0); + } + TEST_CHECK(gauge->map->metric_bucket_count > 64); + value = cfl_kvlist_fetch(gauge->map->otlp_metadata, "family"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + if (value != NULL && value->type == CFL_VARIANT_STRING) { + TEST_CHECK(strcmp(value->data.as_string, "retained") == 0); + } + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_stale_metadata_sidecar_does_not_drop_chunk(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_array *sidecar; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "retained", "retained", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + /* An intermediary removed a family without updating the optional sidecar. */ + sidecar = cfl_array_create(0); + TEST_ASSERT(sidecar != NULL); + TEST_ASSERT(cfl_kvlist_insert_array(ctx->internal_metadata, + "__cmetrics_native_metadata_v1", sidecar) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + TEST_CHECK(cfl_kvlist_fetch(decoded->internal_metadata, + "__cmetrics_native_metadata_v1") == NULL); + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void check_invalid_optional_metadata(const char *payload, size_t payload_size) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_array *sidecar; + struct cfl_array *points; + struct cfl_kvlist *family; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "retained", "retained", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + sidecar = cfl_array_create(1); + points = cfl_array_create(1); + family = cfl_kvlist_create(); + TEST_ASSERT(sidecar != NULL && points != NULL && family != NULL); + TEST_ASSERT(cfl_array_append_null(points) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(family, "points", points) == 0); + TEST_ASSERT(cfl_kvlist_insert_bool(family, "allow_reset", CFL_FALSE) == 0); + TEST_ASSERT(cfl_kvlist_insert_bytes(family, "family", (char *) payload, + payload_size, CFL_FALSE) == 0); + TEST_ASSERT(cfl_array_append_kvlist(sidecar, family) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(ctx->internal_metadata, + "__cmetrics_native_metadata_v1", sidecar) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + TEST_CHECK(gauge->map->otlp_metadata == NULL); + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_invalid_optional_metadata_does_not_drop_chunk(void) +{ + /* Invalid tag, wrong container, non-string key, truncated map, and trailing + * data. Semantic rejections must also cancel MPack's container tracking. */ + check_invalid_optional_metadata("\xc1", 1); + check_invalid_optional_metadata("\x90", 1); + check_invalid_optional_metadata("\x81\x01\xc0", 3); + check_invalid_optional_metadata("\x81", 1); + check_invalid_optional_metadata("\x80\xc0", 2); +} + +static void check_external_reference_metadata(int metadata_mode) +{ + struct cmt *ctx; + struct cmt *decoded = NULL; + struct cmt_gauge *gauge; + struct cfl_kvlist *nested; + struct cfl_array *array; + struct cfl_variant *value; + char text[] = "borrowed metadata"; + char *buffer = NULL; + size_t size = 0; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "references", "references", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + + nested = cfl_kvlist_create(); + array = cfl_array_create(1); + TEST_ASSERT(nested != NULL && array != NULL); + TEST_ASSERT(cfl_kvlist_insert_reference(ctx->external_metadata, "reference", text) == 0); + TEST_ASSERT(cfl_kvlist_insert_reference(nested, "reference", text) == 0); + TEST_ASSERT(cfl_array_append_reference(array, text) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(nested, "array", array) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(ctx->external_metadata, "nested", nested) == 0); + + if (metadata_mode == 1) { + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(gauge->map->metric.otlp_metadata, "flags", 1) == 0); + } + else if (metadata_mode == 2) { + gauge->map->otlp_metadata = cfl_kvlist_create(); + nested = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && nested != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(nested, "description", "family metadata") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", nested) == 0); + } + + result = cmt_encode_msgpack_create(ctx, &buffer, &size); + TEST_CHECK(result == 0); + if (result == 0) { + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + value = cfl_kvlist_fetch(decoded->external_metadata, "reference"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + if (value != NULL && value->type == CFL_VARIANT_STRING) { + TEST_CHECK(strcmp(value->data.as_string, text) == 0); + } + value = cfl_kvlist_fetch(decoded->external_metadata, "nested"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_KVLIST); + if (value != NULL && value->type == CFL_VARIANT_KVLIST) { + nested = value->data.as_kvlist; + value = cfl_kvlist_fetch(nested, "reference"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + if (value != NULL && value->type == CFL_VARIANT_STRING) { + TEST_CHECK(strcmp(value->data.as_string, text) == 0); + } + value = cfl_kvlist_fetch(nested, "array"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_ARRAY); + if (value != NULL && value->type == CFL_VARIANT_ARRAY) { + array = value->data.as_array; + TEST_CHECK(array->entry_count == 1); + if (array->entry_count == 1) { + value = array->entries[0]; + TEST_CHECK(value->type == CFL_VARIANT_STRING); + if (value->type == CFL_VARIANT_STRING) { + TEST_CHECK(strcmp(value->data.as_string, text) == 0); + } + } + } + } + } + } + + /* Encoding must neither replace nor take ownership of the borrowed input. */ + value = cfl_kvlist_fetch(ctx->external_metadata, "reference"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_REFERENCE); + TEST_CHECK(value != NULL && value->data.as_reference == text); + if (decoded != NULL) { + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + TEST_CHECK(strcmp(text, "borrowed metadata") == 0); +} + +static void test_external_references_without_otlp_metadata(void) +{ + check_external_reference_metadata(0); +} + +static void test_external_references_with_point_metadata(void) +{ + check_external_reference_metadata(1); +} + +static void test_external_references_with_family_metadata(void) +{ + check_external_reference_metadata(2); +} + TEST_LIST = { + {"external_references_without_otlp_metadata", test_external_references_without_otlp_metadata}, + {"external_references_with_point_metadata", test_external_references_with_point_metadata}, + {"external_references_with_family_metadata", test_external_references_with_family_metadata}, + {"invalid_optional_metadata_retains_chunk", test_invalid_optional_metadata_does_not_drop_chunk}, + {"stale_metadata_sidecar_does_not_drop_chunk", test_stale_metadata_sidecar_does_not_drop_chunk}, + {"index_resize_preserves_family_metadata", test_index_resize_preserves_family_metadata}, + {"point_metadata_and_unlabeled_windows", test_point_metadata_and_unlabeled_windows}, {"msgpack_abi_legacy_value_only_decode", test_msgpack_abi_legacy_value_only_decode}, {"msgpack_abi_typed_int64_decode", test_msgpack_abi_typed_int64_decode}, {"msgpack_abi_summary_quantiles_reject_mismatch", test_msgpack_abi_summary_quantiles_reject_mismatch}, diff --git a/lib/cmetrics/tests/msgpack_compat.c b/lib/cmetrics/tests/msgpack_compat.c new file mode 100644 index 00000000000..56c4ecb8a68 --- /dev/null +++ b/lib/cmetrics/tests/msgpack_compat.c @@ -0,0 +1,826 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Compatibility of the MessagePack format between Fluent Bit releases. Agents + * exchange it over forward and keep it in filesystem chunks across upgrades + * and rollbacks, and released readers reject any key they do not know. + * + * Fixtures under tests/data/all-types-*.msgpack were written by the CMetrics + * bundled with each Fluent Bit release (see tests/data/all-types-release.c). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "cmt_tests.h" + +static const char *release_fixtures[] = { + "all-types-v5.0.0.msgpack", + "all-types-v5.0.11.msgpack", + "all-types-v5.1.5.msgpack", +}; + +/* Fluent Bit v5.0.0 (CMetrics 2.0.5) does not know "unit", and Fluent Bit 5.0 + * releases write the aggregation type of counters only: their histograms and + * exponential histograms arrive without temporality. */ +#define RELEASE_FIXTURE_HAS_UNIT(index) ((index) > 0) +#define RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index) ((index) > 1) + +static cfl_sds_t read_fixture(const char *name) +{ + char path[1024]; + + snprintf(path, sizeof(path), "%s/%s", CMT_TESTS_DATA_PATH, name); + return read_file(path); +} + +static struct cmt_map *find_map(struct cmt *ctx, int type, const char *name) +{ + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + + switch (type) { + case CMT_COUNTER: + list = &ctx->counters; + break; + case CMT_GAUGE: + list = &ctx->gauges; + break; + case CMT_UNTYPED: + list = &ctx->untypeds; + break; + case CMT_SUMMARY: + list = &ctx->summaries; + break; + case CMT_HISTOGRAM: + list = &ctx->histograms; + break; + default: + list = &ctx->exp_histograms; + break; + } + + cfl_list_foreach(head, list) { + /* Every family struct starts with its opts and map pointers in the + * same order after a type-specific header; use the typed entries. */ + switch (type) { + case CMT_COUNTER: + map = cfl_list_entry(head, struct cmt_counter, _head)->map; + break; + case CMT_GAUGE: + map = cfl_list_entry(head, struct cmt_gauge, _head)->map; + break; + case CMT_UNTYPED: + map = cfl_list_entry(head, struct cmt_untyped, _head)->map; + break; + case CMT_SUMMARY: + map = cfl_list_entry(head, struct cmt_summary, _head)->map; + break; + case CMT_HISTOGRAM: + map = cfl_list_entry(head, struct cmt_histogram, _head)->map; + break; + default: + map = cfl_list_entry(head, struct cmt_exp_histogram, _head)->map; + break; + } + if (strcmp(map->opts->fqname, name) == 0) { + return map; + } + } + return NULL; +} + +static struct cmt_metric *find_sample(struct cmt_map *map, const char *label_value) +{ + struct cfl_list *head; + struct cmt_metric *sample; + struct cmt_map_label *label; + + if (label_value == NULL) { + return map->metric_static_set ? &map->metric : NULL; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + if (label != NULL && label->name != NULL && strcmp(label->name, label_value) == 0) { + return sample; + } + } + return NULL; +} + +static int has_static_label(struct cmt *ctx, const char *key, const char *value) +{ + struct cfl_list *head; + struct cmt_label *label; + + cfl_list_foreach(head, &ctx->static_labels->list) { + label = cfl_list_entry(head, struct cmt_label, _head); + if (strcmp(label->key, key) == 0 && strcmp(label->val, value) == 0) { + return CMT_TRUE; + } + } + return CMT_FALSE; +} + +/* The content written by tests/data/all-types-release.c. */ +static void check_release_content(struct cmt *ctx, int has_unit, int has_temporality, + const char *source) +{ + struct cmt_map *map; + struct cmt_metric *sample; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_exp_histogram_snapshot snapshot; + + TEST_CASE(source); + TEST_CHECK(has_static_label(ctx, "cluster", "prod")); + + map = find_map(ctx, CMT_COUNTER, "fixture_http_requests_total"); + TEST_ASSERT(map != NULL); + TEST_CHECK(map->label_count == 1); + sample = find_sample(map, "GET"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 10); + TEST_CHECK(cmt_metric_get_timestamp(sample) == 1000); + TEST_CHECK(cmt_metric_has_start_timestamp(sample)); + TEST_CHECK(cmt_metric_get_start_timestamp(sample) == 500); + sample = find_sample(map, "POST"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 4); + TEST_CHECK(!cmt_metric_has_start_timestamp(sample)); + + map = find_map(ctx, CMT_COUNTER, "fixture_delta_total"); + TEST_ASSERT(map != NULL); + TEST_CHECK(((struct cmt_counter *) map->parent)->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(cmt_metric_get_value(&map->metric) == 3); + + map = find_map(ctx, CMT_GAUGE, "fixture_temperature"); + TEST_ASSERT(map != NULL); + sample = find_sample(map, "lab"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == -12.5); + sample = find_sample(map, ""); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 21); + + map = find_map(ctx, CMT_GAUGE, "fixture_signed"); + TEST_ASSERT(map != NULL); + TEST_CHECK(cmt_metric_get_value_type(&map->metric) == CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&map->metric) == -9007199254740993LL); + + map = find_map(ctx, CMT_GAUGE, "fixture_unsigned"); + TEST_ASSERT(map != NULL); + TEST_CHECK(cmt_metric_get_value_type(&map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&map->metric) == UINT64_MAX); + + map = find_map(ctx, CMT_UNTYPED, "fixture_untyped"); + TEST_ASSERT(map != NULL); + sample = find_sample(map, "v"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 1.5); + + map = find_map(ctx, CMT_HISTOGRAM, "fixture_latency_seconds"); + TEST_ASSERT(map != NULL); + histogram = map->parent; + TEST_CHECK(histogram->aggregation_type == (has_temporality ? + CMT_AGGREGATION_TYPE_CUMULATIVE : + CMT_AGGREGATION_TYPE_UNSPECIFIED)); + TEST_ASSERT(histogram->buckets != NULL && histogram->buckets->count == 3); + TEST_CHECK(histogram->buckets->upper_bounds[0] == 0.1); + TEST_CHECK(histogram->buckets->upper_bounds[2] == 10.0); + if (has_unit) { + TEST_CHECK(map->unit != NULL && strcmp(map->unit, "s") == 0); + } + sample = find_sample(map, "/"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_hist_get_value(sample, 0) == 1); + TEST_CHECK(cmt_metric_hist_get_value(sample, 1) == 3); + TEST_CHECK(cmt_metric_hist_get_value(sample, 2) == 4); + TEST_CHECK(cmt_metric_hist_get_value(sample, 3) == 5); + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 5); + TEST_CHECK(cmt_metric_hist_get_sum_value(sample) == 6.5); + + map = find_map(ctx, CMT_SUMMARY, "fixture_rpc_seconds"); + TEST_ASSERT(map != NULL); + summary = map->parent; + TEST_ASSERT(summary->quantiles_count == 2); + TEST_CHECK(summary->quantiles[0] == 0.5 && summary->quantiles[1] == 0.9); + sample = find_sample(map, "api"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 0) == 0.2); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 1) == 0.8); + TEST_CHECK(cmt_summary_get_sum_value(sample) == 10); + TEST_CHECK(cmt_summary_get_count_value(sample) == 20); + + map = find_map(ctx, CMT_EXP_HISTOGRAM, "fixture_exp"); + TEST_ASSERT(map != NULL); + exp_histogram = map->parent; + TEST_CHECK(exp_histogram->aggregation_type == (has_temporality ? + CMT_AGGREGATION_TYPE_DELTA : + CMT_AGGREGATION_TYPE_UNSPECIFIED)); + sample = find_sample(map, "read"); + TEST_ASSERT(sample != NULL); + TEST_ASSERT(cmt_metric_exp_hist_get_snapshot(sample, &snapshot) == 0); + TEST_CHECK(snapshot.scale == 2); + TEST_CHECK(snapshot.zero_count == 1); + TEST_CHECK(snapshot.zero_threshold == 0.001); + TEST_CHECK(snapshot.positive_offset == -1 && snapshot.positive_count == 2); + TEST_CHECK(snapshot.positive_count == 2 && + snapshot.positive_buckets[0] == 1 && snapshot.positive_buckets[1] == 2); + TEST_CHECK(snapshot.negative_offset == 0 && snapshot.negative_count == 1); + TEST_CHECK(snapshot.negative_count == 1 && snapshot.negative_buckets[0] == 3); + TEST_CHECK(snapshot.count == 7); + TEST_CHECK(snapshot.sum_set); + cmt_metric_exp_hist_snapshot_destroy(&snapshot); +} + +static struct cmt *decode_all(char *buffer, size_t size, size_t *offset) +{ + struct cmt *decoded = NULL; + int result; + + result = cmt_decode_msgpack_create(&decoded, buffer, size, offset); + TEST_CHECK_(result == CMT_DECODE_MSGPACK_SUCCESS, "decode result %d", result); + return result == CMT_DECODE_MSGPACK_SUCCESS ? decoded : NULL; +} + +/* Chunks written by released agents decode completely in this release, keep + * their content through a re-encode (forward relay) and through cmt_cat. */ +static void test_release_fixtures(void) +{ + size_t index; + size_t offset; + size_t size; + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt *again; + struct cmt *merged; + char *buffer; + char label[128]; + + for (index = 0; index < sizeof(release_fixtures) / sizeof(*release_fixtures); index++) { + fixture = read_fixture(release_fixtures[index]); + TEST_ASSERT(fixture != NULL); + + offset = 0; + decoded = decode_all(fixture, cfl_sds_len(fixture), &offset); + TEST_ASSERT(decoded != NULL); + TEST_CHECK(offset == cfl_sds_len(fixture)); + snprintf(label, sizeof(label), "%s decoded", release_fixtures[index]); + check_release_content(decoded, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + TEST_ASSERT(cmt_encode_msgpack_create(decoded, &buffer, &size) == 0); + offset = 0; + again = decode_all(buffer, size, &offset); + TEST_ASSERT(again != NULL); + snprintf(label, sizeof(label), "%s re-encoded", release_fixtures[index]); + check_release_content(again, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + merged = cmt_create(); + TEST_ASSERT(merged != NULL); + TEST_CHECK(cmt_cat(merged, again) == 0); + snprintf(label, sizeof(label), "%s concatenated", release_fixtures[index]); + check_release_content(merged, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + cmt_destroy(merged); + cmt_decode_msgpack_destroy(again); + cmt_encode_msgpack_destroy(buffer); + cmt_decode_msgpack_destroy(decoded); + cfl_sds_destroy(fixture); + } +} + +/* A forward chunk can hold contexts from agents of different releases. */ +static void test_mixed_release_batch(void) +{ + size_t index; + size_t offset; + size_t count; + size_t size; + cfl_sds_t batch; + cfl_sds_t fixture; + struct cmt *decoded; + char *buffer; + + batch = cfl_sds_create_size(0); + TEST_ASSERT(batch != NULL); + for (index = 0; index < sizeof(release_fixtures) / sizeof(*release_fixtures); index++) { + fixture = read_fixture(release_fixtures[index]); + TEST_ASSERT(fixture != NULL); + batch = cfl_sds_cat(batch, fixture, cfl_sds_len(fixture)); + cfl_sds_destroy(fixture); + } + + /* The current encoder's output for the newest fixture closes the batch. */ + fixture = read_fixture(release_fixtures[2]); + offset = 0; + decoded = decode_all(fixture, cfl_sds_len(fixture), &offset); + TEST_ASSERT(decoded != NULL); + TEST_ASSERT(cmt_encode_msgpack_create(decoded, &buffer, &size) == 0); + batch = cfl_sds_cat(batch, buffer, size); + cmt_encode_msgpack_destroy(buffer); + cmt_decode_msgpack_destroy(decoded); + cfl_sds_destroy(fixture); + + offset = 0; + count = 0; + while (offset < cfl_sds_len(batch)) { + decoded = decode_all(batch, cfl_sds_len(batch), &offset); + TEST_ASSERT(decoded != NULL); + check_release_content(decoded, RELEASE_FIXTURE_HAS_UNIT(count), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(count), "batch entry"); + cmt_decode_msgpack_destroy(decoded); + count++; + } + TEST_CHECK(count == 4); + TEST_CHECK(offset == cfl_sds_len(batch)); + cfl_sds_destroy(batch); +} + +/* Every metric type with labels, static samples, typed values, start + * timestamps, units, OTLP metadata and the native metadata sidecar. */ +static struct cmt *create_full_context(void) +{ + struct cmt *ctx; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *sample; + struct cfl_kvlist *metadata; + double quantiles[] = {0.5, 0.99}; + double quantile_values[] = {1, 2}; + uint64_t buckets[] = {1, 2, 3}; + uint64_t positive[] = {4, 5}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + TEST_ASSERT(cmt_label_add(ctx, "dc", "eu-1") == 0); + + counter = cmt_counter_create(ctx, "ns", "sub", "events_total", "events", 2, + (char *[]) {"a", "b"}); + TEST_ASSERT(counter != NULL); + cmt_counter_allow_reset(counter); + counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + TEST_ASSERT(cmt_counter_set(counter, 10, 5, 2, (char *[]) {"x", "y"}) == 0); + sample = cmt_map_metric_get(&counter->opts, counter->map, 2, (char *[]) {"x", "y"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_start_timestamp(sample, 5); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + counter->map->unit = cfl_sds_create("{event}"); + + gauge = cmt_gauge_create(ctx, "", "", "level", "level", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 10, 0, 0, NULL) == 0); + cmt_metric_set_int64(&gauge->map->metric, 10, INT64_MIN); + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "k", "v") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + + untyped = cmt_untyped_create(ctx, "", "", "raw", "raw", 1, (char *[]) {"k"}); + TEST_ASSERT(untyped != NULL); + TEST_ASSERT(cmt_untyped_set(untyped, 10, NAN, 1, (char *[]) {"nan"}) == 0); + + histogram = cmt_histogram_create(ctx, "", "", "latency", "latency", + cmt_histogram_buckets_create(2, 1.0, 2.0), + 1, (char *[]) {"r"}); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 10, buckets, 4.5, 3, 1, + (char *[]) {"/"}) == 0); + + summary = cmt_summary_create(ctx, "", "", "rpc", "rpc", 2, quantiles, 0, NULL); + TEST_ASSERT(summary != NULL); + TEST_ASSERT(cmt_summary_set_default(summary, 10, quantile_values, 3, 2, 0, NULL) == 0); + + exp_histogram = cmt_exp_histogram_create(ctx, "", "", "exp", "exp", 0, NULL); + TEST_ASSERT(exp_histogram != NULL); + TEST_ASSERT(cmt_exp_histogram_set_default(exp_histogram, 10, -2, 0, 0, 3, 2, positive, + 0, 0, NULL, 0, 0, 9, 0, NULL) == 0); + return ctx; +} + +/* Released decoders (Fluent Bit 5.0.11, 5.1.x) accept exactly these keys; + * Fluent Bit 5.0.0 accepts all of them except "unit". */ +static const char *released_keys[] = { + "aggregation_type", "buckets", "cmetrics", "count", "desc", "exp_histogram", + "external", "hash", "histogram", "labels", "meta", "metrics", "name", + "negative_buckets", "negative_offset", "ns", "opts", "positive_buckets", + "positive_offset", "processing", "quantiles", "quantiles_set", "scale", "ss", + "start_ts", "static_labels", "sum", "summary", "sum_set", "ts", "type", "unit", + "value", "values", "value_type", "value_int64", "value_uint64", "ver", + "zero_count", "zero_threshold", NULL +}; + +static int is_released_key(const char *key, size_t length) +{ + size_t index; + + for (index = 0; released_keys[index] != NULL; index++) { + if (strlen(released_keys[index]) == length && + memcmp(released_keys[index], key, length) == 0) { + return CMT_TRUE; + } + } + return CMT_FALSE; +} + +/* Walk a value; map keys are schema keys except inside the schemaless + * internal ("cmetrics") and external metadata maps, which are skipped. */ +static int walk_keys(mpack_reader_t *reader, int depth) +{ + mpack_tag_t tag; + uint32_t index; + uint32_t count; + char key[64]; + uint32_t length; + int unknown = 0; + + tag = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) { + return -1; + } + switch (mpack_tag_type(&tag)) { + case mpack_type_map: + count = mpack_tag_map_count(&tag); + for (index = 0; index < count; index++) { + tag = mpack_read_tag(reader); + if (mpack_tag_type(&tag) != mpack_type_str) { + return -1; + } + length = mpack_tag_str_length(&tag); + if (length >= sizeof(key)) { + return -1; + } + mpack_read_bytes(reader, key, length); + mpack_done_str(reader); + if (!is_released_key(key, length)) { + TEST_MSG("key not known to released readers: %.*s", (int) length, key); + unknown++; + } + if ((length == 8 && memcmp(key, "cmetrics", 8) == 0) || + (length == 8 && memcmp(key, "external", 8) == 0)) { + mpack_discard(reader); + } + else { + unknown += walk_keys(reader, depth + 1); + } + } + mpack_done_map(reader); + break; + case mpack_type_array: + count = mpack_tag_array_count(&tag); + for (index = 0; index < count; index++) { + unknown += walk_keys(reader, depth + 1); + } + mpack_done_array(reader); + break; + case mpack_type_str: + mpack_skip_bytes(reader, mpack_tag_str_length(&tag)); + mpack_done_str(reader); + break; + case mpack_type_bin: + mpack_skip_bytes(reader, mpack_tag_bin_length(&tag)); + mpack_done_bin(reader); + break; + default: + break; + } + return unknown; +} + +/* New information must travel in the schemaless metadata maps: a new schema + * key makes every released reader reject the whole chunk. */ +static void test_encoder_uses_released_keys_only(void) +{ + struct cmt *ctx; + mpack_reader_t reader; + char *buffer; + size_t size; + int unknown; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + mpack_reader_init_data(&reader, buffer, size); + unknown = walk_keys(&reader, 0); + TEST_CHECK(mpack_reader_destroy(&reader) == mpack_ok); + TEST_CHECK_(unknown == 0, "%d keys unknown to released readers", unknown); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static uint64_t double_bits(double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* Exact values of every scalar type for counters, gauges and untyped. */ +static void test_scalar_extremes(void) +{ + static const uint64_t double_patterns[] = { + 0x7ff8000000000000ULL, /* quiet NaN */ + 0x7ff0000000000002ULL, /* Prometheus staleness marker */ + 0x7ff0000000000000ULL, /* +Inf */ + 0xfff0000000000000ULL, /* -Inf */ + 0x8000000000000000ULL, /* -0.0 */ + 0x0000000000000001ULL, /* smallest denormal */ + 0x7fefffffffffffffULL, /* DBL_MAX */ + }; + static const int64_t signed_values[] = {INT64_MIN, INT64_MAX, -1, 0}; + static const uint64_t unsigned_values[] = {UINT64_MAX, (uint64_t) INT64_MAX + 1, 0}; + struct cmt *ctx; + struct cmt *decoded; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_metric *sample; + struct cmt_map *maps[3]; + char value_label[32]; + char *labels[] = {value_label}; + char *buffer; + size_t size; + size_t offset; + size_t index; + size_t type; + double value; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + counter = cmt_counter_create(ctx, "", "", "c", "c", 1, (char *[]) {"v"}); + gauge = cmt_gauge_create(ctx, "", "", "g", "g", 1, (char *[]) {"v"}); + untyped = cmt_untyped_create(ctx, "", "", "u", "u", 1, (char *[]) {"v"}); + TEST_ASSERT(counter != NULL && gauge != NULL && untyped != NULL); + maps[0] = counter->map; + maps[1] = gauge->map; + maps[2] = untyped->map; + + for (type = 0; type < 3; type++) { + for (index = 0; index < sizeof(double_patterns) / sizeof(*double_patterns); index++) { + snprintf(value_label, sizeof(value_label), "d%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + memcpy(&value, &double_patterns[index], sizeof(value)); + cmt_metric_set_double(sample, 1, value); + } + for (index = 0; index < sizeof(signed_values) / sizeof(*signed_values); index++) { + snprintf(value_label, sizeof(value_label), "i%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_int64(sample, 1, signed_values[index]); + } + for (index = 0; index < sizeof(unsigned_values) / sizeof(*unsigned_values); index++) { + snprintf(value_label, sizeof(value_label), "u%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_uint64(sample, 1, unsigned_values[index]); + } + } + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = decode_all(buffer, size, &offset); + TEST_ASSERT(decoded != NULL); + maps[0] = find_map(decoded, CMT_COUNTER, "c"); + maps[1] = find_map(decoded, CMT_GAUGE, "g"); + maps[2] = find_map(decoded, CMT_UNTYPED, "u"); + for (type = 0; type < 3; type++) { + TEST_ASSERT(maps[type] != NULL); + for (index = 0; index < sizeof(double_patterns) / sizeof(*double_patterns); index++) { + snprintf(value_label, sizeof(value_label), "d%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_DOUBLE); + TEST_CHECK_(double_bits(cmt_metric_get_value(sample)) == double_patterns[index], + "type %zu double %016" PRIx64, type, double_patterns[index]); + } + for (index = 0; index < sizeof(signed_values) / sizeof(*signed_values); index++) { + snprintf(value_label, sizeof(value_label), "i%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_INT64); + TEST_CHECK_(cmt_metric_get_int64_value(sample) == signed_values[index], + "type %zu int64 %" PRId64, type, signed_values[index]); + } + for (index = 0; index < sizeof(unsigned_values) / sizeof(*unsigned_values); index++) { + snprintf(value_label, sizeof(value_label), "u%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK_(cmt_metric_get_uint64_value(sample) == unsigned_values[index], + "type %zu uint64 %" PRIu64, type, unsigned_values[index]); + } + } + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Storage corruption or a broken peer: every prefix and every single byte + * corruption of a full chunk is rejected or decoded, never crashes, and a + * decoded context can be encoded again. Run under Valgrind or ASan. */ +static void test_corrupted_chunks(void) +{ + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + char *copy; + char *again; + size_t size; + size_t again_size; + size_t length; + size_t index; + size_t offset; + int result; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + copy = malloc(size); + TEST_ASSERT(copy != NULL); + + for (length = 0; length < size; length++) { + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, length, &offset); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_decode_msgpack_destroy(decoded); + } + } + + for (index = 0; index < size; index++) { + memcpy(copy, buffer, size); + copy[index] ^= 0xa5; + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, copy, size, &offset); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(offset <= size); + if (cmt_encode_msgpack_create(decoded, &again, &again_size) == 0) { + cmt_encode_msgpack_destroy(again); + } + cmt_decode_msgpack_destroy(decoded); + } + } + + free(copy); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Replace the array header that follows the nth occurrence of a key. */ +static char *replace_array_header(char *buffer, size_t size, const char *key, + const char *header, size_t header_size, size_t *out_size) +{ + size_t index; + size_t key_length; + char *patched; + char encoded_key[32]; + + key_length = strlen(key); + encoded_key[0] = (char) (0xa0 | key_length); + memcpy(&encoded_key[1], key, key_length); + for (index = 0; index + key_length + 2 <= size; index++) { + if (memcmp(&buffer[index], encoded_key, key_length + 1) != 0) { + continue; + } + index += key_length + 1; + /* fixarray headers only: the fixture arrays are small */ + if (((unsigned char) buffer[index] & 0xf0) != 0x90) { + return NULL; + } + patched = malloc(size + header_size); + TEST_ASSERT(patched != NULL); + memcpy(patched, buffer, index); + memcpy(&patched[index], header, header_size); + memcpy(&patched[index + header_size], &buffer[index + 1], size - index - 1); + *out_size = size + header_size - 1; + return patched; + } + return NULL; +} + +/* A declared element count far beyond the remaining input must be rejected + * before any allocation sized from it. */ +static void test_oversized_declared_counts(void) +{ + static const char *keys[] = {"values", "labels", "buckets", "quantiles"}; + static const char huge_array[] = {(char) 0xdd, (char) 0xff, (char) 0xff, + (char) 0xff, (char) 0xff}; + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + char *patched; + size_t size; + size_t patched_size; + size_t offset; + size_t index; + int result; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + for (index = 0; index < sizeof(keys) / sizeof(*keys); index++) { + patched = replace_array_header(buffer, size, keys[index], huge_array, + sizeof(huge_array), &patched_size); + TEST_ASSERT_(patched != NULL, "array key %s found", keys[index]); + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, patched, patched_size, &offset); + TEST_CHECK_(result != CMT_DECODE_MSGPACK_SUCCESS, "%s count rejected", keys[index]); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_decode_msgpack_destroy(decoded); + } + free(patched); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Released readers reject any other format version, so the version cannot + * be raised to signal new content; this release must behave the same. */ +static void test_format_version_is_strict(void) +{ + static const char version_key[] = {(char) 0xa3, 'v', 'e', 'r', 0x02}; + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + size_t size; + size_t index; + size_t offset; + int found = CMT_FALSE; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + for (index = 0; index + sizeof(version_key) <= size; index++) { + if (memcmp(&buffer[index], version_key, sizeof(version_key)) == 0) { + buffer[index + sizeof(version_key) - 1] = 0x03; + found = CMT_TRUE; + break; + } + } + TEST_ASSERT(found); + offset = 0; + decoded = NULL; + TEST_CHECK(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) != + CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(decoded == NULL); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +TEST_LIST = { + {"release_fixtures", test_release_fixtures}, + {"mixed_release_batch", test_mixed_release_batch}, + {"encoder_uses_released_keys_only", test_encoder_uses_released_keys_only}, + {"scalar_extremes", test_scalar_extremes}, + {"corrupted_chunks", test_corrupted_chunks}, + {"oversized_declared_counts", test_oversized_declared_counts}, + {"format_version_is_strict", test_format_version_is_strict}, + {0} +}; diff --git a/lib/cmetrics/tests/msgpack_temporality.c b/lib/cmetrics/tests/msgpack_temporality.c index b4a36e94aa0..e161aa88ed0 100644 --- a/lib/cmetrics/tests/msgpack_temporality.c +++ b/lib/cmetrics/tests/msgpack_temporality.c @@ -37,6 +37,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -174,6 +178,7 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() TEST_CHECK(counter != NULL); counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + counter->allow_reset = CMT_TRUE; offset = 0; result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); @@ -191,6 +196,7 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() TEST_CHECK(decoded_counter != NULL); if (decoded_counter != NULL) { + TEST_CHECK(decoded_counter->allow_reset == CMT_TRUE); TEST_CHECK(decoded_counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); } @@ -202,7 +208,295 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() cmt_destroy(cmt); } +static void test_legacy_histogram_bucket_semantics() +{ + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt_histogram *histogram; + size_t offset = 0; + + fixture = read_file(CMT_TESTS_DATA_PATH "/histogram-v5.1.0.msgpack"); + TEST_ASSERT(fixture != NULL); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, fixture, + cfl_sds_len(fixture), &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + /* Legacy OTLP chunks contain independent bucket counts [5,1,3]. */ + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 0) == 5); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 1) == 6); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 2) == 9); + TEST_CHECK(offset == cfl_sds_len(fixture)); + cmt_destroy(decoded); + cfl_sds_destroy(fixture); +} + +static int buffer_contains(char *buffer, size_t size, const void *needle, size_t length) +{ + size_t index; + + for (index = 0; index + length <= size; index++) { + if (memcmp(buffer + index, needle, length) == 0) { + return CMT_TRUE; + } + } + + return CMT_FALSE; +} + +static void check_histogram_sample(struct cmt_metric *sample, uint64_t *expected, + size_t count) +{ + size_t index; + + for (index = 0; index < count; index++) { + TEST_CHECK(cmt_metric_hist_get_value(sample, index) == expected[index]); + TEST_MSG("bucket %zu: expected %" PRIu64 ", got %" PRIu64, index, + expected[index], cmt_metric_hist_get_value(sample, index)); + } +} + +/* Fluent Bit v5.1.0 natively writes cumulative (Prometheus-style) buckets. */ +static void test_legacy_cumulative_histogram_fixture() +{ + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + size_t offset = 0; + uint64_t cumulative[] = {1, 3, 5, 6}; + uint64_t independent[] = {1, 2, 2, 1}; + size_t index; + + fixture = read_file(CMT_TESTS_DATA_PATH "/histogram-cumulative-v5.1.0.msgpack"); + TEST_ASSERT(fixture != NULL); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, fixture, + cfl_sds_len(fixture), &offset) == 0); + TEST_CHECK(offset == cfl_sds_len(fixture)); + + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(histogram->map->metric_static_set == CMT_TRUE); + check_histogram_sample(&histogram->map->metric, cumulative, 4); + + TEST_ASSERT(cfl_list_size(&histogram->map->metrics) == 1); + sample = cfl_list_entry_first(&histogram->map->metrics, struct cmt_metric, _head); + check_histogram_sample(sample, cumulative, 4); + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 6); + + for (index = 0; index < 4; index++) { + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, index) == independent[index]); + } + + cmt_destroy(decoded); + cfl_sds_destroy(fixture); +} + +/* The wire must stay byte-compatible with v5.1.0 readers: cumulative and + * unmarked, with or without the OTLP metadata sidecar. */ +static void test_histogram_wire_stays_cumulative() +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset; + int pass; + uint64_t counts[] = {5, 6, 9}; + const unsigned char expected[] = {0xa7, 'b','u','c','k','e','t','s',0x93,5,6,9}; + const char marker[] = "__cmetrics_histogram_buckets"; + + for (pass = 0; pass < 2; pass++) { + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, counts, 12.0, 9, 0, NULL) == 0); + + if (pass == 1) { + /* Force the OTLP metadata sidecar path. */ + histogram->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(histogram->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(histogram->map->metric.otlp_metadata, + "flags", 1) == 0); + } + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_CHECK(buffer_contains(buffer, size, expected, sizeof(expected))); + TEST_CHECK(!buffer_contains(buffer, size, marker, sizeof(marker) - 1)); + + offset = 0; + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, counts, 3); + if (pass == 1) { + TEST_CHECK(histogram->map->metric.otlp_metadata != NULL); + } + + cmt_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + } +} + +/* Encode a histogram and splice a bucket marker into the internal metadata + * map, as unreleased writers did; the current encoder never emits it. */ +static int encode_with_marker(uint64_t *counts, uint64_t total, + const char *marker, size_t marker_length, + char **buffer, size_t *size) +{ + struct cmt *ctx; + struct cmt_histogram *histogram; + char *encoded; + char *spliced; + size_t encoded_size; + size_t index; + size_t position; + size_t key_length; + const char header[] = "\xa8" "cmetrics" "\x80"; + const char key[] = "__cmetrics_histogram_buckets"; + + ctx = cmt_create(); + if (ctx == NULL) { + return -1; + } + + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + if (histogram == NULL || + cmt_histogram_set_default(histogram, 100, counts, 12.0, total, 0, NULL) != 0 || + cmt_encode_msgpack_create(ctx, &encoded, &encoded_size) != 0) { + cmt_destroy(ctx); + return -1; + } + cmt_destroy(ctx); + + position = encoded_size; + for (index = 0; index + sizeof(header) - 1 <= encoded_size; index++) { + if (memcmp(encoded + index, header, sizeof(header) - 1) == 0) { + position = index + sizeof(header) - 2; + break; + } + } + if (position == encoded_size) { + cmt_encode_msgpack_destroy(encoded); + return -1; + } + + key_length = sizeof(key) - 1; + spliced = malloc(encoded_size + 2 + key_length + marker_length); + if (spliced == NULL) { + cmt_encode_msgpack_destroy(encoded); + return -1; + } + + /* fixmap(1) { fixstr(key): marker } replaces the empty fixmap. */ + memcpy(spliced, encoded, position); + index = position; + spliced[index++] = (char) 0x81; + spliced[index++] = (char) (0xa0 | key_length); + memcpy(spliced + index, key, key_length); + index += key_length; + memcpy(spliced + index, marker, marker_length); + index += marker_length; + memcpy(spliced + index, encoded + position + 1, encoded_size - position - 1); + index += encoded_size - position - 1; + + cmt_encode_msgpack_destroy(encoded); + *buffer = spliced; + *size = index; + + return 0; +} + +/* Bucket markers written by unreleased builds are advisory: unknown or + * mistyped values must not drop the chunk, and a marker can not force a + * wrong interpretation of the buckets. */ +static void test_histogram_marker_is_ignored() +{ + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset; + size_t index; + int ret; + uint64_t cumulative[] = {5, 6, 9}; + uint64_t independent[] = {5, 1, 3}; + struct { + const char *marker; + size_t length; + uint64_t *counts; + } cases[] = { + {"\xa5" "bogus", 6, cumulative}, + {"\x07", 1, cumulative}, + {"\xc0", 1, cumulative}, + {"\xaa" "per_bucket", 11, cumulative}, + /* spoofed: "cumulative" claimed on independent counts */ + {"\xaa" "cumulative", 11, independent}, + {"\xaa" "cumulative", 11, cumulative}, + {"\xaa" "per_bucket", 11, independent}, + }; + + for (index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) { + buffer = NULL; + size = 0; + ret = encode_with_marker(cases[index].counts, 9, cases[index].marker, + cases[index].length, &buffer, &size); + TEST_ASSERT(ret == 0); + + offset = 0; + decoded = NULL; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_MSG("case %zu: decode returned %d", index, ret); + TEST_CHECK(offset == size); + + if (decoded != NULL) { + histogram = cfl_list_entry_first(&decoded->histograms, + struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, cumulative, 3); + TEST_CHECK(cfl_kvlist_fetch(decoded->internal_metadata, + "__cmetrics_histogram_buckets") == NULL); + cmt_destroy(decoded); + } + + free(buffer); + } +} + +/* Samples whose shape is not consistent with either layout are untouched. */ +static void test_histogram_inconsistent_buckets_untouched() +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset = 0; + uint64_t counts[] = {4, 2, 8}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, counts, 1.0, 11, 0, NULL) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, counts, 3); + cmt_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"legacy_histogram_bucket_semantics", test_legacy_histogram_bucket_semantics}, + {"legacy_cumulative_histogram_fixture", test_legacy_cumulative_histogram_fixture}, + {"histogram_wire_stays_cumulative", test_histogram_wire_stays_cumulative}, + {"histogram_marker_is_ignored", test_histogram_marker_is_ignored}, + {"histogram_inconsistent_buckets_untouched", test_histogram_inconsistent_buckets_untouched}, {"histogram_temporality_survives_msgpack_roundtrip", test_histogram_temporality_survives_msgpack_roundtrip}, {"exp_histogram_temporality_survives_msgpack_roundtrip", diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index f225a7fcfb9..074b8f52517 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -25,6 +26,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -1856,11 +1860,8 @@ static void test_opentelemetry_zero_length_bytes_attribute(void) attribute = roundtrip_dp->attributes[0]; TEST_CHECK(strcmp(attribute->key, "empty_bytes") == 0); TEST_CHECK(attribute->value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); - TEST_CHECK(attribute->value->string_value != NULL); - if (attribute->value->string_value != NULL) { - TEST_CHECK(attribute->value->string_value[0] == '\0'); - } + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE); + TEST_CHECK(attribute->value->bytes_value.len == 0); } } } @@ -2813,7 +2814,1215 @@ static void test_opentelemetry_summary_layout_mismatch(void) cfl_sds_destroy(payload); } +static void test_otlp_unsigned_attribute_preserves_signed_range(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_kvlist *attrs; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *point; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "uint_attributes", "uint attributes", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + attrs = cfl_kvlist_create(); + TEST_ASSERT(attrs != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "small", 3) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "maximum", INT64_MAX) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->metric.otlp_metadata, "attributes", attrs) == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, &gauge->map->metric) == 0); + wire = cmt_encode_opentelemetry_create(ctx); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + point = request->resource_metrics[0]->scope_metrics[0]->metrics[0]->gauge->data_points[0]; + TEST_ASSERT(point->n_attributes == 2); + TEST_CHECK(point->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE); + TEST_CHECK(point->attributes[0]->value->int_value == 3); + TEST_CHECK(point->attributes[1]->value->int_value == INT64_MAX); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + } + cmt_destroy(ctx); +} + +/* Metadata deeper than the msgpack nesting limit stays usable. */ +static void test_otlp_metadata_deep_nesting(void) +{ + int depth; + int found = 0; + struct cfl_list *head; + struct cmt_gauge *entry; + char *buffer; + size_t size; + size_t offset; + struct cmt *ctx; + struct cmt *copy; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *shallow; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + char *labels[] = {"a"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "deep", "deep", 1, (char *[]) {"k"}); + shallow = cmt_gauge_create(ctx, "", "", "shallow", "shallow", 0, NULL); + TEST_ASSERT(gauge != NULL && shallow != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 1000, 7, 1, labels) == 0); + TEST_ASSERT(cmt_gauge_set(shallow, 1000, 9, 0, NULL) == 0); + + /* a point whose metadata nests 40 maps deep */ + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "value") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + sample->otlp_metadata = inner; + + shallow->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(shallow->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(shallow->map->metric.otlp_metadata, + "start_time_unix_nano", 500) == 0); + + /* copying a context copies the deep metadata too */ + copy = cmt_create(); + TEST_ASSERT(copy != NULL); + TEST_CHECK(cmt_cat(copy, ctx) == 0); + gauge = cfl_list_entry_first(©->gauges, struct cmt_gauge, _head); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_CHECK(sample->otlp_metadata != NULL && + cfl_kvlist_fetch(sample->otlp_metadata, "nested") != NULL); + cmt_destroy(copy); + + /* a msgpack round trip skips only the point it cannot read back */ + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = NULL; + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + cfl_list_foreach(head, &decoded->gauges) { + entry = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(entry->opts.name, "shallow") == 0) { + found++; + TEST_CHECK(entry->map->metric.otlp_metadata != NULL && + cfl_kvlist_fetch(entry->map->metric.otlp_metadata, + "start_time_unix_nano") != NULL); + } + else if (strcmp(entry->opts.name, "deep") == 0) { + found++; + sample = cfl_list_entry_first(&entry->map->metrics, struct cmt_metric, _head); + TEST_CHECK(sample->otlp_metadata == NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 7); + } + } + TEST_CHECK(found == 2); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Older readers find point metadata in the external tree, keyed by series. */ +static void test_otlp_metadata_legacy_external_tree(void) +{ + char key[64]; + char *buffer = NULL; + size_t size = 0; + mpack_writer_t writer; + mpack_reader_t reader; + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *family; + struct cfl_kvlist *metadata; + struct cfl_kvlist *external = NULL; + struct cfl_variant *node; + struct cfl_variant *point; + char *labels[] = {"a"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "legacy", "legacy", 1, (char *[]) {"k"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 1000, 7, 1, labels) == 0); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(sample != NULL); + + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "start_time_unix_nano", 500) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + TEST_ASSERT(cfl_kvlist_insert_string(sample->otlp_metadata, "number_value_case", "double") == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, sample) == 0); + + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "unit.source", "otlp") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + + mpack_writer_init_growable(&writer, &buffer, &size); + TEST_ASSERT(cmt_otlp_metadata_pack_external(&writer, ctx) == 0); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok); + mpack_reader_init_data(&reader, buffer, size); + TEST_ASSERT(unpack_cfl_kvlist(&reader, &external) == 0); + mpack_reader_destroy(&reader); + free(buffer); + + node = cfl_kvlist_fetch(external, "otlp"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "metrics"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "gauge"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "legacy"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + family = node->data.as_kvlist; + + node = cfl_kvlist_fetch(family, "metadata"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "unit.source") != NULL); + + node = cfl_kvlist_fetch(family, "datapoints"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + snprintf(key, sizeof(key), "%" PRIx64 ":%" PRIu64, + sample->hash, cmt_metric_get_timestamp(sample)); + point = cfl_kvlist_fetch(node->data.as_kvlist, key); + TEST_ASSERT(point != NULL && point->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(point->data.as_kvlist, "start_time_unix_nano"); + TEST_CHECK(node != NULL && node->data.as_uint64 == 500); + node = cfl_kvlist_fetch(point->data.as_kvlist, "flags"); + TEST_CHECK(node != NULL && node->data.as_uint64 == 1); + /* only the keys older readers use are written */ + TEST_CHECK(cfl_kvlist_fetch(point->data.as_kvlist, "attributes") == NULL); + + /* the context itself is unchanged */ + TEST_CHECK(cfl_kvlist_fetch(ctx->external_metadata, "otlp") == NULL); + + cfl_kvlist_destroy(external); + cmt_destroy(ctx); +} + +/* Typed attributes without an OTLP form keep the label derived attributes. */ +static void test_otlp_unrepresentable_attribute_keeps_point(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_kvlist *attrs; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "huge_attribute", "huge attribute", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + attrs = cfl_kvlist_create(); + TEST_ASSERT(attrs != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "huge", UINT64_MAX) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->metric.otlp_metadata, "attributes", attrs) == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, &gauge->map->metric) == 0); + + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + TEST_ASSERT(request->n_resource_metrics == 1); + metric = request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(strcmp(metric->name, "huge_attribute") == 0); + TEST_ASSERT(metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_double == 42); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_destroy(ctx); +} + +/* Repeated label-less points of one metric are the same series: the last one + * replaces the static sample, as released versions behave. */ +static cfl_sds_t generate_repeated_unlabeled_points_payload() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric gauge_metric; + Opentelemetry__Proto__Metrics__V1__Metric histogram_metric; + Opentelemetry__Proto__Metrics__V1__Gauge gauge; + Opentelemetry__Proto__Metrics__V1__Histogram histogram; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint number_points[2]; + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint histogram_points[2]; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[2]; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *number_point_list[2]; + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *histogram_point_list[2]; + uint64_t first_buckets[] = {5, 1, 3}; + uint64_t second_buckets[] = {1, 1, 1}; + double bounds[] = {1.0, 2.0}; + size_t payload_size; + size_t index; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&gauge_metric); + opentelemetry__proto__metrics__v1__metric__init(&histogram_metric); + opentelemetry__proto__metrics__v1__gauge__init(&gauge); + opentelemetry__proto__metrics__v1__histogram__init(&histogram); + + for (index = 0; index < 2; index++) { + opentelemetry__proto__metrics__v1__number_data_point__init(&number_points[index]); + number_points[index].time_unix_nano = index + 1; + number_points[index].flags = index; + number_points[index].value_case = + OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + number_points[index].as_double = index + 1.0; + number_point_list[index] = &number_points[index]; + + opentelemetry__proto__metrics__v1__histogram_data_point__init(&histogram_points[index]); + histogram_points[index].time_unix_nano = index + 1; + histogram_points[index].n_bucket_counts = 3; + histogram_points[index].bucket_counts = index == 0 ? first_buckets : second_buckets; + histogram_points[index].n_explicit_bounds = 2; + histogram_points[index].explicit_bounds = bounds; + histogram_points[index].count = index == 0 ? 9 : 3; + histogram_points[index].has_sum = 1; + histogram_points[index].sum = 1.0; + histogram_point_list[index] = &histogram_points[index]; + } + + gauge.n_data_points = 2; + gauge.data_points = number_point_list; + gauge_metric.name = "u"; + gauge_metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + gauge_metric.gauge = &gauge; + + histogram.n_data_points = 2; + histogram.data_points = histogram_point_list; + histogram.aggregation_temporality = + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + histogram_metric.name = "h"; + histogram_metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM; + histogram_metric.histogram = &histogram; + + metric_list[0] = &gauge_metric; + metric_list[1] = &histogram_metric; + scope_metrics.n_metrics = 2; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) +{ + cfl_sds_t payload; + cfl_sds_t encoded; + size_t offset; + int result; + struct cfl_list decoded_context_list; + struct cmt *decoded_context; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cfl_variant *flags; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + payload = generate_repeated_unlabeled_points_payload(); + TEST_ASSERT(payload != NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_context_list, payload, + cfl_sds_len(payload), &offset); + TEST_ASSERT(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + + decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); + TEST_ASSERT(decoded_context != NULL); + + gauge = cfl_list_entry_first(&decoded_context->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + TEST_CHECK(gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 2.0); + TEST_CHECK(cmt_metric_get_timestamp(&gauge->map->metric) == 2); + flags = cfl_kvlist_fetch(gauge->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(flags != NULL); + TEST_CHECK(flags->type == CFL_VARIANT_UINT && flags->data.as_uint64 == 1); + + histogram = cfl_list_entry_first(&decoded_context->histograms, struct cmt_histogram, _head); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(cfl_list_size(&histogram->map->metrics) == 0); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 0) == 1); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 1) == 2); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 2) == 3); + TEST_CHECK(cmt_metric_hist_get_count_value(&histogram->map->metric) == 3); + + encoded = cmt_encode_opentelemetry_create(decoded_context); + TEST_ASSERT(encoded != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(encoded), (const uint8_t *) encoded); + TEST_ASSERT(request != NULL); + + metric = find_otlp_metric_by_name(request, "u"); + TEST_ASSERT(metric != NULL && metric->gauge != NULL); + TEST_CHECK(metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_double == 2.0); + + metric = find_otlp_metric_by_name(request, "h"); + TEST_ASSERT(metric != NULL && metric->histogram != NULL); + TEST_CHECK(metric->histogram->n_data_points == 1); + TEST_CHECK(metric->histogram->data_points[0]->n_bucket_counts == 3); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[0] == 1); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[1] == 1); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[2] == 1); + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(encoded); + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); +} + +/* Empty nested AnyValues are optional metadata, not allocation failures. */ +static void test_otlp_nested_empty_attributes_keep_point(void) +{ + int mode; + size_t offset; + size_t size; + int result; + cfl_sds_t payload; + cfl_sds_t wire; + struct cfl_list contexts; + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Common__V1__AnyValue empty; + Opentelemetry__Proto__Common__V1__AnyValue nested; + Opentelemetry__Proto__Common__V1__AnyValue *items[] = {&empty}; + Opentelemetry__Proto__Common__V1__ArrayValue array; + Opentelemetry__Proto__Common__V1__KeyValue entry; + Opentelemetry__Proto__Common__V1__KeyValue *entries[] = {&entry}; + Opentelemetry__Proto__Common__V1__KeyValueList kvlist; + Opentelemetry__Proto__Common__V1__KeyValue *attribute; + Opentelemetry__Proto__Common__V1__AnyValue *original; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + /* An array containing an empty value, a kvlist with no value, and a + * kvlist containing an explicitly empty value must all remain usable. */ + for (mode = 0; mode < 3; mode++) { + payload = generate_gauge_int_otlp_payload_with_many_attributes(2); + TEST_ASSERT(payload != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(payload), (const uint8_t *) payload); + TEST_ASSERT(request != NULL); + cfl_sds_destroy(payload); + attribute = request->resource_metrics[0]->scope_metrics[0]->metrics[0]->gauge->data_points[0]->attributes[0]; + original = attribute->value; + opentelemetry__proto__common__v1__any_value__init(&empty); + opentelemetry__proto__common__v1__any_value__init(&nested); + opentelemetry__proto__common__v1__array_value__init(&array); + opentelemetry__proto__common__v1__key_value__init(&entry); + opentelemetry__proto__common__v1__key_value_list__init(&kvlist); + array.n_values = 1; + array.values = items; + entry.key = "empty"; + entry.value = mode == 2 ? &empty : NULL; + kvlist.n_values = 1; + kvlist.values = entries; + if (mode == 0) { + nested.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + nested.array_value = &array; + } + else { + nested.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + nested.kvlist_value = &kvlist; + } + attribute->value = &nested; + size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(request); + payload = cfl_sds_create_size(size); + TEST_ASSERT(payload != NULL); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(request, (uint8_t *) payload); + cfl_sds_len_set(payload, size); + attribute->value = original; + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, payload, size, &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + ctx = cfl_list_entry_first(&contexts, struct cmt, _head); + TEST_ASSERT(ctx != NULL); + gauge = cfl_list_entry_first(&ctx->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_int64_value(sample) == 1); + TEST_CHECK(cmt_metric_get_timestamp(sample) == 123); + TEST_CHECK(cfl_kvlist_fetch(sample->otlp_metadata, "attributes") == NULL); + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "g_many_attrs"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_int == 1); + TEST_ASSERT(metric->gauge->data_points[0]->n_attributes == 1); + TEST_CHECK(strcmp(metric->gauge->data_points[0]->attributes[0]->key, "attr_001") == 0); + TEST_CHECK(metric->gauge->data_points[0]->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); + TEST_CHECK(strcmp(metric->gauge->data_points[0]->attributes[0]->value->string_value, + "value_001") == 0); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(payload); + } +} + +static void test_otlp_restore_preserves_legacy_only_family(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *legacy; + struct cmt_gauge *native; + struct cfl_kvlist *external = NULL; + struct cfl_variant *node; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + size_t offset = 0; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + legacy = cmt_gauge_create(ctx, "", "", "legacy_only", "legacy", 0, NULL); + native = cmt_gauge_create(ctx, "", "", "native", "native", 0, NULL); + TEST_ASSERT(legacy != NULL && native != NULL); + TEST_ASSERT(cmt_gauge_set(legacy, 100, 7, 0, NULL) == 0); + TEST_ASSERT(cmt_gauge_set(native, 200, 9, 0, NULL) == 0); + legacy->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(legacy->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(legacy->map->metric.otlp_metadata, "flags", 1) == 0); + + /* Build an external legacy tree, then leave this family with no native + * metadata. The other family forces a native sidecar for both maps. */ + mpack_writer_init_growable(&writer, &buffer, &size); + TEST_ASSERT(cmt_otlp_metadata_pack_external(&writer, ctx) == 0); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok); + mpack_reader_init_data(&reader, buffer, size); + TEST_ASSERT(unpack_cfl_kvlist(&reader, &external) == 0); + TEST_ASSERT(mpack_reader_destroy(&reader) == mpack_ok); + free(buffer); + cfl_kvlist_destroy(ctx->external_metadata); + ctx->external_metadata = external; + cfl_kvlist_destroy(legacy->map->metric.otlp_metadata); + legacy->map->metric.otlp_metadata = NULL; + native->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(native->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(native->map->metric.otlp_metadata, "flags", 2) == 0); + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + node = cfl_kvlist_fetch(decoded->external_metadata, "otlp"); + TEST_ASSERT(node != NULL); + node = cfl_kvlist_fetch(node->data.as_kvlist, "metrics"); + TEST_ASSERT(node != NULL); + node = cfl_kvlist_fetch(node->data.as_kvlist, "gauge"); + TEST_ASSERT(node != NULL); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "legacy_only") != NULL); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "native") == NULL); + + wire = cmt_encode_opentelemetry_create(decoded); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "legacy_only"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->flags == 1); + metric = find_otlp_metric_by_name(request, "native"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->flags == 2); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* + * Released readers decode the sidecar and the legacy tree as opaque context + * metadata and write them back unchanged, even after editing points. These + * helpers capture both as such a relay would see them. + */ +static struct cfl_kvlist *relay_snapshot(struct cmt *ctx, int internal) +{ + struct cfl_kvlist *copy = NULL; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + int result; + + mpack_writer_init_growable(&writer, &buffer, &size); + if (internal) { + result = cmt_otlp_metadata_pack(&writer, ctx); + } + else { + result = cmt_otlp_metadata_pack_external(&writer, ctx); + } + if (mpack_writer_destroy(&writer) != mpack_ok || result != 0) { + free(buffer); + return NULL; + } + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + if (mpack_reader_destroy(&reader) != mpack_ok || result != 0) { + cfl_kvlist_destroy(copy); + copy = NULL; + } + free(buffer); + return copy; +} + +/* Replace native metadata with the captured opaque copies, as a released + * relay holds nothing else. */ +static void relay_install(struct cmt *ctx, struct cmt_map *map, + struct cfl_kvlist *internal, struct cfl_kvlist *external) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = NULL; + cfl_kvlist_destroy(map->metric.otlp_metadata); + map->metric.otlp_metadata = NULL; + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; + } + cfl_kvlist_destroy(ctx->internal_metadata); + ctx->internal_metadata = internal; + cfl_kvlist_destroy(ctx->external_metadata); + ctx->external_metadata = external; +} + +/* Remove the point identifiers, producing a v2.3.0 sidecar. */ +static void relay_strip_point_ids(struct cfl_kvlist *internal) +{ + struct cfl_variant *families; + size_t index; + + families = cfl_kvlist_fetch(internal, "__cmetrics_native_metadata_v1"); + TEST_ASSERT(families != NULL && families->type == CFL_VARIANT_ARRAY); + for (index = 0; index < families->data.as_array->entry_count; index++) { + cfl_kvlist_remove(families->data.as_array->entries[index]->data.as_kvlist, + "point_ids"); + } +} + +static struct cmt_gauge *relay_gauge(struct cmt *ctx, uint64_t flags_a, uint64_t flags_b, + int family_metadata) +{ + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *metadata; + char *values[2][1] = {{"a"}, {"b"}}; + uint64_t flags[2]; + int index; + + flags[0] = flags_a; + flags[1] = flags_b; + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < 2; index++) { + TEST_ASSERT(cmt_gauge_set(gauge, 100, index + 1, 1, values[index]) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, values[index], CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags[index]) == 0); + } + if (family_metadata) { + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "owner", "relay") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + } + return gauge; +} + +static struct cmt *relay_round_trip(struct cmt *ctx, char **buffer) +{ + struct cmt *decoded = NULL; + size_t size; + size_t offset = 0; + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, *buffer, size, &offset) == 0); + TEST_CHECK(offset == size); + return decoded; +} + +static int relay_otlp_flags(struct cmt *ctx, char *site, uint32_t *flags) +{ + cfl_sds_t wire; + size_t index; + int found = CMT_FALSE; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *point; + + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "relay"); + TEST_ASSERT(metric != NULL && metric->gauge != NULL); + for (index = 0; index < metric->gauge->n_data_points; index++) { + point = metric->gauge->data_points[index]; + if (point->n_attributes == 1 && + strcmp(point->attributes[0]->value->string_value, site) == 0) { + *flags = point->flags; + found = CMT_TRUE; + } + } + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + return found; +} + +/* A released relay replaces series "a" with "c" without updating the sidecar. + * Metadata recorded for "a" must not be attached to another series. */ +static void test_otlp_restore_binds_points_by_identity(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 0, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(cmt_map_metric_get(&gauge->opts, gauge->map, 1, + (char *[]) {"a"}, CMT_FALSE)); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 3, 1, (char *[]) {"c"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + TEST_CHECK(relay_otlp_flags(decoded, "c", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A released relay records a new observation for "b" without updating the + * sidecar. Metadata of the previous observation must not be restored. */ +static void test_otlp_restore_drops_stale_observation(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 0, 1, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 200, 9, 1, (char *[]) {"b"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A relay that reverses a large family must not make restoration scan the + * identifiers once per point, and every point keeps its own metadata. */ +static void test_otlp_restore_binds_reordered_points(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cfl_variant *flags; + struct cfl_list reversed; + struct cfl_list *head; + struct cfl_list *tmp; + struct cmt_map_label *label; + char value[16]; + char *values[1] = {value}; + char *buffer; + int index; + int bound = 0; + const int count = 2000; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < count; index++) { + snprintf(value, sizeof(value), "%d", index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, index, 1, values) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, values, CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", index) == 0); + } + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + + cfl_list_init(&reversed); + cfl_list_foreach_safe(head, tmp, &gauge->map->metrics) { + cfl_list_del(head); + cfl_list_prepend(head, &reversed); + } + cfl_list_foreach_safe(head, tmp, &reversed) { + cfl_list_del(head); + cfl_list_add(head, &gauge->map->metrics); + } + + decoded = relay_round_trip(ctx, &buffer); + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + flags = cfl_kvlist_fetch(sample->otlp_metadata, "flags"); + if (flags != NULL && + (uint64_t) (flags->type == CFL_VARIANT_UINT ? flags->data.as_uint64 : + (uint64_t) flags->data.as_int64) == strtoull(label->name, NULL, 10)) { + bound++; + } + } + TEST_CHECK(bound == count); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* The series hash concatenates label values without boundaries, so these + * two series share it. A released relay drops the first: its metadata must + * not move to the survivor. */ +static void test_otlp_restore_binds_points_by_label_values(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cmt_metric *second; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cfl_variant *flags; + char *buffer; + char *a[] = {"ab", "c"}; + char *b[] = {"a", "bc"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 2, (char *[]) {"x", "y"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 1, 2, a) == 0); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 2, 2, b) == 0); + first = cmt_map_metric_get(&gauge->opts, gauge->map, 2, a, CMT_FALSE); + second = cmt_map_metric_get(&gauge->opts, gauge->map, 2, b, CMT_FALSE); + TEST_ASSERT(first != NULL && second != NULL && first != second); + TEST_ASSERT(first->hash == second->hash); + first->otlp_metadata = cfl_kvlist_create(); + second->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(first->otlp_metadata != NULL && second->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(first->otlp_metadata, "flags", 1) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(second->otlp_metadata, "flags", 0) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(first); + + decoded = relay_round_trip(ctx, &buffer); + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 1); + second = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + flags = cfl_kvlist_fetch(second->otlp_metadata, "flags"); + TEST_ASSERT(flags != NULL); + TEST_CHECK((flags->type == CFL_VARIANT_UINT && flags->data.as_uint64 == 0) || + (flags->type == CFL_VARIANT_INT && flags->data.as_int64 == 0)); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* One window of series "a" at time 100, as an OTLP point with its own + * start time. */ +static struct cmt *relay_window(uint64_t start, uint64_t flags, double value) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, value, 1, (char *[]) {"a"}) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"a"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_start_timestamp(sample, start); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags) == 0); + return ctx; +} + +/* Series "a" with point metadata nested beyond the decoder's depth limit, + * relayed by a released reader, optionally after changing its value at the + * same time. */ +static struct cmt *relay_unreadable(int change_value, char **buffer) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cmt *decoded; + int depth; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 0, CMT_FALSE); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"a"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "v") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + TEST_ASSERT(cfl_kvlist_insert_kvlist(sample->otlp_metadata, "nested", inner) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + if (change_value) { + TEST_ASSERT(cmt_gauge_set(gauge, 100, 99, 1, (char *[]) {"a"}) == 0); + } + decoded = relay_round_trip(ctx, buffer); + cmt_destroy(ctx); + return decoded; +} + +enum shared_key_sidecar { + SHARED_KEY_CURRENT, /* restored */ + SHARED_KEY_V230, /* no identifiers, point count changed: skipped */ + SHARED_KEY_NEW_FAMILY /* relay added a family: rejected */ +}; + +/* A released relay adds a series whose labels share the series hash of a + * recorded point, at the same time. That point's legacy entry, its only + * readable metadata, can belong to either: neither may take it, whether the + * native sidecar is restored, skipped or rejected. */ +static void check_shared_legacy_key(enum shared_key_sidecar sidecar) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *added; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + cfl_sds_t wire; + char *buffer; + char *a[] = {"ab", "c"}; + char *b[] = {"a", "bc"}; + size_t index; + int depth; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 2, (char *[]) {"x", "y"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 1, 2, a) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 2, a, CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "v") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + TEST_ASSERT(cfl_kvlist_insert_kvlist(sample->otlp_metadata, "nested", inner) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + if (sidecar == SHARED_KEY_V230) { + relay_strip_point_ids(internal); + } + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 2, 2, b) == 0); + TEST_ASSERT(cmt_map_metric_get(&gauge->opts, gauge->map, 2, b, CMT_FALSE)->hash == + sample->hash); + if (sidecar == SHARED_KEY_NEW_FAMILY) { + added = cmt_gauge_create(ctx, "", "", "added", "added", 0, NULL); + TEST_ASSERT(added != NULL); + TEST_ASSERT(cmt_gauge_set(added, 100, 3, 0, NULL) == 0); + } + + decoded = relay_round_trip(ctx, &buffer); + wire = cmt_encode_opentelemetry_create(decoded); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "relay"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 2); + for (index = 0; index < metric->gauge->n_data_points; index++) { + TEST_CHECK_(metric->gauge->data_points[index]->flags == 0, + "sidecar %d: point %g takes no shared legacy flags", sidecar, + metric->gauge->data_points[index]->as_double); + } + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_otlp_restore_drops_shared_legacy_key(void) +{ + check_shared_legacy_key(SHARED_KEY_CURRENT); + check_shared_legacy_key(SHARED_KEY_V230); + check_shared_legacy_key(SHARED_KEY_NEW_FAMILY); +} + +/* Native point metadata that cannot be read back leaves the point its + * legacy entry, but only while its identity still matches: a changed value + * must not take the flags recorded for the old one. */ +static void test_otlp_restore_keeps_legacy_when_native_unreadable(void) +{ + struct cmt *decoded; + char *buffer; + uint32_t flags = 0; + + decoded = relay_unreadable(CMT_FALSE, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 1); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + + decoded = relay_unreadable(CMT_TRUE, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); +} + +/* A released relay changes the value of series "b" without changing its + * time. The metadata recorded for the old value must not describe the new + * one, from the sidecar or from the legacy tree. */ +static void test_otlp_restore_drops_changed_value(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 1, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 1, (char *[]) {"b"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 1); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Two windows of one series differ only by start time. A released relay + * drops the first: its metadata must not move to the survivor. */ +static void test_otlp_restore_binds_points_by_start_time(void) +{ + struct cmt *ctx; + struct cmt *window; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + window = relay_window(10, 1, 1.0); + TEST_ASSERT(cmt_cat(ctx, window) == 0); + cmt_destroy(window); + window = relay_window(20, 0, 2.0); + TEST_ASSERT(cmt_cat(ctx, window) == 0); + cmt_destroy(window); + + gauge = cfl_list_entry_first(&ctx->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 2); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + first = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(cmt_metric_get_start_timestamp(first) == 10); + cmt_map_metric_destroy(first); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A v2.3.0 sidecar binds by position. When a released relay removed a series, + * its points are not restored, and the legacy tree must still provide what + * released readers provide, even though family metadata was restored. */ +static void test_otlp_restore_keeps_unbound_legacy_points(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *result; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 0, 1, CMT_TRUE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_strip_point_ids(internal); + + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(cmt_map_metric_get(&gauge->opts, gauge->map, 1, + (char *[]) {"a"}, CMT_FALSE)); + + decoded = relay_round_trip(ctx, &buffer); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_CHECK(result->map->otlp_metadata != NULL); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 1); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"otlp_restore_binds_points_by_identity", test_otlp_restore_binds_points_by_identity}, + {"otlp_restore_drops_stale_observation", test_otlp_restore_drops_stale_observation}, + {"otlp_restore_drops_changed_value", test_otlp_restore_drops_changed_value}, + {"otlp_restore_keeps_legacy_when_native_unreadable", test_otlp_restore_keeps_legacy_when_native_unreadable}, + {"otlp_restore_drops_shared_legacy_key", test_otlp_restore_drops_shared_legacy_key}, + {"otlp_restore_binds_points_by_start_time", test_otlp_restore_binds_points_by_start_time}, + {"otlp_restore_binds_points_by_label_values", test_otlp_restore_binds_points_by_label_values}, + {"otlp_restore_binds_reordered_points", test_otlp_restore_binds_reordered_points}, + {"otlp_restore_keeps_unbound_legacy_points", test_otlp_restore_keeps_unbound_legacy_points}, + {"otlp_nested_empty_attributes_keep_point", test_otlp_nested_empty_attributes_keep_point}, + {"otlp_restore_preserves_legacy_only_family", test_otlp_restore_preserves_legacy_only_family}, + {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, + {"otlp_metadata_legacy_external_tree", test_otlp_metadata_legacy_external_tree}, + {"otlp_metadata_deep_nesting", test_otlp_metadata_deep_nesting}, + {"repeated_unlabeled_points_last_wins", test_opentelemetry_repeated_unlabeled_points_last_wins}, + {"otlp_unsigned_attribute_signed_range", test_otlp_unsigned_attribute_preserves_signed_range}, {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, {"opentelemetry_encode_multi_resource_scope_containers", test_opentelemetry_encode_multi_resource_scope_containers}, {"opentelemetry_encode_scope_attributes_cleanup", test_opentelemetry_encode_scope_attributes_cleanup}, diff --git a/lib/cmetrics/tests/otlp_pipeline.c b/lib/cmetrics/tests/otlp_pipeline.c new file mode 100644 index 00000000000..0dba4c2ca12 --- /dev/null +++ b/lib/cmetrics/tests/otlp_pipeline.c @@ -0,0 +1,1794 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * OTLP traffic through the transformations Fluent Bit applies to it: + * + * direct : in_opentelemetry -> out_opentelemetry in one process + * chunk : OTLP -> MessagePack chunk -> OTLP (storage, output processors) + * forward : two MessagePack hops (agent -> forward -> aggregator -> OTLP) + * cat : chunk decoded and merged with cmt_cat (batching, processors) + * + * Every data point of the input request is reduced to a canonical line that + * includes its resource, scope, metric and point fields with exact bit + * patterns for doubles and typed, order independent attributes. A pipeline is + * lossless when the output request produces the same multiset of lines. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +/* Builder and canonicalization allocations: abort rather than flood the + * check log, these are test infrastructure, not behavior under test. */ +static void *require(void *pointer) +{ + if (pointer == NULL) { + abort(); + } + return pointer; +} + +typedef Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest otlp_request; +typedef Opentelemetry__Proto__Metrics__V1__ResourceMetrics otlp_resource_metrics; +typedef Opentelemetry__Proto__Metrics__V1__ScopeMetrics otlp_scope_metrics; +typedef Opentelemetry__Proto__Metrics__V1__Metric otlp_metric; +typedef Opentelemetry__Proto__Metrics__V1__NumberDataPoint otlp_number_point; +typedef Opentelemetry__Proto__Metrics__V1__HistogramDataPoint otlp_histogram_point; +typedef Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint otlp_exp_point; +typedef Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets otlp_exp_buckets; +typedef Opentelemetry__Proto__Metrics__V1__SummaryDataPoint otlp_summary_point; +typedef Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile otlp_quantile; +typedef Opentelemetry__Proto__Metrics__V1__Exemplar otlp_exemplar; +typedef Opentelemetry__Proto__Common__V1__KeyValue otlp_kv; +typedef Opentelemetry__Proto__Common__V1__AnyValue otlp_any; + +#define TEMPORALITY_DELTA \ + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_DELTA +#define TEMPORALITY_CUMULATIVE \ + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE + +enum pipeline { + PIPELINE_DIRECT = 0, + PIPELINE_CHUNK, + PIPELINE_FORWARD, + PIPELINE_CAT, + PIPELINE_COUNT +}; + +static const char *pipeline_names[] = {"direct", "chunk", "forward", "cat"}; + +/* Every builder allocation is released at once after a case. */ +static void **allocations; +static size_t allocation_count; +static size_t allocation_size; + +static void *test_alloc(size_t size) +{ + void **grown; + void *memory; + + if (allocation_count == allocation_size) { + allocation_size = allocation_size ? allocation_size * 2 : 64; + grown = require(realloc(allocations, allocation_size * sizeof(*allocations))); + allocations = grown; + } + memory = require(calloc(1, size ? size : 1)); + allocations[allocation_count++] = memory; + return memory; +} + +static void test_alloc_release(void) +{ + size_t index; + + for (index = 0; index < allocation_count; index++) { + free(allocations[index]); + } + free(allocations); + allocations = NULL; + allocation_count = 0; + allocation_size = 0; +} + +static void **pointer_list(size_t count, va_list arguments) +{ + void **list; + size_t index; + + list = test_alloc(sizeof(void *) * count); + for (index = 0; index < count; index++) { + list[index] = va_arg(arguments, void *); + } + return list; +} + +/* AnyValue and KeyValue builders */ + +static otlp_any *any_string(const char *value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; + any->string_value = (char *) value; + return any; +} + +static otlp_any *any_int(int64_t value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE; + any->int_value = value; + return any; +} + +static otlp_any *any_double(double value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE; + any->double_value = value; + return any; +} + +static otlp_any *any_bool(int value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE; + any->bool_value = value; + return any; +} + +static otlp_any *any_bytes(const char *value, size_t length) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE; + any->bytes_value.data = (uint8_t *) value; + any->bytes_value.len = length; + return any; +} + +static otlp_any *any_array(size_t count, ...) +{ + va_list arguments; + otlp_any *any = test_alloc(sizeof(*any)); + Opentelemetry__Proto__Common__V1__ArrayValue *array = test_alloc(sizeof(*array)); + + opentelemetry__proto__common__v1__any_value__init(any); + opentelemetry__proto__common__v1__array_value__init(array); + va_start(arguments, count); + array->values = (otlp_any **) pointer_list(count, arguments); + va_end(arguments); + array->n_values = count; + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + any->array_value = array; + return any; +} + +static otlp_any *any_kvlist(size_t count, ...) +{ + va_list arguments; + otlp_any *any = test_alloc(sizeof(*any)); + Opentelemetry__Proto__Common__V1__KeyValueList *list = test_alloc(sizeof(*list)); + + opentelemetry__proto__common__v1__any_value__init(any); + opentelemetry__proto__common__v1__key_value_list__init(list); + va_start(arguments, count); + list->values = (otlp_kv **) pointer_list(count, arguments); + va_end(arguments); + list->n_values = count; + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + any->kvlist_value = list; + return any; +} + +static otlp_kv *kv(const char *key, otlp_any *value) +{ + otlp_kv *pair = test_alloc(sizeof(*pair)); + + opentelemetry__proto__common__v1__key_value__init(pair); + pair->key = (char *) key; + pair->value = value; + return pair; +} + +static otlp_kv **kv_list(size_t count, ...) +{ + va_list arguments; + otlp_kv **list; + + va_start(arguments, count); + list = (otlp_kv **) pointer_list(count, arguments); + va_end(arguments); + return list; +} + +/* Data point builders */ + +static otlp_number_point *number_int(uint64_t start, uint64_t time, int64_t value, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_number_point *point = test_alloc(sizeof(*point)); + + opentelemetry__proto__metrics__v1__number_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + point->as_int = value; + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_number_point *number_double(uint64_t start, uint64_t time, double value, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_number_point *point = number_int(start, time, 0, attribute_count, attributes); + + point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + point->as_double = value; + return point; +} + +static otlp_exemplar *exemplar_double(uint64_t time, double value, const char *trace_id, + const char *span_id, size_t attribute_count, + otlp_kv **attributes) +{ + otlp_exemplar *exemplar = test_alloc(sizeof(*exemplar)); + + opentelemetry__proto__metrics__v1__exemplar__init(exemplar); + exemplar->time_unix_nano = time; + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_DOUBLE; + exemplar->as_double = value; + if (trace_id != NULL) { + exemplar->trace_id.data = (uint8_t *) trace_id; + exemplar->trace_id.len = 16; + } + if (span_id != NULL) { + exemplar->span_id.data = (uint8_t *) span_id; + exemplar->span_id.len = 8; + } + exemplar->n_filtered_attributes = attribute_count; + exemplar->filtered_attributes = attributes; + return exemplar; +} + +static otlp_exemplar *exemplar_int(uint64_t time, int64_t value) +{ + otlp_exemplar *exemplar = exemplar_double(time, 0, NULL, NULL, 0, NULL); + + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT; + exemplar->as_int = value; + return exemplar; +} + +static otlp_exemplar **exemplar_list(size_t count, ...) +{ + va_list arguments; + otlp_exemplar **list; + + va_start(arguments, count); + list = (otlp_exemplar **) pointer_list(count, arguments); + va_end(arguments); + return list; +} + +static otlp_histogram_point *histogram_point(uint64_t start, uint64_t time, + size_t bound_count, const double *bounds, + const uint64_t *counts, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_histogram_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__histogram_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->n_explicit_bounds = bound_count; + point->explicit_bounds = test_alloc(sizeof(double) * (bound_count + 1)); + if (bound_count > 0) { + memcpy(point->explicit_bounds, bounds, sizeof(double) * bound_count); + } + point->n_bucket_counts = bound_count + 1; + point->bucket_counts = test_alloc(sizeof(uint64_t) * (bound_count + 1)); + memcpy(point->bucket_counts, counts, sizeof(uint64_t) * (bound_count + 1)); + for (index = 0; index <= bound_count; index++) { + point->count += counts[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_exp_buckets *exp_buckets(int32_t offset, size_t count, const uint64_t *counts) +{ + otlp_exp_buckets *buckets = test_alloc(sizeof(*buckets)); + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(buckets); + buckets->offset = offset; + buckets->n_bucket_counts = count; + buckets->bucket_counts = test_alloc(sizeof(uint64_t) * (count + 1)); + memcpy(buckets->bucket_counts, counts, sizeof(uint64_t) * count); + return buckets; +} + +static otlp_exp_point *exp_point(uint64_t start, uint64_t time, int32_t scale, + uint64_t zero_count, double zero_threshold, + otlp_exp_buckets *positive, otlp_exp_buckets *negative, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_exp_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->scale = scale; + point->zero_count = zero_count; + point->zero_threshold = zero_threshold; + point->positive = positive; + point->negative = negative; + point->count = zero_count; + for (index = 0; positive != NULL && index < positive->n_bucket_counts; index++) { + point->count += positive->bucket_counts[index]; + } + for (index = 0; negative != NULL && index < negative->n_bucket_counts; index++) { + point->count += negative->bucket_counts[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_summary_point *summary_point(uint64_t start, uint64_t time, uint64_t count, + double sum, size_t quantile_count, + const double *quantiles, const double *values, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_summary_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__summary_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->count = count; + point->sum = sum; + point->n_quantile_values = quantile_count; + point->quantile_values = test_alloc(sizeof(otlp_quantile *) * (quantile_count + 1)); + for (index = 0; index < quantile_count; index++) { + point->quantile_values[index] = test_alloc(sizeof(otlp_quantile)); + opentelemetry__proto__metrics__v1__summary_data_point__value_at_quantile__init( + point->quantile_values[index]); + point->quantile_values[index]->quantile = quantiles[index]; + point->quantile_values[index]->value = values[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +/* Metric builders: points are passed as a count and a variadic list. */ + +static otlp_metric *metric_base(const char *name, const char *description, const char *unit) +{ + otlp_metric *metric = test_alloc(sizeof(*metric)); + + opentelemetry__proto__metrics__v1__metric__init(metric); + metric->name = (char *) name; + metric->description = (char *) (description ? description : ""); + metric->unit = (char *) (unit ? unit : ""); + return metric; +} + +static otlp_metric *metric_gauge(const char *name, const char *description, const char *unit, + size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, description, unit); + Opentelemetry__Proto__Metrics__V1__Gauge *gauge = test_alloc(sizeof(*gauge)); + + opentelemetry__proto__metrics__v1__gauge__init(gauge); + va_start(arguments, count); + gauge->data_points = (otlp_number_point **) pointer_list(count, arguments); + va_end(arguments); + gauge->n_data_points = count; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + metric->gauge = gauge; + return metric; +} + +static otlp_metric *metric_sum(const char *name, const char *description, const char *unit, + int temporality, int monotonic, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, description, unit); + Opentelemetry__Proto__Metrics__V1__Sum *sum = test_alloc(sizeof(*sum)); + + opentelemetry__proto__metrics__v1__sum__init(sum); + va_start(arguments, count); + sum->data_points = (otlp_number_point **) pointer_list(count, arguments); + va_end(arguments); + sum->n_data_points = count; + sum->aggregation_temporality = temporality; + sum->is_monotonic = monotonic; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM; + metric->sum = sum; + return metric; +} + +static otlp_metric *metric_histogram(const char *name, const char *unit, int temporality, + size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "histogram", unit); + Opentelemetry__Proto__Metrics__V1__Histogram *histogram = test_alloc(sizeof(*histogram)); + + opentelemetry__proto__metrics__v1__histogram__init(histogram); + va_start(arguments, count); + histogram->data_points = (otlp_histogram_point **) pointer_list(count, arguments); + va_end(arguments); + histogram->n_data_points = count; + histogram->aggregation_temporality = temporality; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM; + metric->histogram = histogram; + return metric; +} + +static otlp_metric *metric_exp_histogram(const char *name, int temporality, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "exponential histogram", "s"); + Opentelemetry__Proto__Metrics__V1__ExponentialHistogram *histogram = + test_alloc(sizeof(*histogram)); + + opentelemetry__proto__metrics__v1__exponential_histogram__init(histogram); + va_start(arguments, count); + histogram->data_points = (otlp_exp_point **) pointer_list(count, arguments); + va_end(arguments); + histogram->n_data_points = count; + histogram->aggregation_temporality = temporality; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM; + metric->exponential_histogram = histogram; + return metric; +} + +static otlp_metric *metric_summary(const char *name, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "summary", "ms"); + Opentelemetry__Proto__Metrics__V1__Summary *summary = test_alloc(sizeof(*summary)); + + opentelemetry__proto__metrics__v1__summary__init(summary); + va_start(arguments, count); + summary->data_points = (otlp_summary_point **) pointer_list(count, arguments); + va_end(arguments); + summary->n_data_points = count; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY; + metric->summary = summary; + return metric; +} + +static otlp_scope_metrics *scope(const char *name, const char *version, + size_t attribute_count, otlp_kv **attributes, + size_t count, ...) +{ + va_list arguments; + otlp_scope_metrics *scope_metrics = test_alloc(sizeof(*scope_metrics)); + Opentelemetry__Proto__Common__V1__InstrumentationScope *instrumentation; + + opentelemetry__proto__metrics__v1__scope_metrics__init(scope_metrics); + if (name != NULL) { + instrumentation = test_alloc(sizeof(*instrumentation)); + opentelemetry__proto__common__v1__instrumentation_scope__init(instrumentation); + instrumentation->name = (char *) name; + instrumentation->version = (char *) (version ? version : ""); + instrumentation->n_attributes = attribute_count; + instrumentation->attributes = attributes; + scope_metrics->scope = instrumentation; + } + va_start(arguments, count); + scope_metrics->metrics = (otlp_metric **) pointer_list(count, arguments); + va_end(arguments); + scope_metrics->n_metrics = count; + return scope_metrics; +} + +static otlp_resource_metrics *resource(size_t attribute_count, otlp_kv **attributes, + size_t count, ...) +{ + va_list arguments; + otlp_resource_metrics *resource_metrics = test_alloc(sizeof(*resource_metrics)); + Opentelemetry__Proto__Resource__V1__Resource *resource_value = + test_alloc(sizeof(*resource_value)); + + opentelemetry__proto__metrics__v1__resource_metrics__init(resource_metrics); + opentelemetry__proto__resource__v1__resource__init(resource_value); + resource_value->n_attributes = attribute_count; + resource_value->attributes = attributes; + resource_metrics->resource = resource_value; + va_start(arguments, count); + resource_metrics->scope_metrics = (otlp_scope_metrics **) pointer_list(count, arguments); + va_end(arguments); + resource_metrics->n_scope_metrics = count; + return resource_metrics; +} + +static otlp_request *request(size_t count, ...) +{ + va_list arguments; + otlp_request *value = test_alloc(sizeof(*value)); + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(value); + va_start(arguments, count); + value->resource_metrics = (otlp_resource_metrics **) pointer_list(count, arguments); + va_end(arguments); + value->n_resource_metrics = count; + return value; +} + +/* Canonical form */ + +/* cmt_cat merges metric families only; context metadata (resource, scope) + * is not merged by design, so the cat pipeline compares families alone. */ +static int canon_include_context = CMT_TRUE; + +struct lines { + cfl_sds_t *entries; + size_t count; + size_t size; +}; + +static void lines_add(struct lines *lines, cfl_sds_t line) +{ + cfl_sds_t *grown; + + if (lines->count == lines->size) { + lines->size = lines->size ? lines->size * 2 : 16; + grown = require(realloc(lines->entries, lines->size * sizeof(*grown))); + lines->entries = grown; + } + lines->entries[lines->count++] = line; +} + +static void lines_destroy(struct lines *lines) +{ + size_t index; + + for (index = 0; index < lines->count; index++) { + cfl_sds_destroy(lines->entries[index]); + } + free(lines->entries); + memset(lines, 0, sizeof(*lines)); +} + +static cfl_sds_t sds_printf(cfl_sds_t buffer, const char *format, ...) +{ + va_list arguments; + char stack[512]; + int length; + + va_start(arguments, format); + length = vsnprintf(stack, sizeof(stack), format, arguments); + va_end(arguments); + if (length < 0 || (size_t) length >= sizeof(stack)) { + abort(); + } + return require(cfl_sds_cat(buffer, stack, length)); +} + +static cfl_sds_t canon_double(cfl_sds_t buffer, double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return sds_printf(buffer, "%016" PRIx64, bits); +} + +static cfl_sds_t canon_bytes(cfl_sds_t buffer, const uint8_t *data, size_t length) +{ + size_t index; + + for (index = 0; index < length; index++) { + buffer = sds_printf(buffer, "%02x", data[index]); + } + return buffer; +} + +static int compare_strings(const void *left, const void *right) +{ + return strcmp(*(const char **) left, *(const char **) right); +} + +static cfl_sds_t canon_attributes(cfl_sds_t buffer, size_t count, otlp_kv **attributes); + +static cfl_sds_t canon_any(cfl_sds_t buffer, otlp_any *value) +{ + size_t index; + + if (value == NULL) { + return cfl_sds_cat(buffer, "none", 4); + } + switch (value->value_case) { + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + buffer = cfl_sds_cat(buffer, "s:", 2); + return cfl_sds_cat(buffer, value->string_value, strlen(value->string_value)); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + return sds_printf(buffer, "i:%" PRId64, value->int_value); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + buffer = cfl_sds_cat(buffer, "d:", 2); + return canon_double(buffer, value->double_value); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + return sds_printf(buffer, "b:%d", value->bool_value ? 1 : 0); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + buffer = cfl_sds_cat(buffer, "x:", 2); + return canon_bytes(buffer, value->bytes_value.data, value->bytes_value.len); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + buffer = cfl_sds_cat(buffer, "[", 1); + for (index = 0; index < value->array_value->n_values; index++) { + buffer = canon_any(buffer, value->array_value->values[index]); + buffer = cfl_sds_cat(buffer, ",", 1); + } + return cfl_sds_cat(buffer, "]", 1); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + return canon_attributes(buffer, value->kvlist_value->n_values, + value->kvlist_value->values); + default: + return cfl_sds_cat(buffer, "unset", 5); + } +} + +/* Attributes are a set in OTLP: canonicalize independently of order. */ +static cfl_sds_t canon_attributes(cfl_sds_t buffer, size_t count, otlp_kv **attributes) +{ + cfl_sds_t *entries; + size_t index; + + entries = require(calloc(count + 1, sizeof(*entries))); + for (index = 0; index < count; index++) { + entries[index] = cfl_sds_create(attributes[index]->key); + entries[index] = cfl_sds_cat(entries[index], "=", 1); + entries[index] = canon_any(entries[index], attributes[index]->value); + } + qsort(entries, count, sizeof(*entries), compare_strings); + buffer = cfl_sds_cat(buffer, "{", 1); + for (index = 0; index < count; index++) { + buffer = cfl_sds_cat(buffer, entries[index], cfl_sds_len(entries[index])); + buffer = cfl_sds_cat(buffer, ";", 1); + cfl_sds_destroy(entries[index]); + } + free(entries); + return cfl_sds_cat(buffer, "}", 1); +} + +static cfl_sds_t canon_exemplars(cfl_sds_t buffer, size_t count, otlp_exemplar **exemplars) +{ + size_t index; + otlp_exemplar *exemplar; + + buffer = cfl_sds_cat(buffer, " ex=[", 5); + for (index = 0; index < count; index++) { + exemplar = exemplars[index]; + buffer = sds_printf(buffer, "t%" PRIu64 ",", exemplar->time_unix_nano); + if (exemplar->value_case == OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT) { + buffer = sds_printf(buffer, "i%" PRId64, exemplar->as_int); + } + else { + buffer = cfl_sds_cat(buffer, "d", 1); + buffer = canon_double(buffer, exemplar->as_double); + } + buffer = cfl_sds_cat(buffer, ",tr", 3); + buffer = canon_bytes(buffer, exemplar->trace_id.data, exemplar->trace_id.len); + buffer = cfl_sds_cat(buffer, ",sp", 3); + buffer = canon_bytes(buffer, exemplar->span_id.data, exemplar->span_id.len); + buffer = canon_attributes(buffer, exemplar->n_filtered_attributes, + exemplar->filtered_attributes); + buffer = cfl_sds_cat(buffer, ";", 1); + } + return cfl_sds_cat(buffer, "]", 1); +} + +static cfl_sds_t canon_point_common(cfl_sds_t buffer, size_t attribute_count, + otlp_kv **attributes, uint64_t start, uint64_t time, + uint32_t flags) +{ + buffer = cfl_sds_cat(buffer, " attrs=", 7); + buffer = canon_attributes(buffer, attribute_count, attributes); + return sds_printf(buffer, " st=%" PRIu64 " t=%" PRIu64 " flags=%u", start, time, flags); +} + +static cfl_sds_t canon_optional_double(cfl_sds_t buffer, const char *name, + int present, double value) +{ + buffer = sds_printf(buffer, " %s=", name); + if (!present) { + return cfl_sds_cat(buffer, "-", 1); + } + return canon_double(buffer, value); +} + +static void canon_metric(struct lines *lines, cfl_sds_t prefix, otlp_metric *metric) +{ + size_t index; + size_t bucket; + cfl_sds_t line; + otlp_number_point **numbers = NULL; + size_t number_count = 0; + otlp_number_point *number; + otlp_histogram_point *histogram; + otlp_exp_point *exp; + otlp_summary_point *summary; + + prefix = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + prefix = sds_printf(prefix, "|metric name=%s desc=%s unit=%s meta=", + metric->name, metric->description, metric->unit); + prefix = canon_attributes(prefix, metric->n_metadata, metric->metadata); + + switch (metric->data_case) { + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE: + prefix = cfl_sds_cat(prefix, " gauge", 6); + numbers = metric->gauge->data_points; + number_count = metric->gauge->n_data_points; + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM: + prefix = sds_printf(prefix, " sum temporality=%d monotonic=%d", + metric->sum->aggregation_temporality, + metric->sum->is_monotonic ? 1 : 0); + numbers = metric->sum->data_points; + number_count = metric->sum->n_data_points; + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM: + prefix = sds_printf(prefix, " histogram temporality=%d", + metric->histogram->aggregation_temporality); + for (index = 0; index < metric->histogram->n_data_points; index++) { + histogram = metric->histogram->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, histogram->n_attributes, histogram->attributes, + histogram->start_time_unix_nano, + histogram->time_unix_nano, histogram->flags); + line = sds_printf(line, " count=%" PRIu64 " bounds=", histogram->count); + for (bucket = 0; bucket < histogram->n_explicit_bounds; bucket++) { + line = canon_double(line, histogram->explicit_bounds[bucket]); + line = cfl_sds_cat(line, ",", 1); + } + line = cfl_sds_cat(line, " buckets=", 9); + for (bucket = 0; bucket < histogram->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", histogram->bucket_counts[bucket]); + } + line = canon_optional_double(line, "sum", histogram->has_sum, histogram->sum); + line = canon_optional_double(line, "min", histogram->has_min, histogram->min); + line = canon_optional_double(line, "max", histogram->has_max, histogram->max); + line = canon_exemplars(line, histogram->n_exemplars, histogram->exemplars); + lines_add(lines, line); + } + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM: + prefix = sds_printf(prefix, " exp_histogram temporality=%d", + metric->exponential_histogram->aggregation_temporality); + for (index = 0; index < metric->exponential_histogram->n_data_points; index++) { + exp = metric->exponential_histogram->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, exp->n_attributes, exp->attributes, + exp->start_time_unix_nano, exp->time_unix_nano, + exp->flags); + line = sds_printf(line, " count=%" PRIu64 " scale=%d zero=%" PRIu64 " zt=", + exp->count, exp->scale, exp->zero_count); + line = canon_double(line, exp->zero_threshold); + /* Absent and empty bucket ranges carry the same information. */ + if (exp->positive != NULL && exp->positive->n_bucket_counts > 0) { + line = sds_printf(line, " pos@%d:", exp->positive->offset); + for (bucket = 0; bucket < exp->positive->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", exp->positive->bucket_counts[bucket]); + } + } + if (exp->negative != NULL && exp->negative->n_bucket_counts > 0) { + line = sds_printf(line, " neg@%d:", exp->negative->offset); + for (bucket = 0; bucket < exp->negative->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", exp->negative->bucket_counts[bucket]); + } + } + line = canon_optional_double(line, "sum", exp->has_sum, exp->sum); + line = canon_optional_double(line, "min", exp->has_min, exp->min); + line = canon_optional_double(line, "max", exp->has_max, exp->max); + line = canon_exemplars(line, exp->n_exemplars, exp->exemplars); + lines_add(lines, line); + } + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY: + prefix = cfl_sds_cat(prefix, " summary", 8); + for (index = 0; index < metric->summary->n_data_points; index++) { + summary = metric->summary->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, summary->n_attributes, summary->attributes, + summary->start_time_unix_nano, + summary->time_unix_nano, summary->flags); + line = sds_printf(line, " count=%" PRIu64 " sum=", summary->count); + line = canon_double(line, summary->sum); + line = cfl_sds_cat(line, " q=", 3); + for (bucket = 0; bucket < summary->n_quantile_values; bucket++) { + line = canon_double(line, summary->quantile_values[bucket]->quantile); + line = cfl_sds_cat(line, ":", 1); + line = canon_double(line, summary->quantile_values[bucket]->value); + line = cfl_sds_cat(line, ",", 1); + } + lines_add(lines, line); + } + break; + default: + prefix = cfl_sds_cat(prefix, " unset", 6); + break; + } + + for (index = 0; index < number_count; index++) { + number = numbers[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, number->n_attributes, number->attributes, + number->start_time_unix_nano, number->time_unix_nano, + number->flags); + if (number->value_case == OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT) { + line = sds_printf(line, " int=%" PRId64, number->as_int); + } + else { + line = cfl_sds_cat(line, " double=", 8); + line = canon_double(line, number->as_double); + } + line = canon_exemplars(line, number->n_exemplars, number->exemplars); + lines_add(lines, line); + } + cfl_sds_destroy(prefix); +} + +static void canon_request(struct lines *lines, otlp_request *value) +{ + size_t resource_index; + size_t scope_index; + size_t metric_index; + otlp_resource_metrics *resource_metrics; + otlp_scope_metrics *scope_metrics; + cfl_sds_t resource_prefix; + cfl_sds_t scope_prefix; + + for (resource_index = 0; resource_index < value->n_resource_metrics; resource_index++) { + resource_metrics = value->resource_metrics[resource_index]; + resource_prefix = cfl_sds_create("resource="); + if (!canon_include_context) { + resource_prefix = cfl_sds_cat(resource_prefix, "*", 1); + } + else if (resource_metrics->resource != NULL) { + resource_prefix = canon_attributes(resource_prefix, + resource_metrics->resource->n_attributes, + resource_metrics->resource->attributes); + resource_prefix = sds_printf(resource_prefix, " dropped=%u", + resource_metrics->resource->dropped_attributes_count); + } + resource_prefix = sds_printf(resource_prefix, " schema=%s", + !canon_include_context ? "*" : + resource_metrics->schema_url ? + resource_metrics->schema_url : ""); + for (scope_index = 0; scope_index < resource_metrics->n_scope_metrics; scope_index++) { + scope_metrics = resource_metrics->scope_metrics[scope_index]; + scope_prefix = cfl_sds_create_len(resource_prefix, cfl_sds_len(resource_prefix)); + scope_prefix = cfl_sds_cat(scope_prefix, "|scope=", 7); + if (!canon_include_context) { + scope_prefix = cfl_sds_cat(scope_prefix, "*", 1); + } + else if (scope_metrics->scope != NULL) { + scope_prefix = sds_printf(scope_prefix, "%s@%s dropped=%u ", + scope_metrics->scope->name, + scope_metrics->scope->version, + scope_metrics->scope->dropped_attributes_count); + scope_prefix = canon_attributes(scope_prefix, + scope_metrics->scope->n_attributes, + scope_metrics->scope->attributes); + } + scope_prefix = sds_printf(scope_prefix, " schema=%s", + !canon_include_context ? "*" : + scope_metrics->schema_url ? scope_metrics->schema_url : ""); + for (metric_index = 0; metric_index < scope_metrics->n_metrics; metric_index++) { + canon_metric(lines, scope_prefix, scope_metrics->metrics[metric_index]); + } + cfl_sds_destroy(scope_prefix); + } + cfl_sds_destroy(resource_prefix); + } +} + +/* Pipelines */ + +static struct cmt *msgpack_hop(struct cmt *context, int destroy_source) +{ + struct cmt *decoded = NULL; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + result = cmt_encode_msgpack_create(context, &buffer, &size); + TEST_CHECK(result == 0); + if (result != 0) { + return NULL; + } + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + cmt_encode_msgpack_destroy(buffer); + if (destroy_source) { + cmt_decode_msgpack_destroy(context); + } + return result == CMT_DECODE_MSGPACK_SUCCESS ? decoded : NULL; +} + +/* Run one pipeline on a decoded context and append the OTLP output lines. */ +static int run_pipeline(struct lines *lines, struct cmt *context, int pipeline) +{ + struct cmt *current; + struct cmt *merged; + cfl_sds_t wire; + otlp_request *output; + + current = context; + merged = NULL; + if (pipeline != PIPELINE_DIRECT) { + current = msgpack_hop(context, CMT_FALSE); + if (current == NULL) { + return -1; + } + } + if (pipeline == PIPELINE_FORWARD) { + current = msgpack_hop(current, CMT_TRUE); + if (current == NULL) { + return -1; + } + } + if (pipeline == PIPELINE_CAT) { + merged = cmt_create(); + TEST_ASSERT(merged != NULL); + TEST_CHECK(cmt_cat(merged, current) == 0); + cmt_decode_msgpack_destroy(current); + current = merged; + } + + wire = cmt_encode_opentelemetry_create(current); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + output = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_CHECK(output != NULL); + if (output != NULL) { + canon_request(lines, output); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + output, NULL); + } + cmt_encode_opentelemetry_destroy(wire); + } + + if (merged != NULL) { + cmt_destroy(merged); + } + else if (current != context) { + cmt_decode_msgpack_destroy(current); + } + return wire != NULL ? 0 : -1; +} + +static cfl_sds_t pack_request(otlp_request *value) +{ + size_t size; + uint8_t *buffer; + cfl_sds_t payload; + + size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(value); + buffer = require(calloc(1, size + 1)); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(value, buffer); + payload = cfl_sds_create_len((char *) buffer, size); + free(buffer); + TEST_ASSERT(payload != NULL); + return payload; +} + +static void compare_lines(struct lines *expected, struct lines *actual, const char *pipeline) +{ + size_t index; + int equal; + + if (expected->count > 1) { + qsort(expected->entries, expected->count, sizeof(cfl_sds_t), compare_strings); + } + if (actual->count > 1) { + qsort(actual->entries, actual->count, sizeof(cfl_sds_t), compare_strings); + } + equal = expected->count == actual->count; + for (index = 0; equal && index < expected->count; index++) { + equal = strcmp(expected->entries[index], actual->entries[index]) == 0; + } + TEST_CHECK_(equal, "pipeline %s reproduces every data point", pipeline); + if (!equal) { + for (index = 0; index < expected->count; index++) { + TEST_MSG("expected: %s", expected->entries[index]); + } + for (index = 0; index < actual->count; index++) { + TEST_MSG("actual: %s", actual->entries[index]); + } + } +} + +/* + * Decode the request as in_opentelemetry does and check that every requested + * pipeline reproduces it. The canonical lines of 'expected' default to the + * input itself; a case passes its own expectation when a transformation is + * intentionally lossy. + */ +static void check_pipelines(otlp_request *input, otlp_request *expected, unsigned int pipelines) +{ + struct cfl_list contexts; + struct cfl_list *head; + struct lines expected_lines; + struct lines actual_lines; + struct cmt *context; + cfl_sds_t payload; + size_t offset; + int pipeline; + int result; + + payload = pack_request(input); + + for (pipeline = 0; pipeline < PIPELINE_COUNT; pipeline++) { + if ((pipelines & (1u << pipeline)) == 0) { + continue; + } + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, payload, cfl_sds_len(payload), + &offset); + TEST_CHECK_(result == CMT_DECODE_OPENTELEMETRY_SUCCESS, + "OTLP decode (result %d)", result); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + break; + } + canon_include_context = pipeline != PIPELINE_CAT; + memset(&expected_lines, 0, sizeof(expected_lines)); + memset(&actual_lines, 0, sizeof(actual_lines)); + canon_request(&expected_lines, expected != NULL ? expected : input); + cfl_list_foreach(head, &contexts) { + context = cfl_list_entry(head, struct cmt, _head); + TEST_CHECK(run_pipeline(&actual_lines, context, pipeline) == 0); + } + compare_lines(&expected_lines, &actual_lines, pipeline_names[pipeline]); + canon_include_context = CMT_TRUE; + lines_destroy(&expected_lines); + lines_destroy(&actual_lines); + cmt_decode_opentelemetry_destroy(&contexts); + } + + cfl_sds_destroy(payload); +} + +#define ALL_PIPELINES ((1u << PIPELINE_COUNT) - 1) + +/* Cases */ + +#define RESOURCE_ATTRIBUTES \ + 2, kv_list(2, kv("service.name", any_string("checkout")), \ + kv("host.name", any_string("node-1"))) + +static otlp_request *single_metric(otlp_metric *metric) +{ + return request(1, resource(RESOURCE_ATTRIBUTES, 1, + scope("io.opentelemetry.runtime", "1.2.0", 0, NULL, 1, metric))); +} + +static otlp_kv **attrs1(const char *key, otlp_any *value) +{ + return kv_list(1, kv(key, value)); +} + +static double double_from_bits(uint64_t bits) +{ + double value; + + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static void run_case(otlp_request *input) +{ + check_pipelines(input, NULL, ALL_PIPELINES); + test_alloc_release(); +} + +/* Counter: monotonic cumulative sum with integer and double points. */ +static void test_sum_monotonic_cumulative(void) +{ + run_case(single_metric( + metric_sum("http.server.requests", "requests", "{request}", TEMPORALITY_CUMULATIVE, 1, 2, + number_int(1000, 2000, 42, 1, attrs1("http.route", any_string("/a"))), + number_int(1000, 2000, 7, 1, attrs1("http.route", any_string("/b")))))); +} + +/* UpDownCounter: non-monotonic sums must not become monotonic counters. */ +static void test_sum_non_monotonic(void) +{ + run_case(single_metric( + metric_sum("queue.items", "items", "{item}", TEMPORALITY_CUMULATIVE, 0, 2, + number_int(1000, 2000, -5, 1, attrs1("queue", any_string("in"))), + number_double(1000, 2000, -0.25, 1, attrs1("queue", any_string("out")))))); +} + +/* SDKs configured for delta temporality (common with some backends). */ +static void test_sum_delta(void) +{ + run_case(single_metric( + metric_sum("bytes.sent", "bytes", "By", TEMPORALITY_DELTA, 1, 1, + number_int(1000, 2000, 512, 1, attrs1("peer", any_string("db")))))); +} + +static void test_gauge_integer_extremes(void) +{ + run_case(single_metric( + metric_gauge("extremes", "integer extremes", "1", 4, + number_int(0, 100, INT64_MIN, 1, attrs1("v", any_string("min"))), + number_int(0, 100, INT64_MAX, 1, attrs1("v", any_string("max"))), + number_int(0, 100, -1, 1, attrs1("v", any_string("minus_one"))), + number_int(0, 100, 9007199254740993LL, 1, attrs1("v", any_string("2^53+1")))))); +} + +/* Prometheus receivers emit staleness markers (0x7ff0000000000002) and + * applications report NaN, infinities and negative zero. */ +static void test_gauge_special_doubles(void) +{ + run_case(single_metric( + metric_gauge("specials", "special doubles", "1", 6, + number_double(0, 100, double_from_bits(0x7ff8000000000000ULL), 1, + attrs1("v", any_string("nan"))), + number_double(0, 100, double_from_bits(0x7ff0000000000002ULL), 1, + attrs1("v", any_string("stale"))), + number_double(0, 100, INFINITY, 1, attrs1("v", any_string("+inf"))), + number_double(0, 100, -INFINITY, 1, attrs1("v", any_string("-inf"))), + number_double(0, 100, -0.0, 1, attrs1("v", any_string("-0"))), + number_double(0, 100, 4.9406564584124654e-324, 1, attrs1("v", any_string("denormal")))))); +} + +static void test_point_flags_and_timestamps(void) +{ + otlp_number_point *missing; + + missing = number_double(10, 20, 0, 1, attrs1("v", any_string("no_value"))); + missing->flags = 1; + run_case(single_metric( + metric_gauge("timestamps", "flags and timestamps", "1", 3, + missing, + number_double(0, 9223372036854775813ULL, 1, 1, attrs1("v", any_string("late"))), + number_double(500, 400, 2, 1, attrs1("v", any_string("start_after_time")))))); +} + +static void test_histogram_explicit_bounds(void) +{ + static const double bounds[] = {0.005, 0.01, 0.1, 1, 10}; + static const uint64_t counts_a[] = {1, 2, 3, 4, 5, 6}; + static const uint64_t counts_b[] = {0, 0, 0, 0, 0, 9}; + otlp_histogram_point *a; + otlp_histogram_point *b; + + a = histogram_point(1000, 2000, 5, bounds, counts_a, 1, + attrs1("http.route", any_string("/a"))); + a->has_sum = 1; + a->sum = 123.5; + a->has_min = 1; + a->min = 0.001; + a->has_max = 1; + a->max = 42; + a->n_exemplars = 2; + a->exemplars = exemplar_list(2, + exemplar_double(1500, 0.007, "0123456789abcdef", "01234567", 1, + attrs1("user", any_string("u1"))), + exemplar_int(1600, 3)); + b = histogram_point(1000, 2000, 5, bounds, counts_b, 1, + attrs1("http.route", any_string("/b"))); + b->has_sum = 1; + b->sum = 900; + run_case(single_metric(metric_histogram("http.server.duration", "s", + TEMPORALITY_CUMULATIVE, 2, a, b))); +} + +static void test_histogram_delta_without_sum(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t counts[] = {3, 0, 1}; + + run_case(single_metric(metric_histogram("latency.delta", "ms", TEMPORALITY_DELTA, 1, + histogram_point(1000, 2000, 2, bounds, counts, 1, + attrs1("op", any_string("get")))))); +} + +/* + * A histogram with no explicit bounds has a single +Inf bucket (SDK views use + * it to keep only count and sum). CMetrics cannot represent it: the request is + * rejected. Accepting it would also emit a zero-bucket MessagePack histogram + * that every released reader rejects, so this is pinned until decided. + */ +static void test_histogram_without_bounds_is_rejected(void) +{ + static const uint64_t counts[] = {17}; + struct cfl_list contexts; + cfl_sds_t payload; + size_t offset = 0; + int result; + + payload = pack_request(single_metric(metric_histogram("single.bucket", "s", + TEMPORALITY_CUMULATIVE, 1, histogram_point(1000, 2000, 0, NULL, counts, 0, NULL)))); + result = cmt_decode_opentelemetry_create(&contexts, payload, cfl_sds_len(payload), &offset); + TEST_CHECK(result != CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(payload); + test_alloc_release(); +} + +static void test_exponential_histogram_layouts(void) +{ + static const uint64_t positive[] = {1, 0, 4, 2}; + static const uint64_t negative[] = {3, 1}; + static const uint64_t fine[] = {5}; + otlp_exp_point *coarse; + otlp_exp_point *precise; + otlp_exp_point *zeros; + + coarse = exp_point(1000, 2000, -3, 2, 1e-9, exp_buckets(-5, 4, positive), + exp_buckets(2, 2, negative), 1, attrs1("layout", any_string("coarse"))); + coarse->has_sum = 1; + coarse->sum = -12.5; + coarse->has_min = 1; + coarse->min = -40; + coarse->has_max = 1; + coarse->max = 300; + precise = exp_point(1000, 2000, 20, 0, 0, exp_buckets(1048576, 1, fine), NULL, 1, + attrs1("layout", any_string("precise"))); + precise->has_sum = 1; + precise->sum = 10; + zeros = exp_point(1000, 2000, 0, 6, 0.5, NULL, NULL, 1, + attrs1("layout", any_string("zeros"))); + run_case(single_metric(metric_exp_histogram("rpc.duration", TEMPORALITY_DELTA, 3, + coarse, precise, zeros))); +} + +static void test_summary_quantiles(void) +{ + static const double quantiles[] = {0, 0.5, 0.99, 1}; + static const double values[] = {0.1, 2.5, 9.75, 12}; + + run_case(single_metric(metric_summary("gc.pause", 2, + summary_point(1000, 2000, 40, 100.5, 4, quantiles, values, 1, + attrs1("gen", any_string("young"))), + summary_point(1000, 2000, 2, 30, 4, quantiles, values, 1, + attrs1("gen", any_string("old")))))); +} + +static void test_summary_without_quantiles(void) +{ + run_case(single_metric(metric_summary("requests.summary", 1, + summary_point(1000, 2000, 5, 12.5, 0, NULL, NULL, 1, + attrs1("route", any_string("/")))))); +} + +/* Attribute values of every OTLP type keep their type end to end. */ +static void test_typed_attributes(void) +{ + run_case(single_metric(metric_gauge("typed", "typed attributes", "1", 1, + number_double(0, 100, 1, 9, kv_list(9, + kv("string", any_string("value")), + kv("empty", any_string("")), + kv("unicode", any_string("caf\xc3\xa9 \xe2\x82\xac")), + kv("int", any_int(-42)), + kv("double", any_double(2.5)), + kv("bool", any_bool(1)), + kv("bytes", any_bytes("\x00\xff\x10", 3)), + kv("array", any_array(3, any_string("a"), any_int(1), any_bool(0))), + kv("map", any_kvlist(2, kv("inner", any_string("x")), kv("n", any_double(0.5))))))))); +} + +static otlp_request *type_identity_request(int collapsed) +{ + if (collapsed) { + return single_metric(metric_gauge("identity", "type identity", "1", 1, + number_double(0, 100, 3, 1, attrs1("k", any_double(7))))); + } + return single_metric(metric_gauge("identity", "type identity", "1", 3, + number_double(0, 100, 1, 1, attrs1("k", any_int(7))), + number_double(0, 100, 2, 1, attrs1("k", any_string("7"))), + number_double(0, 100, 3, 1, attrs1("k", any_double(7))))); +} + +/* + * int 7, string "7" and double 7 are three OTLP series and stay three through + * OTLP and MessagePack. cmt_cat matches series by their string labels, which + * are identical, so it merges them and the last point wins. Pinned: series + * identity in cmt_cat does not consider attribute types. + */ +static void test_attribute_type_identity(void) +{ + check_pipelines(type_identity_request(CMT_FALSE), NULL, + ALL_PIPELINES & ~(1u << PIPELINE_CAT)); + check_pipelines(type_identity_request(CMT_FALSE), type_identity_request(CMT_TRUE), + 1u << PIPELINE_CAT); + test_alloc_release(); +} + +/* Native metadata as a released relay carries it: opaque context metadata + * written back unchanged, whatever it does to the points. */ +static struct cfl_kvlist *relay_copy(struct cmt *context, int internal) +{ + struct cfl_kvlist *copy = NULL; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + int result; + + mpack_writer_init_growable(&writer, &buffer, &size); + result = internal ? cmt_otlp_metadata_pack(&writer, context) : + cmt_otlp_metadata_pack_external(&writer, context); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok && result == 0); + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + TEST_ASSERT(mpack_reader_destroy(&reader) == mpack_ok && result == 0); + free(buffer); + return copy; +} + +/* Distribution points that differ only by attribute type, count and sum, + * but not by distribution, stay distinct through every hop. */ +static void test_distribution_type_identity(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t buckets_a[] = {1, 0, 1}; + static const uint64_t buckets_b[] = {0, 2, 0}; + static const double quantiles[] = {0.5}; + static const double quantile_a[] = {1}; + static const double quantile_b[] = {2}; + static const uint64_t positive_a[] = {1, 1}; + static const uint64_t positive_b[] = {2, 0}; + unsigned int mask = ALL_PIPELINES & ~(1u << PIPELINE_CAT); + otlp_histogram_point *histogram[2]; + otlp_summary_point *summary[2]; + otlp_exp_point *exp[2]; + int index; + + histogram[0] = histogram_point(0, 100, 2, bounds, buckets_a, 1, attrs1("k", any_int(7))); + histogram[1] = histogram_point(0, 100, 2, bounds, buckets_b, 1, attrs1("k", any_string("7"))); + summary[0] = summary_point(0, 100, 2, 3, 1, quantiles, quantile_a, 1, attrs1("k", any_int(7))); + summary[1] = summary_point(0, 100, 2, 3, 1, quantiles, quantile_b, 1, + attrs1("k", any_string("7"))); + exp[0] = exp_point(0, 100, 0, 0, 0, exp_buckets(0, 2, positive_a), NULL, 1, + attrs1("k", any_int(7))); + exp[1] = exp_point(0, 100, 0, 0, 0, exp_buckets(0, 2, positive_b), NULL, 1, + attrs1("k", any_string("7"))); + for (index = 0; index < 2; index++) { + histogram[index]->has_sum = 1; + histogram[index]->sum = 3; + exp[index]->has_sum = 1; + exp[index]->sum = 3; + } + histogram[0]->flags = 1; + summary[0]->flags = 1; + exp[0]->flags = 1; + + check_pipelines(single_metric(metric_histogram("latency", "s", TEMPORALITY_CUMULATIVE, 2, + histogram[0], histogram[1])), NULL, mask); + check_pipelines(single_metric(metric_summary("rpc", 2, summary[0], summary[1])), NULL, mask); + check_pipelines(single_metric(metric_exp_histogram("exp", TEMPORALITY_CUMULATIVE, 2, + exp[0], exp[1])), NULL, mask); + test_alloc_release(); +} + +struct typed_relay { + struct cfl_list contexts; + struct cmt *decoded; + char *buffer; + cfl_sds_t payload; + cfl_sds_t wire; + otlp_request *output; +}; + +/* Points of one series and time that differ only by attribute type: integer + * 7, string "7" and double 7.0. */ +static otlp_request *typed_points(const double *values, const uint32_t *flags) +{ + otlp_request *request; + otlp_metric *metric; + size_t index; + + request = single_metric(metric_gauge("identity", "type identity", "1", 3, + number_double(0, 100, values[0], 1, attrs1("k", any_int(7))), + number_double(0, 100, values[1], 1, attrs1("k", any_string("7"))), + number_double(0, 100, values[2], 1, attrs1("k", any_double(7))))); + metric = request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + for (index = 0; index < 3; index++) { + metric->gauge->data_points[index]->flags = flags[index]; + } + return request; +} + +/* Decode the request, then act as a released relay: keep native metadata as + * opaque context metadata and either drop the point at drop_index or swap + * the first two points, then decode the relayed chunk and export it. */ +static void typed_relay_run(struct typed_relay *relay, otlp_request *input, + int drop_index, int swap) +{ + struct cfl_list *head; + struct cfl_list *first; + struct cmt *context; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + size_t size; + size_t offset = 0; + int index = 0; + + memset(relay, 0, sizeof(*relay)); + relay->payload = pack_request(input); + TEST_ASSERT(cmt_decode_opentelemetry_create(&relay->contexts, relay->payload, + cfl_sds_len(relay->payload), &offset) == + CMT_DECODE_OPENTELEMETRY_SUCCESS); + context = cfl_list_entry_first(&relay->contexts, struct cmt, _head); + gauge = cfl_list_entry_first(&context->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 3); + + internal = relay_copy(context, CMT_TRUE); + external = relay_copy(context, CMT_FALSE); + cfl_kvlist_destroy(gauge->map->otlp_metadata); + gauge->map->otlp_metadata = NULL; + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; + } + cfl_kvlist_destroy(context->internal_metadata); + context->internal_metadata = internal; + cfl_kvlist_destroy(context->external_metadata); + context->external_metadata = external; + + if (swap) { + first = gauge->map->metrics.next; + cfl_list_del(first); + cfl_list_add_after(first, gauge->map->metrics.next, &gauge->map->metrics); + } + cfl_list_foreach(head, &gauge->map->metrics) { + if (index++ == drop_index) { + cmt_map_metric_destroy(cfl_list_entry(head, struct cmt_metric, _head)); + break; + } + } + + TEST_ASSERT(cmt_encode_msgpack_create(context, &relay->buffer, &size) == 0); + offset = 0; + TEST_ASSERT(cmt_decode_msgpack_create(&relay->decoded, relay->buffer, size, &offset) == 0); + relay->wire = cmt_encode_opentelemetry_create(relay->decoded); + TEST_ASSERT(relay->wire != NULL); + relay->output = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(relay->wire), (const uint8_t *) relay->wire); + TEST_ASSERT(relay->output != NULL); +} + +static void typed_relay_destroy(struct typed_relay *relay) +{ + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + relay->output, NULL); + cmt_encode_opentelemetry_destroy(relay->wire); + cmt_decode_msgpack_destroy(relay->decoded); + cmt_encode_msgpack_destroy(relay->buffer); + cmt_decode_opentelemetry_destroy(&relay->contexts); + cfl_sds_destroy(relay->payload); +} + +/* Each exported point carries the flags and attribute type recorded for its + * value, or, when values cannot tell points apart, neither. */ +static void check_typed_points(struct typed_relay *relay, size_t expected_points, + const double *values, const uint32_t *flags, + int distinguishable) +{ + static const int cases[] = { + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE, + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE, + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE + }; + otlp_metric *metric; + otlp_number_point *point; + size_t index; + size_t source; + + metric = relay->output->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(metric->gauge->n_data_points == expected_points); + for (index = 0; index < metric->gauge->n_data_points; index++) { + point = metric->gauge->data_points[index]; + TEST_ASSERT(point->n_attributes == 1); + if (!distinguishable) { + TEST_CHECK_(point->flags == 0, "point %g carries no recorded flags", + point->as_double); + TEST_CHECK_(point->attributes[0]->value->value_case == cases[1], + "point %g keeps its label-derived attribute", point->as_double); + continue; + } + for (source = 0; source < 3 && values[source] != point->as_double; source++); + TEST_ASSERT(source < 3); + TEST_CHECK_(point->flags == flags[source], "point %g keeps its own flags", + point->as_double); + TEST_CHECK_(point->attributes[0]->value->value_case == cases[source], + "point %g keeps its own attribute type", point->as_double); + } +} + +/* Points that differ only by attribute type share labels; their values tell + * them apart through any relay edit. Points identical in value too cannot be + * told apart, so none of them may take metadata recorded for another, from + * the native sidecar or from the legacy tree. */ +static void test_attribute_type_identity_after_relay(void) +{ + static const double distinct[] = {1, 2, 3}; + static const double equal[] = {5, 5, 5}; + static const uint32_t first_flagged[] = {1, 0, 0}; + static const uint32_t last_flagged[] = {0, 0, 1}; + struct typed_relay relay; + + /* drop the first point */ + typed_relay_run(&relay, typed_points(distinct, first_flagged), 0, CMT_FALSE); + check_typed_points(&relay, 2, distinct, first_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + + /* swap the first two points: count and identities are unchanged */ + typed_relay_run(&relay, typed_points(distinct, first_flagged), -1, CMT_TRUE); + check_typed_points(&relay, 3, distinct, first_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + typed_relay_run(&relay, typed_points(equal, first_flagged), -1, CMT_TRUE); + check_typed_points(&relay, 3, equal, first_flagged, CMT_FALSE); + typed_relay_destroy(&relay); + + /* drop the flagged last point: its legacy entry shares the survivors' + * series hash and time */ + typed_relay_run(&relay, typed_points(distinct, last_flagged), 2, CMT_FALSE); + check_typed_points(&relay, 2, distinct, last_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + typed_relay_run(&relay, typed_points(equal, last_flagged), 2, CMT_FALSE); + check_typed_points(&relay, 2, equal, last_flagged, CMT_FALSE); + typed_relay_destroy(&relay); + + test_alloc_release(); +} + +/* SDKs do not order attributes; series with the same keys in a different + * order must keep their own values. */ +static void test_attribute_order(void) +{ + run_case(single_metric(metric_gauge("order", "attribute order", "1", 2, + number_double(0, 100, 1, 2, kv_list(2, kv("a", any_string("1")), kv("b", any_string("2")))), + number_double(0, 100, 2, 2, kv_list(2, kv("b", any_string("3")), kv("a", any_string("4"))))))); +} + +/* Points of one metric with different attribute key sets. */ +static void test_attribute_key_sets(void) +{ + run_case(single_metric(metric_gauge("key_sets", "key sets", "1", 3, + number_double(0, 100, 1, 0, NULL), + number_double(0, 100, 2, 1, attrs1("a", any_string("x"))), + number_double(0, 100, 3, 2, kv_list(2, kv("a", any_string("x")), + kv("b", any_string("y"))))))); +} + +/* A collector forwarding several services in one request. */ +static void test_multiple_resources_and_scopes(void) +{ + run_case(request(2, + resource(1, attrs1("service.name", any_string("cart")), 2, + scope("lib.http", "1.0", 0, NULL, 1, + metric_gauge("shared.name", "from http", "1", 1, + number_double(0, 100, 1, 0, NULL))), + scope("lib.db", "2.0", 1, attrs1("db.system", any_string("postgresql")), 1, + metric_sum("shared.name", "from db", "1", TEMPORALITY_CUMULATIVE, 1, 1, + number_int(0, 100, 5, 0, NULL)))), + resource(1, attrs1("service.name", any_string("payment")), 1, + scope("lib.http", "1.0", 0, NULL, 1, + metric_gauge("shared.name", "from http", "1", 1, + number_double(0, 100, 2, 0, NULL)))))); +} + +static otlp_request *details_request(const char *description) +{ + otlp_request *input; + + input = single_metric(metric_gauge("details", description, "", 1, + number_double(0, 100, 1, 0, NULL))); + input->resource_metrics[0]->schema_url = "https://opentelemetry.io/schemas/1.24.0"; + input->resource_metrics[0]->resource->dropped_attributes_count = 3; + input->resource_metrics[0]->scope_metrics[0]->schema_url = + "https://opentelemetry.io/schemas/1.21.0"; + input->resource_metrics[0]->scope_metrics[0]->scope->dropped_attributes_count = 1; + input->resource_metrics[0]->scope_metrics[0]->scope->n_attributes = 1; + input->resource_metrics[0]->scope_metrics[0]->scope->attributes = + attrs1("scope.kind", any_string("auto")); + return input; +} + +/* Schema URLs, dropped counts and scope attributes. CMetrics families require + * a description, so an empty one is emitted as "-" (pinned). */ +static void test_resource_and_scope_details(void) +{ + check_pipelines(details_request(""), details_request("-"), ALL_PIPELINES); + test_alloc_release(); +} + +static void test_metric_metadata(void) +{ + otlp_metric *metric; + + metric = metric_gauge("with.metadata", "has metadata", "By", 1, + number_double(0, 100, 1, 0, NULL)); + metric->n_metadata = 2; + metric->metadata = kv_list(2, kv("prometheus.type", any_string("gauge")), + kv("weight", any_int(3))); + run_case(single_metric(metric)); +} + +static void test_number_exemplars(void) +{ + otlp_number_point *point; + + point = number_int(1000, 2000, 10, 1, attrs1("route", any_string("/"))); + point->n_exemplars = 2; + point->exemplars = exemplar_list(2, + exemplar_int(1500, 4), + exemplar_double(1600, 6.5, "fedcba9876543210", "76543210", 2, + kv_list(2, kv("user", any_string("u2")), kv("n", any_int(1))))); + run_case(single_metric(metric_sum("exemplars", "with exemplars", "1", + TEMPORALITY_CUMULATIVE, 1, 1, point))); +} + +/* Large cardinality as produced by busy collectors. */ +static void test_many_series(void) +{ + otlp_metric *metric; + char *values; + size_t index; + + metric = metric_gauge("many.series", "cardinality", "1", 0); + metric->gauge->n_data_points = 2000; + metric->gauge->data_points = test_alloc(sizeof(otlp_number_point *) * 2000); + values = test_alloc(2000 * 8); + for (index = 0; index < 2000; index++) { + snprintf(&values[index * 8], 8, "%zu", index); + metric->gauge->data_points[index] = + number_double(0, 100, (double) index, 1, + attrs1("series", any_string(&values[index * 8]))); + } + run_case(single_metric(metric)); +} + +/* Every prefix and single byte corruption of a real request must be handled + * without crashing; whatever decodes must also survive a MessagePack hop. */ +static void test_truncated_and_corrupted_requests(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t counts[] = {1, 2, 3}; + struct cfl_list contexts; + struct cfl_list *head; + struct cmt *context; + struct cmt *decoded; + cfl_sds_t payload; + cfl_sds_t copy; + size_t length; + size_t offset; + int result; + + payload = pack_request(request(1, resource(RESOURCE_ATTRIBUTES, 1, + scope("s", "1", 1, attrs1("k", any_string("v")), 3, + metric_sum("c", "d", "1", TEMPORALITY_CUMULATIVE, 1, 1, + number_int(1, 2, 3, 1, attrs1("a", any_int(1)))), + metric_histogram("h", "s", TEMPORALITY_CUMULATIVE, 1, + histogram_point(1, 2, 2, bounds, counts, 1, attrs1("a", any_bool(1)))), + metric_exp_histogram("e", TEMPORALITY_DELTA, 1, + exp_point(1, 2, 1, 1, 0, exp_buckets(-1, 3, counts), NULL, 0, NULL)))))); + + for (length = 0; length <= cfl_sds_len(payload) * 2; length++) { + copy = cfl_sds_create_len(payload, length <= cfl_sds_len(payload) ? + length : cfl_sds_len(payload)); + if (length > cfl_sds_len(payload)) { + copy[length - cfl_sds_len(payload) - 1] ^= 0x5a; + } + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, copy, cfl_sds_len(copy), &offset); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_list_foreach(head, &contexts) { + context = cfl_list_entry(head, struct cmt, _head); + decoded = msgpack_hop(context, CMT_FALSE); + if (decoded != NULL) { + cmt_decode_msgpack_destroy(decoded); + } + } + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(copy); + } + cfl_sds_destroy(payload); + test_alloc_release(); +} + +/* Containers without data must not produce data points or fail. */ +static void test_empty_containers(void) +{ + otlp_request *input; + + input = request(3, + resource(RESOURCE_ATTRIBUTES, 0), + resource(RESOURCE_ATTRIBUTES, 1, scope("empty", "1", 0, NULL, 0)), + resource(RESOURCE_ATTRIBUTES, 1, scope("s", "1", 0, NULL, 1, + metric_gauge("no.points", "", "", 0)))); + check_pipelines(input, request(0), ALL_PIPELINES); + test_alloc_release(); +} + +TEST_LIST = { + {"sum_monotonic_cumulative", test_sum_monotonic_cumulative}, + {"sum_non_monotonic", test_sum_non_monotonic}, + {"sum_delta", test_sum_delta}, + {"gauge_integer_extremes", test_gauge_integer_extremes}, + {"gauge_special_doubles", test_gauge_special_doubles}, + {"point_flags_and_timestamps", test_point_flags_and_timestamps}, + {"histogram_explicit_bounds", test_histogram_explicit_bounds}, + {"histogram_delta_without_sum", test_histogram_delta_without_sum}, + {"histogram_without_bounds_is_rejected", test_histogram_without_bounds_is_rejected}, + {"exponential_histogram_layouts", test_exponential_histogram_layouts}, + {"summary_quantiles", test_summary_quantiles}, + {"summary_without_quantiles", test_summary_without_quantiles}, + {"typed_attributes", test_typed_attributes}, + {"attribute_type_identity", test_attribute_type_identity}, + {"attribute_type_identity_after_relay", test_attribute_type_identity_after_relay}, + {"distribution_type_identity", test_distribution_type_identity}, + {"attribute_order", test_attribute_order}, + {"attribute_key_sets", test_attribute_key_sets}, + {"multiple_resources_and_scopes", test_multiple_resources_and_scopes}, + {"resource_and_scope_details", test_resource_and_scope_details}, + {"metric_metadata", test_metric_metadata}, + {"number_exemplars", test_number_exemplars}, + {"many_series", test_many_series}, + {"truncated_and_corrupted_requests", test_truncated_and_corrupted_requests}, + {"empty_containers", test_empty_containers}, + {0} +};