From 3dbdd8d143af4607cdb8e1a8215c492eb7c34bb1 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:05 +0400 Subject: [PATCH 01/51] chore: treat images as binary and ignore build and run output Co-Authored-By: Claude Opus 5.5 --- .dockerignore | 9 +++++++++ .gitattributes | 5 +++++ .gitignore | 20 ++++++++++++++------ 3 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f8e0ec2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +build +results +.venv +**/node_modules +**/__pycache__ +.mypy_cache +.ruff_cache +.pytest_cache diff --git a/.gitattributes b/.gitattributes index 6313b56..2b2f8c9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,6 @@ * text=auto eol=lf + +*.png binary +*.jpg binary +*.gif binary +*.pdf binary diff --git a/.gitignore b/.gitignore index 56a8e4b..9ce7c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,15 @@ -# Build output and benchmark results -*.o -/energy_explorer -/tests/test_sorts -/data.csv -/*.png +# Build output +/build/ + +# Harness runs made for development, smoke tests and rehearsals. Study data goes in data/runs/. +/results/ + +# Python /.venv/ +__pycache__/ +.mypy_cache/ +.ruff_cache/ +.pytest_cache/ + +# Node.js +node_modules/ From c9888ee5b85b0d24b6b3222a410b888038ae6dd6 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:06 +0400 Subject: [PATCH 02/51] refactor: move the sorts and the input generator into src/ The sorts keep their exact behaviour and counts (the tests now check the counts Algorithm-Energy-C published for 4,000 random integers). Counters are 64-bit, and the xorshift32 generator, the input orders and an FNV-1a input hash move out of main.c into src/inputs.c so the harness and the workers can share them. Co-Authored-By: Claude Opus 5.5 --- src/inputs.c | 64 +++++++++++++++++++++++++++ src/inputs.h | 35 +++++++++++++++ sorts.c => src/sorts.c | 12 ++++++ sorts.h => src/sorts.h | 11 +++-- tests/check.h | 30 +++++++++++++ tests/test_inputs.c | 98 ++++++++++++++++++++++++++++++++++++++++++ tests/test_sorts.c | 60 +++++++++++++++++--------- 7 files changed, 287 insertions(+), 23 deletions(-) create mode 100644 src/inputs.c create mode 100644 src/inputs.h rename sorts.c => src/sorts.c (88%) rename sorts.h => src/sorts.h (56%) create mode 100644 tests/check.h create mode 100644 tests/test_inputs.c diff --git a/src/inputs.c b/src/inputs.c new file mode 100644 index 0000000..b3f07f3 --- /dev/null +++ b/src/inputs.c @@ -0,0 +1,64 @@ +#include "inputs.h" + +#include + +static const char *const ORDER_NAMES[ORDER_COUNT] = {"random", "sorted", "reversed", "nearly_sorted"}; + +uint32_t xorshift32_next(uint32_t *state) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} + +const char *order_name(t_order order) { + return (unsigned)order < ORDER_COUNT ? ORDER_NAMES[order] : "unknown"; +} + +int order_from_name(const char *name, t_order *order) { + for (int i = 0; i < ORDER_COUNT; i++) { + if (strcmp(name, ORDER_NAMES[i]) == 0) { + *order = (t_order)i; + return 0; + } + } + return -1; +} + +static void swap_ints(int *a, int *b) { + int tmp = *a; + *a = *b; + *b = tmp; +} + +void input_fill(int *arr, size_t n, t_order order, uint32_t seed) { + uint32_t state = seed; + + for (size_t i = 0; i < n; i++) + arr[i] = order == ORDER_REVERSED ? (int)(n - 1 - i) : (int)i; + if (order == ORDER_RANDOM) { + for (size_t i = n; i > 1; i--) /* Fisher-Yates shuffle */ + swap_ints(&arr[i - 1], &arr[xorshift32_next(&state) % i]); + } else if (order == ORDER_NEARLY_SORTED) { + for (size_t k = 0; k < n / 20; k++) { /* swap 5% of the elements with random partners */ + size_t a = xorshift32_next(&state) % n; + size_t b = xorshift32_next(&state) % n; + swap_ints(&arr[a], &arr[b]); + } + } +} + +uint32_t input_hash(const int *arr, size_t n) { + uint32_t hash = 2166136261u; + + for (size_t i = 0; i < n; i++) { + uint32_t value = (uint32_t)arr[i]; + for (int byte = 0; byte < 4; byte++) { + hash ^= (value >> (8 * byte)) & 0xFFu; + hash *= 16777619u; + } + } + return hash; +} diff --git a/src/inputs.h b/src/inputs.h new file mode 100644 index 0000000..d746d6e --- /dev/null +++ b/src/inputs.h @@ -0,0 +1,35 @@ +#ifndef INPUTS_H +#define INPUTS_H + +#include +#include + +/* The seed from Marsaglia's xorshift paper. Every language uses it by default. */ +#define INPUT_DEFAULT_SEED 2463534242u +#define INPUT_MAX_N 100000 + +typedef enum e_order { + ORDER_RANDOM, + ORDER_SORTED, + ORDER_REVERSED, + ORDER_NEARLY_SORTED, + ORDER_COUNT +} t_order; + +/* xorshift32 (shifts 13, 17, 5). The Python and Node.js workers implement the same + generator, and the parity tests check that all three produce identical sequences. */ +uint32_t xorshift32_next(uint32_t *state); + +const char *order_name(t_order order); +/* Returns 0 and sets *order when the name is known, -1 otherwise. */ +int order_from_name(const char *name, t_order *order); + +/* Fills arr with a permutation of 0..n-1 in the given order. The generator starts from + seed, so equal arguments always give equal arrays. seed must not be 0. */ +void input_fill(int *arr, size_t n, t_order order, uint32_t seed); + +/* 32-bit FNV-1a over each value as four little-endian bytes. Workers report it so the + harness can check that every language sorted exactly the same input. */ +uint32_t input_hash(const int *arr, size_t n); + +#endif diff --git a/sorts.c b/src/sorts.c similarity index 88% rename from sorts.c rename to src/sorts.c index 3666a35..166bb5e 100644 --- a/sorts.c +++ b/src/sorts.c @@ -1,5 +1,7 @@ #include "sorts.h" +#include + static void swap(int *a, int *b, t_metrics *m) { int tmp = *a; *a = *b; @@ -77,3 +79,13 @@ void quick_sort(int *arr, size_t n, t_metrics *m) { if (n > 1) quick_sort_range(arr, 0, n - 1, m); } + +t_sort_fn sort_by_name(const char *name) { + if (strcmp(name, "bubble") == 0) + return bubble_sort; + if (strcmp(name, "insertion") == 0) + return insertion_sort; + if (strcmp(name, "quick") == 0) + return quick_sort; + return NULL; +} diff --git a/sorts.h b/src/sorts.h similarity index 56% rename from sorts.h rename to src/sorts.h index cbf835b..b17810b 100644 --- a/sorts.h +++ b/src/sorts.h @@ -2,11 +2,13 @@ #define SORTS_H #include +#include -/* Work done by one sort: comparisons between elements, and writes into the array. */ +/* Work done by one sort: comparisons between elements, and writes into the array. + 64-bit so that bubble sort on the largest allowed input cannot overflow. */ typedef struct s_metrics { - unsigned long comparisons; - unsigned long writes; + uint64_t comparisons; + uint64_t writes; } t_metrics; typedef void (*t_sort_fn)(int *arr, size_t n, t_metrics *m); @@ -15,4 +17,7 @@ void bubble_sort(int *arr, size_t n, t_metrics *m); void insertion_sort(int *arr, size_t n, t_metrics *m); void quick_sort(int *arr, size_t n, t_metrics *m); +/* Looks an algorithm up by name ("bubble", "insertion" or "quick"); NULL if unknown. */ +t_sort_fn sort_by_name(const char *name); + #endif diff --git a/tests/check.h b/tests/check.h new file mode 100644 index 0000000..14993a2 --- /dev/null +++ b/tests/check.h @@ -0,0 +1,30 @@ +#ifndef CHECK_H +#define CHECK_H + +/* A minimal test helper: CHECK records a failure and carries on, so one run reports every + broken check. Each test file is its own program, so the counters can be static. */ + +#include + +static int checks; +static int failures; + +#define CHECK(condition, name) \ + do { \ + checks++; \ + if (!(condition)) { \ + failures++; \ + fprintf(stderr, "%s:%d: %s: check failed: %s\n", __FILE__, __LINE__, name, #condition); \ + } \ + } while (0) + +static inline int check_report(const char *suite) { + if (failures > 0) { + fprintf(stderr, "%s: %d of %d checks failed\n", suite, failures, checks); + return 1; + } + printf("%s: all %d checks passed\n", suite, checks); + return 0; +} + +#endif diff --git a/tests/test_inputs.c b/tests/test_inputs.c new file mode 100644 index 0000000..6dfd6f3 --- /dev/null +++ b/tests/test_inputs.c @@ -0,0 +1,98 @@ +#include +#include + +#include "check.h" +#include "inputs.h" + +/* The first outputs from the seed in Marsaglia's "Xorshift RNGs" (2003) and from seed 1, + computed independently with a short Python script. */ +static void test_known_sequence(void) { + static const uint32_t expected[] = {723471715u, 2497366906u, 2064144800u, 2008045182u, 3532304609u}; + static const uint32_t from_one[] = {270369u, 67634689u, 2647435461u}; + uint32_t state = INPUT_DEFAULT_SEED; + + for (size_t i = 0; i < sizeof expected / sizeof *expected; i++) + CHECK(xorshift32_next(&state) == expected[i], "xorshift32 from the default seed"); + state = 1; + for (size_t i = 0; i < sizeof from_one / sizeof *from_one; i++) + CHECK(xorshift32_next(&state) == from_one[i], "xorshift32 from seed 1"); +} + +static int is_permutation(const int *arr, size_t n) { + char *seen = calloc(n, 1); + int ok = seen != NULL; + + for (size_t i = 0; ok && i < n; i++) { + if (arr[i] < 0 || (size_t)arr[i] >= n || seen[arr[i]]) + ok = 0; + else + seen[arr[i]] = 1; + } + free(seen); + return ok; +} + +static void test_orders(void) { + enum { N = 1000 }; + static int a[N], b[N]; + + input_fill(a, N, ORDER_SORTED, INPUT_DEFAULT_SEED); + CHECK(a[0] == 0 && a[N - 1] == N - 1 && is_permutation(a, N), "sorted is 0..n-1"); + input_fill(a, N, ORDER_REVERSED, INPUT_DEFAULT_SEED); + CHECK(a[0] == N - 1 && a[N - 1] == 0 && is_permutation(a, N), "reversed is n-1..0"); + + input_fill(a, N, ORDER_RANDOM, INPUT_DEFAULT_SEED); + CHECK(is_permutation(a, N), "random is a permutation"); + input_fill(b, N, ORDER_RANDOM, INPUT_DEFAULT_SEED); + CHECK(memcmp(a, b, sizeof a) == 0, "same seed, same random input"); + input_fill(b, N, ORDER_RANDOM, 7u); + CHECK(memcmp(a, b, sizeof a) != 0, "different seed, different random input"); + + input_fill(a, N, ORDER_NEARLY_SORTED, INPUT_DEFAULT_SEED); + size_t displaced = 0; + for (size_t i = 0; i < N; i++) + if (a[i] != (int)i) + displaced++; + CHECK(is_permutation(a, N), "nearly sorted is a permutation"); + CHECK(displaced > 0 && displaced <= 2 * (N / 20), "nearly sorted moves at most 10% of the elements"); + + int one = -1; + input_fill(&one, 1, ORDER_RANDOM, INPUT_DEFAULT_SEED); + CHECK(one == 0, "a single element"); + input_fill(&one, 1, ORDER_NEARLY_SORTED, INPUT_DEFAULT_SEED); + CHECK(one == 0, "a single nearly sorted element"); +} + +static void test_order_names(void) { + t_order order; + + for (int i = 0; i < ORDER_COUNT; i++) { + CHECK(order_from_name(order_name((t_order)i), &order) == 0 && order == (t_order)i, "order names round-trip"); + } + CHECK(order_from_name("shuffled", &order) == -1, "unknown order"); + CHECK(order_from_name("", &order) == -1, "empty order"); +} + +/* Reference values computed independently with a short Python script. */ +static void test_hash(void) { + static const int one_two_three[] = {1, 2, 3}; + int ten[10]; + int zero = 0; + int minus_one = -1; + + for (int i = 0; i < 10; i++) + ten[i] = i; + CHECK(input_hash(NULL, 0) == 0x811c9dc5u, "FNV-1a offset basis for no input"); + CHECK(input_hash(&zero, 1) == 0x4b95f515u, "hash of [0]"); + CHECK(input_hash(one_two_three, 3) == 0x794671b5u, "hash of [1, 2, 3]"); + CHECK(input_hash(ten, 10) == 0x3a210ed4u, "hash of 0..9"); + CHECK(input_hash(&minus_one, 1) == 0xe3160fb1u, "hash of [-1] uses the two's complement bytes"); +} + +int main(void) { + test_known_sequence(); + test_orders(); + test_order_names(); + test_hash(); + return check_report("inputs"); +} diff --git a/tests/test_sorts.c b/tests/test_sorts.c index 68d7d21..7c00a37 100644 --- a/tests/test_sorts.c +++ b/tests/test_sorts.c @@ -1,22 +1,10 @@ -#include #include #include +#include "check.h" +#include "inputs.h" #include "sorts.h" -static int checks; -static int failures; - -#define CHECK(condition, name) \ - do { \ - checks++; \ - if (!(condition)) { \ - failures++; \ - fprintf(stderr, "%s:%d: %s: check failed: %s\n", __FILE__, __LINE__, name, \ - #condition); \ - } \ - } while (0) - typedef struct s_case { const char *name; t_sort_fn sort; @@ -85,14 +73,46 @@ static void test_known_counts(void) { CHECK(m.comparisons == 4950, "quick, reversed input"); } +/* The counts Algorithm-Energy-C published for 4,000 random integers. The input generator + and the sorts moved into new files; these checks prove they still do exactly the same work. */ +static void test_counts_unchanged_since_algorithm_energy_c(void) { + static int arr[4000]; + t_metrics m; + + input_fill(arr, 4000, ORDER_RANDOM, INPUT_DEFAULT_SEED); + m = (t_metrics){0, 0}; + bubble_sort(arr, 4000, &m); + CHECK(m.comparisons == 7998000 && m.writes == 8003836, "bubble, 4000 random"); + + input_fill(arr, 4000, ORDER_RANDOM, INPUT_DEFAULT_SEED); + m = (t_metrics){0, 0}; + insertion_sort(arr, 4000, &m); + CHECK(m.comparisons == 4005907 && m.writes == 4005905, "insertion, 4000 random"); + + input_fill(arr, 4000, ORDER_RANDOM, INPUT_DEFAULT_SEED); + m = (t_metrics){0, 0}; + quick_sort(arr, 4000, &m); + CHECK(m.comparisons == 55119 && m.writes == 56984, "quick, 4000 random"); + + input_fill(arr, 4000, ORDER_NEARLY_SORTED, INPUT_DEFAULT_SEED); + m = (t_metrics){0, 0}; + insertion_sort(arr, 4000, &m); + CHECK(m.comparisons == 533531 && m.writes == 533520, "insertion, 4000 nearly sorted"); +} + +static void test_lookup_by_name(void) { + CHECK(sort_by_name("bubble") == bubble_sort, "bubble by name"); + CHECK(sort_by_name("insertion") == insertion_sort, "insertion by name"); + CHECK(sort_by_name("quick") == quick_sort, "quick by name"); + CHECK(sort_by_name("merge") == NULL, "unknown name"); + CHECK(sort_by_name("none") == NULL, "none is not a sort"); +} + int main(void) { for (size_t i = 0; i < sizeof SORTS / sizeof *SORTS; i++) test_matches_qsort(&SORTS[i]); test_known_counts(); - if (failures > 0) { - fprintf(stderr, "%d of %d checks failed\n", failures, checks); - return 1; - } - printf("All %d checks passed\n", checks); - return 0; + test_counts_unchanged_since_algorithm_energy_c(); + test_lookup_by_name(); + return check_report("sorts"); } From 6fb666fe79c93d5c8c5274b88bfc108297f3f980 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:06 +0400 Subject: [PATCH 03/51] feat: read RAPL energy from powercap, with wraparound and clear failures Package and DRAM zones are found under /sys/class/powercap, read with pread, and summed over sockets. One counter wrap is handled with max_energy_range_uj. A missing, unreadable or corrupt counter makes energy unavailable with a reason; it is never reported as zero. Fake powercap trees are stored as manifests because zone names contain colons. Co-Authored-By: Claude Opus 5.5 --- src/rapl.c | 300 ++++++++++++++++++ src/rapl.h | 83 +++++ tests/fixture.c | 79 +++++ tests/fixture.h | 16 + tests/fixtures/powercap/README.md | 20 ++ tests/fixtures/powercap/amd-package-only.txt | 4 + tests/fixtures/powercap/bad-max-range.txt | 3 + tests/fixtures/powercap/counter-above-max.txt | 3 + tests/fixtures/powercap/dram-only.txt | 3 + tests/fixtures/powercap/laptop-intel.txt | 19 ++ tests/fixtures/powercap/no-rapl.txt | 1 + tests/fixtures/powercap/server-two-socket.txt | 13 + tests/integration/powercap.py | 45 +++ tests/test_rapl.c | 195 ++++++++++++ 14 files changed, 784 insertions(+) create mode 100644 src/rapl.c create mode 100644 src/rapl.h create mode 100644 tests/fixture.c create mode 100644 tests/fixture.h create mode 100644 tests/fixtures/powercap/README.md create mode 100644 tests/fixtures/powercap/amd-package-only.txt create mode 100644 tests/fixtures/powercap/bad-max-range.txt create mode 100644 tests/fixtures/powercap/counter-above-max.txt create mode 100644 tests/fixtures/powercap/dram-only.txt create mode 100644 tests/fixtures/powercap/laptop-intel.txt create mode 100644 tests/fixtures/powercap/no-rapl.txt create mode 100644 tests/fixtures/powercap/server-two-socket.txt create mode 100644 tests/integration/powercap.py create mode 100644 tests/test_rapl.c diff --git a/src/rapl.c b/src/rapl.c new file mode 100644 index 0000000..6d0df97 --- /dev/null +++ b/src/rapl.c @@ -0,0 +1,300 @@ +#define _POSIX_C_SOURCE 200809L /* pread, dirent */ + +#include "rapl.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_ZONES 64 +#define ZONE_PREFIX "intel-rapl:" + +int rapl_parse_u64(const char *text, uint64_t *out) { + const char *p = text; + uint64_t value = 0; + + if (*p < '0' || *p > '9') + return -1; + while (*p >= '0' && *p <= '9') { + uint64_t digit = (uint64_t)(*p - '0'); + if (value > (UINT64_MAX - digit) / 10) + return -1; + value = value * 10 + digit; + p++; + } + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') + p++; + if (*p != '\0') + return -1; + *out = value; + return 0; +} + +/* The hardware counter is 32 bits wide and max_energy_range_uj is its largest value in + microjoules, so a wrap loses the single step from the maximum back to zero: at most one + energy unit, about 15 to 61 microjoules depending on the processor. */ +int rapl_delta(uint64_t start, uint64_t end, uint64_t max_range_uj, uint64_t *delta) { + if (start > max_range_uj || end > max_range_uj) + return -1; + *delta = end >= start ? end - start : (max_range_uj - start) + end; + return 0; +} + +/* Accepts "intel-rapl:N" and "intel-rapl:N:M" with decimal N and M. */ +static int zone_depth(const char *zone) { + size_t prefix = strlen(ZONE_PREFIX); + int depth = 0; + + if (strncmp(zone, ZONE_PREFIX, prefix) != 0) + return -1; + const char *p = zone + prefix; + for (;;) { + if (*p < '0' || *p > '9') + return -1; + while (*p >= '0' && *p <= '9') + p++; + depth++; + if (*p == '\0') + return depth; + if (*p != ':' || depth == 2) + return -1; + p++; + } +} + +int rapl_classify(const char *zone, const char *name, t_rapl_kind *kind) { + int depth = zone_depth(zone); + + if (depth < 0) + return -1; + if (depth == 1 && strncmp(name, "package", 7) == 0) { + *kind = RAPL_PACKAGE; + return 0; + } + if (strcmp(name, "dram") == 0) { + *kind = RAPL_DRAM; + return 0; + } + return -1; +} + +static void set_reason(t_rapl *rapl, const char *format, ...) { + va_list args; + + va_start(args, format); + vsnprintf(rapl->reason, sizeof rapl->reason, format, args); + va_end(args); +} + +/* Reads a small sysfs file into buf, without the trailing newline. */ +static int read_small_file(const char *path, char *buf, size_t size) { + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + ssize_t n = read(fd, buf, size - 1); + int saved = errno; + close(fd); + if (n < 0) { + errno = saved; + return -1; + } + buf[n] = '\0'; + while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r' || buf[n - 1] == ' ')) + buf[--n] = '\0'; + return 0; +} + +static int build_path(char *out, size_t size, const char *root, const char *zone, const char *file) { + int written = snprintf(out, size, "%s/%s/%s", root, zone, file); + return written < 0 || (size_t)written >= size ? -1 : 0; +} + +static int compare_names(const void *a, const void *b) { + return strcmp((const char *)a, (const char *)b); +} + +/* Collects candidate zone directory names, sorted so that domains appear in a stable order. */ +static int list_zones(t_rapl *rapl, char zones[][RAPL_ZONE_LEN], size_t *count) { + DIR *dir = opendir(rapl->root); + struct dirent *entry; + + *count = 0; + if (dir == NULL) { + if (errno == ENOENT) + set_reason(rapl, + "%s does not exist, so this system exposes no RAPL counters " + "(WSL2, Docker Desktop, most virtual machines and CI runners do not)", + rapl->root); + else + set_reason(rapl, "cannot open %s: %s", rapl->root, strerror(errno)); + return -1; + } + while ((entry = readdir(dir)) != NULL) { + if (zone_depth(entry->d_name) < 0 || strlen(entry->d_name) >= RAPL_ZONE_LEN) + continue; + if (*count == MAX_ZONES) + break; + memcpy(zones[*count], entry->d_name, strlen(entry->d_name) + 1); + (*count)++; + } + closedir(dir); + qsort(zones, *count, RAPL_ZONE_LEN, compare_names); + return 0; +} + +static int open_domain(t_rapl *rapl, const char *zone, t_rapl_kind kind, const char *name) { + t_rapl_domain *domain = &rapl->domains[rapl->count]; + char path[RAPL_PATH_LEN]; + char text[64]; + + if (rapl->count == RAPL_MAX_DOMAINS) { + set_reason(rapl, "more than %d RAPL package and DRAM zones under %s", RAPL_MAX_DOMAINS, + rapl->root); + return -1; + } + if (build_path(path, sizeof path, rapl->root, zone, "max_energy_range_uj") != 0 || + read_small_file(path, text, sizeof text) != 0) { + set_reason(rapl, "cannot read %s/%s/max_energy_range_uj: %s", rapl->root, zone, strerror(errno)); + return -1; + } + if (rapl_parse_u64(text, &domain->max_range_uj) != 0 || domain->max_range_uj == 0) { + set_reason(rapl, "%s/%s/max_energy_range_uj does not hold a positive integer", rapl->root, zone); + return -1; + } + if (build_path(path, sizeof path, rapl->root, zone, "energy_uj") != 0) { + set_reason(rapl, "path too long under %s", rapl->root); + return -1; + } + domain->fd = open(path, O_RDONLY | O_CLOEXEC); + if (domain->fd < 0) { + if (errno == EACCES || errno == EPERM) + set_reason(rapl, + "permission denied reading %s. Since the fix for CVE-2020-8694 only root can read " + "RAPL counters; see docs/bare-metal.md", + path); + else + set_reason(rapl, "cannot open %s: %s", path, strerror(errno)); + return -1; + } + memcpy(domain->zone, zone, strlen(zone) + 1); + snprintf(domain->name, sizeof domain->name, "%s", name); + domain->kind = kind; + rapl->count++; + + /* A counter that opens but cannot be read or parsed is as useless as a missing one. */ + ssize_t n = pread(domain->fd, text, sizeof text - 1, 0); + uint64_t first; + if (n <= 0) { + set_reason(rapl, "cannot read %s: %s", path, n < 0 ? strerror(errno) : "empty file"); + return -1; + } + text[n] = '\0'; + if (rapl_parse_u64(text, &first) != 0 || first > domain->max_range_uj) { + set_reason(rapl, "%s does not hold a valid counter value", path); + return -1; + } + return 0; +} + +int rapl_open(t_rapl *rapl, const char *root) { + char zones[MAX_ZONES][RAPL_ZONE_LEN]; + size_t zone_count; + int packages = 0; + + memset(rapl, 0, sizeof *rapl); + for (size_t i = 0; i < RAPL_MAX_DOMAINS; i++) + rapl->domains[i].fd = -1; + if (root == NULL) + root = RAPL_DEFAULT_ROOT; + if (strlen(root) >= sizeof rapl->root - RAPL_ZONE_LEN - 32) { + set_reason(rapl, "powercap root path is too long"); + return -1; + } + memcpy(rapl->root, root, strlen(root) + 1); + if (list_zones(rapl, zones, &zone_count) != 0) + return -1; + + for (size_t z = 0; z < zone_count; z++) { + char path[RAPL_PATH_LEN]; + char name[RAPL_NAME_LEN]; + t_rapl_kind kind; + + if (build_path(path, sizeof path, rapl->root, zones[z], "name") != 0 || + read_small_file(path, name, sizeof name) != 0) + continue; /* not a zone we can identify */ + if (rapl_classify(zones[z], name, &kind) != 0) + continue; + if (open_domain(rapl, zones[z], kind, name) != 0) { + rapl_close(rapl); + return -1; + } + if (kind == RAPL_PACKAGE) + packages++; + else + rapl->has_dram = 1; + } + if (packages == 0) { + set_reason(rapl, + "no RAPL package zone (intel-rapl:N named package-N) under %s; this CPU, " + "kernel or virtual machine does not expose RAPL", + rapl->root); + rapl_close(rapl); + return -1; + } + return 0; +} + +int rapl_read(const t_rapl *rapl, t_rapl_sample *sample) { + for (size_t i = 0; i < rapl->count; i++) { + char text[64]; + ssize_t n = pread(rapl->domains[i].fd, text, sizeof text - 1, 0); + if (n <= 0) { + if (n == 0) + errno = EIO; + return -1; + } + text[n] = '\0'; + if (rapl_parse_u64(text, &sample->uj[i]) != 0) { + errno = EINVAL; + return -1; + } + } + return 0; +} + +int rapl_energy_between(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl_sample *b, + t_rapl_energy *energy) { + energy->package_uj = 0; + energy->dram_uj = 0; + for (size_t i = 0; i < rapl->count; i++) { + uint64_t delta; + if (rapl_delta(a->uj[i], b->uj[i], rapl->domains[i].max_range_uj, &delta) != 0) + return -1; + if (rapl->domains[i].kind == RAPL_PACKAGE) + energy->package_uj += delta; + else + energy->dram_uj += delta; + } + return 0; +} + +double rapl_net_uj(uint64_t gross_uj, uint64_t idle_uj, uint64_t run_ns, uint64_t idle_ns) { + if (idle_ns == 0) + return (double)gross_uj; + return (double)gross_uj - (double)idle_uj * ((double)run_ns / (double)idle_ns); +} + +void rapl_close(t_rapl *rapl) { + for (size_t i = 0; i < RAPL_MAX_DOMAINS; i++) { + if (rapl->domains[i].fd >= 0) + close(rapl->domains[i].fd); + rapl->domains[i].fd = -1; + } + rapl->count = 0; + rapl->has_dram = 0; +} diff --git a/src/rapl.h b/src/rapl.h new file mode 100644 index 0000000..2dbbad5 --- /dev/null +++ b/src/rapl.h @@ -0,0 +1,83 @@ +#ifndef RAPL_H +#define RAPL_H + +/* Energy counters from the Linux powercap interface (Intel RAPL, and AMD on kernels whose + intel_rapl driver supports the CPU). Each zone directory, for example + /sys/class/powercap/intel-rapl:0, holds: + name "package-0", "dram", "core", "uncore" or "psys" + energy_uj a counter in microjoules that wraps to 0 + max_energy_range_uj the value at which energy_uj wraps + Only package and DRAM zones are read; core and uncore are inside the package, and psys + covers more than the processor. */ + +#include +#include + +#define RAPL_DEFAULT_ROOT "/sys/class/powercap" +#define RAPL_MAX_DOMAINS 16 +#define RAPL_ZONE_LEN 64 +#define RAPL_NAME_LEN 64 +#define RAPL_PATH_LEN 512 +#define RAPL_REASON_LEN 512 + +typedef enum e_rapl_kind { RAPL_PACKAGE, RAPL_DRAM } t_rapl_kind; + +typedef struct s_rapl_domain { + char zone[RAPL_ZONE_LEN]; /* directory name, for example "intel-rapl:0:2" */ + char name[RAPL_NAME_LEN]; /* contents of the name file, for example "dram" */ + t_rapl_kind kind; + uint64_t max_range_uj; + int fd; /* energy_uj, kept open and re-read with pread */ +} t_rapl_domain; + +typedef struct s_rapl { + char root[RAPL_PATH_LEN]; + size_t count; + t_rapl_domain domains[RAPL_MAX_DOMAINS]; + int has_dram; + char reason[RAPL_REASON_LEN]; /* why energy is unavailable, when rapl_open fails */ +} t_rapl; + +typedef struct s_rapl_sample { + uint64_t uj[RAPL_MAX_DOMAINS]; +} t_rapl_sample; + +/* Energy between two samples, summed over sockets. */ +typedef struct s_rapl_energy { + uint64_t package_uj; + uint64_t dram_uj; /* 0 when there is no DRAM zone; check has_dram */ +} t_rapl_energy; + +/* Parses a sysfs counter: decimal digits, then optional whitespace (sysfs ends with "\n"). + Returns 0 on success, -1 on anything else, including overflow. */ +int rapl_parse_u64(const char *text, uint64_t *out); + +/* Energy used between two readings of one counter that wraps at max_range_uj. Handles one + wrap; the harness keeps every measured interval far shorter than a wrap period. Returns -1 + if either reading is above max_range_uj (a corrupt reading), 0 otherwise. */ +int rapl_delta(uint64_t start, uint64_t end, uint64_t max_range_uj, uint64_t *delta); + +/* Classifies a zone directory name and the contents of its name file. + Returns 0 and sets *kind for a package or DRAM zone, -1 for anything to skip. */ +int rapl_classify(const char *zone, const char *name, t_rapl_kind *kind); + +/* Finds and opens the package and DRAM zones under root (RAPL_DEFAULT_ROOT when NULL). + Returns 0 on success. Returns -1 when energy cannot be measured, with the reason in + rapl->reason; nothing is left open in that case. */ +int rapl_open(t_rapl *rapl, const char *root); + +/* Reads every open counter. Returns 0, or -1 with errno set. */ +int rapl_read(const t_rapl *rapl, t_rapl_sample *sample); + +/* Energy from a to b. Returns 0, or -1 if a counter reading was corrupt. */ +int rapl_energy_between(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl_sample *b, + t_rapl_energy *energy); + +/* Energy left after removing the idle baseline: gross minus idle energy scaled from the + idle window to the run's duration. Can be negative when the run used no more power than + idle; the value is kept, not clamped, so the noise stays visible. */ +double rapl_net_uj(uint64_t gross_uj, uint64_t idle_uj, uint64_t run_ns, uint64_t idle_ns); + +void rapl_close(t_rapl *rapl); + +#endif diff --git a/tests/fixture.c b/tests/fixture.c new file mode 100644 index 0000000..90414ae --- /dev/null +++ b/tests/fixture.c @@ -0,0 +1,79 @@ +#define _XOPEN_SOURCE 700 /* mkdtemp, nftw */ + +#include "fixture.h" + +#include +#include +#include +#include +#include +#include +#include + +static int make_parents(char *path) { + for (char *p = path + 1; *p != '\0'; p++) { + if (*p != '/') + continue; + *p = '\0'; + if (mkdir(path, 0755) != 0 && errno != EEXIST) + return -1; + *p = '/'; + } + return 0; +} + +int fixture_write(const char *root, const char *relative, const char *contents) { + char path[1024]; + + if (snprintf(path, sizeof path, "%s/%s", root, relative) >= (int)sizeof path || make_parents(path) != 0) + return -1; + FILE *f = fopen(path, "w"); + if (f == NULL) + return -1; + fprintf(f, "%s\n", contents); + return fclose(f); +} + +int fixture_build(const char *manifest, char *root, size_t size) { + const char *tmp = getenv("TMPDIR"); + char line[512]; + + if (snprintf(root, size, "%s/greenbench-powercap-XXXXXX", tmp != NULL && *tmp ? tmp : "/tmp") >= (int)size || + mkdtemp(root) == NULL) + return -1; + FILE *f = fopen(manifest, "r"); + if (f == NULL) { + fixture_remove(root); + return -1; + } + while (fgets(line, sizeof line, f) != NULL) { + line[strcspn(line, "\r\n")] = '\0'; + if (line[0] == '\0' || line[0] == '#') + continue; + char *space = strpbrk(line, " \t"); + if (space == NULL) + continue; + *space = '\0'; + char *contents = space + 1; + while (*contents == ' ' || *contents == '\t') + contents++; + if (fixture_write(root, line, contents) != 0) { + fclose(f); + fixture_remove(root); + return -1; + } + } + fclose(f); + return 0; +} + +static int remove_entry(const char *path, const struct stat *st, int type, struct FTW *ftw) { + (void)st; + (void)type; + (void)ftw; + return remove(path); +} + +void fixture_remove(const char *root) { + nftw(root, remove_entry, 16, FTW_DEPTH | FTW_PHYS); +} diff --git a/tests/fixture.h b/tests/fixture.h new file mode 100644 index 0000000..7fb075a --- /dev/null +++ b/tests/fixture.h @@ -0,0 +1,16 @@ +#ifndef FIXTURE_H +#define FIXTURE_H + +#include + +/* Builds the fake powercap tree described by a manifest (see tests/fixtures/powercap) in a + new temporary directory, and writes that directory's path to root. Returns 0 or -1. */ +int fixture_build(const char *manifest, char *root, size_t size); + +/* Overwrites one file inside a built tree, as the kernel does when a counter moves. */ +int fixture_write(const char *root, const char *relative, const char *contents); + +/* Removes a built tree. */ +void fixture_remove(const char *root); + +#endif diff --git a/tests/fixtures/powercap/README.md b/tests/fixtures/powercap/README.md new file mode 100644 index 0000000..50dcefd --- /dev/null +++ b/tests/fixtures/powercap/README.md @@ -0,0 +1,20 @@ +# Fake powercap trees + +Real powercap zone directories are named like `intel-rapl:0`, and a colon is not allowed in +a Windows file name. So each fake tree is stored as a manifest instead: one file per line, +its path relative to the powercap root, then whitespace, then its contents. The C tests +(`tests/fixture.c`) and the Python tests (`tests/integration/powercap.py`) build the tree in a +temporary directory before pointing the code at it. + +The values are invented but shaped like real sysfs output: counters in microjoules and +`max_energy_range_uj` values of the size real processors report. + +| Manifest | What it models | +| :--- | :--- | +| `laptop-intel.txt` | One package with core, uncore and DRAM subzones, a psys zone and an MMIO duplicate of the package | +| `server-two-socket.txt` | Two packages, each with a DRAM subzone | +| `amd-package-only.txt` | One package and no DRAM zone, as AMD processors report | +| `no-rapl.txt` | A powercap directory with no RAPL zones | +| `bad-max-range.txt` | A package whose `max_energy_range_uj` is not a number | +| `counter-above-max.txt` | A package whose counter is above its wrap value | +| `dram-only.txt` | A DRAM zone without a package zone | diff --git a/tests/fixtures/powercap/amd-package-only.txt b/tests/fixtures/powercap/amd-package-only.txt new file mode 100644 index 0000000..ddfe020 --- /dev/null +++ b/tests/fixtures/powercap/amd-package-only.txt @@ -0,0 +1,4 @@ +intel-rapl/enabled 1 +intel-rapl:0/name package-0 +intel-rapl:0/energy_uj 3000000000 +intel-rapl:0/max_energy_range_uj 65532610987 diff --git a/tests/fixtures/powercap/bad-max-range.txt b/tests/fixtures/powercap/bad-max-range.txt new file mode 100644 index 0000000..cba2cf9 --- /dev/null +++ b/tests/fixtures/powercap/bad-max-range.txt @@ -0,0 +1,3 @@ +intel-rapl:0/name package-0 +intel-rapl:0/energy_uj 1000 +intel-rapl:0/max_energy_range_uj not-a-number diff --git a/tests/fixtures/powercap/counter-above-max.txt b/tests/fixtures/powercap/counter-above-max.txt new file mode 100644 index 0000000..d3e7c68 --- /dev/null +++ b/tests/fixtures/powercap/counter-above-max.txt @@ -0,0 +1,3 @@ +intel-rapl:0/name package-0 +intel-rapl:0/energy_uj 70000000000 +intel-rapl:0/max_energy_range_uj 65532610987 diff --git a/tests/fixtures/powercap/dram-only.txt b/tests/fixtures/powercap/dram-only.txt new file mode 100644 index 0000000..ac81a87 --- /dev/null +++ b/tests/fixtures/powercap/dram-only.txt @@ -0,0 +1,3 @@ +intel-rapl:0:0/name dram +intel-rapl:0:0/energy_uj 1000 +intel-rapl:0:0/max_energy_range_uj 65712999613 diff --git a/tests/fixtures/powercap/laptop-intel.txt b/tests/fixtures/powercap/laptop-intel.txt new file mode 100644 index 0000000..5b25782 --- /dev/null +++ b/tests/fixtures/powercap/laptop-intel.txt @@ -0,0 +1,19 @@ +intel-rapl/enabled 1 +intel-rapl:0/name package-0 +intel-rapl:0/energy_uj 81234567890 +intel-rapl:0/max_energy_range_uj 262143328850 +intel-rapl:0:0/name core +intel-rapl:0:0/energy_uj 40123456789 +intel-rapl:0:0/max_energy_range_uj 262143328850 +intel-rapl:0:1/name uncore +intel-rapl:0:1/energy_uj 1234567 +intel-rapl:0:1/max_energy_range_uj 262143328850 +intel-rapl:0:2/name dram +intel-rapl:0:2/energy_uj 9876543210 +intel-rapl:0:2/max_energy_range_uj 65712999613 +intel-rapl:1/name psys +intel-rapl:1/energy_uj 123456789012 +intel-rapl:1/max_energy_range_uj 262143328850 +intel-rapl-mmio:0/name package-0 +intel-rapl-mmio:0/energy_uj 81234567000 +intel-rapl-mmio:0/max_energy_range_uj 262143328850 diff --git a/tests/fixtures/powercap/no-rapl.txt b/tests/fixtures/powercap/no-rapl.txt new file mode 100644 index 0000000..2710bb9 --- /dev/null +++ b/tests/fixtures/powercap/no-rapl.txt @@ -0,0 +1 @@ +dtpm/enabled 0 diff --git a/tests/fixtures/powercap/server-two-socket.txt b/tests/fixtures/powercap/server-two-socket.txt new file mode 100644 index 0000000..6137fc8 --- /dev/null +++ b/tests/fixtures/powercap/server-two-socket.txt @@ -0,0 +1,13 @@ +intel-rapl/enabled 1 +intel-rapl:0/name package-0 +intel-rapl:0/energy_uj 150000000000 +intel-rapl:0/max_energy_range_uj 262143328850 +intel-rapl:0:0/name dram +intel-rapl:0:0/energy_uj 20000000000 +intel-rapl:0:0/max_energy_range_uj 65712999613 +intel-rapl:1/name package-1 +intel-rapl:1/energy_uj 160000000000 +intel-rapl:1/max_energy_range_uj 262143328850 +intel-rapl:1:0/name dram +intel-rapl:1:0/energy_uj 21000000000 +intel-rapl:1:0/max_energy_range_uj 65712999613 diff --git a/tests/integration/powercap.py b/tests/integration/powercap.py new file mode 100644 index 0000000..3d5ddbe --- /dev/null +++ b/tests/integration/powercap.py @@ -0,0 +1,45 @@ +"""Build a fake powercap tree from a manifest in tests/fixtures/powercap. + +Each manifest line is a path relative to the powercap root, whitespace, then the file's +contents. Zone directories are named like ``intel-rapl:0``, which Windows cannot store, so +the trees are kept as manifests and built on demand. + + python3 tests/integration/powercap.py tests/fixtures/powercap/laptop-intel.txt /tmp/powercap +""" + +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + + +def build(manifest: Path, root: Path) -> Path: + """Create the tree described by ``manifest`` under ``root``, replacing anything there.""" + if root.exists(): + shutil.rmtree(root) + root.mkdir(parents=True) + for number, raw in enumerate(manifest.read_text(encoding="utf-8").splitlines(), start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + parts = line.split(maxsplit=1) + if len(parts) != 2: + raise ValueError(f"{manifest}:{number}: expected a path and its contents") + relative, contents = parts + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents + "\n", encoding="utf-8") + return root + + +def main(argv: list[str]) -> int: + if len(argv) != 3: + print("usage: powercap.py MANIFEST OUTPUT_DIR", file=sys.stderr) + return 2 + build(Path(argv[1]), Path(argv[2])) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/test_rapl.c b/tests/test_rapl.c new file mode 100644 index 0000000..fbe44ca --- /dev/null +++ b/tests/test_rapl.c @@ -0,0 +1,195 @@ +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include + +#include "check.h" +#include "fixture.h" +#include "rapl.h" + +#define FIXTURES "tests/fixtures/powercap/" + +static void test_parse(void) { + uint64_t v = 0; + + CHECK(rapl_parse_u64("0", &v) == 0 && v == 0, "zero"); + CHECK(rapl_parse_u64("262143328850\n", &v) == 0 && v == 262143328850u, "sysfs value with newline"); + CHECK(rapl_parse_u64("42 \r\n", &v) == 0 && v == 42, "trailing whitespace"); + CHECK(rapl_parse_u64("18446744073709551615", &v) == 0 && v == UINT64_MAX, "largest 64-bit value"); + CHECK(rapl_parse_u64("18446744073709551616", &v) == -1, "overflow is rejected"); + CHECK(rapl_parse_u64("", &v) == -1, "empty"); + CHECK(rapl_parse_u64("\n", &v) == -1, "newline only"); + CHECK(rapl_parse_u64("-5", &v) == -1, "negative"); + CHECK(rapl_parse_u64(" 5", &v) == -1, "leading space"); + CHECK(rapl_parse_u64("12abc", &v) == -1, "trailing junk"); + CHECK(rapl_parse_u64("12 34", &v) == -1, "two numbers"); +} + +static void test_delta(void) { + uint64_t d = 0; + const uint64_t max = 262143328850u; + + CHECK(rapl_delta(100, 250, max, &d) == 0 && d == 150, "no wrap"); + CHECK(rapl_delta(500, 500, max, &d) == 0 && d == 0, "no change"); + CHECK(rapl_delta(max - 100, 50, max, &d) == 0 && d == 150, "wrap: 100 to the top, then 50"); + CHECK(rapl_delta(max, 0, max, &d) == 0 && d == 0, "exactly at the top, then zero"); + CHECK(rapl_delta(0, max, max, &d) == 0 && d == max, "a full range without wrapping"); + CHECK(rapl_delta(max + 1, 5, max, &d) == -1, "start above the range is corrupt"); + CHECK(rapl_delta(5, max + 1, max, &d) == -1, "end above the range is corrupt"); +} + +static void test_classify(void) { + t_rapl_kind kind; + + CHECK(rapl_classify("intel-rapl:0", "package-0", &kind) == 0 && kind == RAPL_PACKAGE, "package zone"); + CHECK(rapl_classify("intel-rapl:1", "package-1", &kind) == 0 && kind == RAPL_PACKAGE, "second socket"); + CHECK(rapl_classify("intel-rapl:0:2", "dram", &kind) == 0 && kind == RAPL_DRAM, "DRAM subzone"); + CHECK(rapl_classify("intel-rapl:2", "dram", &kind) == 0 && kind == RAPL_DRAM, "top-level DRAM zone"); + CHECK(rapl_classify("intel-rapl:0:0", "core", &kind) == -1, "core is inside the package"); + CHECK(rapl_classify("intel-rapl:0:1", "uncore", &kind) == -1, "uncore is inside the package"); + CHECK(rapl_classify("intel-rapl:1", "psys", &kind) == -1, "psys covers more than the processor"); + CHECK(rapl_classify("intel-rapl:0:0", "package-0", &kind) == -1, "a package must be a top-level zone"); + CHECK(rapl_classify("intel-rapl-mmio:0", "package-0", &kind) == -1, "MMIO duplicate is skipped"); + CHECK(rapl_classify("intel-rapl", "package-0", &kind) == -1, "control type directory"); + CHECK(rapl_classify("intel-rapl:", "package-0", &kind) == -1, "no zone number"); + CHECK(rapl_classify("intel-rapl:0:1:2", "dram", &kind) == -1, "too deep"); + CHECK(rapl_classify("intel-rapl:x", "package-0", &kind) == -1, "not a number"); +} + +/* Opens a manifest; returns rapl_open's result and leaves the tree path in root. */ +static int open_fixture(const char *manifest, t_rapl *rapl, char *root, size_t size) { + if (fixture_build(manifest, root, size) != 0) { + fprintf(stderr, "cannot build fixture %s\n", manifest); + return -2; + } + return rapl_open(rapl, root); +} + +static void test_laptop(void) { + t_rapl rapl; + char root[512]; + + CHECK(open_fixture(FIXTURES "laptop-intel.txt", &rapl, root, sizeof root) == 0, "laptop opens"); + CHECK(rapl.count == 2, "package and DRAM only: no core, uncore, psys or MMIO"); + CHECK(strcmp(rapl.domains[0].zone, "intel-rapl:0") == 0 && rapl.domains[0].kind == RAPL_PACKAGE, + "package first"); + CHECK(strcmp(rapl.domains[1].zone, "intel-rapl:0:2") == 0 && rapl.domains[1].kind == RAPL_DRAM, "then DRAM"); + CHECK(rapl.has_dram, "has DRAM"); + CHECK(rapl.domains[1].max_range_uj == 65712999613u, "DRAM range read"); + + t_rapl_sample a, b; + t_rapl_energy e; + CHECK(rapl_read(&rapl, &a) == 0 && a.uj[0] == 81234567890u && a.uj[1] == 9876543210u, "first reading"); + + /* Move the counters the way the kernel would: the package by 5 J, DRAM across its wrap. */ + fixture_write(root, "intel-rapl:0/energy_uj", "81239567890"); + fixture_write(root, "intel-rapl:0:2/energy_uj", "1000"); + fixture_write(root, "intel-rapl:0:2/max_energy_range_uj", "1"); /* ignored: read once at open */ + CHECK(rapl_read(&rapl, &b) == 0, "second reading"); + CHECK(rapl_energy_between(&rapl, &a, &b, &e) == 0, "energy between readings"); + CHECK(e.package_uj == 5000000u, "package: 5 J"); + CHECK(e.dram_uj == (65712999613u - 9876543210u) + 1000u, "DRAM: wrapped once"); + + fixture_write(root, "intel-rapl:0/energy_uj", "garbage"); + CHECK(rapl_read(&rapl, &b) == -1, "an unreadable counter is an error, not zero"); + rapl_close(&rapl); + fixture_remove(root); +} + +static void test_two_sockets(void) { + t_rapl rapl; + char root[512]; + + CHECK(open_fixture(FIXTURES "server-two-socket.txt", &rapl, root, sizeof root) == 0, "server opens"); + CHECK(rapl.count == 4 && rapl.has_dram, "two packages, two DRAM zones"); + + t_rapl_sample a, b; + t_rapl_energy e; + CHECK(rapl_read(&rapl, &a) == 0, "first reading"); + fixture_write(root, "intel-rapl:0/energy_uj", "150001000000"); + fixture_write(root, "intel-rapl:1/energy_uj", "160002000000"); + fixture_write(root, "intel-rapl:0:0/energy_uj", "20000300000"); + fixture_write(root, "intel-rapl:1:0/energy_uj", "21000400000"); + CHECK(rapl_read(&rapl, &b) == 0 && rapl_energy_between(&rapl, &a, &b, &e) == 0, "second reading"); + CHECK(e.package_uj == 3000000u, "packages summed over sockets"); + CHECK(e.dram_uj == 700000u, "DRAM summed over sockets"); + rapl_close(&rapl); + fixture_remove(root); +} + +static void test_amd(void) { + t_rapl rapl; + char root[512]; + + CHECK(open_fixture(FIXTURES "amd-package-only.txt", &rapl, root, sizeof root) == 0, "AMD opens"); + CHECK(rapl.count == 1 && !rapl.has_dram, "package without DRAM"); + rapl_close(&rapl); + fixture_remove(root); +} + +static void expect_unavailable(const char *manifest, const char *needle, const char *name) { + t_rapl rapl; + char root[512]; + + CHECK(open_fixture(manifest, &rapl, root, sizeof root) == -1, name); + CHECK(strstr(rapl.reason, needle) != NULL, name); + CHECK(rapl.count == 0, name); + fixture_remove(root); +} + +static void test_unavailable(void) { + t_rapl rapl; + + CHECK(rapl_open(&rapl, "/nonexistent/greenbench/powercap") == -1, "missing root"); + CHECK(strstr(rapl.reason, "does not exist") != NULL && strstr(rapl.reason, "WSL2") != NULL, + "missing root explains where RAPL is absent"); + expect_unavailable(FIXTURES "no-rapl.txt", "no RAPL package zone", "no RAPL zones"); + expect_unavailable(FIXTURES "dram-only.txt", "no RAPL package zone", "DRAM without a package"); + expect_unavailable(FIXTURES "bad-max-range.txt", "max_energy_range_uj", "bad max range"); + expect_unavailable(FIXTURES "counter-above-max.txt", "valid counter value", "counter above its range"); +} + +/* Since the fix for CVE-2020-8694, energy_uj is readable by root only. Root reads it anyway, + so this check only means something for an ordinary user, which is how CI runs it. */ +static void test_permission_denied(void) { + t_rapl rapl; + char root[512]; + char path[600]; + + if (geteuid() == 0) { + printf("rapl: skipping the permission check because the tests run as root\n"); + return; + } + if (fixture_build(FIXTURES "amd-package-only.txt", root, sizeof root) != 0) { + CHECK(0, "build fixture"); + return; + } + snprintf(path, sizeof path, "%s/intel-rapl:0/energy_uj", root); + chmod(path, 0); + CHECK(rapl_open(&rapl, root) == -1, "unreadable counter"); + CHECK(strstr(rapl.reason, "permission denied") != NULL && strstr(rapl.reason, "root") != NULL, + "the reason names the permission problem"); + fixture_remove(root); +} + +static void test_net(void) { + CHECK(rapl_net_uj(1000, 400, 100, 100) == 600.0, "same-length idle window"); + CHECK(rapl_net_uj(1000, 400, 100, 200) == 800.0, "idle window twice as long is scaled down"); + CHECK(rapl_net_uj(300, 400, 100, 100) == -100.0, "negative results are kept, not clamped"); + CHECK(rapl_net_uj(1000, 400, 100, 0) == 1000.0, "no idle window leaves the gross value"); +} + +int main(void) { + test_parse(); + test_delta(); + test_classify(); + test_laptop(); + test_two_sockets(); + test_amd(); + test_unavailable(); + test_permission_denied(); + test_net(); + return check_report("rapl"); +} From a2c657541a6a0299bcea5b0a4001937e99adfe35 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:07 +0400 Subject: [PATCH 04/51] feat: add the C worker and the one-line result protocol Every worker takes the same commands (run, sequence, input) and prints the same result line: counts for one sort, its own loop time and a hash of its input, so the harness can check that every language did the same work. Co-Authored-By: Claude Opus 5.5 --- src/protocol.c | 176 +++++++++++++++++++++++++++++ src/protocol.h | 38 +++++++ src/worker.c | 257 ++++++++++++++++++++++++++++++++++++++++++ tests/test_protocol.c | 73 ++++++++++++ 4 files changed, 544 insertions(+) create mode 100644 src/protocol.c create mode 100644 src/protocol.h create mode 100644 src/worker.c create mode 100644 tests/test_protocol.c diff --git a/src/protocol.c b/src/protocol.c new file mode 100644 index 0000000..82a6156 --- /dev/null +++ b/src/protocol.c @@ -0,0 +1,176 @@ +#include "protocol.h" + +#include +#include +#include +#include + +enum e_field { + F_LANGUAGE, + F_ALGORITHM, + F_ORDER, + F_N, + F_SEED, + F_REPS, + F_COMPARISONS, + F_WRITES, + F_LOOP_NS, + F_INPUT_HASH, + F_COUNT +}; + +static const char *const FIELDS[F_COUNT] = {"language", "algorithm", "order", "n", "seed", + "reps", "comparisons", "writes", "loop_ns", "input_hash"}; + +int protocol_format(char *buf, size_t size, const t_worker_result *r) { + int written = snprintf(buf, size, + "result language=%s algorithm=%s order=%s n=%" PRIu64 " seed=%" PRIu64 + " reps=%" PRIu64 " comparisons=%" PRIu64 " writes=%" PRIu64 " loop_ns=%" PRIu64 + " input_hash=%08" PRIx32 "\n", + r->language, r->algorithm, r->order, r->n, r->seed, r->reps, r->comparisons, + r->writes, r->loop_ns, r->input_hash); + return written < 0 || (size_t)written >= size ? -1 : written; +} + +static void fail(char *err, size_t err_size, const char *format, ...) { + va_list args; + + va_start(args, format); + vsnprintf(err, err_size, format, args); + va_end(args); +} + +static int parse_decimal(const char *text, uint64_t *out) { + uint64_t value = 0; + + if (*text == '\0') + return -1; + for (const char *p = text; *p != '\0'; p++) { + if (*p < '0' || *p > '9') + return -1; + uint64_t digit = (uint64_t)(*p - '0'); + if (value > (UINT64_MAX - digit) / 10) + return -1; + value = value * 10 + digit; + } + *out = value; + return 0; +} + +static int parse_hash(const char *text, uint32_t *out) { + uint32_t value = 0; + + if (strlen(text) != 8) + return -1; + for (const char *p = text; *p != '\0'; p++) { + uint32_t digit; + if (*p >= '0' && *p <= '9') + digit = (uint32_t)(*p - '0'); + else if (*p >= 'a' && *p <= 'f') + digit = (uint32_t)(*p - 'a' + 10); + else + return -1; + value = value << 4 | digit; + } + *out = value; + return 0; +} + +static int parse_name(const char *text, char *out) { + size_t length = strlen(text); + + if (length == 0 || length >= PROTOCOL_NAME_LEN) + return -1; + for (const char *p = text; *p != '\0'; p++) + if (!((*p >= 'a' && *p <= 'z') || *p == '_')) + return -1; + memcpy(out, text, length + 1); + return 0; +} + +static int store(int field, const char *value, t_worker_result *r) { + uint64_t *numbers[F_COUNT] = {NULL, NULL, NULL, &r->n, &r->seed, &r->reps, &r->comparisons, &r->writes, + &r->loop_ns, NULL}; + + switch (field) { + case F_LANGUAGE: + return parse_name(value, r->language); + case F_ALGORITHM: + return parse_name(value, r->algorithm); + case F_ORDER: + return parse_name(value, r->order); + case F_INPUT_HASH: + return parse_hash(value, &r->input_hash); + default: + return parse_decimal(value, numbers[field]); + } +} + +int protocol_parse(const char *line, t_worker_result *r, char *err, size_t err_size) { + char buf[1024]; + int seen[F_COUNT] = {0}; + size_t length = strlen(line); + + while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r')) + length--; + if (length >= sizeof buf) { + fail(err, err_size, "result line is too long"); + return -1; + } + memcpy(buf, line, length); + buf[length] = '\0'; + memset(r, 0, sizeof *r); + + char *token = buf; + int first = 1; + while (*token != '\0') { + char *end = strchr(token, ' '); + if (end != NULL) + *end = '\0'; + if (first) { + if (strcmp(token, "result") != 0) { + fail(err, err_size, "expected a line starting with 'result', got '%.60s'", line); + return -1; + } + first = 0; + } else if (*token != '\0') { + char *equals = strchr(token, '='); + int field = -1; + if (equals == NULL) { + fail(err, err_size, "field '%.40s' has no value", token); + return -1; + } + *equals = '\0'; + for (int i = 0; i < F_COUNT; i++) + if (strcmp(token, FIELDS[i]) == 0) + field = i; + if (field < 0) { + fail(err, err_size, "unknown field '%.40s'", token); + return -1; + } + if (seen[field]) { + fail(err, err_size, "field '%s' appears twice", FIELDS[field]); + return -1; + } + seen[field] = 1; + if (store(field, equals + 1, r) != 0) { + fail(err, err_size, "field '%s' has an invalid value '%.40s'", FIELDS[field], equals + 1); + return -1; + } + } + if (end == NULL) + break; + token = end + 1; + } + if (first) { + fail(err, err_size, "empty result line"); + return -1; + } + for (int i = 0; i < F_COUNT; i++) { + if (!seen[i]) { + fail(err, err_size, "missing field '%s'", FIELDS[i]); + return -1; + } + } + return 0; +} diff --git a/src/protocol.h b/src/protocol.h new file mode 100644 index 0000000..b6e9ca9 --- /dev/null +++ b/src/protocol.h @@ -0,0 +1,38 @@ +#ifndef PROTOCOL_H +#define PROTOCOL_H + +/* The one line a worker prints after "worker run", in every language: + + result language=c algorithm=quick order=random n=1000 seed=2463534242 reps=40 + comparisons=11039 writes=11846 loop_ns=5123400 input_hash=5f0e3c21 + + (on one line). comparisons and writes are for a single sort; loop_ns covers all reps + sorts, timed by the worker with its own monotonic clock; input_hash is the FNV-1a hash of + the generated input as eight hex digits. */ + +#include +#include + +#define PROTOCOL_NAME_LEN 16 + +typedef struct s_worker_result { + char language[PROTOCOL_NAME_LEN]; + char algorithm[PROTOCOL_NAME_LEN]; + char order[PROTOCOL_NAME_LEN]; + uint64_t n; + uint64_t seed; + uint64_t reps; + uint64_t comparisons; + uint64_t writes; + uint64_t loop_ns; + uint32_t input_hash; +} t_worker_result; + +/* Formats a result line, with a trailing newline. Returns the length, or -1 if it does not fit. */ +int protocol_format(char *buf, size_t size, const t_worker_result *result); + +/* Parses a result line. Every field must appear exactly once. Returns 0, or -1 with a + message in err. */ +int protocol_parse(const char *line, t_worker_result *result, char *err, size_t err_size); + +#endif diff --git a/src/worker.c b/src/worker.c new file mode 100644 index 0000000..fd00272 --- /dev/null +++ b/src/worker.c @@ -0,0 +1,257 @@ +/* greenbench-worker: the C worker. The Python and Node.js workers accept the same + commands and print the same output, byte for byte: + + greenbench-worker run --algorithm NAME --order NAME --n N [--seed S] [--reps K] + greenbench-worker sequence --count C [--seed S] + greenbench-worker input --order NAME --n N [--seed S] + + run generates the input, sorts a fresh copy of it K times, checks the result and prints + one result line (see protocol.h). Algorithm "none" generates the input and copies it K + times without sorting; with K = 0 it measures what starting the process costs. + sequence prints the first C numbers of the xorshift32 generator, and input prints the + generated input, one number per line; the parity tests compare them across languages. */ + +#define _GNU_SOURCE /* CLOCK_MONOTONIC_RAW */ + +#include +#include +#include +#include +#include + +#include "inputs.h" +#include "protocol.h" +#include "sorts.h" + +#define EXIT_USAGE 2 +#define MAX_REPS 100000000u +#define MAX_COUNT 1000000u + +typedef struct s_args { + const char *algorithm; + const char *order; + const char *n; + const char *seed; + const char *reps; + const char *count; +} t_args; + +static int usage(const char *message, const char *detail) { + fprintf(stderr, "greenbench-worker: %s%s\n", message, detail); + fprintf(stderr, "usage: greenbench-worker run --algorithm NAME --order NAME --n N [--seed S] [--reps K]\n" + " greenbench-worker sequence --count C [--seed S]\n" + " greenbench-worker input --order NAME --n N [--seed S]\n"); + return EXIT_USAGE; +} + +/* Decimal digits only, within [min, max]. */ +static int parse_number(const char *text, uint64_t min, uint64_t max, uint64_t *out) { + uint64_t value = 0; + + if (text == NULL || *text == '\0') + return -1; + for (const char *p = text; *p != '\0'; p++) { + if (*p < '0' || *p > '9') + return -1; + uint64_t digit = (uint64_t)(*p - '0'); + if (value > (max - digit) / 10) + return -1; + value = value * 10 + digit; + } + if (value < min) + return -1; + *out = value; + return 0; +} + +static int parse_args(int argc, char **argv, t_args *args) { + memset(args, 0, sizeof *args); + for (int i = 0; i < argc; i += 2) { + const char **slot = NULL; + if (strcmp(argv[i], "--algorithm") == 0) + slot = &args->algorithm; + else if (strcmp(argv[i], "--order") == 0) + slot = &args->order; + else if (strcmp(argv[i], "--n") == 0) + slot = &args->n; + else if (strcmp(argv[i], "--seed") == 0) + slot = &args->seed; + else if (strcmp(argv[i], "--reps") == 0) + slot = &args->reps; + else if (strcmp(argv[i], "--count") == 0) + slot = &args->count; + if (slot == NULL) + return usage("unknown argument ", argv[i]); + if (i + 1 >= argc) + return usage("missing value for ", argv[i]); + if (*slot != NULL) + return usage("repeated argument ", argv[i]); + *slot = argv[i + 1]; + } + return 0; +} + +static uint64_t now_ns(void) { + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec; +} + +static int parse_seed(const t_args *args, uint32_t *seed) { + uint64_t value = INPUT_DEFAULT_SEED; + + if (args->seed != NULL && parse_number(args->seed, 1, UINT32_MAX, &value) != 0) + return usage("--seed must be a whole number from 1 to 4294967295, got ", args->seed); + *seed = (uint32_t)value; + return 0; +} + +static int parse_input_args(const t_args *args, t_order *order, size_t *n, uint32_t *seed) { + uint64_t value; + + if (args->order == NULL || order_from_name(args->order, order) != 0) + return usage("--order must be random, sorted, reversed or nearly_sorted, got ", + args->order ? args->order : "nothing"); + if (parse_number(args->n, 1, INPUT_MAX_N, &value) != 0) + return usage("--n must be a whole number from 1 to 100000, got ", args->n ? args->n : "nothing"); + *n = (size_t)value; + return parse_seed(args, seed); +} + +static int command_sequence(const t_args *args) { + uint64_t count; + uint32_t seed; + + if (args->algorithm || args->order || args->n || args->reps) + return usage("sequence takes only --count and --seed", ""); + if (parse_number(args->count, 1, MAX_COUNT, &count) != 0) + return usage("--count must be a whole number from 1 to 1000000, got ", args->count ? args->count : "nothing"); + if (parse_seed(args, &seed) != 0) + return EXIT_USAGE; + uint32_t state = seed; + for (uint64_t i = 0; i < count; i++) + printf("%" PRIu32 "\n", xorshift32_next(&state)); + return 0; +} + +static int command_input(const t_args *args) { + t_order order; + size_t n; + uint32_t seed; + + if (args->algorithm || args->reps || args->count) + return usage("input takes only --order, --n and --seed", ""); + if (parse_input_args(args, &order, &n, &seed) != 0) + return EXIT_USAGE; + int *arr = malloc(n * sizeof *arr); + if (arr == NULL) + return fprintf(stderr, "greenbench-worker: out of memory\n"), 1; + input_fill(arr, n, order, seed); + for (size_t i = 0; i < n; i++) + printf("%d\n", arr[i]); + free(arr); + return 0; +} + +static int command_run(const t_args *args) { + t_order order; + size_t n; + uint32_t seed; + uint64_t reps = 1; + t_sort_fn sort = NULL; + int is_none; + + if (args->count) + return usage("run does not take --count", ""); + if (args->algorithm == NULL) + return usage("--algorithm must be bubble, insertion, quick or none, got ", "nothing"); + is_none = strcmp(args->algorithm, "none") == 0; + if (!is_none && (sort = sort_by_name(args->algorithm)) == NULL) + return usage("--algorithm must be bubble, insertion, quick or none, got ", args->algorithm); + if (parse_input_args(args, &order, &n, &seed) != 0) + return EXIT_USAGE; + if (args->reps != NULL && parse_number(args->reps, is_none ? 0 : 1, MAX_REPS, &reps) != 0) + return usage(is_none ? "--reps must be a whole number from 0 to 100000000, got " + : "--reps must be a whole number from 1 to 100000000, got ", + args->reps); + + int *input = malloc(n * sizeof *input); + int *work = malloc(n * sizeof *work); + if (input == NULL || work == NULL) { + free(input); + free(work); + fprintf(stderr, "greenbench-worker: out of memory\n"); + return 1; + } + input_fill(input, n, order, seed); + + t_metrics first = {0, 0}; + int consistent = 1; + uint64_t start = now_ns(); + for (uint64_t r = 0; r < reps; r++) { + t_metrics metrics = {0, 0}; + memcpy(work, input, n * sizeof *work); + if (sort != NULL) + sort(work, n, &metrics); + if (r == 0) + first = metrics; + else if (metrics.comparisons != first.comparisons || metrics.writes != first.writes) + consistent = 0; + } + uint64_t loop_ns = now_ns() - start; + + int status = 0; + if (!consistent) { + fprintf(stderr, "greenbench-worker: operation counts changed between repetitions\n"); + status = 1; + } + /* The input is a permutation of 0..n-1, so a correct sort leaves work[i] == i. */ + for (size_t i = 0; sort != NULL && reps > 0 && status == 0 && i < n; i++) { + if (work[i] != (int)i) { + fprintf(stderr, "greenbench-worker: %s sort left the %s input of %zu elements unsorted\n", + args->algorithm, args->order, n); + status = 1; + } + } + if (status == 0) { + t_worker_result result; + char line[512]; + memset(&result, 0, sizeof result); + snprintf(result.language, sizeof result.language, "c"); + snprintf(result.algorithm, sizeof result.algorithm, "%s", args->algorithm); + snprintf(result.order, sizeof result.order, "%s", order_name(order)); + result.n = n; + result.seed = seed; + result.reps = reps; + result.comparisons = first.comparisons; + result.writes = first.writes; + result.loop_ns = loop_ns; + result.input_hash = input_hash(input, n); + if (protocol_format(line, sizeof line, &result) < 0) { + fprintf(stderr, "greenbench-worker: result line too long\n"); + status = 1; + } else { + fputs(line, stdout); + } + } + free(input); + free(work); + return status; +} + +int main(int argc, char **argv) { + t_args args; + + if (argc < 2) + return usage("missing command", ""); + if (parse_args(argc - 2, argv + 2, &args) != 0) + return EXIT_USAGE; + if (strcmp(argv[1], "run") == 0) + return command_run(&args); + if (strcmp(argv[1], "sequence") == 0) + return command_sequence(&args); + if (strcmp(argv[1], "input") == 0) + return command_input(&args); + return usage("unknown command ", argv[1]); +} diff --git a/tests/test_protocol.c b/tests/test_protocol.c new file mode 100644 index 0000000..7c5537d --- /dev/null +++ b/tests/test_protocol.c @@ -0,0 +1,73 @@ +#include + +#include "check.h" +#include "protocol.h" + +static const char *const GOOD = "result language=python algorithm=quick order=nearly_sorted n=1000 seed=2463534242 " + "reps=40 comparisons=11039 writes=11846 loop_ns=5123400 input_hash=0badf00d\n"; + +static void test_round_trip(void) { + t_worker_result r, back; + char line[512]; + char err[256]; + + memset(&r, 0, sizeof r); + strcpy(r.language, "c"); + strcpy(r.algorithm, "none"); + strcpy(r.order, "random"); + r.n = 64; + r.seed = 1; + r.reps = 0; + r.loop_ns = 17; + r.input_hash = 0x0000abcdu; + CHECK(protocol_format(line, sizeof line, &r) > 0, "format"); + CHECK(strstr(line, "input_hash=0000abcd\n") != NULL, "hash is eight hex digits, then a newline"); + CHECK(protocol_parse(line, &back, err, sizeof err) == 0, "parse what was formatted"); + CHECK(memcmp(&r, &back, sizeof r) == 0, "round trip preserves every field"); + CHECK(protocol_format(line, 10, &r) == -1, "too small a buffer is reported"); +} + +static void test_parse_good(void) { + t_worker_result r; + char err[256]; + + CHECK(protocol_parse(GOOD, &r, err, sizeof err) == 0, "a worker line parses"); + CHECK(strcmp(r.language, "python") == 0 && strcmp(r.algorithm, "quick") == 0 && + strcmp(r.order, "nearly_sorted") == 0, + "names"); + CHECK(r.n == 1000 && r.seed == 2463534242u && r.reps == 40 && r.comparisons == 11039 && r.writes == 11846 && + r.loop_ns == 5123400, + "numbers"); + CHECK(r.input_hash == 0x0badf00du, "hash"); +} + +static void expect_error(const char *line, const char *needle, const char *name) { + t_worker_result r; + char err[256] = ""; + + CHECK(protocol_parse(line, &r, err, sizeof err) == -1, name); + CHECK(strstr(err, needle) != NULL, name); +} + +static void test_parse_bad(void) { + expect_error("", "empty", "empty line"); + expect_error("Traceback (most recent call last):", "starting with 'result'", "not a result line"); + expect_error("result language=c algorithm=quick order=random n=1 seed=1 reps=1 comparisons=0 writes=0 loop_ns=1", + "missing field 'input_hash'", "missing field"); + expect_error("result language=c language=c", "appears twice", "duplicate field"); + expect_error("result colour=blue", "unknown field 'colour'", "unknown field"); + expect_error("result n", "has no value", "no value"); + expect_error("result n=-1", "invalid value", "negative number"); + expect_error("result n=12x", "invalid value", "junk after a number"); + expect_error("result input_hash=123", "invalid value", "short hash"); + expect_error("result input_hash=0BADF00D", "invalid value", "upper-case hash"); + expect_error("result language=C++", "invalid value", "invalid name"); + expect_error("result loop_ns=99999999999999999999999", "invalid value", "overflow"); +} + +int main(void) { + test_round_trip(); + test_parse_good(); + test_parse_bad(); + return check_report("protocol"); +} From e57293e2f3b1fa888209555f99680e71b3242d4d Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:07 +0400 Subject: [PATCH 05/51] feat: add the measurement harness greenbench run starts one worker process per measured run and reads RAPL and CLOCK_MONOTONIC_RAW just before and after it. It calibrates sorts per process to a minimum loop time, measures an idle window of the same length after each run, shuffles every run into one seeded random order, can pin to a CPU, checks that all languages report identical inputs and counts, and writes a tidy runs.csv with meta.json. greenbench probe checks the counters. Co-Authored-By: Claude Opus 5.5 --- Makefile | 94 ++++- main.c | 151 ------- src/catalog.c | 34 ++ src/catalog.h | 20 + src/greenbench.c | 972 +++++++++++++++++++++++++++++++++++++++++++ src/options.c | 439 +++++++++++++++++++ src/options.h | 65 +++ src/output.c | 81 ++++ src/output.h | 41 ++ src/plan.c | 69 +++ src/plan.h | 45 ++ src/spawn.c | 133 ++++++ src/spawn.h | 26 ++ src/timing.c | 26 ++ src/timing.h | 13 + tests/test_options.c | 139 +++++++ tests/test_output.c | 112 +++++ tests/test_plan.c | 115 +++++ tests/test_spawn.c | 65 +++ 19 files changed, 2467 insertions(+), 173 deletions(-) delete mode 100644 main.c create mode 100644 src/catalog.c create mode 100644 src/catalog.h create mode 100644 src/greenbench.c create mode 100644 src/options.c create mode 100644 src/options.h create mode 100644 src/output.c create mode 100644 src/output.h create mode 100644 src/plan.c create mode 100644 src/plan.h create mode 100644 src/spawn.c create mode 100644 src/spawn.h create mode 100644 src/timing.c create mode 100644 src/timing.h create mode 100644 tests/test_options.c create mode 100644 tests/test_output.c create mode 100644 tests/test_plan.c create mode 100644 tests/test_spawn.c diff --git a/Makefile b/Makefile index d1dec68..73faa14 100644 --- a/Makefile +++ b/Makefile @@ -1,40 +1,90 @@ CC ?= cc CFLAGS ?= -O2 +LDFLAGS ?= WARNINGS := -Wall -Wextra -Wpedantic -Wshadow -Wconversion -Werror ALL_CFLAGS = -std=c11 $(WARNINGS) $(CFLAGS) - -NAME := energy_explorer -TEST_BIN := tests/test_sorts +BUILD ?= build +# Use the project's virtual environment when there is one (see README), else python3. +PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) +NPM ?= npm SANITIZE := -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer -.PHONY: all test sanitize results clean fclean re +# Runs the analysis package from analysis/ without installing it. +PYRUN := PYTHONPATH=analysis $(PYTHON) + +LIB := sorts inputs rapl catalog options plan protocol output spawn timing +LIB_OBJ := $(LIB:%=$(BUILD)/obj/%.o) +HEADERS := $(wildcard src/*.h) +HARNESS := $(BUILD)/greenbench +WORKER := $(BUILD)/greenbench-worker +TESTS := test_sorts test_inputs test_rapl test_options test_plan test_protocol test_output test_spawn +TEST_BIN := $(TESTS:%=$(BUILD)/tests/%) -all: $(NAME) +.PHONY: all test sanitize smoke parity integration check-python check-node lint-shell check clean study -$(NAME): main.o sorts.o - $(CC) $(ALL_CFLAGS) $(LDFLAGS) -o $@ main.o sorts.o +all: $(HARNESS) $(WORKER) -%.o: %.c sorts.h +$(BUILD)/obj $(BUILD)/tests: + mkdir -p $@ + +$(BUILD)/obj/%.o: src/%.c $(HEADERS) | $(BUILD)/obj $(CC) $(ALL_CFLAGS) -c -o $@ $< -$(TEST_BIN): tests/test_sorts.c sorts.c sorts.h - $(CC) $(ALL_CFLAGS) -I. $(LDFLAGS) -o $@ tests/test_sorts.c sorts.c +$(BUILD)/obj/fixture.o: tests/fixture.c tests/fixture.h | $(BUILD)/obj + $(CC) $(ALL_CFLAGS) -c -o $@ $< + +$(HARNESS): $(BUILD)/obj/greenbench.o $(LIB_OBJ) + $(CC) $(ALL_CFLAGS) $(LDFLAGS) -o $@ $^ + +$(WORKER): $(BUILD)/obj/worker.o $(BUILD)/obj/sorts.o $(BUILD)/obj/inputs.o $(BUILD)/obj/protocol.o + $(CC) $(ALL_CFLAGS) $(LDFLAGS) -o $@ $^ + +$(BUILD)/tests/%: tests/%.c tests/check.h $(LIB_OBJ) $(BUILD)/obj/fixture.o $(HEADERS) | $(BUILD)/tests + $(CC) $(ALL_CFLAGS) -Isrc -Itests $(LDFLAGS) -o $@ $< $(LIB_OBJ) $(BUILD)/obj/fixture.o +# C unit tests. Run from the repository root: some read tests/fixtures. test: $(TEST_BIN) - ./$(TEST_BIN) + @for t in $(TEST_BIN); do ./$$t || exit 1; done -sanitize: fclean - $(MAKE) test all CFLAGS="$(SANITIZE)" LDFLAGS="$(SANITIZE)" - ./$(NAME) 100 300 > /dev/null +# Unit tests and a short C-only harness run under AddressSanitizer and UBSan. The harness +# reads a fake powercap tree, so the energy and idle-baseline code runs too. +sanitize: + $(MAKE) BUILD=build/sanitize CFLAGS="$(SANITIZE)" LDFLAGS="$(SANITIZE)" all test + $(PYTHON) tests/integration/powercap.py tests/fixtures/powercap/server-two-socket.txt build/sanitize/powercap + build/sanitize/greenbench run --profile smoke --languages c --quiet --energy require \ + --powercap-root build/sanitize/powercap --out build/sanitize/smoke -results: $(NAME) - ./$(NAME) > data.csv - python3 visualize.py data.csv +# A short run of every language with energy off, then the report. Works anywhere. +smoke: all + $(HARNESS) run --profile smoke --energy off --out results/smoke + $(PYRUN) -m greenbench_analysis report --runs results/smoke/runs.csv --out results/smoke/results.md \ + --img-dir results/smoke/img --withhold-timings -clean: - rm -f *.o $(TEST_BIN) +# The three workers must generate identical sequences and inputs and do identical work. +parity: all + $(PYRUN) -m pytest -q tests/integration/test_parity.py + +# End-to-end tests of the harness binary, the workers and the report. +integration: all + $(PYRUN) -m pytest -q tests/integration + +check-python: + $(PYRUN) -m ruff check . + $(PYRUN) -m ruff format --check . + $(PYRUN) -m mypy + $(PYRUN) -m pytest -q analysis/tests workers/python/tests + +check-node: + cd workers/node && $(NPM) ci --no-audit --no-fund && $(NPM) run lint && $(NPM) run typecheck && $(NPM) test -fclean: clean - rm -f $(NAME) data.csv comparisons.png time.png orders.png +lint-shell: + shellcheck scripts/*.sh -re: fclean all +check: all test check-python check-node integration + +# The full study on bare-metal Linux; see docs/bare-metal.md. +study: + ./scripts/run-study.sh + +clean: + rm -rf build results/smoke diff --git a/main.c b/main.c deleted file mode 100644 index deeff60..0000000 --- a/main.c +++ /dev/null @@ -1,151 +0,0 @@ -#define _POSIX_C_SOURCE 200809L /* clock_gettime */ - -#include -#include -#include -#include -#include - -#include "sorts.h" - -#define RUNS 5 /* timed runs per input; the median is reported */ -#define MAX_SIZES 16 -#define MAX_N 100000 - -typedef struct s_algorithm { - const char *name; - t_sort_fn sort; -} t_algorithm; - -static const t_algorithm ALGORITHMS[] = { - {"bubble", bubble_sort}, - {"insertion", insertion_sort}, - {"quick", quick_sort}, -}; - -static const char *const ORDERS[] = {"random", "sorted", "reversed", "nearly_sorted"}; - -/* xorshift32: the same sequence on every platform, so every run sees the same inputs. */ -static unsigned int next_random(unsigned int *state) { - unsigned int x = *state; - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - *state = x; - return x; -} - -static void fill(int *arr, size_t n, const char *order, unsigned int *state) { - for (size_t i = 0; i < n; i++) - arr[i] = strcmp(order, "reversed") == 0 ? (int)(n - 1 - i) : (int)i; - if (strcmp(order, "random") == 0) { - for (size_t i = n; i > 1; i--) { /* Fisher-Yates shuffle */ - size_t j = next_random(state) % i; - int tmp = arr[i - 1]; - arr[i - 1] = arr[j]; - arr[j] = tmp; - } - } else if (strcmp(order, "nearly_sorted") == 0) { - for (size_t k = 0; k < n / 20; k++) { /* swap 5% of the elements with random partners */ - size_t a = next_random(state) % n; - size_t b = next_random(state) % n; - int tmp = arr[a]; - arr[a] = arr[b]; - arr[b] = tmp; - } - } -} - -static int is_sorted(const int *arr, size_t n) { - for (size_t i = 1; i < n; i++) - if (arr[i - 1] > arr[i]) - return 0; - return 1; -} - -static double seconds_now(void) { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; -} - -static int compare_doubles(const void *a, const void *b) { - double x = *(const double *)a; - double y = *(const double *)b; - return (x > y) - (x < y); -} - -static int parse_sizes(int argc, char **argv, size_t *sizes, size_t *count) { - static const size_t defaults[] = {500, 1000, 2000, 4000}; - - if (argc < 2) { - *count = sizeof defaults / sizeof *defaults; - memcpy(sizes, defaults, sizeof defaults); - return 0; - } - if (argc - 1 > MAX_SIZES) - return -1; - for (int i = 1; i < argc; i++) { - char *end; - errno = 0; - unsigned long value = strtoul(argv[i], &end, 10); - if (end == argv[i] || *end != '\0' || errno != 0 || value < 1 || value > MAX_N) - return -1; - sizes[i - 1] = (size_t)value; - } - *count = (size_t)(argc - 1); - return 0; -} - -int main(int argc, char **argv) { - size_t sizes[MAX_SIZES]; - size_t n_sizes; - - if (parse_sizes(argc, argv, sizes, &n_sizes) != 0) { - fprintf(stderr, "Usage: %s [SIZE...] (up to %d sizes, each 1 to %d; default 500 1000 2000 4000)\n", - argv[0], MAX_SIZES, MAX_N); - return 2; - } - - printf("algorithm,order,n,median_seconds,comparisons,writes\n"); - for (size_t s = 0; s < n_sizes; s++) { - size_t n = sizes[s]; - int *source = malloc(n * sizeof *source); - int *work = malloc(n * sizeof *work); - if (source == NULL || work == NULL) { - fprintf(stderr, "%s: out of memory\n", argv[0]); - free(source); - free(work); - return 1; - } - for (size_t o = 0; o < sizeof ORDERS / sizeof *ORDERS; o++) { - unsigned int state = 2463534242u; /* fixed seed: identical input for every algorithm */ - fill(source, n, ORDERS[o], &state); - for (size_t a = 0; a < sizeof ALGORITHMS / sizeof *ALGORITHMS; a++) { - double times[RUNS]; - t_metrics metrics = {0, 0}; - for (int run = 0; run < RUNS; run++) { - memcpy(work, source, n * sizeof *work); - metrics.comparisons = 0; - metrics.writes = 0; - double start = seconds_now(); - ALGORITHMS[a].sort(work, n, &metrics); - times[run] = seconds_now() - start; - if (!is_sorted(work, n)) { - fprintf(stderr, "%s: %s sort left %s input of %zu unsorted\n", argv[0], - ALGORITHMS[a].name, ORDERS[o], n); - free(source); - free(work); - return 1; - } - } - qsort(times, RUNS, sizeof *times, compare_doubles); - printf("%s,%s,%zu,%.9f,%lu,%lu\n", ALGORITHMS[a].name, ORDERS[o], n, times[RUNS / 2], - metrics.comparisons, metrics.writes); - } - } - free(source); - free(work); - } - return 0; -} diff --git a/src/catalog.c b/src/catalog.c new file mode 100644 index 0000000..05e197a --- /dev/null +++ b/src/catalog.c @@ -0,0 +1,34 @@ +#include "catalog.h" + +#include + +static const char *const LANGUAGE_NAMES[LANG_COUNT] = {"c", "python", "node"}; +static const char *const ALGORITHM_NAMES[ALGO_COUNT] = {"bubble", "insertion", "quick", "none"}; + +const char *language_name(t_language language) { + return (unsigned)language < LANG_COUNT ? LANGUAGE_NAMES[language] : "unknown"; +} + +int language_from_name(const char *name, t_language *language) { + for (int i = 0; i < LANG_COUNT; i++) { + if (strcmp(name, LANGUAGE_NAMES[i]) == 0) { + *language = (t_language)i; + return 0; + } + } + return -1; +} + +const char *algorithm_name(t_algorithm algorithm) { + return (unsigned)algorithm < ALGO_COUNT ? ALGORITHM_NAMES[algorithm] : "unknown"; +} + +int algorithm_from_name(const char *name, t_algorithm *algorithm) { + for (int i = 0; i < ALGO_NONE; i++) { + if (strcmp(name, ALGORITHM_NAMES[i]) == 0) { + *algorithm = (t_algorithm)i; + return 0; + } + } + return -1; +} diff --git a/src/catalog.h b/src/catalog.h new file mode 100644 index 0000000..4e1b79a --- /dev/null +++ b/src/catalog.h @@ -0,0 +1,20 @@ +#ifndef CATALOG_H +#define CATALOG_H + +/* The languages and algorithms the harness knows, and their names on the command line, + in the worker protocol and in the CSV. */ + +typedef enum e_language { LANG_C, LANG_PYTHON, LANG_NODE, LANG_COUNT } t_language; + +/* ALGO_NONE is the startup trial: the worker generates its input and exits without + sorting, which measures what starting the process costs in each language. */ +typedef enum e_algorithm { ALGO_BUBBLE, ALGO_INSERTION, ALGO_QUICK, ALGO_NONE, ALGO_COUNT } t_algorithm; + +const char *language_name(t_language language); +int language_from_name(const char *name, t_language *language); + +const char *algorithm_name(t_algorithm algorithm); +/* Accepts the sorting algorithms only; "none" is internal to the harness. */ +int algorithm_from_name(const char *name, t_algorithm *algorithm); + +#endif diff --git a/src/greenbench.c b/src/greenbench.c new file mode 100644 index 0000000..4eff565 --- /dev/null +++ b/src/greenbench.c @@ -0,0 +1,972 @@ +/* greenbench: runs the sorting workers in C, Python and Node.js on identical inputs, in a + randomised order, and records time, energy and operation counts for every run. + + Each measured run starts one worker process. The harness reads the RAPL counters and + CLOCK_MONOTONIC_RAW just before starting it and just after it exits, so time and energy + cover the same interval. With the idle baseline on, the harness then sleeps for the same + duration and reads the counters again, so the idle energy can be subtracted. */ + +#define _GNU_SOURCE /* sched_setaffinity, CPU_SET */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "catalog.h" +#include "inputs.h" +#include "options.h" +#include "output.h" +#include "plan.h" +#include "protocol.h" +#include "rapl.h" +#include "spawn.h" +#include "timing.h" + +#define GREENBENCH_VERSION "2.0.0" +#define EXIT_USAGE 2 +#define EXIT_ENERGY 3 +#define TEXT_LEN 256 +#define ERROR_LEN 1024 + +#ifdef __clang__ +#define COMPILER_VERSION "clang " __clang_version__ +#elif defined(__GNUC__) +#define COMPILER_VERSION "gcc " __VERSION__ +#else +#define COMPILER_VERSION "unknown" +#endif + +typedef struct s_measurement { + t_worker_result result; + uint64_t wall_ns; + t_rapl_energy energy; +} t_measurement; + +typedef struct s_consistency { + int seen; + uint32_t value; + uint64_t comparisons; + uint64_t writes; + t_language first; +} t_consistency; + +typedef struct s_context { + t_options options; + t_rapl rapl; + int energy_ok; + const char *energy_status; + char energy_reason[RAPL_REASON_LEN]; + int baseline; + char c_worker[PATH_MAX]; + char python_worker[PATH_MAX]; + char node_worker[PATH_MAX]; + char versions[LANG_COUNT][TEXT_LEN]; + t_trial *trials; + size_t trial_count; + t_consistency hashes[ORDER_COUNT][OPTIONS_MAX_SIZES]; + t_consistency counts[ALGO_COUNT][ORDER_COUNT][OPTIONS_MAX_SIZES]; + t_spawn_result spawn; + char started_utc[32]; + char meta_path[PATH_MAX]; + char command[2048]; /* the harness's own command line, for meta.json */ + char csv_path[PATH_MAX]; +} t_context; + +static const char *g_program = "greenbench"; +static volatile sig_atomic_t g_interrupted; + +static void on_interrupt(int signal_number) { + (void)signal_number; + g_interrupted = 1; +} + +/* Ctrl-C or SIGTERM stops the run after the current worker, so meta.json still records how + many runs were written. */ +static void catch_interrupts(void) { + struct sigaction action; + + memset(&action, 0, sizeof action); + action.sa_handler = on_interrupt; + sigemptyset(&action.sa_mask); + sigaction(SIGINT, &action, NULL); + sigaction(SIGTERM, &action, NULL); +} + +static int interrupted(void) { + return g_interrupted != 0; +} + +static void error_message(const char *format, ...) { + va_list args; + + fprintf(stderr, "%s: ", g_program); + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + fputc('\n', stderr); +} + +static void print_usage(FILE *out) { + fprintf(out, + "Usage: greenbench run [OPTIONS] measure the sorting workers\n" + " greenbench probe [OPTIONS] check whether RAPL energy counters can be read\n" + " greenbench --version\n" + "\n" + "Options for run (defaults are the study profile):\n" + " --profile NAME study (30 runs, sizes 250,500,1000,2000) or smoke (3 runs, sizes 64,256)\n" + " --languages LIST c,python,node\n" + " --algorithms LIST bubble,insertion,quick\n" + " --orders LIST random,sorted,reversed,nearly_sorted\n" + " --sizes LIST input sizes, each 1 to %d\n" + " --runs N measured runs per trial (the study uses at least %d)\n" + " --warmup N unmeasured runs per trial before measuring; the first calibrates\n" + " --min-loop-ms N sort for at least N ms per process, well above the ~1 ms RAPL update\n" + " --max-reps N never sort more than N times in one process\n" + " --no-startup skip the startup trials (processes that exit without sorting)\n" + " --seed N input seed, identical for every language (default %u)\n" + " --order-seed N seed for the randomised run order\n" + " --energy MODE auto (measure when possible), require (fail without RAPL) or off\n" + " --idle-baseline MODE auto (on when energy is measured), on or off\n" + " --powercap-root DIR where to look for RAPL zones (default %s)\n" + " --cpu N pin the harness and every worker to CPU N (Linux)\n" + " --out DIR write runs.csv and meta.json here (default results/latest)\n" + " --python CMD Python interpreter (default python3)\n" + " --node CMD Node.js executable (default node)\n" + " --c-worker PATH, --python-worker PATH, --node-worker PATH\n" + " worker locations (default: found from the executable's location)\n" + " --label TEXT free text stored in meta.json\n" + " --dry-run print the plan and exit\n" + " --quiet no progress output\n" + "\n" + "Options for probe:\n" + " --powercap-root DIR as above\n" + " --sample-ms N measure idle power for N ms (default 1000; 0 skips it)\n" + "\n" + "Exit status: 0 success, 1 measurement error, 2 usage error, 3 energy required or unavailable.\n", + INPUT_MAX_N, OPTIONS_MIN_STUDY_RUNS, INPUT_DEFAULT_SEED, RAPL_DEFAULT_ROOT); +} + +/* ---- paths and environment ------------------------------------------------------------ */ + +static int file_exists(const char *path, int mode) { + return access(path, mode) == 0; +} + +static int join_path(char *out, size_t size, const char *dir, const char *rest) { + int written = snprintf(out, size, "%s/%s", dir, rest); + return written < 0 || (size_t)written >= size ? -1 : 0; +} + +/* Directory holding the running executable, from /proc/self/exe or argv[0]. */ +static void executable_dir(const char *argv0, char *out, size_t size) { + char path[PATH_MAX]; + ssize_t n = readlink("/proc/self/exe", path, sizeof path - 1); + + if (n > 0) + path[n] = '\0'; + else + snprintf(path, sizeof path, "%s", argv0); + char *slash = strrchr(path, '/'); + if (slash == NULL) + snprintf(out, size, "."); + else if (slash == path) + snprintf(out, size, "/"); + else { + *slash = '\0'; + snprintf(out, size, "%s", path); + } +} + +/* Walks up from the executable's directory to the repository holding the workers. */ +static int find_repository(const char *start, char *out, size_t size) { + char dir[PATH_MAX]; + char probe[PATH_MAX]; + + snprintf(dir, sizeof dir, "%s", start); + for (int level = 0; level < 5; level++) { + if (join_path(probe, sizeof probe, dir, "workers/python/greenbench_worker.py") == 0 && + file_exists(probe, R_OK)) { + char resolved[PATH_MAX]; + snprintf(out, size, "%s", realpath(dir, resolved) != NULL ? resolved : dir); + return 0; + } + size_t length = strlen(dir); + if (length + 3 >= sizeof dir) + break; + memcpy(dir + length, "/..", 4); + } + return -1; +} + +static int resolve_workers(t_context *ctx, const char *argv0) { + t_options *o = &ctx->options; + char exe_dir[PATH_MAX]; + char repo[PATH_MAX]; + int have_repo; + + executable_dir(argv0, exe_dir, sizeof exe_dir); + have_repo = find_repository(exe_dir, repo, sizeof repo) == 0; + + if (o->c_worker != NULL) + snprintf(ctx->c_worker, sizeof ctx->c_worker, "%s", o->c_worker); + else if (join_path(ctx->c_worker, sizeof ctx->c_worker, exe_dir, "greenbench-worker") != 0) + return error_message("path too long: %s", exe_dir), -1; + if (o->python_worker != NULL) + snprintf(ctx->python_worker, sizeof ctx->python_worker, "%s", o->python_worker); + else if (have_repo) + join_path(ctx->python_worker, sizeof ctx->python_worker, repo, "workers/python/greenbench_worker.py"); + if (o->node_worker != NULL) + snprintf(ctx->node_worker, sizeof ctx->node_worker, "%s", o->node_worker); + else if (have_repo) + join_path(ctx->node_worker, sizeof ctx->node_worker, repo, "workers/node/worker.js"); + + for (size_t i = 0; i < o->language_count; i++) { + switch (o->languages[i]) { + case LANG_C: + if (!file_exists(ctx->c_worker, X_OK)) + return error_message("C worker not found at %s; run 'make' first or pass --c-worker", ctx->c_worker), -1; + break; + case LANG_PYTHON: + if (ctx->python_worker[0] == '\0' || !file_exists(ctx->python_worker, R_OK)) + return error_message("Python worker not found%s%s; pass --python-worker", + ctx->python_worker[0] ? " at " : "", ctx->python_worker), + -1; + break; + case LANG_NODE: + if (ctx->node_worker[0] == '\0' || !file_exists(ctx->node_worker, R_OK)) + return error_message("Node.js worker not found%s%s; pass --node-worker", + ctx->node_worker[0] ? " at " : "", ctx->node_worker), + -1; + break; + default: + break; + } + } + return 0; +} + +/* First line of a command's output, for recording interpreter versions. */ +static int command_version(t_context *ctx, const char *command, char *out, size_t size) { + char *argv[] = {(char *)command, "--version", NULL}; + + if (spawn_capture(argv, &ctx->spawn) != 0) { + spawn_describe_failure(&ctx->spawn, out, size); + return -1; + } + const char *text = ctx->spawn.out[0] != '\0' ? ctx->spawn.out : ctx->spawn.err; + size_t length = strcspn(text, "\r\n"); + snprintf(out, size, "%.*s", (int)length, text); + return 0; +} + +static int record_versions(t_context *ctx) { + t_options *o = &ctx->options; + + for (size_t i = 0; i < o->language_count; i++) { + t_language language = o->languages[i]; + if (language == LANG_C) { + snprintf(ctx->versions[language], TEXT_LEN, "%s", COMPILER_VERSION); + continue; + } + const char *command = language == LANG_PYTHON ? o->python : o->node; + if (command_version(ctx, command, ctx->versions[language], TEXT_LEN) != 0) { + error_message("cannot run '%s --version' (%s); install it, pass --%s, or leave %s out of --languages", + command, ctx->versions[language], language_name(language), language_name(language)); + return -1; + } + } + return 0; +} + +static void utc_now(char *out, size_t size) { + time_t now = time(NULL); + struct tm tm; + + gmtime_r(&now, &tm); + strftime(out, size, "%Y-%m-%dT%H:%M:%SZ", &tm); +} + +static int make_dirs(const char *path) { + char buf[PATH_MAX]; + + if (snprintf(buf, sizeof buf, "%s", path) >= (int)sizeof buf) + return -1; + for (char *p = buf + 1; *p != '\0'; p++) { + if (*p != '/') + continue; + *p = '\0'; + if (mkdir(buf, 0777) != 0 && errno != EEXIST) + return -1; + *p = '/'; + } + if (mkdir(buf, 0777) != 0 && errno != EEXIST) + return -1; + return 0; +} + +static void read_first_line(const char *path, char *out, size_t size) { + FILE *f = fopen(path, "r"); + + out[0] = '\0'; + if (f == NULL) + return; + if (fgets(out, (int)size, f) != NULL) + out[strcspn(out, "\r\n")] = '\0'; + fclose(f); +} + +static void cpu_model(char *out, size_t size) { + char line[512]; + FILE *f = fopen("/proc/cpuinfo", "r"); + + out[0] = '\0'; + if (f == NULL) + return; + while (fgets(line, sizeof line, f) != NULL) { + if (strncmp(line, "model name", 10) == 0) { + char *colon = strchr(line, ':'); + if (colon != NULL) { + colon++; + while (*colon == ' ' || *colon == '\t') + colon++; + colon[strcspn(colon, "\r\n")] = '\0'; + snprintf(out, size, "%s", colon); + } + break; + } + } + fclose(f); +} + +/* ---- metadata ------------------------------------------------------------------------- */ + +static void json_key_string(FILE *f, const char *indent, const char *key, const char *value, int comma) { + fprintf(f, "%s\"%s\": ", indent, key); + if (value == NULL || value[0] == '\0') + fputs("null", f); + else + output_json_string(f, value); + fputs(comma ? ",\n" : "\n", f); +} + +static void json_list(FILE *f, const char *key, const char *const *names, size_t count) { + fprintf(f, " \"%s\": [", key); + for (size_t i = 0; i < count; i++) + fprintf(f, "%s\"%s\"", i ? ", " : "", names[i]); + fputs("],\n", f); +} + +static int write_meta(const t_context *ctx, const char *status, uint64_t measured_runs) { + const t_options *o = &ctx->options; + char tmp_path[PATH_MAX + 8]; + char finished[32] = ""; + char text[TEXT_LEN]; + struct utsname uts; + const char *names[OPTIONS_MAX_SIZES]; + + snprintf(tmp_path, sizeof tmp_path, "%s.tmp", ctx->meta_path); + FILE *f = fopen(tmp_path, "w"); + if (f == NULL) + return error_message("cannot write %s: %s", tmp_path, strerror(errno)), -1; + if (strcmp(status, "complete") == 0) + utc_now(finished, sizeof finished); + + fputs("{\n", f); + fprintf(f, " \"tool\": \"greenbench\",\n \"version\": \"%s\",\n \"csv_schema\": %d,\n", GREENBENCH_VERSION, + OUTPUT_CSV_SCHEMA); + json_key_string(f, " ", "status", status, 1); + json_key_string(f, " ", "started_utc", ctx->started_utc, 1); + json_key_string(f, " ", "finished_utc", finished, 1); + json_key_string(f, " ", "label", o->label, 1); + json_key_string(f, " ", "command", ctx->command, 1); + fprintf(f, " \"measured_runs\": %" PRIu64 ",\n", measured_runs); + fputs(" \"clock\": \"CLOCK_MONOTONIC_RAW\",\n", f); + + fputs(" \"host\": {\n", f); + if (uname(&uts) == 0) { + json_key_string(f, " ", "sysname", uts.sysname, 1); + json_key_string(f, " ", "release", uts.release, 1); + json_key_string(f, " ", "machine", uts.machine, 1); + } + cpu_model(text, sizeof text); + json_key_string(f, " ", "cpu_model", text, 1); + fprintf(f, " \"online_cpus\": %ld,\n", sysconf(_SC_NPROCESSORS_ONLN)); + char governor_path[128]; + snprintf(governor_path, sizeof governor_path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", + o->cpu >= 0 ? o->cpu : 0); + read_first_line(governor_path, text, sizeof text); + json_key_string(f, " ", "scaling_governor", text, 1); + read_first_line("/sys/devices/system/cpu/intel_pstate/no_turbo", text, sizeof text); + json_key_string(f, " ", "intel_pstate_no_turbo", text, 1); + read_first_line("/sys/devices/system/cpu/cpufreq/boost", text, sizeof text); + json_key_string(f, " ", "cpufreq_boost", text, 0); + fputs(" },\n", f); + + fputs(" \"energy\": {\n", f); + json_key_string(f, " ", "status", ctx->energy_status, 1); + json_key_string(f, " ", "reason", ctx->energy_reason, 1); + json_key_string(f, " ", "powercap_root", o->powercap_root, 1); + fprintf(f, " \"idle_baseline\": %s,\n", ctx->baseline ? "true" : "false"); + fputs(" \"domains\": [", f); + for (size_t i = 0; ctx->energy_ok && i < ctx->rapl.count; i++) { + const t_rapl_domain *d = &ctx->rapl.domains[i]; + fprintf(f, "%s\n {\"zone\": ", i ? "," : ""); + output_json_string(f, d->zone); + fputs(", \"name\": ", f); + output_json_string(f, d->name); + fprintf(f, ", \"kind\": \"%s\", \"max_energy_range_uj\": %" PRIu64 "}", + d->kind == RAPL_PACKAGE ? "package" : "dram", d->max_range_uj); + } + fputs(ctx->energy_ok && ctx->rapl.count > 0 ? "\n ]\n" : "]\n", f); + fputs(" },\n", f); + + fputs(" \"settings\": {\n", f); + json_key_string(f, " ", "profile", o->profile, 1); + for (size_t i = 0; i < o->language_count; i++) + names[i] = language_name(o->languages[i]); + json_list(f, "languages", names, o->language_count); + for (size_t i = 0; i < o->algorithm_count; i++) + names[i] = algorithm_name(o->algorithms[i]); + json_list(f, "algorithms", names, o->algorithm_count); + for (size_t i = 0; i < o->order_count; i++) + names[i] = order_name(o->orders[i]); + json_list(f, "orders", names, o->order_count); + fputs(" \"sizes\": [", f); + for (size_t i = 0; i < o->size_count; i++) + fprintf(f, "%s%zu", i ? ", " : "", o->sizes[i]); + fputs("],\n", f); + fprintf(f, + " \"runs\": %u,\n \"warmup\": %u,\n \"seed\": %" PRIu32 ",\n \"order_seed\": %" PRIu32 + ",\n \"min_loop_ns\": %" PRIu64 ",\n \"max_reps\": %" PRIu64 ",\n \"startup_trials\": %s,\n" + " \"cpu\": %d\n", + o->runs, o->warmup, o->seed, o->order_seed, o->min_loop_ns, o->max_reps, + o->startup_trials ? "true" : "false", o->cpu); + fputs(" },\n", f); + + fputs(" \"workers\": {\n", f); + for (size_t i = 0; i < o->language_count; i++) { + t_language language = o->languages[i]; + const char *path = language == LANG_C ? ctx->c_worker + : language == LANG_PYTHON ? ctx->python_worker + : ctx->node_worker; + fprintf(f, " \"%s\": {\"path\": ", language_name(language)); + output_json_string(f, path); + fputs(", \"version\": ", f); + output_json_string(f, ctx->versions[language]); + fprintf(f, "}%s\n", i + 1 < o->language_count ? "," : ""); + } + fputs(" },\n", f); + + fputs(" \"calibration\": [", f); + int first = 1; + for (size_t i = 0; i < ctx->trial_count && strcmp(status, "planned") != 0; i++) { + const t_trial *t = &ctx->trials[i]; + fprintf(f, "%s\n {\"language\": \"%s\", \"algorithm\": \"%s\", \"order\": \"%s\", \"n\": %zu, " + "\"inner_reps\": %" PRIu64 "}", + first ? "" : ",", language_name(t->language), algorithm_name(t->algorithm), order_name(t->order), + t->n, t->inner_reps); + first = 0; + } + fputs(first ? "]\n" : "\n ]\n", f); + fputs("}\n", f); + + if (fclose(f) != 0 || rename(tmp_path, ctx->meta_path) != 0) + return error_message("cannot write %s: %s", ctx->meta_path, strerror(errno)), -1; + return 0; +} + +/* ---- running workers ------------------------------------------------------------------ */ + +static int size_index(const t_options *o, size_t n) { + for (size_t i = 0; i < o->size_count; i++) + if (o->sizes[i] == n) + return (int)i; + return -1; +} + +static void describe_trial(const t_trial *t, char *out, size_t size) { + snprintf(out, size, "%s %s %s n=%zu", language_name(t->language), algorithm_name(t->algorithm), + order_name(t->order), t->n); +} + +static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measurement *m) { + const t_options *o = &ctx->options; + char n_text[24], seed_text[24], reps_text[24], what[TEXT_LEN], err[ERROR_LEN]; + char *argv[16]; + int k = 0; + t_rapl_sample before, after; + + snprintf(n_text, sizeof n_text, "%zu", t->n); + snprintf(seed_text, sizeof seed_text, "%" PRIu32, o->seed); + snprintf(reps_text, sizeof reps_text, "%" PRIu64, reps); + switch (t->language) { + case LANG_C: + argv[k++] = ctx->c_worker; + break; + case LANG_PYTHON: + argv[k++] = (char *)o->python; + argv[k++] = ctx->python_worker; + break; + default: + argv[k++] = (char *)o->node; + argv[k++] = ctx->node_worker; + break; + } + argv[k++] = "run"; + argv[k++] = "--algorithm"; + argv[k++] = (char *)algorithm_name(t->algorithm); + argv[k++] = "--order"; + argv[k++] = (char *)order_name(t->order); + argv[k++] = "--n"; + argv[k++] = n_text; + argv[k++] = "--seed"; + argv[k++] = seed_text; + argv[k++] = "--reps"; + argv[k++] = reps_text; + argv[k] = NULL; + + describe_trial(t, what, sizeof what); + if (ctx->energy_ok && rapl_read(&ctx->rapl, &before) != 0) + return error_message("reading the RAPL counters failed: %s", strerror(errno)), -1; + uint64_t start = timing_now_ns(); + int rc = spawn_capture(argv, &ctx->spawn); + uint64_t end = timing_now_ns(); + if (ctx->energy_ok && rapl_read(&ctx->rapl, &after) != 0) + return error_message("reading the RAPL counters failed: %s", strerror(errno)), -1; + + if (rc != 0 && g_interrupted) + return -1; /* the worker got the same Ctrl-C; command_run reports it */ + if (rc != 0) { + spawn_describe_failure(&ctx->spawn, err, sizeof err); + return error_message("worker for %s (%s): %s", what, argv[0], err), -1; + } + if (protocol_parse(ctx->spawn.out, &m->result, err, sizeof err) != 0) + return error_message("worker for %s printed an unexpected result: %s", what, err), -1; + const t_worker_result *r = &m->result; + if (strcmp(r->language, language_name(t->language)) != 0 || + strcmp(r->algorithm, algorithm_name(t->algorithm)) != 0 || strcmp(r->order, order_name(t->order)) != 0 || + r->n != t->n || r->seed != o->seed || r->reps != reps) + return error_message("worker for %s reported a different configuration: %s", what, ctx->spawn.out), -1; + m->wall_ns = end - start; + m->energy.package_uj = 0; + m->energy.dram_uj = 0; + if (ctx->energy_ok && rapl_energy_between(&ctx->rapl, &before, &after, &m->energy) != 0) + return error_message("a RAPL counter reading was above its max_energy_range_uj"), -1; + return 0; +} + +/* Every language must see the same input and do the same work on it. */ +static int check_consistency(t_context *ctx, const t_trial *t, const t_worker_result *r) { + int s = size_index(&ctx->options, t->n); + t_consistency *hash = &ctx->hashes[t->order][s]; + char what[TEXT_LEN]; + + describe_trial(t, what, sizeof what); + if (!hash->seen) { + hash->seen = 1; + hash->value = r->input_hash; + hash->first = t->language; + } else if (hash->value != r->input_hash) { + return error_message("%s: input hash %08" PRIx32 " differs from %08" PRIx32 " reported by the %s worker; " + "the input generators disagree", + what, r->input_hash, hash->value, language_name(hash->first)), + -1; + } + if (t->algorithm == ALGO_NONE) + return 0; + t_consistency *count = &ctx->counts[t->algorithm][t->order][s]; + if (!count->seen) { + count->seen = 1; + count->comparisons = r->comparisons; + count->writes = r->writes; + count->first = t->language; + } else if (count->comparisons != r->comparisons || count->writes != r->writes) { + return error_message("%s: %" PRIu64 " comparisons and %" PRIu64 " writes, but the %s worker reported %" PRIu64 + " and %" PRIu64 "; the implementations disagree", + what, r->comparisons, r->writes, language_name(count->first), count->comparisons, + count->writes), + -1; + } + return 0; +} + +static void format_duration(uint64_t ns, char *out, size_t size) { + uint64_t seconds = ns / 1000000000u; + + if (seconds >= 3600) + snprintf(out, size, "%" PRIu64 " h %" PRIu64 " min", seconds / 3600, seconds % 3600 / 60); + else if (seconds >= 60) + snprintf(out, size, "%" PRIu64 " min %" PRIu64 " s", seconds / 60, seconds % 60); + else + snprintf(out, size, "%" PRIu64 " s", seconds); +} + +/* Progress on standard error: rewritten in place about once a second on a terminal, and a + new line every 30 seconds otherwise, so that logs stay readable. */ +typedef struct s_progress { + int tty; + uint64_t last; +} t_progress; + +static int progress_due(t_progress *p, int final) { + uint64_t now = timing_now_ns(); + uint64_t interval = p->tty ? 1000000000u : 30000000000u; + + if (p->last != 0 && !final && now - p->last < interval) + return 0; + p->last = now; + return 1; +} + +static int warm_up(t_context *ctx) { + const t_options *o = &ctx->options; + size_t *order = malloc(ctx->trial_count * sizeof *order); + t_progress progress = {isatty(STDERR_FILENO), 0}; + + if (order == NULL) + return error_message("out of memory"), -1; + for (size_t i = 0; i < ctx->trial_count; i++) + order[i] = i; + plan_shuffle(order, ctx->trial_count, sizeof *order, o->order_seed ^ 0x9E3779B9u); + + for (size_t i = 0; i < ctx->trial_count; i++) { + t_trial *t = &ctx->trials[order[i]]; + uint64_t reps = t->algorithm == ALGO_NONE ? 0 : 1; + t_measurement m; + + for (unsigned w = 0; w < o->warmup; w++) { + if (interrupted() || run_worker(ctx, t, reps, &m) != 0 || check_consistency(ctx, t, &m.result) != 0) { + free(order); + return -1; + } + t->warm_wall_ns = m.wall_ns; + if (t->algorithm != ALGO_NONE) { + uint64_t next = plan_calibrate(m.result.loop_ns, reps, o->min_loop_ns, o->max_reps); + /* Estimate the next run's length for the progress report. */ + uint64_t overhead = m.wall_ns > m.result.loop_ns ? m.wall_ns - m.result.loop_ns : 0; + t->warm_wall_ns = overhead + m.result.loop_ns / reps * next; + reps = next; + } + } + t->inner_reps = reps; + if (!o->quiet && progress_due(&progress, i + 1 == ctx->trial_count)) + fprintf(stderr, "%swarm-up and calibration: %zu/%zu trials%s", progress.tty ? "\r" : "", i + 1, + ctx->trial_count, progress.tty ? "" : "\n"); + } + if (!o->quiet && progress.tty) + fputc('\n', stderr); + free(order); + return 0; +} + +static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { + const t_options *o = &ctx->options; + size_t total = ctx->trial_count * o->runs; + t_run_item *items = malloc(total * sizeof *items); + uint64_t estimate = 0; + t_progress progress = {isatty(STDERR_FILENO), 0}; + uint64_t started = timing_now_ns(); + char text[64]; + + if (items == NULL) + return error_message("out of memory"), -1; + plan_runs(ctx->trial_count, o->runs, items); + plan_shuffle(items, total, sizeof *items, o->order_seed); + for (size_t i = 0; i < total; i++) + estimate += ctx->trials[items[i].trial].warm_wall_ns * (ctx->baseline ? 2u : 1u); + if (!o->quiet) { + format_duration(estimate, text, sizeof text); + fprintf(stderr, "measuring %zu runs in random order; estimated time %s\n", total, text); + } + + for (size_t i = 0; i < total; i++) { + const t_trial *t = &ctx->trials[items[i].trial]; + t_measurement m; + t_run_record record; + + if (interrupted() || run_worker(ctx, t, t->inner_reps, &m) != 0 || check_consistency(ctx, t, &m.result) != 0) { + free(items); + return -1; + } + memset(&record, 0, sizeof record); + record.sequence = i + 1; + record.trial = t; + record.repetition = items[i].repetition; + record.seed = o->seed; + record.wall_ns = m.wall_ns; + record.loop_ns = m.result.loop_ns; + record.comparisons = m.result.comparisons; + record.writes = m.result.writes; + record.input_hash = m.result.input_hash; + record.energy_status = ctx->energy_status; + record.has_energy = ctx->energy_ok; + record.has_dram = ctx->rapl.has_dram; + record.package_uj = m.energy.package_uj; + record.dram_uj = m.energy.dram_uj; + + if (ctx->baseline) { + t_rapl_sample before, after; + t_rapl_energy idle = {0, 0}; + if (ctx->energy_ok && rapl_read(&ctx->rapl, &before) != 0) + return free(items), error_message("reading the RAPL counters failed: %s", strerror(errno)), -1; + uint64_t idle_start = timing_now_ns(); + timing_sleep_ns(m.wall_ns); + uint64_t idle_end = timing_now_ns(); + if (ctx->energy_ok && (rapl_read(&ctx->rapl, &after) != 0 || + rapl_energy_between(&ctx->rapl, &before, &after, &idle) != 0)) + return free(items), error_message("reading the RAPL counters failed during the idle window"), -1; + record.has_idle = 1; + record.idle_ns = idle_end - idle_start; + record.idle_package_uj = idle.package_uj; + record.idle_dram_uj = idle.dram_uj; + } + output_csv_row(csv, &record); + if (fflush(csv) != 0) { + free(items); + return error_message("cannot write %s: %s", ctx->csv_path, strerror(errno)), -1; + } + *measured = i + 1; + + if (!o->quiet && progress_due(&progress, i + 1 == total)) { + uint64_t elapsed = timing_now_ns() - started; + uint64_t left = (uint64_t)((double)elapsed / (double)(i + 1) * (double)(total - i - 1)); + format_duration(left, text, sizeof text); + fprintf(stderr, "%s[%zu/%zu %3zu%%] about %s left%s", progress.tty ? "\r" : "", i + 1, total, + (i + 1) * 100 / total, text, progress.tty ? " " : "\n"); + } + } + if (!o->quiet && progress.tty) + fputc('\n', stderr); + free(items); + return 0; +} + +static int pin_cpu(int cpu) { +#ifdef __linux__ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET((size_t)cpu, &set); + if (sched_setaffinity(0, sizeof set, &set) != 0) + return error_message("cannot pin to CPU %d: %s", cpu, strerror(errno)), -1; + return 0; +#else + (void)cpu; + return error_message("--cpu is only supported on Linux"), -1; +#endif +} + +static void setup_energy(t_context *ctx) { + const t_options *o = &ctx->options; + + if (o->energy == ENERGY_OFF) { + ctx->energy_status = "off"; + snprintf(ctx->energy_reason, sizeof ctx->energy_reason, "turned off with --energy off"); + } else if (rapl_open(&ctx->rapl, o->powercap_root) == 0) { + ctx->energy_ok = 1; + ctx->energy_status = "ok"; + } else { + ctx->energy_status = "unavailable"; + snprintf(ctx->energy_reason, sizeof ctx->energy_reason, "%s", ctx->rapl.reason); + } + ctx->baseline = o->baseline == BASELINE_ON || (o->baseline == BASELINE_AUTO && ctx->energy_ok); +} + +/* "greenbench run ARGS", quoting any argument that the shell would split or expand. */ +static void record_command(t_context *ctx, int argc, char **argv) { + size_t used = (size_t)snprintf(ctx->command, sizeof ctx->command, "greenbench run"); + + for (int i = 0; i < argc && used < sizeof ctx->command; i++) { + int plain = argv[i][0] != '\0' && strspn(argv[i], "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789-_=,./:+") == strlen(argv[i]); + int written = snprintf(ctx->command + used, sizeof ctx->command - used, plain ? " %s" : " '%s'", argv[i]); + if (written < 0) + break; + used += (size_t)written; + } +} + +static void cleanup(t_context *ctx) { + if (ctx->energy_ok) + rapl_close(&ctx->rapl); + ctx->energy_ok = 0; + free(ctx->trials); + ctx->trials = NULL; +} + +static int command_run(int argc, char **argv, const char *argv0) { + static t_context ctx; + char err[ERROR_LEN]; + uint64_t measured = 0; + t_options *o = &ctx.options; + + options_defaults(o); + record_command(&ctx, argc, argv); + if (options_parse(argc, argv, o, err, sizeof err) != 0) { + error_message("%s", err); + fprintf(stderr, "Try 'greenbench --help'.\n"); + return EXIT_USAGE; + } + if (o->help) { + print_usage(stdout); + return 0; + } + ctx.trial_count = plan_trial_count(o); + ctx.trials = calloc(ctx.trial_count, sizeof *ctx.trials); + if (ctx.trials == NULL) + return error_message("out of memory"), 1; + plan_trials(o, ctx.trials); + + if (o->dry_run) { + uint64_t processes = (uint64_t)ctx.trial_count * (o->warmup + o->runs); + printf("profile %s: %zu trials (%zu languages x %zu algorithms%s x %zu orders x %zu sizes)\n", o->profile, + ctx.trial_count, o->language_count, o->algorithm_count, o->startup_trials ? " + startup" : "", + o->order_count, o->size_count); + printf("%u warm-up and %u measured runs per trial: %" PRIu64 " worker processes, %zu CSV rows\n", + o->warmup, o->runs, processes, ctx.trial_count * o->runs); + cleanup(&ctx); + return 0; + } + if (o->runs < OPTIONS_MIN_STUDY_RUNS && !o->quiet) + fprintf(stderr, "%s: note: %u measured runs per trial; the study uses at least %d\n", g_program, o->runs, + OPTIONS_MIN_STUDY_RUNS); + + if (resolve_workers(&ctx, argv0) != 0 || record_versions(&ctx) != 0) { + cleanup(&ctx); + return EXIT_USAGE; + } + setup_energy(&ctx); + if (!ctx.energy_ok && o->energy == ENERGY_REQUIRE) { + error_message("energy unavailable: %s", ctx.energy_reason); + cleanup(&ctx); + return EXIT_ENERGY; + } + if (!o->quiet) { + if (ctx.energy_ok) + fprintf(stderr, "energy: reading %zu RAPL zone(s) under %s%s\n", ctx.rapl.count, o->powercap_root, + ctx.baseline ? ", with an idle baseline after every run" : ""); + else if (o->energy == ENERGY_OFF) + fprintf(stderr, "energy: off; measuring time and operation counts only\n"); + else + fprintf(stderr, "energy unavailable: %s\nMeasuring time and operation counts only.\n", ctx.energy_reason); + } + if (o->cpu >= 0 && pin_cpu(o->cpu) != 0) { + cleanup(&ctx); + return EXIT_USAGE; + } + + if (make_dirs(o->out_dir) != 0 || join_path(ctx.csv_path, sizeof ctx.csv_path, o->out_dir, "runs.csv") != 0 || + join_path(ctx.meta_path, sizeof ctx.meta_path, o->out_dir, "meta.json") != 0) { + error_message("cannot create output directory %s: %s", o->out_dir, strerror(errno)); + cleanup(&ctx); + return 1; + } + utc_now(ctx.started_utc, sizeof ctx.started_utc); + int fd = open(ctx.csv_path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0666); + FILE *csv = fd >= 0 ? fdopen(fd, "w") : NULL; + if (csv == NULL || write_meta(&ctx, "planned", 0) != 0) { + error_message("cannot write %s: %s", ctx.csv_path, strerror(errno)); + if (csv != NULL) + fclose(csv); + cleanup(&ctx); + return 1; + } + output_csv_header(csv); + + catch_interrupts(); + int status = warm_up(&ctx) == 0 && write_meta(&ctx, "running", 0) == 0 && measure(&ctx, csv, &measured) == 0 + ? 0 + : 1; + if (fclose(csv) != 0 && status == 0) { + error_message("cannot write %s: %s", ctx.csv_path, strerror(errno)); + status = 1; + } + if (g_interrupted) + error_message("interrupted after %" PRIu64 " measured runs; they are in %s", measured, ctx.csv_path); + if (write_meta(&ctx, status == 0 ? "complete" : g_interrupted ? "interrupted" : "failed", measured) != 0) + status = 1; + if (status == 0 && !o->quiet) + fprintf(stderr, "wrote %s and %s\n", ctx.csv_path, ctx.meta_path); + cleanup(&ctx); + return status; +} + +static int command_probe(int argc, char **argv) { + const char *root = RAPL_DEFAULT_ROOT; + uint64_t sample_ms = 1000; + t_rapl rapl; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--powercap-root") == 0 && i + 1 < argc && argv[i + 1][0] != '\0') { + root = argv[++i]; + } else if (strcmp(argv[i], "--sample-ms") == 0 && i + 1 < argc && + options_parse_u64(argv[i + 1], 0, 60000, &sample_ms) == 0) { + i++; + } else if (strcmp(argv[i], "--help") == 0) { + print_usage(stdout); + return 0; + } else { + error_message("probe: unexpected or invalid argument '%s'", argv[i]); + fprintf(stderr, "Try 'greenbench --help'.\n"); + return EXIT_USAGE; + } + } + printf("powercap root: %s\n", root); + if (rapl_open(&rapl, root) != 0) { + printf("energy: unavailable\nreason: %s\n", rapl.reason); + return EXIT_ENERGY; + } + printf("energy: available\n"); + for (size_t i = 0; i < rapl.count; i++) + printf(" %-16s %-12s %-8s wraps at %" PRIu64 " uJ\n", rapl.domains[i].zone, rapl.domains[i].name, + rapl.domains[i].kind == RAPL_PACKAGE ? "package" : "dram", rapl.domains[i].max_range_uj); + if (sample_ms > 0) { + t_rapl_sample before, after; + t_rapl_energy energy; + if (rapl_read(&rapl, &before) != 0) + return rapl_close(&rapl), error_message("reading the counters failed: %s", strerror(errno)), 1; + uint64_t start = timing_now_ns(); + timing_sleep_ns(sample_ms * 1000000u); + uint64_t elapsed = timing_now_ns() - start; + if (rapl_read(&rapl, &after) != 0 || rapl_energy_between(&rapl, &before, &after, &energy) != 0) + return rapl_close(&rapl), error_message("reading the counters failed"), 1; + double seconds = (double)elapsed / 1e9; + printf("idle power over %.2f s: package %.2f W", seconds, (double)energy.package_uj / 1e6 / seconds); + if (rapl.has_dram) + printf(", dram %.2f W", (double)energy.dram_uj / 1e6 / seconds); + putchar('\n'); + if (energy.package_uj == 0) + printf("warning: the package counter did not change; the readings may be frozen\n"); + } + rapl_close(&rapl); + return 0; +} + +int main(int argc, char **argv) { + if (argc < 2 || strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0) { + print_usage(argc < 2 ? stderr : stdout); + return argc < 2 ? EXIT_USAGE : 0; + } + if (strcmp(argv[1], "--version") == 0) { + printf("greenbench %s\n", GREENBENCH_VERSION); + return 0; + } + if (strcmp(argv[1], "run") == 0) + return command_run(argc - 2, argv + 2, argv[0]); + if (strcmp(argv[1], "probe") == 0) + return command_probe(argc - 2, argv + 2); + error_message("unknown command '%s'", argv[1]); + fprintf(stderr, "Try 'greenbench --help'.\n"); + return EXIT_USAGE; +} diff --git a/src/options.c b/src/options.c new file mode 100644 index 0000000..82116e0 --- /dev/null +++ b/src/options.c @@ -0,0 +1,439 @@ +#include "options.h" + +#include +#include +#include + +#include "rapl.h" + +#define DEFAULT_ORDER_SEED 1234567u +#define LIST_BUFFER 512 +#define NAME_BUFFER 32 + +static const size_t STUDY_SIZES[] = {250, 500, 1000, 2000}; +static const size_t SMOKE_SIZES[] = {64, 256}; + +static void fail(char *err, size_t err_size, const char *format, ...) { + va_list args; + + va_start(args, format); + vsnprintf(err, err_size, format, args); + va_end(args); +} + +static void select_everything(t_options *o) { + o->language_count = 0; + for (int i = 0; i < LANG_COUNT; i++) + o->languages[o->language_count++] = (t_language)i; + o->algorithm_count = 0; + for (int i = 0; i < ALGO_NONE; i++) + o->algorithms[o->algorithm_count++] = (t_algorithm)i; + o->order_count = 0; + for (int i = 0; i < ORDER_COUNT; i++) + o->orders[o->order_count++] = (t_order)i; +} + +static void set_sizes(t_options *o, const size_t *sizes, size_t count) { + memcpy(o->sizes, sizes, count * sizeof *sizes); + o->size_count = count; +} + +int options_apply_profile(t_options *o, const char *name) { + if (strcmp(name, "study") == 0) { + o->profile = "study"; + select_everything(o); + set_sizes(o, STUDY_SIZES, sizeof STUDY_SIZES / sizeof *STUDY_SIZES); + o->runs = OPTIONS_MIN_STUDY_RUNS; + o->warmup = 3; + o->min_loop_ns = 200000000u; /* 200 ms: about 200 RAPL updates */ + o->max_reps = 1000000u; + return 0; + } + if (strcmp(name, "smoke") == 0) { + o->profile = "smoke"; + select_everything(o); + set_sizes(o, SMOKE_SIZES, sizeof SMOKE_SIZES / sizeof *SMOKE_SIZES); + o->runs = 3; + o->warmup = 1; + o->min_loop_ns = 2000000u; /* 2 ms: enough to exercise calibration, too short for energy */ + o->max_reps = 100000u; + return 0; + } + return -1; +} + +void options_defaults(t_options *o) { + memset(o, 0, sizeof *o); + options_apply_profile(o, "study"); + o->seed = INPUT_DEFAULT_SEED; + o->order_seed = DEFAULT_ORDER_SEED; + o->startup_trials = 1; + o->energy = ENERGY_AUTO; + o->baseline = BASELINE_AUTO; + o->powercap_root = RAPL_DEFAULT_ROOT; + o->cpu = -1; + o->out_dir = "results/latest"; + o->python = "python3"; + o->node = "node"; +} + +int options_parse_u64(const char *text, uint64_t min, uint64_t max, uint64_t *out) { + uint64_t value = 0; + + if (*text == '\0') + return -1; + for (const char *p = text; *p != '\0'; p++) { + if (*p < '0' || *p > '9') + return -1; + uint64_t digit = (uint64_t)(*p - '0'); + if (value > (UINT64_MAX - digit) / 10) + return -1; + value = value * 10 + digit; + } + if (value < min || value > max) + return -1; + *out = value; + return 0; +} + +/* Splits a comma-separated list into tokens inside buffer. Returns the count, or -1. */ +static int split_list(const char *text, char *buffer, size_t buffer_size, char **tokens, size_t max_tokens) { + size_t count = 0; + size_t length = strlen(text); + + if (length == 0 || length >= buffer_size) + return -1; + memcpy(buffer, text, length + 1); + char *start = buffer; + for (;;) { + char *comma = strchr(start, ','); + if (comma != NULL) + *comma = '\0'; + if (*start == '\0' || count == max_tokens) + return -1; + tokens[count++] = start; + if (comma == NULL) + break; + start = comma + 1; + } + return (int)count; +} + +int options_parse_sizes(const char *text, size_t *sizes, size_t *count, char *err, size_t err_size) { + char buffer[LIST_BUFFER]; + char *tokens[OPTIONS_MAX_SIZES]; + int n = split_list(text, buffer, sizeof buffer, tokens, OPTIONS_MAX_SIZES); + + if (n < 0) { + fail(err, err_size, "--sizes: expected a comma-separated list of up to %d sizes", OPTIONS_MAX_SIZES); + return -1; + } + for (int i = 0; i < n; i++) { + uint64_t value; + if (options_parse_u64(tokens[i], 1, INPUT_MAX_N, &value) != 0) { + fail(err, err_size, "--sizes: '%s' is not a whole number from 1 to %d", tokens[i], INPUT_MAX_N); + return -1; + } + for (int j = 0; j < i; j++) { + if (sizes[j] == (size_t)value) { + fail(err, err_size, "--sizes: %s is listed twice", tokens[i]); + return -1; + } + } + sizes[i] = (size_t)value; + } + *count = (size_t)n; + return 0; +} + +static int parse_languages(const char *text, t_options *o, char *err, size_t err_size) { + char buffer[LIST_BUFFER]; + char *tokens[LANG_COUNT]; + int n = split_list(text, buffer, sizeof buffer, tokens, LANG_COUNT); + + if (n < 0) { + fail(err, err_size, "--languages: expected a comma-separated list of c, python and node"); + return -1; + } + for (int i = 0; i < n; i++) { + if (language_from_name(tokens[i], &o->languages[i]) != 0) { + fail(err, err_size, "--languages: unknown language '%s' (expected c, python or node)", tokens[i]); + return -1; + } + for (int j = 0; j < i; j++) { + if (o->languages[j] == o->languages[i]) { + fail(err, err_size, "--languages: %s is listed twice", tokens[i]); + return -1; + } + } + } + o->language_count = (size_t)n; + return 0; +} + +static int parse_algorithms(const char *text, t_options *o, char *err, size_t err_size) { + char buffer[LIST_BUFFER]; + char *tokens[ALGO_NONE]; + int n = split_list(text, buffer, sizeof buffer, tokens, ALGO_NONE); + + if (n < 0) { + fail(err, err_size, "--algorithms: expected a comma-separated list of bubble, insertion and quick"); + return -1; + } + for (int i = 0; i < n; i++) { + if (algorithm_from_name(tokens[i], &o->algorithms[i]) != 0) { + fail(err, err_size, "--algorithms: unknown algorithm '%s' (expected bubble, insertion or quick)", + tokens[i]); + return -1; + } + for (int j = 0; j < i; j++) { + if (o->algorithms[j] == o->algorithms[i]) { + fail(err, err_size, "--algorithms: %s is listed twice", tokens[i]); + return -1; + } + } + } + o->algorithm_count = (size_t)n; + return 0; +} + +static int parse_orders(const char *text, t_options *o, char *err, size_t err_size) { + char buffer[LIST_BUFFER]; + char *tokens[ORDER_COUNT]; + int n = split_list(text, buffer, sizeof buffer, tokens, ORDER_COUNT); + + if (n < 0) { + fail(err, err_size, "--orders: expected a comma-separated list of random, sorted, reversed and " + "nearly_sorted"); + return -1; + } + for (int i = 0; i < n; i++) { + if (order_from_name(tokens[i], &o->orders[i]) != 0) { + fail(err, err_size, + "--orders: unknown order '%s' (expected random, sorted, reversed or nearly_sorted)", tokens[i]); + return -1; + } + for (int j = 0; j < i; j++) { + if (o->orders[j] == o->orders[i]) { + fail(err, err_size, "--orders: %s is listed twice", tokens[i]); + return -1; + } + } + } + o->order_count = (size_t)n; + return 0; +} + +static int parse_number(const char *option, const char *text, uint64_t min, uint64_t max, uint64_t *out, + char *err, size_t err_size) { + if (options_parse_u64(text, min, max, out) != 0) { + fail(err, err_size, "--%s: '%s' is not a whole number from %llu to %llu", option, text, + (unsigned long long)min, (unsigned long long)max); + return -1; + } + return 0; +} + +static int parse_choice(const char *option, const char *text, const char *const *choices, int count, + int *out, char *err, size_t err_size) { + for (int i = 0; i < count; i++) { + if (strcmp(text, choices[i]) == 0) { + *out = i; + return 0; + } + } + fail(err, err_size, "--%s: unknown value '%s' (expected %s, %s or %s)", option, text, choices[0], + choices[1], choices[2]); + return -1; +} + +static const char *const FLAGS[] = {"no-startup", "dry-run", "quiet", "help"}; +static const char *const VALUED[] = {"profile", "languages", "algorithms", "orders", "sizes", + "runs", "warmup", "seed", "order-seed", "min-loop-ms", + "max-reps", "cpu", "energy", "idle-baseline", + "powercap-root", "out", "python", "node", "c-worker", + "python-worker", "node-worker", "label"}; + +static int is_valued(const char *name) { + for (size_t i = 0; i < sizeof VALUED / sizeof *VALUED; i++) + if (strcmp(name, VALUED[i]) == 0) + return 1; + return 0; +} + +static int is_flag(const char *name) { + for (size_t i = 0; i < sizeof FLAGS / sizeof *FLAGS; i++) + if (strcmp(name, FLAGS[i]) == 0) + return 1; + return 0; +} + +static int apply_flag(const char *name, t_options *o) { + if (strcmp(name, "no-startup") == 0) + o->startup_trials = 0; + else if (strcmp(name, "dry-run") == 0) + o->dry_run = 1; + else if (strcmp(name, "quiet") == 0) + o->quiet = 1; + else + o->help = 1; + return 0; +} + +static int require_text(const char *name, const char *value, const char **out, char *err, size_t err_size) { + if (*value == '\0') { + fail(err, err_size, "--%s: value must not be empty", name); + return -1; + } + *out = value; + return 0; +} + +static int apply_option(const char *name, const char *value, t_options *o, char *err, size_t err_size) { + static const char *const ENERGY[] = {"auto", "require", "off"}; + static const char *const BASELINE[] = {"auto", "on", "off"}; + uint64_t number; + int choice; + + if (strcmp(name, "profile") == 0) + return 0; /* applied before everything else */ + if (strcmp(name, "languages") == 0) + return parse_languages(value, o, err, err_size); + if (strcmp(name, "algorithms") == 0) + return parse_algorithms(value, o, err, err_size); + if (strcmp(name, "orders") == 0) + return parse_orders(value, o, err, err_size); + if (strcmp(name, "sizes") == 0) + return options_parse_sizes(value, o->sizes, &o->size_count, err, err_size); + if (strcmp(name, "runs") == 0) { + if (parse_number(name, value, 1, 100000, &number, err, err_size) != 0) + return -1; + o->runs = (unsigned)number; + return 0; + } + if (strcmp(name, "warmup") == 0) { + if (parse_number(name, value, 1, 1000, &number, err, err_size) != 0) + return -1; + o->warmup = (unsigned)number; + return 0; + } + if (strcmp(name, "seed") == 0 || strcmp(name, "order-seed") == 0) { + if (parse_number(name, value, 1, UINT32_MAX, &number, err, err_size) != 0) + return -1; + if (strcmp(name, "seed") == 0) + o->seed = (uint32_t)number; + else + o->order_seed = (uint32_t)number; + return 0; + } + if (strcmp(name, "min-loop-ms") == 0) { + if (parse_number(name, value, 0, 600000, &number, err, err_size) != 0) + return -1; + o->min_loop_ns = number * 1000000u; + return 0; + } + if (strcmp(name, "max-reps") == 0) { + if (parse_number(name, value, 1, 100000000, &number, err, err_size) != 0) + return -1; + o->max_reps = number; + return 0; + } + if (strcmp(name, "cpu") == 0) { + if (parse_number(name, value, 0, 4095, &number, err, err_size) != 0) + return -1; + o->cpu = (int)number; + return 0; + } + if (strcmp(name, "energy") == 0) { + if (parse_choice(name, value, ENERGY, 3, &choice, err, err_size) != 0) + return -1; + o->energy = (t_energy_mode)choice; + return 0; + } + if (strcmp(name, "idle-baseline") == 0) { + if (parse_choice(name, value, BASELINE, 3, &choice, err, err_size) != 0) + return -1; + o->baseline = (t_baseline_mode)choice; + return 0; + } + if (strcmp(name, "powercap-root") == 0) + return require_text(name, value, &o->powercap_root, err, err_size); + if (strcmp(name, "out") == 0) + return require_text(name, value, &o->out_dir, err, err_size); + if (strcmp(name, "python") == 0) + return require_text(name, value, &o->python, err, err_size); + if (strcmp(name, "node") == 0) + return require_text(name, value, &o->node, err, err_size); + if (strcmp(name, "c-worker") == 0) + return require_text(name, value, &o->c_worker, err, err_size); + if (strcmp(name, "python-worker") == 0) + return require_text(name, value, &o->python_worker, err, err_size); + if (strcmp(name, "node-worker") == 0) + return require_text(name, value, &o->node_worker, err, err_size); + if (strcmp(name, "label") == 0) + return require_text(name, value, &o->label, err, err_size); + fail(err, err_size, "unknown option --%s", name); + return -1; +} + +/* Splits "--name" or "--name=value". Returns -1 if arg is not an option. */ +static int split_option(const char *arg, char *name, size_t name_size, const char **inline_value) { + if (strncmp(arg, "--", 2) != 0 || arg[2] == '\0') + return -1; + const char *body = arg + 2; + const char *equals = strchr(body, '='); + size_t length = equals != NULL ? (size_t)(equals - body) : strlen(body); + if (length == 0 || length >= name_size) + return -1; + memcpy(name, body, length); + name[length] = '\0'; + *inline_value = equals != NULL ? equals + 1 : NULL; + return 0; +} + +int options_parse(int argc, char *const argv[], t_options *o, char *err, size_t err_size) { + char name[NAME_BUFFER]; + const char *inline_value; + + /* First pass: the profile, so that explicit options override it. */ + for (int i = 0; i < argc; i++) { + if (split_option(argv[i], name, sizeof name, &inline_value) != 0 || strcmp(name, "profile") != 0) + continue; + const char *value = inline_value; + if (value == NULL && i + 1 < argc) + value = argv[i + 1]; + if (value == NULL || options_apply_profile(o, value) != 0) { + fail(err, err_size, "--profile: expected study or smoke"); + return -1; + } + } + for (int i = 0; i < argc; i++) { + if (split_option(argv[i], name, sizeof name, &inline_value) != 0) { + fail(err, err_size, "unexpected argument '%s'", argv[i]); + return -1; + } + if (is_flag(name)) { + if (inline_value != NULL) { + fail(err, err_size, "--%s does not take a value", name); + return -1; + } + apply_flag(name, o); + continue; + } + if (!is_valued(name)) { + fail(err, err_size, "unknown option --%s", name); + return -1; + } + const char *value = inline_value; + if (value == NULL) { + if (i + 1 >= argc) { + fail(err, err_size, "--%s needs a value", name); + return -1; + } + value = argv[++i]; + } + if (apply_option(name, value, o, err, err_size) != 0) + return -1; + } + return 0; +} diff --git a/src/options.h b/src/options.h new file mode 100644 index 0000000..a186e45 --- /dev/null +++ b/src/options.h @@ -0,0 +1,65 @@ +#ifndef OPTIONS_H +#define OPTIONS_H + +#include +#include + +#include "catalog.h" +#include "inputs.h" + +#define OPTIONS_MAX_SIZES 16 +#define OPTIONS_MIN_STUDY_RUNS 30 + +typedef enum e_energy_mode { ENERGY_AUTO, ENERGY_REQUIRE, ENERGY_OFF } t_energy_mode; +typedef enum e_baseline_mode { BASELINE_AUTO, BASELINE_ON, BASELINE_OFF } t_baseline_mode; + +typedef struct s_options { + const char *profile; + t_language languages[LANG_COUNT]; + size_t language_count; + t_algorithm algorithms[ALGO_NONE]; + size_t algorithm_count; + t_order orders[ORDER_COUNT]; + size_t order_count; + size_t sizes[OPTIONS_MAX_SIZES]; + size_t size_count; + unsigned runs; /* measured runs per trial */ + unsigned warmup; /* unmeasured runs per trial; the first also calibrates */ + uint32_t seed; /* input generator seed, the same for every language */ + uint32_t order_seed; /* seed for the randomised run order */ + uint64_t min_loop_ns; /* each measured sort loop should last at least this long */ + uint64_t max_reps; /* upper bound on sorts per process */ + int startup_trials; /* also measure processes that exit without sorting */ + t_energy_mode energy; + t_baseline_mode baseline; + const char *powercap_root; + int cpu; /* CPU to pin to, or -1 */ + const char *out_dir; + const char *python; + const char *node; + const char *c_worker; /* NULL: next to the greenbench executable */ + const char *python_worker; /* NULL: workers/python/greenbench_worker.py in the repository */ + const char *node_worker; /* NULL: workers/node/worker.js in the repository */ + const char *label; + int dry_run; + int quiet; + int help; +} t_options; + +/* Sets the defaults, which are the "study" profile. */ +void options_defaults(t_options *options); + +/* Applies a named profile ("study" or "smoke"). Returns -1 for an unknown name. */ +int options_apply_profile(t_options *options, const char *name); + +/* Parses the arguments after "greenbench run". A --profile option is applied first, so + other options override it wherever they appear. Returns 0, or -1 with a message in err. */ +int options_parse(int argc, char *const argv[], t_options *options, char *err, size_t err_size); + +/* Parses a comma-separated list of sizes (each 1 to INPUT_MAX_N, no duplicates). */ +int options_parse_sizes(const char *text, size_t *sizes, size_t *count, char *err, size_t err_size); + +/* Parses a decimal integer in [min, max] with nothing else in the string. */ +int options_parse_u64(const char *text, uint64_t min, uint64_t max, uint64_t *out); + +#endif diff --git a/src/output.c b/src/output.c new file mode 100644 index 0000000..ab727ab --- /dev/null +++ b/src/output.c @@ -0,0 +1,81 @@ +#include "output.h" + +#include + +#include "rapl.h" + +void output_csv_header(FILE *file) { + fputs("sequence,language,algorithm,order,n,seed,repetition,inner_reps,wall_ns,loop_ns,comparisons,writes," + "input_hash,energy_status,pkg_uj,dram_uj,idle_ns,idle_pkg_uj,idle_dram_uj,net_pkg_uj,net_dram_uj\n", + file); +} + +void output_csv_row(FILE *file, const t_run_record *r) { + const t_trial *t = r->trial; + + fprintf(file, "%" PRIu64 ",%s,%s,%s,%zu,%" PRIu32 ",%u,%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 + ",%08" PRIx32 ",%s,", + r->sequence, language_name(t->language), algorithm_name(t->algorithm), order_name(t->order), t->n, + r->seed, r->repetition, t->inner_reps, r->wall_ns, r->loop_ns, r->comparisons, r->writes, + r->input_hash, r->energy_status); + /* pkg_uj, dram_uj */ + if (r->has_energy) + fprintf(file, "%" PRIu64 ",", r->package_uj); + else + fputc(',', file); + if (r->has_energy && r->has_dram) + fprintf(file, "%" PRIu64 ",", r->dram_uj); + else + fputc(',', file); + /* idle_ns, idle_pkg_uj, idle_dram_uj */ + if (r->has_idle) + fprintf(file, "%" PRIu64 ",", r->idle_ns); + else + fputc(',', file); + int has_idle_energy = r->has_idle && r->has_energy; + if (has_idle_energy) + fprintf(file, "%" PRIu64 ",", r->idle_package_uj); + else + fputc(',', file); + if (has_idle_energy && r->has_dram) + fprintf(file, "%" PRIu64 ",", r->idle_dram_uj); + else + fputc(',', file); + /* net_pkg_uj, net_dram_uj */ + if (has_idle_energy) + fprintf(file, "%.1f,", rapl_net_uj(r->package_uj, r->idle_package_uj, r->wall_ns, r->idle_ns)); + else + fputc(',', file); + if (has_idle_energy && r->has_dram) + fprintf(file, "%.1f", rapl_net_uj(r->dram_uj, r->idle_dram_uj, r->wall_ns, r->idle_ns)); + fputc('\n', file); +} + +void output_json_string(FILE *file, const char *s) { + fputc('"', file); + for (const unsigned char *p = (const unsigned char *)s; *p != '\0'; p++) { + switch (*p) { + case '"': + fputs("\\\"", file); + break; + case '\\': + fputs("\\\\", file); + break; + case '\n': + fputs("\\n", file); + break; + case '\r': + fputs("\\r", file); + break; + case '\t': + fputs("\\t", file); + break; + default: + if (*p < 0x20) + fprintf(file, "\\u%04x", (unsigned)*p); + else + fputc(*p, file); + } + } + fputc('"', file); +} diff --git a/src/output.h b/src/output.h new file mode 100644 index 0000000..7ea33a3 --- /dev/null +++ b/src/output.h @@ -0,0 +1,41 @@ +#ifndef OUTPUT_H +#define OUTPUT_H + +/* The tidy CSV: one row per measured run, one column per variable. Energy columns are + left empty, never zero, when energy was not measured. */ + +#include +#include + +#include "plan.h" + +#define OUTPUT_CSV_SCHEMA 1 + +typedef struct s_run_record { + uint64_t sequence; /* position in the randomised run order, from 1 */ + const t_trial *trial; + unsigned repetition; + uint32_t seed; + uint64_t wall_ns; /* whole worker process, measured by the harness */ + uint64_t loop_ns; /* sort loop only, measured by the worker */ + uint64_t comparisons; + uint64_t writes; + uint32_t input_hash; + const char *energy_status; /* "ok", "unavailable" or "off" */ + int has_energy; + int has_dram; + uint64_t package_uj; + uint64_t dram_uj; + int has_idle; /* an idle window of the same length followed the run */ + uint64_t idle_ns; + uint64_t idle_package_uj; + uint64_t idle_dram_uj; +} t_run_record; + +void output_csv_header(FILE *file); +void output_csv_row(FILE *file, const t_run_record *record); + +/* Writes s as a JSON string literal, quotes included. */ +void output_json_string(FILE *file, const char *s); + +#endif diff --git a/src/plan.c b/src/plan.c new file mode 100644 index 0000000..bd28e71 --- /dev/null +++ b/src/plan.c @@ -0,0 +1,69 @@ +#include "plan.h" + +#include + +size_t plan_trial_count(const t_options *o) { + size_t per_language = o->order_count * o->size_count; + size_t algorithms = o->algorithm_count + (o->startup_trials ? 1 : 0); + return o->language_count * algorithms * per_language; +} + +void plan_trials(const t_options *o, t_trial *trials) { + size_t k = 0; + size_t algorithm_slots = o->algorithm_count + (o->startup_trials ? 1 : 0); + + for (size_t l = 0; l < o->language_count; l++) + for (size_t a = 0; a < algorithm_slots; a++) + for (size_t r = 0; r < o->order_count; r++) + for (size_t s = 0; s < o->size_count; s++) { + t_trial *t = &trials[k++]; + memset(t, 0, sizeof *t); + t->language = o->languages[l]; + t->algorithm = a < o->algorithm_count ? o->algorithms[a] : ALGO_NONE; + t->order = o->orders[r]; + t->n = o->sizes[s]; + t->inner_reps = t->algorithm == ALGO_NONE ? 0 : 1; + } +} + +void plan_runs(size_t trial_count, unsigned runs, t_run_item *items) { + size_t k = 0; + + for (size_t t = 0; t < trial_count; t++) + for (unsigned r = 1; r <= runs; r++) { + items[k].trial = t; + items[k].repetition = r; + k++; + } +} + +/* The modulo bias of drawing an index with "% i" is below i / 2^32, far too small to matter + for a run order. */ +void plan_shuffle(void *base, size_t count, size_t size, uint32_t seed) { + unsigned char *bytes = base; + unsigned char tmp[64]; + uint32_t state = seed; + + if (size > sizeof tmp) + return; + for (size_t i = count; i > 1; i--) { + size_t j = xorshift32_next(&state) % i; + if (j == i - 1) + continue; + memcpy(tmp, bytes + (i - 1) * size, size); + memcpy(bytes + (i - 1) * size, bytes + j * size, size); + memcpy(bytes + j * size, tmp, size); + } +} + +uint64_t plan_calibrate(uint64_t loop_ns, uint64_t reps, uint64_t min_loop_ns, uint64_t max_reps) { + if (reps == 0) + reps = 1; + uint64_t per_sort = loop_ns / reps; + if (per_sort == 0) + per_sort = 1; + uint64_t wanted = min_loop_ns / per_sort + (uint64_t)(min_loop_ns % per_sort != 0); + if (wanted < 1) + wanted = 1; + return wanted > max_reps ? max_reps : wanted; +} diff --git a/src/plan.h b/src/plan.h new file mode 100644 index 0000000..184269f --- /dev/null +++ b/src/plan.h @@ -0,0 +1,45 @@ +#ifndef PLAN_H +#define PLAN_H + +#include +#include + +#include "catalog.h" +#include "inputs.h" +#include "options.h" + +/* One configuration to measure: a language running an algorithm on one input. */ +typedef struct s_trial { + t_language language; + t_algorithm algorithm; + t_order order; + size_t n; + uint64_t inner_reps; /* sorts per process, set by calibration; 0 for startup trials */ + uint64_t warm_wall_ns; /* duration of the last warm-up run, used for the time estimate */ +} t_trial; + +/* One measured run: the trial it belongs to and its repetition number (1-based). */ +typedef struct s_run_item { + size_t trial; + unsigned repetition; +} t_run_item; + +/* Number of trials the options describe: languages x algorithms x orders x sizes, plus one + startup trial per language, order and size when startup trials are on. */ +size_t plan_trial_count(const t_options *options); + +/* Fills trials (which must hold plan_trial_count entries) in a fixed nested order. */ +void plan_trials(const t_options *options, t_trial *trials); + +/* Lists every measured run, trial by trial: runs items per trial. */ +void plan_runs(size_t trial_count, unsigned runs, t_run_item *items); + +/* Shuffles count elements of the given size in place (Fisher-Yates driven by xorshift32), + so the same seed always gives the same order. Elements must be at most 64 bytes. */ +void plan_shuffle(void *base, size_t count, size_t size, uint32_t seed); + +/* Sorts per process so that one process's sort loop lasts at least min_loop_ns, given that + reps sorts took loop_ns. Always at least 1 and at most max_reps. */ +uint64_t plan_calibrate(uint64_t loop_ns, uint64_t reps, uint64_t min_loop_ns, uint64_t max_reps); + +#endif diff --git a/src/spawn.c b/src/spawn.c new file mode 100644 index 0000000..8becefb --- /dev/null +++ b/src/spawn.c @@ -0,0 +1,133 @@ +#define _POSIX_C_SOURCE 200809L + +#include "spawn.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +/* Appends what is available on fd to buf, discarding anything beyond its capacity. + Returns 1 at end of file, 0 otherwise. */ +static int drain(int fd, char *buf, size_t *used) { + char chunk[1024]; + ssize_t n = read(fd, chunk, sizeof chunk); + + if (n < 0) + return errno == EINTR || errno == EAGAIN ? 0 : 1; + if (n == 0) + return 1; + size_t room = SPAWN_OUTPUT_LEN - 1 - *used; + size_t take = (size_t)n < room ? (size_t)n : room; + memcpy(buf + *used, chunk, take); + *used += take; + buf[*used] = '\0'; + return 0; +} + +int spawn_capture(char *const argv[], t_spawn_result *result) { + int out_pipe[2]; + int err_pipe[2]; + posix_spawn_file_actions_t actions; + pid_t pid; + size_t out_used = 0; + size_t err_used = 0; + int status; + + memset(result, 0, sizeof *result); + if (pipe(out_pipe) != 0) + return result->start_error = errno, -1; + if (pipe(err_pipe) != 0) { + result->start_error = errno; + close(out_pipe[0]); + close(out_pipe[1]); + return -1; + } + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, out_pipe[1], STDOUT_FILENO); + posix_spawn_file_actions_adddup2(&actions, err_pipe[1], STDERR_FILENO); + posix_spawn_file_actions_addclose(&actions, out_pipe[0]); + posix_spawn_file_actions_addclose(&actions, err_pipe[0]); + posix_spawn_file_actions_addclose(&actions, out_pipe[1]); + posix_spawn_file_actions_addclose(&actions, err_pipe[1]); + int rc = posix_spawnp(&pid, argv[0], &actions, NULL, argv, environ); + posix_spawn_file_actions_destroy(&actions); + close(out_pipe[1]); + close(err_pipe[1]); + if (rc != 0) { + result->start_error = rc; + close(out_pipe[0]); + close(err_pipe[0]); + return -1; + } + result->started = 1; + + struct pollfd fds[2] = {{out_pipe[0], POLLIN, 0}, {err_pipe[0], POLLIN, 0}}; + int open_count = 2; + while (open_count > 0) { + if (poll(fds, 2, -1) < 0) { + if (errno == EINTR) + continue; + break; + } + for (int i = 0; i < 2; i++) { + if (fds[i].fd < 0 || fds[i].revents == 0) + continue; + int done = i == 0 ? drain(fds[i].fd, result->out, &out_used) : drain(fds[i].fd, result->err, &err_used); + if (done) { + close(fds[i].fd); + fds[i].fd = -1; + open_count--; + } + } + } + for (int i = 0; i < 2; i++) + if (fds[i].fd >= 0) + close(fds[i].fd); + + while (waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) { + result->start_error = errno; + return -1; + } + } + if (WIFEXITED(status)) { + result->exited = 1; + result->exit_code = WEXITSTATUS(status); + return result->exit_code == 0 ? 0 : -1; + } + if (WIFSIGNALED(status)) + result->signal = WTERMSIG(status); + return -1; +} + +void spawn_describe_failure(const t_spawn_result *result, char *buf, size_t size) { + char detail[512]; + size_t length = strlen(result->err); + + /* Keep the last line of standard error: that is usually the actual message. */ + while (length > 0 && (result->err[length - 1] == '\n' || result->err[length - 1] == '\r')) + length--; + size_t start = length; + while (start > 0 && result->err[start - 1] != '\n') + start--; + snprintf(detail, sizeof detail, "%.*s", (int)(length - start), result->err + start); + + if (!result->started) + snprintf(buf, size, "could not start: %s", strerror(result->start_error)); + else if (result->exited) + snprintf(buf, size, "exited with status %d%s%s", result->exit_code, detail[0] ? ": " : "", detail); + else if (result->signal != 0) + snprintf(buf, size, "killed by signal %d (%s)%s%s", result->signal, strsignal(result->signal), + detail[0] ? ": " : "", detail); + else + snprintf(buf, size, "failed: %s", strerror(result->start_error)); +} diff --git a/src/spawn.h b/src/spawn.h new file mode 100644 index 0000000..e44668a --- /dev/null +++ b/src/spawn.h @@ -0,0 +1,26 @@ +#ifndef SPAWN_H +#define SPAWN_H + +#include + +#define SPAWN_OUTPUT_LEN 4096 + +typedef struct s_spawn_result { + int started; /* 0 if the program could not be started at all */ + int start_error; /* errno from posix_spawnp when started is 0 */ + int exited; /* 1 if the process exited normally */ + int exit_code; /* its exit status when exited is 1 */ + int signal; /* the signal that ended it otherwise */ + char out[SPAWN_OUTPUT_LEN]; /* standard output, truncated if longer */ + char err[SPAWN_OUTPUT_LEN]; /* standard error, truncated if longer */ +} t_spawn_result; + +/* Runs argv[0] (searched in PATH) with the given arguments, captures its standard output + and standard error, and waits for it to finish. Returns 0 if it ran and exited with + status 0, -1 otherwise; result says what happened. */ +int spawn_capture(char *const argv[], t_spawn_result *result); + +/* Describes a failed spawn in one line, for error messages. */ +void spawn_describe_failure(const t_spawn_result *result, char *buf, size_t size); + +#endif diff --git a/src/timing.c b/src/timing.c new file mode 100644 index 0000000..a165924 --- /dev/null +++ b/src/timing.c @@ -0,0 +1,26 @@ +#define _GNU_SOURCE /* CLOCK_MONOTONIC_RAW */ + +#include "timing.h" + +#include +#include +#include +#include + +uint64_t timing_now_ns(void) { + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts) != 0) { + perror("clock_gettime(CLOCK_MONOTONIC_RAW)"); + exit(1); + } + return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec; +} + +void timing_sleep_ns(uint64_t ns) { + struct timespec request = {(time_t)(ns / 1000000000u), (long)(ns % 1000000000u)}; + struct timespec remaining; + + while (nanosleep(&request, &remaining) != 0 && errno == EINTR) + request = remaining; +} diff --git a/src/timing.h b/src/timing.h new file mode 100644 index 0000000..2602f7f --- /dev/null +++ b/src/timing.h @@ -0,0 +1,13 @@ +#ifndef TIMING_H +#define TIMING_H + +#include + +/* Nanoseconds from CLOCK_MONOTONIC_RAW: never adjusted by NTP, so an interval measured + with it is not stretched or shrunk while the clock is being corrected. */ +uint64_t timing_now_ns(void); + +/* Sleeps for at least ns nanoseconds, resuming after signals. */ +void timing_sleep_ns(uint64_t ns); + +#endif diff --git a/tests/test_options.c b/tests/test_options.c new file mode 100644 index 0000000..ee6b2d4 --- /dev/null +++ b/tests/test_options.c @@ -0,0 +1,139 @@ +#include + +#include "check.h" +#include "options.h" + +static char err[512]; + +static int parse(t_options *o, int argc, char *const argv[]) { + options_defaults(o); + err[0] = '\0'; + return options_parse(argc, argv, o, err, sizeof err); +} + +static void test_defaults_are_the_study(void) { + t_options o; + + options_defaults(&o); + CHECK(strcmp(o.profile, "study") == 0, "study profile"); + CHECK(o.language_count == 3 && o.algorithm_count == 3 && o.order_count == 4, "everything selected"); + CHECK(o.size_count == 4 && o.sizes[0] == 250 && o.sizes[3] == 2000, "study sizes"); + CHECK(o.runs >= OPTIONS_MIN_STUDY_RUNS, "at least 30 measured runs"); + CHECK(o.warmup >= 1, "warm-up runs"); + CHECK(o.seed == INPUT_DEFAULT_SEED, "default seed"); + CHECK(o.energy == ENERGY_AUTO && o.baseline == BASELINE_AUTO, "energy and baseline automatic"); + CHECK(o.cpu == -1, "not pinned"); + CHECK(o.startup_trials == 1, "startup trials on"); +} + +static void test_profile_and_overrides(void) { + t_options o; + char *const smoke[] = {"--profile", "smoke"}; + char *const override_first[] = {"--runs", "7", "--profile", "smoke"}; + char *const inline_values[] = {"--profile=smoke", "--sizes=10,20,30", "--min-loop-ms=0"}; + + CHECK(parse(&o, 2, smoke) == 0 && strcmp(o.profile, "smoke") == 0 && o.runs == 3, "smoke profile"); + CHECK(parse(&o, 4, override_first) == 0 && o.runs == 7, "options override the profile wherever they appear"); + CHECK(parse(&o, 3, inline_values) == 0 && o.size_count == 3 && o.sizes[2] == 30 && o.min_loop_ns == 0, + "--name=value form"); +} + +static void test_lists(void) { + t_options o; + char *const langs[] = {"--languages", "node,c"}; + char *const algos[] = {"--algorithms", "quick"}; + char *const orders[] = {"--orders", "nearly_sorted,random"}; + + CHECK(parse(&o, 2, langs) == 0 && o.language_count == 2 && o.languages[0] == LANG_NODE && + o.languages[1] == LANG_C, + "languages in the given order"); + CHECK(parse(&o, 2, algos) == 0 && o.algorithm_count == 1 && o.algorithms[0] == ALGO_QUICK, "one algorithm"); + CHECK(parse(&o, 2, orders) == 0 && o.order_count == 2 && o.orders[0] == ORDER_NEARLY_SORTED, "orders"); +} + +static void test_values(void) { + t_options o; + char *const all[] = {"--runs", "31", "--warmup", "2", "--seed", "99", + "--order-seed", "5", "--max-reps", "10", "--cpu", "3", + "--energy", "require", "--idle-baseline", "off", "--out", "x/y", + "--python", "python3.12", "--node", "/usr/bin/node", "--label", "laptop test", + "--no-startup", "--dry-run", "--quiet", "--powercap-root", "/tmp/pc"}; + int argc = (int)(sizeof all / sizeof *all); + + CHECK(parse(&o, argc, all) == 0, "every option parses"); + CHECK(o.runs == 31 && o.warmup == 2 && o.seed == 99 && o.order_seed == 5 && o.max_reps == 10, "numbers"); + CHECK(o.cpu == 3 && o.energy == ENERGY_REQUIRE && o.baseline == BASELINE_OFF, "choices"); + CHECK(strcmp(o.out_dir, "x/y") == 0 && strcmp(o.python, "python3.12") == 0 && + strcmp(o.node, "/usr/bin/node") == 0 && strcmp(o.label, "laptop test") == 0 && + strcmp(o.powercap_root, "/tmp/pc") == 0, + "text"); + CHECK(o.startup_trials == 0 && o.dry_run == 1 && o.quiet == 1, "flags"); +} + +static void expect_error(int argc, char *const argv[], const char *needle, const char *name) { + t_options o; + + CHECK(parse(&o, argc, argv) == -1, name); + CHECK(strstr(err, needle) != NULL, name); +} + +static void test_errors(void) { + char *const a[] = {"--runs", "0"}; + char *const b[] = {"--runs", "ten"}; + char *const c[] = {"--languages", "c,ruby"}; + char *const d[] = {"--languages", "c,c"}; + char *const e[] = {"--sizes", "10,,20"}; + char *const f[] = {"--sizes", "100001"}; + char *const g[] = {"--sizes", "5,5"}; + char *const h[] = {"--seed", "0"}; + char *const i[] = {"--seed", "4294967296"}; + char *const j[] = {"--energy", "maybe"}; + char *const k[] = {"--bogus"}; + char *const l[] = {"--runs"}; + char *const m[] = {"--quiet=yes"}; + char *const n[] = {"stray"}; + char *const p[] = {"--profile", "huge"}; + char *const q[] = {"--out", ""}; + char *const r[] = {"--orders", "sideways"}; + char *const s[] = {"--algorithms", "none"}; + + expect_error(2, a, "--runs", "zero runs"); + expect_error(2, b, "not a whole number", "runs not a number"); + expect_error(2, c, "unknown language 'ruby'", "unknown language"); + expect_error(2, d, "listed twice", "duplicate language"); + expect_error(2, e, "--sizes", "empty list item"); + expect_error(2, f, "from 1 to 100000", "size too large"); + expect_error(2, g, "listed twice", "duplicate size"); + expect_error(2, h, "--seed", "seed 0 would make xorshift32 output only zeros"); + expect_error(2, i, "--seed", "seed above 32 bits"); + expect_error(2, j, "auto, require or off", "unknown energy mode"); + expect_error(1, k, "unknown option --bogus", "unknown option"); + expect_error(1, l, "--runs needs a value", "missing value"); + expect_error(1, m, "does not take a value", "flag with a value"); + expect_error(1, n, "unexpected argument 'stray'", "positional argument"); + expect_error(2, p, "--profile", "unknown profile"); + expect_error(2, q, "must not be empty", "empty text value"); + expect_error(2, r, "unknown order", "unknown order"); + expect_error(2, s, "unknown algorithm 'none'", "none is internal"); +} + +static void test_parse_u64(void) { + uint64_t v; + + CHECK(options_parse_u64("0", 0, 10, &v) == 0 && v == 0, "zero"); + CHECK(options_parse_u64("10", 0, 10, &v) == 0 && v == 10, "upper bound"); + CHECK(options_parse_u64("11", 0, 10, &v) == -1, "above the bound"); + CHECK(options_parse_u64("+1", 0, 10, &v) == -1, "sign"); + CHECK(options_parse_u64("", 0, 10, &v) == -1, "empty"); + CHECK(options_parse_u64("99999999999999999999999", 0, UINT64_MAX, &v) == -1, "overflow"); +} + +int main(void) { + test_defaults_are_the_study(); + test_profile_and_overrides(); + test_lists(); + test_values(); + test_errors(); + test_parse_u64(); + return check_report("options"); +} diff --git a/tests/test_output.c b/tests/test_output.c new file mode 100644 index 0000000..76ef969 --- /dev/null +++ b/tests/test_output.c @@ -0,0 +1,112 @@ +#include +#include + +#include "check.h" +#include "output.h" + +static void capture_row(const t_run_record *r, char *out, size_t size) { + FILE *f = tmpfile(); + size_t n; + + output_csv_row(f, r); + rewind(f); + n = fread(out, 1, size - 1, f); + out[n] = '\0'; + fclose(f); +} + +static void test_header(void) { + char buf[1024]; + FILE *f = tmpfile(); + + output_csv_header(f); + rewind(f); + size_t n = fread(buf, 1, sizeof buf - 1, f); + buf[n] = '\0'; + fclose(f); + const char *start = "sequence,language,algorithm,order,n,seed,repetition,inner_reps,wall_ns,loop_ns,"; + CHECK(strncmp(buf, start, strlen(start)) == 0, "header starts with the identifiers"); + int commas = 0; + for (const char *p = buf; *p; p++) + commas += *p == ','; + CHECK(commas == 20, "21 columns"); +} + +static t_trial trial = {LANG_PYTHON, ALGO_QUICK, ORDER_RANDOM, 1000, 40, 0}; + +static t_run_record base_record(void) { + t_run_record r; + + memset(&r, 0, sizeof r); + r.sequence = 17; + r.trial = &trial; + r.repetition = 3; + r.seed = 2463534242u; + r.wall_ns = 250000000; + r.loop_ns = 200000000; + r.comparisons = 11039; + r.writes = 11846; + r.input_hash = 0x0badf00du; + return r; +} + +static void test_row_without_energy(void) { + char row[1024]; + t_run_record r = base_record(); + + r.energy_status = "unavailable"; + capture_row(&r, row, sizeof row); + CHECK(strcmp(row, "17,python,quick,random,1000,2463534242,3,40,250000000,200000000,11039,11846,0badf00d," + "unavailable,,,,,,,\n") == 0, + "energy cells are empty, not zero"); +} + +static void test_row_with_energy_and_baseline(void) { + char row[1024]; + t_run_record r = base_record(); + + r.energy_status = "ok"; + r.has_energy = 1; + r.has_dram = 1; + r.package_uj = 5000000; + r.dram_uj = 400000; + r.has_idle = 1; + r.idle_ns = 500000000; /* twice the run's length */ + r.idle_package_uj = 2000000; + r.idle_dram_uj = 200000; + capture_row(&r, row, sizeof row); + CHECK(strstr(row, ",ok,5000000,400000,500000000,2000000,200000,4000000.0,300000.0\n") != NULL, + "net energy subtracts idle energy scaled to the run's length"); +} + +static void test_row_energy_without_dram_or_baseline(void) { + char row[1024]; + t_run_record r = base_record(); + + r.energy_status = "ok"; + r.has_energy = 1; + r.package_uj = 123; + capture_row(&r, row, sizeof row); + CHECK(strstr(row, ",ok,123,,,,,,\n") != NULL, "no DRAM zone and no idle baseline leave those cells empty"); +} + +static void test_json_string(void) { + char buf[256]; + FILE *f = tmpfile(); + + output_json_string(f, "Intel(R) \"Core\" \\ i7\n\t\x01"); + rewind(f); + size_t n = fread(buf, 1, sizeof buf - 1, f); + buf[n] = '\0'; + fclose(f); + CHECK(strcmp(buf, "\"Intel(R) \\\"Core\\\" \\\\ i7\\n\\t\\u0001\"") == 0, "JSON escaping"); +} + +int main(void) { + test_header(); + test_row_without_energy(); + test_row_with_energy_and_baseline(); + test_row_energy_without_dram_or_baseline(); + test_json_string(); + return check_report("output"); +} diff --git a/tests/test_plan.c b/tests/test_plan.c new file mode 100644 index 0000000..573bc29 --- /dev/null +++ b/tests/test_plan.c @@ -0,0 +1,115 @@ +#include +#include + +#include "check.h" +#include "plan.h" + +static void test_trials(void) { + t_options o; + + options_defaults(&o); + size_t count = plan_trial_count(&o); + CHECK(count == 3 * (3 + 1) * 4 * 4, "3 languages x (3 algorithms + startup) x 4 orders x 4 sizes"); + o.startup_trials = 0; + CHECK(plan_trial_count(&o) == 3 * 3 * 4 * 4, "without startup trials"); + o.startup_trials = 1; + + t_trial *trials = calloc(count, sizeof *trials); + plan_trials(&o, trials); + size_t startup = 0; + for (size_t i = 0; i < count; i++) { + if (trials[i].algorithm == ALGO_NONE) { + startup++; + CHECK(trials[i].inner_reps == 0, "startup trials never sort"); + } else { + CHECK(trials[i].inner_reps == 1, "sorting trials start at one sort per process"); + } + } + CHECK(startup == 3 * 4 * 4, "one startup trial per language, order and size"); + CHECK(trials[0].language == LANG_C && trials[0].algorithm == ALGO_BUBBLE && trials[0].order == ORDER_RANDOM && + trials[0].n == 250, + "first trial"); + CHECK(trials[count - 1].language == LANG_NODE && trials[count - 1].algorithm == ALGO_NONE && + trials[count - 1].n == 2000, + "last trial"); + + /* Every (language, algorithm, order, size) appears exactly once. */ + int unique = 1; + for (size_t i = 0; i < count && unique; i++) + for (size_t j = i + 1; j < count; j++) + if (trials[i].language == trials[j].language && trials[i].algorithm == trials[j].algorithm && + trials[i].order == trials[j].order && trials[i].n == trials[j].n) + unique = 0; + CHECK(unique, "no duplicate trials"); + free(trials); +} + +static int item_order_changed(const t_run_item *a, const t_run_item *b, size_t n) { + for (size_t i = 0; i < n; i++) + if (a[i].trial != b[i].trial || a[i].repetition != b[i].repetition) + return 1; + return 0; +} + +static void test_runs_and_shuffle(void) { + enum { TRIALS = 50, RUNS = 30, TOTAL = TRIALS * RUNS }; + static t_run_item planned[TOTAL], shuffled[TOTAL], again[TOTAL], other[TOTAL]; + static int seen[TRIALS][RUNS + 1]; + + plan_runs(TRIALS, RUNS, planned); + CHECK(planned[0].trial == 0 && planned[0].repetition == 1, "first run"); + CHECK(planned[TOTAL - 1].trial == TRIALS - 1 && planned[TOTAL - 1].repetition == RUNS, "last run"); + + memcpy(shuffled, planned, sizeof planned); + plan_shuffle(shuffled, TOTAL, sizeof *shuffled, 1234567u); + memcpy(again, planned, sizeof planned); + plan_shuffle(again, TOTAL, sizeof *again, 1234567u); + memcpy(other, planned, sizeof planned); + plan_shuffle(other, TOTAL, sizeof *other, 7654321u); + + CHECK(!item_order_changed(shuffled, again, TOTAL), "same seed, same order"); + CHECK(item_order_changed(shuffled, other, TOTAL), "different seed, different order"); + CHECK(item_order_changed(shuffled, planned, TOTAL), "the order is actually shuffled"); + + int complete = 1; + for (size_t i = 0; i < TOTAL; i++) + seen[shuffled[i].trial][shuffled[i].repetition]++; + for (size_t t = 0; t < TRIALS; t++) + for (size_t r = 1; r <= RUNS; r++) + if (seen[t][r] != 1) + complete = 0; + CHECK(complete, "every run appears exactly once after shuffling"); + + /* Trials should be spread through the run, not clustered: the first tenth of the + shuffled order should already contain most trials. */ + int present[TRIALS] = {0}; + size_t distinct = 0; + for (size_t i = 0; i < TOTAL / 10; i++) + if (!present[shuffled[i].trial]++) + distinct++; + CHECK(distinct > TRIALS * 8 / 10, "trials are interleaved"); + + size_t one = 42; + plan_shuffle(&one, 1, sizeof one, 1u); + CHECK(one == 42, "a single element stays put"); + plan_shuffle(NULL, 0, sizeof one, 1u); +} + +static void test_calibrate(void) { + const uint64_t ms = 1000000u; + + CHECK(plan_calibrate(1 * ms, 1, 200 * ms, 1000000) == 200, "1 ms per sort needs 200 sorts for 200 ms"); + CHECK(plan_calibrate(30 * ms, 10, 200 * ms, 1000000) == 67, "3 ms per sort rounds up to 67"); + CHECK(plan_calibrate(500 * ms, 1, 200 * ms, 1000000) == 1, "a slow sort runs once"); + CHECK(plan_calibrate(0, 1, 200 * ms, 1000000) == 1000000, "an unmeasurably fast sort hits the cap"); + CHECK(plan_calibrate(1000, 1, 200 * ms, 50) == 50, "the cap applies"); + CHECK(plan_calibrate(5 * ms, 0, 200 * ms, 1000000) == 40, "zero reps is treated as one"); + CHECK(plan_calibrate(5 * ms, 1, 0, 1000000) == 1, "no minimum means one sort"); +} + +int main(void) { + test_trials(); + test_runs_and_shuffle(); + test_calibrate(); + return check_report("plan"); +} diff --git a/tests/test_spawn.c b/tests/test_spawn.c new file mode 100644 index 0000000..bf93993 --- /dev/null +++ b/tests/test_spawn.c @@ -0,0 +1,65 @@ +#include +#include + +#include "check.h" +#include "spawn.h" + +static void test_success(void) { + t_spawn_result r; + char *argv[] = {"sh", "-c", "echo hello; echo warning >&2", NULL}; + + CHECK(spawn_capture(argv, &r) == 0, "a successful command"); + CHECK(r.started && r.exited && r.exit_code == 0, "status"); + CHECK(strcmp(r.out, "hello\n") == 0, "standard output captured"); + CHECK(strcmp(r.err, "warning\n") == 0, "standard error captured separately"); +} + +static void test_failure(void) { + t_spawn_result r; + char why[256]; + char *argv[] = {"sh", "-c", "echo first >&2; echo 'the real problem' >&2; exit 3", NULL}; + + CHECK(spawn_capture(argv, &r) == -1, "a failing command"); + CHECK(r.exited && r.exit_code == 3, "exit status kept"); + spawn_describe_failure(&r, why, sizeof why); + CHECK(strcmp(why, "exited with status 3: the real problem") == 0, "the last line of standard error is shown"); +} + +static void test_missing_program(void) { + t_spawn_result r; + char why[256]; + char *argv[] = {"greenbench-no-such-program", NULL}; + + CHECK(spawn_capture(argv, &r) == -1, "a missing program"); + CHECK(!r.started, "never started"); + spawn_describe_failure(&r, why, sizeof why); + CHECK(strstr(why, "could not start") != NULL, "described as not starting"); +} + +static void test_signal(void) { + t_spawn_result r; + char why[256]; + char *argv[] = {"sh", "-c", "kill -TERM $$", NULL}; + + CHECK(spawn_capture(argv, &r) == -1, "a killed command"); + CHECK(!r.exited && r.signal == SIGTERM, "signal recorded"); + spawn_describe_failure(&r, why, sizeof why); + CHECK(strstr(why, "killed by signal") != NULL, "described as killed"); +} + +static void test_large_output(void) { + t_spawn_result r; + char *argv[] = {"sh", "-c", "i=0; while [ $i -lt 20000 ]; do echo 0123456789; i=$((i+1)); done", NULL}; + + CHECK(spawn_capture(argv, &r) == 0, "more output than a pipe holds does not block"); + CHECK(strlen(r.out) == SPAWN_OUTPUT_LEN - 1, "output is truncated to the buffer"); +} + +int main(void) { + test_success(); + test_failure(); + test_missing_program(); + test_signal(); + test_large_output(); + return check_report("spawn"); +} From 21b932ed8c677b657488e7930035238b99efc5d3 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:43 +0400 Subject: [PATCH 06/51] feat: add the Python worker and the Python tooling A line-for-line port of the C worker using only the standard library, so it runs on the Python a Linux distribution ships (3.10 or later). pyproject.toml configures ruff, mypy --strict and pytest; requirements files are compiled with hashes from the .in files. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 28 + requirements-dev.in | 6 + requirements-dev.txt | 818 +++++++++++++++++++++ requirements.in | 5 + requirements.txt | 513 ++++++++++++- workers/python/greenbench_worker.py | 295 ++++++++ workers/python/tests/test_python_worker.py | 131 ++++ 7 files changed, 1795 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements.in create mode 100644 workers/python/greenbench_worker.py create mode 100644 workers/python/tests/test_python_worker.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4b63b60 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +# Tool settings for the Python code: the analysis package (analysis/), the Python worker +# (workers/python/) and the integration tests (tests/integration/). Dependencies are pinned +# with hashes in requirements.txt and requirements-dev.txt, compiled from the .in files. + +[tool.ruff] +line-length = 120 +target-version = "py310" +extend-exclude = ["workers/node", "build", "results"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP", "N", "SIM", "C4", "PT", "RUF", "PTH"] +ignore = ["RUF001", "RUF002", "RUF003"] # "µ" in units and prose is intended + +[tool.ruff.lint.isort] +known-first-party = ["greenbench_analysis", "greenbench_worker", "powercap", "synthetic", "workers"] + +[tool.mypy] +python_version = "3.12" +strict = true +files = ["analysis", "workers/python", "tests/integration"] +mypy_path = ["analysis", "analysis/tests", "workers/python", "tests/integration"] +explicit_package_bases = true +warn_unreachable = true + +[tool.pytest.ini_options] +pythonpath = ["analysis", "analysis/tests", "workers/python", "tests/integration"] +testpaths = ["analysis/tests", "workers/python/tests", "tests/integration"] +addopts = ["-ra", "--strict-markers", "--import-mode=importlib"] diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..1b432b4 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,6 @@ +# Test and lint tools. Regenerate requirements-dev.txt with: +# uv pip compile --universal --python-version 3.12 --generate-hashes requirements-dev.in -o requirements-dev.txt +-r requirements.in +pytest==9.1.1 +mypy==2.3.1 +ruff==0.16.9 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..23428ae --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,818 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --python-version 3.12 --generate-hashes requirements-dev.in -o requirements-dev.txt +ast-serialize==0.11.2 \ + --hash=sha256:00119a8fb8c1dc0f1fab023f4d8071fa49e3b0208ee54d589fd463c16ab0124e \ + --hash=sha256:00bbf1f6669f813b48925b759f7ae4591067d456d443924055cab386e7e0a719 \ + --hash=sha256:08eda88a0f290a36c38cab33df8bf7e35eb95bc802ca5beb2c8fcda471a7d10c \ + --hash=sha256:0d01f61352c96370febf6c0dbd488dee9183a731fb2702170da9163ae317cded \ + --hash=sha256:0de02520c11391a026e62987a9aa2c3c2ff01545155059ddf0c4bdf2c5ecbe9f \ + --hash=sha256:13b13afe32e845c86a573497729e1b7ddeb26c572c78bf50ece51da23b8fad5e \ + --hash=sha256:1844ed9a487fb3de7325c52ddb33f2918b66b65cd54d3f8d83d23785ffe99fa4 \ + --hash=sha256:1d6ad94edbe93bf1dabc06c9f37d55b898fdabc456aa6d7ced5e23c14f795f32 \ + --hash=sha256:2fa3be25f7f5351b1b39c9f8a52779b2dbf21199efbae564b4746422e8edca4e \ + --hash=sha256:2fdf31a0bb85ea2575cc91669f005e6647d2efed491231c4dc1497bc9a5b3aa6 \ + --hash=sha256:3b78e6fdef3b06c86ed263e1962fee5a7b9d2d158e738b212d13b2c605ee12f5 \ + --hash=sha256:40b2801cf2221bd922d9f69d2f0ebc373c3db47207315d525b2d87fa161a2af4 \ + --hash=sha256:43b51e6ebe6549bf21416c3c78ee886147b80875a87cc6f69e303dde0d75be0b \ + --hash=sha256:554d117cb916d8032d85007c654d179efbbfd446174c048062778136a922944f \ + --hash=sha256:57c0f5cb0021a5beb1e5e4d6e840ae2f23a28909703ef4d256a144cc1ad3d437 \ + --hash=sha256:59c25f47524efa052971b860e128b1add0c94ede7dd16b2962952c85c3582365 \ + --hash=sha256:6061a54f39e82a9f2cbcb9c268fc441890e4818a6636473caa4f4063254e0750 \ + --hash=sha256:75a1c7f46b9c19fc0ae01ca6fd076301628faa2ed7a8edbd55c6353c483946a3 \ + --hash=sha256:76cc294246e60a914326b4ca88c6a5ea89c064906614aaf1537ce82f09e9449f \ + --hash=sha256:7aaaffc32905159774a107d3cf33dad59bd41b7a0d1bc9885532186753ee7439 \ + --hash=sha256:7f1823275b246f9c7d373be6879e4eec09686948895d4ad083f4b27fd7e4da70 \ + --hash=sha256:8532f20916fa3189d4d785ef2a62d93c4d651ec9c5bffda66d2fc36898351f34 \ + --hash=sha256:85fbb01e83967a126d71f679f2b9528ef0912cb0854aa1a4657314c34e255b57 \ + --hash=sha256:89499a439955931281986e97ca4dd3c064bf0d2e0027c0017344eb86667733a1 \ + --hash=sha256:8a5ffa70e76191dcf240d3c43e20c93b3bfd26f54d89148c762d57837f5bcd2c \ + --hash=sha256:8d62a47714c8bc432b9fabcc29989c815c5da17327d35151f2fd0d85c2a7a5ff \ + --hash=sha256:8df32ad4ff7843734a6c2f067ee974f6d3109ee5a2c3e1a9d2f79347bd282a9a \ + --hash=sha256:976a5bd75845d22f4b52905ddf53ab669ef1b14dba7735f5512841a2ef2b5450 \ + --hash=sha256:9d80a81ec84660422579bdb8e789f656a794b48c7a1ae1261f6bd8bc1897d17d \ + --hash=sha256:a0fd40c668b0fa19b8fdb61d9e63d547e2e19cfbfe053a51ef0b6c37070298a8 \ + --hash=sha256:a586be418eb70a9f1396cea29ddac8f4b9bf277fb73ea2340db31e218bc00f32 \ + --hash=sha256:a7004ba572f09be34342ccb98dcd4bad5707d3d81adc8cb4c3f685d2a2c51bbc \ + --hash=sha256:a9ffa8a197a721f07a352d0be6185f5b3e6f9aaebfdb66169ed652108531ae3b \ + --hash=sha256:ab924ba260efd7509492f272d4e236d24564033f20c005d7c63a107c6a76fc85 \ + --hash=sha256:abdb3e49ba053c3486ac1263bee9f16cc9a4a8abd9f8c90bfc21e3669f3ad9d1 \ + --hash=sha256:af8c003ce721b0099dd55cef4ba733500fc3054ea0cc8565d8957aaf7cccdeb4 \ + --hash=sha256:b17869f4ba261a5fa468a753328a548f4dbaf74b4eadae9e28aff66df7f1425b \ + --hash=sha256:b4e4558956b6a0fb35e18fba58f7d1810b1f2c0e6b52352572cd5dfb6b4ef33a \ + --hash=sha256:b9065dd23131a23b41f5bab3bf4e9b3c350a3fe8e36e8200eded9b729fcea484 \ + --hash=sha256:bfbe47a3a7c368f28836e78b2440a3643ac0ec4c67d9fe53588e1448f0a3d35d \ + --hash=sha256:c58bb119b73657fdc5569692f316e1e25ca114bd62f7782eb527c6be438ba3a9 \ + --hash=sha256:cae5addfbb54cc1d47fe947ef9138e9d83849ed1cbc72b819cf36d96a2315b07 \ + --hash=sha256:cb073bfa15742699d408ac50f60878383b5665ae1791d1b6799ea6f08633cd77 \ + --hash=sha256:cd320a5c4f1f2742af97eea22954f776379175c5ef2504801e9a155f2ff9a4d7 \ + --hash=sha256:d60515335750d431e462af6e722bb55720a5e7827192777bddfd9c4376065a4d \ + --hash=sha256:d70556a2f9230a44c99a655774cde823f056efc34466eabfb4085f0cb1ea9f99 \ + --hash=sha256:daadf1c3e0224621607ffe16f1379e4bd372271ed2e1db8a67878f0bab3ef7e4 \ + --hash=sha256:dab599cbdcb7b45b18c41fad746645580b3a24357082b7f0e8921cd373804f27 \ + --hash=sha256:ec1c20f89c3e0d83576e3c06f79375ce936266591fe0d5fd969914af3185cbaa \ + --hash=sha256:ee732ae167e686d1d3c00f98d7d82b23138304694f0441b14d7ddf9c0f8a921c \ + --hash=sha256:efa819d7c14c8e4153dcd84671826331538be7cbe460383fc6386f5eea5bd234 \ + --hash=sha256:f3109fe4805384effc8d0f8e41fbf875aa8f389af91b4348c1cfb60ea6e4cb82 \ + --hash=sha256:f3a367e0e05ed2d1b747ceb07aa728a8c204cc008b589127e9bd4f40053d7575 \ + --hash=sha256:f6a8dfc5ab204a706f6e5d39c6f77c18c27ef084fa2081803a64a9160ce89277 \ + --hash=sha256:f739e0b601be7300c5697a2573d9200bd1db74b34ab111ef9537b9d5dcd7f106 \ + --hash=sha256:fd666cebd6ab3b3c0fd348a6202c26e18a401ee34293c3804d3472266bc146f6 \ + --hash=sha256:feb16d9c2a720e0120c58dd5d6e7b3c7c86b43249b60a3bc212bcb8fa031e2dd + # via mypy +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via pytest +contourpy==1.4.0 \ + --hash=sha256:018227e134b73090b06f911e773a526c77c9af426be29065500ce096a5accd4c \ + --hash=sha256:072a702e2e178f4fcf96f04775d0c059e4c917925abf0d3208ebb6865df3c23d \ + --hash=sha256:0c7a4c2716a4e98342221954416a836cca77c996c14ddbf22b5f01d5d93ca09c \ + --hash=sha256:107ec46f7aa1664266d69b1181aadc953d58d39f7d8d648fa5b795d4a060b0de \ + --hash=sha256:181bea01bc742734ae672fa00c717d855dd35e1f029536406538c52e7b0cd73d \ + --hash=sha256:186ba929df36d61b6127da2e89cd1357e4cb79aca647381ab1a6feb0b152877b \ + --hash=sha256:196759a3e4db60e1546167e361090ebb6d199aa3aafc9e34ea17a5ac3b23e814 \ + --hash=sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1 \ + --hash=sha256:20da2f86bfaed60dbba729fe738c8241b1dd22b6cf7e8cbbf30df290bd04ba28 \ + --hash=sha256:2deb580178ca19437a84bd77e4bc2cd91a8ccad212413a83c273900868b5978c \ + --hash=sha256:2e741a39dfc96babe1722561e81351aeec104526c57dbe91c167c1db606c2d55 \ + --hash=sha256:3863ef2e2b13fe93f8c0ebb08ee400cb07153b8b0e91c5acb26e4537f283634f \ + --hash=sha256:3e0a2392533e5e2abdf277c951047516acc8d2f3c42b2dd9dc34c907c9ebabf2 \ + --hash=sha256:404dbcd9233513dfd1323b66ea593fef90e899f658f6b8a9ed8e93bd0ca669db \ + --hash=sha256:417be048f7e122cefbe2a34c51a1f2b0410f8eb35796f794d45475ddb7872d9d \ + --hash=sha256:423bd5f4b3f11d54a8c597234e513382e3454bb106b8ec7ae32ba4ed63f8d99b \ + --hash=sha256:432e89f835cf01f8d2123a130a3126b17ee6e0348e4d62c0973872b30aa873bf \ + --hash=sha256:43c3ccbb32c6294b183dcc8e8c46dacf5ecef809497e3d48a5be298eef185ad0 \ + --hash=sha256:43d80072a32299bf945de0a6dd4e034ea0162e832261de2e07a274a5adbba9c9 \ + --hash=sha256:4bffcb2e8cc5c0e837631acab59dd04265fdac376550e7c3e310523877c68cb7 \ + --hash=sha256:510f7d93d94cf6ebf4e2cd0640bea16ab8affac7547230175b6745b596ce0599 \ + --hash=sha256:5450f091ac1be0be3ad3a2a3b3f23b5e443e78c670ced4fd347d626f92a28fd2 \ + --hash=sha256:5545ff38229c15d11d822fc6f7415b932da9f4975f53e305b7b96ebfb01f889c \ + --hash=sha256:5a4c89c7f38a0d7a94356d74e3391073c4325c4d47ad2fd065c3fe7dbae16c0c \ + --hash=sha256:61624f6722480aa7e168fd746164e7bfdf48f8ccb6542f1613741200dc37e2b1 \ + --hash=sha256:618137ec5778fb76d494c9ec854b104a377fddba0d128bef7136cce2c3bf0df8 \ + --hash=sha256:630d32f06cedf37b7f1dd2b6a12e4492826bd477a4c9c7b2e2fee6ce3a2cea76 \ + --hash=sha256:64039341e2d8804f1a13bda8c69083eab935167bbd7e856cf096241f17d5bd45 \ + --hash=sha256:6507a016976a75a15aee47809a34605de6244ccc056e6164bf9fb14b27eecad4 \ + --hash=sha256:6ced1670fafee703b8277ab1645072a226e74f167166eae146fb743916119b67 \ + --hash=sha256:6e697d94e69f499ff6bebb899cae97a58d5d14f0e1fe9568b43a0248d2f9af8c \ + --hash=sha256:6f963ea9f0d68d2b6a02d861e3c0aae09ae25ba21f243fcf281e9fed468b078d \ + --hash=sha256:738c44fa71735a617f36da58e32810512407d283d5d1bb5b1cecb00d7eeba7cf \ + --hash=sha256:758e7496cec3195fc28a28945eab3ceef348c47fb95c7afc2eca2f3901209c8c \ + --hash=sha256:78ed5c5f962b3e156109f0531b174a65ba80d13cc681a70227b12524c162e184 \ + --hash=sha256:79b06c60d5e569ce12c5da8f0c51217cdc484386e2349fa717c30fafb56d2871 \ + --hash=sha256:7f861af508fca2384eef392bc1d8377cfa349b2b854cb11b45d89c75780e2561 \ + --hash=sha256:80c7fc8e7ac217777ce7ba2db3cf8478652b5a4b335283111012ff9eaac9d842 \ + --hash=sha256:86d05cec773c9507a3950122e0e40ce77c23c75ceeb2fc189514e71893cbb34b \ + --hash=sha256:875f42444c9cf48d56f724f2637e60d0f73b3b12c9041e1484580a233edf9591 \ + --hash=sha256:90feb8da006803a95ba573fddc9536357d0c3faddc3a4c9e22816ca49af52878 \ + --hash=sha256:912c6afaa106e2f74ba22b30416b77ef7eb94e3bdc5a4df51ab147845dba9b6d \ + --hash=sha256:96019f059bcb3774acc0104be3360a57b36d33eaaa2e1d7f77c800ca8e07618b \ + --hash=sha256:9c0e07c691f3b3321913ed9b8161c50ea5f77fadd006f52f5e755359b0dcbfc5 \ + --hash=sha256:a03b32c2e7eda8b17757c022162c13c86f5c3d6f937dddf9ba3c3ff7b513953b \ + --hash=sha256:a1c8a74744fa746eeaa12f0f9ed7f8b4add9d8f1b14d95e3b5e133675eac888f \ + --hash=sha256:a3e67bc1a6a4618a1dac7c3053a9ffece5ddfa2046b2670b342cf430bb0b87d1 \ + --hash=sha256:a58b5f130afec61a093d743cdda435cc4047748b6791627e34a68f41ec3a9b07 \ + --hash=sha256:a5e2bb871b90cd7a52bdee63bf80ef7acb46398d27efc66d7679240016efc5aa \ + --hash=sha256:abd0f7e51ecc52bf8c95713bd80b1c17c516a71fa391c54475308a4377903c2b \ + --hash=sha256:b7794ea07cab575633daad8d6e963b662053391022294d3aacc1d9ba9ef54114 \ + --hash=sha256:b9dc493e7924e5aa32d89d0b8e50986280f3d4b284968a792787bd7bcffd3cad \ + --hash=sha256:be85d160e7c795113c2a93254dde2fc7e86b375322570eee78b420969426973c \ + --hash=sha256:bee6ff96653e0807cd0f8eaef00174a031e3a2effa5c20f49a5a8d574b05af23 \ + --hash=sha256:c76f3a5318164db7d9401132fc1b5364b784c613c93fa506e3b5e6d1bf353ec8 \ + --hash=sha256:c8be3392b84cf43373a488874dc1ebf46f60784a70977f92586e18019d71c7ed \ + --hash=sha256:c927ec747633e68c9920bcaad48b0bcefe648812b822f18c34e2d0e2270a3d2a \ + --hash=sha256:cc87f2ffa84a49a77d76cb792f28a48ff3b05a222aa0b2bddbd839ba78e1d5ca \ + --hash=sha256:dd59df9e2fb0aff8bd7fd1adb9f349b7c835245a9d0f18c2f2deeb537190f2b1 \ + --hash=sha256:ddf5a2596d716fd3793434844caf31abc7a40b1e8718431420c89858268fb909 \ + --hash=sha256:de503609fdb71f597634ca64fb4bd2f13d27c97f140e912b1e8ed93544840df0 \ + --hash=sha256:e21d1d4a9db5b3a793649c7ab0ddb4697f2818929eec871a7b9c45fce7b2a1ea \ + --hash=sha256:e439ab450c93455feb0218532ded3e946ec7a9b5f459069f122cfa44b89229f6 \ + --hash=sha256:ea09dc704029930cbd097f75cb0cc1db9852adbcf1a151fac7e9559f7bcbfd19 \ + --hash=sha256:ef47fef9a912c77e3d84b014f701e5a9340f25703e9ed3bcebe37f91fb69dc49 \ + --hash=sha256:ef9440f6f8506246269a82734f5ff9e2e4c5e775b3996fc883cc491c5257eca6 \ + --hash=sha256:f1219a8898523cba821085f2da8a1b962cc696325763f09d7f93538ba43b2d70 \ + --hash=sha256:f863c6100bf926cf47d13f3cd75f9bb8ebd98aaab230eeb4468b91f98f39a6b3 \ + --hash=sha256:fa1b787362a3856e63dd2b89449f6c384d55ee1beb8f94f4bcc871b40f02462c \ + --hash=sha256:fc9feef8f1f001c5b87decadc67c4a5d1eebb62ca39c4763d1237ff62cf2b707 + # via matplotlib +cycler==0.12.1 \ + --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ + --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c + # via matplotlib +fonttools==4.66.0 \ + --hash=sha256:02117d05dddb51e39b5c5a6eed0702ed8e0e1c341cb138101b2446eb479858f8 \ + --hash=sha256:03922992966a1830b94a750961d1ccfe1a7375d792ca61077a5afb719811788a \ + --hash=sha256:056641fcedeaad24b92e343e35e9fd1337200c102450678894b17928482d010d \ + --hash=sha256:0640e69b00e089d6b82e0a54e3a03af84b775f1f0b2cbecc1b0d7fe3980a194a \ + --hash=sha256:0cf33d1041364ebb7a9958db757e213c6d2a238eb41755fbf935db028c2db20f \ + --hash=sha256:107aae7c38f05561fef5103baa98a44400b5f4c69d87839c9a5f8125405e7d9a \ + --hash=sha256:135ce8738a6341fbefeedc061e20c2889350e84405abea7e61b1c9d87e1a1f2f \ + --hash=sha256:15315302c620e6ce8582934e07932dcf442fc3f59ab208d30564200b9a2f7ea4 \ + --hash=sha256:1d120ea0f5260b04e9b5ac0d9239efde4c23d990347d5d5889cdd37221726fec \ + --hash=sha256:221defad3b1949c1fdde2558d1fe780d42f926b683a71a46a4495a0e6a6614bd \ + --hash=sha256:2b278e2abe596872b3c9fe611f956ded85588537974e48e2adbe2d6ee159e099 \ + --hash=sha256:35684b562df7154d7a0ebfa512db9199c7fb0a93b5abfb59dac9d022dbee7aaa \ + --hash=sha256:379b94fe10651d26caf398eb646b3ecde886a9c504d3cce5e46164d58a78e32d \ + --hash=sha256:3889fdbe63e85dbf2a0d9d7c9d90746827ce60af2cdd24f0bb4738380a9540eb \ + --hash=sha256:398bfc5ce9055e0b91a6189f8cf462d4230b38f522b2ceaecb4f34cbf4fb5480 \ + --hash=sha256:3ea59aaec135c96c4cff20d79fdf1d10623993712a77990585a42cfd6a42bc5c \ + --hash=sha256:4513753259f06316bdc2039c9bf3129e74923c1cfad0722b4728a9ce19beeeb2 \ + --hash=sha256:4bdaf80590fcccb1b9d8e3b8ca72225defb14d03b79788f59781ce1bb66049b9 \ + --hash=sha256:5dda4087a7f8f0b985995489b3c0115bd117f18c0fc236013a0e78e837bb9ac2 \ + --hash=sha256:6371e7ed43cc9e23550fe4a1b72af3205275f28dcecac8c09b80a4582e326802 \ + --hash=sha256:6627f48130cdbe04c7e1189bff4192886ecda757f575626ee8468f6580eaecf3 \ + --hash=sha256:67ea5af3ca60e1e5c9b2841b1f6dba5ef16aab581bf463abb68515447154ab61 \ + --hash=sha256:6b11180c4166a4fc353bb02b147322b7c454927bba4f5040ccdef109a73f9051 \ + --hash=sha256:700c448f190cc7bbd9f059bd96131f4770e90938c9692550b142153bea73b14c \ + --hash=sha256:8009e736e1491c569392b2e9782c30dcb6ac879def102c8c2e06add8d2adc10f \ + --hash=sha256:850d50b347f31667d3a277a6e66a4219236db9d98afadb2993b8addf9bd4f593 \ + --hash=sha256:85e2960a29e6882567b981ba408b5a5ca2d8865b8f67240e1b420348997a13be \ + --hash=sha256:8739d77810aeb6d2ecd55cc2e4864d98a00c317fb6131a6d22ca58b3f50cf22a \ + --hash=sha256:886f3029ea10362c592d2e1a5326a44645b85a2a0be6fdb7c9edbdb60cedb0ab \ + --hash=sha256:8af85dfc2ad564f95b3e650024578bb2268bc4d827d418aab9650bbde4b6926a \ + --hash=sha256:8ea6265535b25f8254868f5a212a1c9dbd88c2c4050f3df18d9174949df0d6d9 \ + --hash=sha256:9b0198962bbba64a88bf15dc5d95e4bab8c5208dede22a10fb06280406ed9874 \ + --hash=sha256:a5a0624aaefb0a29e4ed7500a98732a1b0ec2d3040fe86cc8837f82e3fe4f446 \ + --hash=sha256:a9a45a23b020e667499fb2dbce0f1373b1b1b8cf88e47f3031e69ef3ea21f3ee \ + --hash=sha256:ada87643b20a8fa5763e6dd4fa0f5e2901cbdbe0833deb6d5c5a9c3186782b94 \ + --hash=sha256:aea0cc5609a5f2a2500f91bd6e1989fdd22da0f225be00dc1c4cec775838a7d0 \ + --hash=sha256:b02686b2e47115b2378ab6156abd984c776c6faa4f56898fbcc1c30ee387937d \ + --hash=sha256:b432b6b54a0f3be118c6243312c826aa086aa9bcb150160be4f0bc8a21ea3aaa \ + --hash=sha256:b46500e4d6f5708a9127ea12902ba54cb0ec594fc6bec649afc53154a1125b2b \ + --hash=sha256:ba65eb3e2c46ad0922d84ef78e287c2f48b7a8cae4420728bf3b8e583282d6d7 \ + --hash=sha256:bc7b7ddc1a1f46c363354304e9a8dd93722e4a6a24f785015650898dacf40df9 \ + --hash=sha256:c144e68572af1dfcb05fe065827d0920d4118b265504928e1d2262e3701a25a1 \ + --hash=sha256:c3a66c749b69e9abd92e519ccbdea7210522b0fffdba3f6492a8fab461d4708a \ + --hash=sha256:c519433e8632284dbe64ee4c81b609a17e205bc1afff285553723964880d1c89 \ + --hash=sha256:ce818581070527883e3678b92b1ac7ed3569e48c80f51b89b264f97d84be2608 \ + --hash=sha256:cf0033f343cb1592fb24ae3eaf8b58156f208884ac16991094293ee94c585654 \ + --hash=sha256:cffa168522a0e057d3a5628c8ebd43ddf66a34613f42e116bffb85b0b32ce5a0 \ + --hash=sha256:d2d10f89c4bf7cd42b84695da617f9130fa594647ee74673bf8c8d001fa04ef9 \ + --hash=sha256:dbee639a23c4e067aedfbcf84d2e466a71a1a1156ad04d6d09023791ed167495 \ + --hash=sha256:e4677025ee40b3b1420387ca85f55545b8095f7596c3d5d59552dbee19c91f07 \ + --hash=sha256:ee4f85d1341af630d787514eef1a0e1ad883f9d6c11a4c1465faeb9cd0bb5ced \ + --hash=sha256:ee7b9f6c835ea1a9beb5a35c8eb0c62b6a3290f8105d3d0c29a0a18c90f9b8ba \ + --hash=sha256:ef0610dfe7bb5bf574d9bdad6f597403ebc9807d124ac6f148d7604b2609be98 \ + --hash=sha256:f014bbf05f30731bf8b9f5c7d2b4c0e4f28f375926f95b187f85e0bbceb16029 \ + --hash=sha256:f2032ac005e14c56e774fd19f2bc1c22c796c0c3b39d91741c0e3f9575944d38 \ + --hash=sha256:fa995d96ced196388d8ed7a67ee001b181459d230e24bb68b193ae859f5609da \ + --hash=sha256:fd3dcb69d65c05a2fffa5d9b743b77e0cdc13832665ba435065493dad2b57f83 \ + --hash=sha256:fd9297a5f670f3e59ae697289ddad5902d3c4a86188e83e7823bde06334a560c + # via matplotlib +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +kiwisolver==1.5.1 \ + --hash=sha256:007a5553dfc4f4e8d184f588a0200e2cd4b63a59cc8796df3c39909e679dc7a0 \ + --hash=sha256:0324cd2567259b7a095f6cf18a52b0ffc6f3de9e69528ff1bc0e7a37bd43ff1a \ + --hash=sha256:0627b9bceb9c3cdcf12b8a18655eedfed2692b038df27423383c120d0b7dc2d6 \ + --hash=sha256:06a6917674de9e0fe3f66f5430787f59a9f2ddb64af9b714eaec547e29ef5c19 \ + --hash=sha256:072bdb15a3c19a5b5dbc8f8fb1f4e1884bf4f3507eeb4cc6334401274d37a5c0 \ + --hash=sha256:0a4faea5c6db201c6a21391d2ac926ea97acf7dacdbc3c417189e1adb1a00837 \ + --hash=sha256:0ba9527afc80ae3d7814ed98b6572d02bf85eaf48065678342c5f0c6dab7a8c7 \ + --hash=sha256:0d8924877ce22e17326a99a418c3c82037da078df3c6a260b13eca677444e6e7 \ + --hash=sha256:0ebdef3eae5336568147c39a55be6a2036ffde53faa9ca2d978989ae7c2da12c \ + --hash=sha256:1209042a623ddfda5497e4066c7b77651dde8e1d3a9dd97599dc7e97f3b9b78c \ + --hash=sha256:16895f553ee6620a827d2da56b871f835fb70b9216cca5d188e885caf6e3bd23 \ + --hash=sha256:17851e5dad4484be0cbccbde3b15331deae036de9aebd45eed964487802b172f \ + --hash=sha256:1798e83840c3f627246104c4d8a9639c60fa068adf9ce92b61791781fa8a68c1 \ + --hash=sha256:18170a77ddfecf40ec60d0928268dc95880c881864e015a8f34094ed18b9b9ad \ + --hash=sha256:186884a58486651e3c217b6acea0a53eaa9498fdd472057c46f2f0fb5c25aad5 \ + --hash=sha256:18a0cfb124546a4c2e6087c5f3029c7f44b37c85b142e0ced71f73a7599ac208 \ + --hash=sha256:1983f0974a750a6f6556f368ba11105d1d8369c735b944747c9f12ae5aea7aae \ + --hash=sha256:1a7587dc335f2c0f5bd577fd0540bd16c66006bdb60f759a1059f025e6c4f071 \ + --hash=sha256:1acc7e5b7ef05e9da8bb70cd6c7c4513090213d2e1ad9720f599f0bf6c52aec5 \ + --hash=sha256:1d852545c4d0e35a72728d072cbaa59e2fa7dd84bdf01e068d670dd0ceb58eb6 \ + --hash=sha256:1ed0f5e49d0ceff8b72190824d9e59c062fbbc02c231b853112c78474b3f5ec2 \ + --hash=sha256:1fff05e239575b1481b6ed1a782f6fad616efbf1f0b1f44e6e85c4dfe426e483 \ + --hash=sha256:21e46b23a2da695c364124817bc01d970effd5483147f8d66a6a7167e3f6b851 \ + --hash=sha256:22d5e5aaad6be121f2515765e3b1c444352cb8eb4c86510801db8f2e50757316 \ + --hash=sha256:2551cf9917af48ee7c4b29cc82320489508cf96fd26a51f6fc124de661cd44c7 \ + --hash=sha256:255605693a483db7bd5c79f60437f7bf658f7f520d61aa42722e32257c941951 \ + --hash=sha256:26e8268480be5061d509e29669d59103c067a26377a56491630ece11762e3858 \ + --hash=sha256:27add358abe374ebaa3b8763ef380bc99051b5a4b18d94878366a9e4f59efef0 \ + --hash=sha256:2ae70bc59790d2af72a3f76f24b272403e135070340281108b447cb77ea70819 \ + --hash=sha256:2e10ae1bba1899188b33557c10d73affcc12033edd18adddb57d209039976a4c \ + --hash=sha256:3221f78211074f561c44ca42eac0619828171bec15a2c4cf6f7747d07df76e8e \ + --hash=sha256:34633ecf50d16187ab8e5528b7a2530f2feb4e23f300db4672538b51cfc5cd38 \ + --hash=sha256:34ec467940442c9943016fb2d4c81d1ba84351eeca2f1a78f8bc87f1ba0d414c \ + --hash=sha256:37f801b5d7cc0e5a548921308e059fd2b057bb42972b591cfa3049f95423c4ed \ + --hash=sha256:38f6e0deb4d0a4615efe0c4efc5990b06ae450ab50a0b321c0b078b6d238c083 \ + --hash=sha256:3c24cd69455e1b00ddf770c13b6e2c33e07d6dc3f2d34add0bf9277c5c6bbd46 \ + --hash=sha256:3cc210010fd2f438a3ed430b45f1b501fd13a8618bf984dc2c5ce5b69b78752e \ + --hash=sha256:3fa5855898f6d3d01b72ccd48a2d65cbdee301251603fefe34e2025bddba219c \ + --hash=sha256:416ba7ff9f233b7036689bb5a3783537e838ad483f63558d2a800f75afe738b1 \ + --hash=sha256:431dc224a1a92a5c8f582d96e505196a3b5997a7271076678da2dfde67b77e9a \ + --hash=sha256:43844c1a7ad6d723d5b5b4c4fc7f5bd399c40e288120d16257c7c9e8765c6e85 \ + --hash=sha256:44b8faef94f1857e77fa0238f3390ff1ac51d2ea20a487e2e452a59fd2b5f5ca \ + --hash=sha256:470d420f98d368d6f010633a20659b544c5fdfa5329e6b70219f2ef08fd4a7ef \ + --hash=sha256:482676e5bd48d70ac99d9fc78863469845421e01184fa83f1f9366dc49f7e974 \ + --hash=sha256:4d4ca09bf13cff792b1884f64b98ee6c2467930d632233be25c56b442d99f10e \ + --hash=sha256:5025e36fb4fb275cef0a4e30dbb11cb4ae61d1c83deb90189cb5d7e4cafd6b55 \ + --hash=sha256:509735237ae0d849e8a843551d423d2500d2e0a9ac1611a145658b29c0fb9f85 \ + --hash=sha256:534f02c1abb31ed6dbd3515545285c330b2f12d00fdb1fdb71658b9ca5a13a6a \ + --hash=sha256:5978c3340f16a35c30f8ab2fa7bcf559973c55f1a5ef6970e1f621acf3c4db13 \ + --hash=sha256:5b973887ff782cfd6b67c9904ad8ca542e0bc5e4961503408b423b5a688b4d38 \ + --hash=sha256:5c490db2168a508088f59140dd392556a54b8bd1048fc6383c8baff13c359673 \ + --hash=sha256:5d142e352eb13facc7dd047489aebdff6ba78576c239f1ea04931979caaf0567 \ + --hash=sha256:5daa1f19e097050b9c4d9a78fcc9263cb96c9dfae08037ddc1b7c4ad1889f2a2 \ + --hash=sha256:61e9a64c7635095a6bfe483e2ff055d437c59bd45f3617a228b37277f0185d62 \ + --hash=sha256:63fb7294b768f444eb4b068965f2662f28c2fd4161e23bd60fcf3ff27b74c046 \ + --hash=sha256:685929988b208a911f1285e2f8ed54210b0d681a3dc0f03e00d599d291986e7e \ + --hash=sha256:6a797a1cefc8b9c93170db580337e1fe3d011ad18b1299943231279406342048 \ + --hash=sha256:6b92f60017dda7d877fdc546438b5e28f31c523264f49cf5a48c1d0ce1a0dfbc \ + --hash=sha256:70ed9a45c7484d2b30cdacf60d220f494a1763b9fec1ad03285c6553fa0889f2 \ + --hash=sha256:719a35fa1156db3640555f95ebb94f60a444e64d1c69626b0edef5df78eba225 \ + --hash=sha256:74ad5c3dad54a4641b4c28cd15ded70899d04459c6c7aeacafea716be97cce6d \ + --hash=sha256:74ea337e0ec3f6f342a36a4f1b5cd94dd9affddcd28ba9aae2905af932ee8c6b \ + --hash=sha256:75d9b1cf8258462dbdc1eeda718c96ea7f079324c09067f6daabfcf37712b7fe \ + --hash=sha256:77a4c8187a5948d7f8795adb765a3c7b553d07d86d88e43038fc32fc1fb9a3f3 \ + --hash=sha256:7824b5e8bdbf0bccb4ccd37bbb115849a1dc45437fb4de8351385ed07c437ee0 \ + --hash=sha256:7d38b0c279c3032e8c9cc013b405c6df8e1668dbf15465779aa7f15f61201812 \ + --hash=sha256:7e9c01d3dd7ceba4d1d436cc021d40d592466e40b9bc7f5d83dc4e98a5c9cd8c \ + --hash=sha256:7fd82debf43c6acd0a94359d232f6bb516ee13f269a7993736a9ac9f988bb5d9 \ + --hash=sha256:824c3d763a05ea9e9003610145186b0e9848c7584a5575c79bac5a8e7cd80bad \ + --hash=sha256:828f75af2b0080c8a972e75f649ab46af008e92c6104a57a759157200b835b75 \ + --hash=sha256:83f78128fa28705fa85d01c59771c72fe81c11bd0e6155edbb9f818983a7d761 \ + --hash=sha256:876bbfd276473d3daffe30e8c975df4ed9429967b41a6cb362dbb5155b6f13ad \ + --hash=sha256:886fc26012f0e8b5f69d1cfe6d711f6b11f194621539bf8e6bb1c25c5dc82724 \ + --hash=sha256:8a34616dc2521cc8dc1d7d081734da63539f021ac0450ce950908340c6e7aa2f \ + --hash=sha256:8a708a47ade1fe19e8371d5da076bac0dd4b0a5a7985ad6c637f7f7e361b6baa \ + --hash=sha256:8af9b142ad719ae3a911ebf616bc4b78b32bbab84d6a40d3ad2f129670509957 \ + --hash=sha256:8bf4df63592c2a66b4f8edc5df2544998c288aa02f96ce0acd880cd1de8c8127 \ + --hash=sha256:8de6f2a4ce7e7bd27d23dd94abf0ccafe0e0e5cc9c764b0577191f2c25f08f26 \ + --hash=sha256:8f8fddb8e323bd6eee4e54e69a39243beab22689070f4c66b472c4cc88bb89d8 \ + --hash=sha256:8fca690b00c4c48f6c2a547b0160ed511357093a4e4c9b47e0fadf3128066d89 \ + --hash=sha256:9506e892bcc3b409831d363c6f53e5985e1c8d1f6f6b0256d00358684ff85378 \ + --hash=sha256:958254518717542d02d0688d0d20cbf771da5e415e6f49543f92481c850a4540 \ + --hash=sha256:95a02752aa032eef4aed01cda6d9b687c669bd0396bf4519eef8bba22a286720 \ + --hash=sha256:96c30002424670b5e1e46495c2b8cbffef39cf77c1d79e76462029d50339785b \ + --hash=sha256:98b208a7cc42c803445ef551d6753cc42a5ea13e9cab1ee66cd8b9cb70195330 \ + --hash=sha256:9b3092d8992a1d69b7a59c3e39f35e1b9be327a17f68a7c35fc17329e337d6f2 \ + --hash=sha256:9e51c119992ea8820706871c30a4642ec76de20ae82f9b50b9a45517d8e9f810 \ + --hash=sha256:a5716a33bfabb2c6ce27b6cf03253467b3804f83e215f4d202685cf93c6c9874 \ + --hash=sha256:a5a00665d1a0e26763a7338d7e911d4598fbc1d50dd0d6b7919b7dc6c5d6569f \ + --hash=sha256:a5ca5aebae78a0bc13c1943af4af615d4966c5b650b05d5aa83b50e427196fee \ + --hash=sha256:a7b85b2cc6ea45e5f7e8c9a30bc9fabd47cda09106cbb4b967335c3e6c43b69d \ + --hash=sha256:a83ee7107df13abe42a54a6654670eef9bb39425cf2e27f65e0007465e1286ab \ + --hash=sha256:aa7d00b1700966d2917e54d278aba86897890ca9276dd8b76cf6446b6c181b92 \ + --hash=sha256:ab620eb663952455271ac37f9aaad86b73c969c02f11f53cea405b38e96a4300 \ + --hash=sha256:ad8b9671348d7c8716715652ae11f85ed0eb99e265a2df2ca490577d69860b2c \ + --hash=sha256:aefe930d113798330e9462f7874542977869c0613cba3262e2de3a8d5dee8f3a \ + --hash=sha256:b03af77d77e50edba2030fd5f7c352ff209314b09030a3cba7c14edf9a09a444 \ + --hash=sha256:b390aec180a7c054919c04898835e1c77bced23ea8383eb2c570213bf25d1a86 \ + --hash=sha256:b3d78f7bb2b9d9a30345be1474b9aaa8685430b54afb51ba3639b5c6c11e9ed6 \ + --hash=sha256:b5664603a253efd3a75716d793d1d3a6a82723b61dc6db767b2460bbbeec4c0f \ + --hash=sha256:b69602970994a2ed8bbfa78c2f0394a7435226c6040489702d9f0a0ad0c07052 \ + --hash=sha256:b6ae6a0328f0bc035741820fdeecdcd67bf4694eee03972e843663107122f450 \ + --hash=sha256:bad20d4c69c851c982a1e3606f4c293edfd5a87885786c50082412240c4b1ffd \ + --hash=sha256:bb7c99f0673c03017a3ee01e54a5c2617a05468b11eabe513b0080e063ed95b1 \ + --hash=sha256:bebb89489b279b2f5661bbbb2abcc87bcd4a46607bb4a5c966f04f1db6b8df9a \ + --hash=sha256:bfd1de989b3330420e29de39352f5c049905c9e3ee67233a50d550e3d652c148 \ + --hash=sha256:c2306e8bb53601979fcb3fa09cc65e031876d9ae01eff2fcbcd7a84ef94d5bc1 \ + --hash=sha256:c3a4e41e3096bf1f0f1b76e2ffd6d828d6547f574f702d59bdbef7acfa59db9c \ + --hash=sha256:c6834b92dd2428e2dd85ef3d85f723d3c12f20aaf43a2ddd4f944ca25d833408 \ + --hash=sha256:c90d3022d8a94778939cda8638c6c8da8fa757b8958dad7ec868ce29c87681b8 \ + --hash=sha256:ca307d6c259e5c98d3cb9ade55342b47a6839762caf2536f3d7b46ee660cc82e \ + --hash=sha256:ca7f6fe0f37ca978a1e5eb7a3a68e6413f417e78e838324947ffd420202b198b \ + --hash=sha256:cb6fae641357ed2f6e533c0d3c6504a4a5703621a50c89459e46051d56b61140 \ + --hash=sha256:cdaeeb6c350106df6bf9d873395973e5f066a9713200b72cd64f55d0a3eafab6 \ + --hash=sha256:cea20da04494e662b83c872683bf4ff2345206043d036315ed0e924b652e7294 \ + --hash=sha256:cea90547bfd93807e0013a004dc76552be44fad3bc1cc2b38610a9e889ed098f \ + --hash=sha256:d09037ca068d784ebc4aec290ef952ca27ac15dd9c0b5801a88c6e1096b83e6b \ + --hash=sha256:d27c2123977cb9269c30a49ba45f03a4323017ef693e19db4ec9dbe1299a3002 \ + --hash=sha256:d50de98e8d807dc31822fff96f50293163a62418eb65487a21b42713d72ed0b7 \ + --hash=sha256:d66a64dd5dec136040ec2ae94aa026a912ee60fdd45bc28d3db30037fd809e88 \ + --hash=sha256:d79308fa689fac89cbcfbd4dbfc80b5f95c54c5a7fd4d194be221f9d33d026e6 \ + --hash=sha256:da3275833be0edbaf4830fae08bae3dc7219f40ce0c37eaa6c25825957e06612 \ + --hash=sha256:dc1a26b8e53395a01c2c611e58602fa47461f136fba7cd5542e6db6d64be1839 \ + --hash=sha256:dc23390afe9f4ef9ac3bcc72a03a56eebbde03f4c571a32cb38f859cff9a6524 \ + --hash=sha256:e05c2f7925f1d88778e53cb44f14e0223204a3bdd09a41664750363acfb1f2ef \ + --hash=sha256:e12dfea7f5fc2a34a9080efbf79c4c44eb380ec5b9c6fea09407e08f0d1e941d \ + --hash=sha256:e4e4523d6f336708d732516e6cfca7796cf3d96c9474eb5aecf6165f2f1fefc3 \ + --hash=sha256:e4e49f7e1a4e7191bdf9dc67a974db714501b1fc52c24324103d06a86abd5c08 \ + --hash=sha256:e68e151428b5384f766cd25739bf77c7e4a3dc93b5ded7a12118d9fbfdf78ab6 \ + --hash=sha256:e8e4d953faaded9ec7ede36824e9814082d22d4c7b1eafbfa079ecba8cd0d076 \ + --hash=sha256:ee9df1f0d77b9c6e94f4ac0fec533fbddd5ea3a327807f18d7b069ae019ded80 \ + --hash=sha256:f0a887b6565bbfe80efde2b7f6e8890d7d9bbdb11bdb17028a3690c32fe0621f \ + --hash=sha256:f0f4a42db92d6ec7677ab9d12830a2a8ec145a9c6d15db2b593466bc875c78d7 \ + --hash=sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a \ + --hash=sha256:f1d56ec54d257d05e0b50f5780d967540cd07beeaf9e5f645b26d50cce79f4d8 \ + --hash=sha256:f4167e87b397f273dc2356fcf1eaf50a6bac51e6105f45103ef7129c8efb0255 \ + --hash=sha256:f76fc85bd054c806960f917ec0f329e24e436f1712267d90588e4c39890caa63 \ + --hash=sha256:f942903fde7363d1d879057ec5de01310efda2597161784d752fa9953a01a71a \ + --hash=sha256:f9b1c4900736e489a812c529100de4b8fb617d4db075e931e213c57424b83d9b \ + --hash=sha256:fc271a6f0a2126958f4090e5507b9da5848927dae331f8f763bd4aa642b3d2cd \ + --hash=sha256:febcce10f2bcdbb80b4ea919238a6a4ac13dbc4c7cadbe8d5d75c3682f8b5404 + # via matplotlib +librt==0.15.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:04d5387b908676c0b8d5d2f5fb58373b4ea382d81f7a6f0fab8ea2a462bb4738 \ + --hash=sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302 \ + --hash=sha256:0a15cb554761247d84a3ec0cbdf4078d70725384f0e4662c0fa3b26266eb60ad \ + --hash=sha256:0da0d94cb802f32a0524653e7201f2cef72d5f700a5407678f5290483d4fcd08 \ + --hash=sha256:0e2d0c0acf5b0ada7d045912b7cf787c21315c95b38b1fa939ef72d45d366b3d \ + --hash=sha256:0f0ee3644d951f31055ad07d77d92520e84505dd7a432cc4cd501dd70ee06785 \ + --hash=sha256:1172c6ad2a88b646e7fe3b480e3fac4ab4418b3443fd8a4061fdd531e0622fc7 \ + --hash=sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9 \ + --hash=sha256:1a1a8cd430c7dd0c083f455cb1b328d7fc682b05c31b940906f7845bdff80881 \ + --hash=sha256:1cd3b721f24c206398b9e26da3c3a9c011e6e89d06f318ba8ebefc30f1003890 \ + --hash=sha256:1e47b8ba865d7ede071a91a7163073bbaeb72541f1ef8a07d512c45c7b5007f2 \ + --hash=sha256:1f4ef2e71db33df4309167ed7f1520c4fae5e611226e159fa9cf33f93e6ddb3d \ + --hash=sha256:2067ff438048cead9d223ca5675bae2a25e520a7c3e6c1498bf9c6892d22caab \ + --hash=sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13 \ + --hash=sha256:234d8d394721fa0d786af15ebf1f3fb7f3ed82fd1cd0cde45c2f247b5d4281d2 \ + --hash=sha256:256237037a3ab001ae8d9803b2d43562a4c3aa38739843694349e4d5ebb0fd56 \ + --hash=sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b \ + --hash=sha256:29c4cab9df457b19672c39be7f384ebb2bc925c4e2684b8780c222b43eb36389 \ + --hash=sha256:2cfd1a81a648806e6a7717be4cc4d1bb392fa229752bf8444ba365e381e984d6 \ + --hash=sha256:2fde98cf1fc4bac144ce23c2c4c017b924ba714509ea9334977b0b27050c837d \ + --hash=sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d \ + --hash=sha256:355e3a4c725225a14262004fc1872a552b9d3634b4f791a0dfc80804aafbfd55 \ + --hash=sha256:3722a099730704c9a3d70c879fc0f51daec25fe5f1555672d97bc595abeafb95 \ + --hash=sha256:38c0c7d4b6fc06c3324b3f9162c8391bfc4fd9dde53afe1033ce7edb48d5a714 \ + --hash=sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416 \ + --hash=sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54 \ + --hash=sha256:3e79f05e4a08b4d880342673312bbc895b56df7765605796f15902eb5367d3ae \ + --hash=sha256:3ff5893a2c23d886aa9ce786de5ac6ddc74aeeaf90743682b74d920e117d2e28 \ + --hash=sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d \ + --hash=sha256:4388184646efe2054911c5b00a1077d6d1ee86a95b7e8ba96dc7850a809f3f40 \ + --hash=sha256:4a6369168d371207339b1e50d4532b06a7121586141f82599505a3f315751d47 \ + --hash=sha256:4bbcc257e3babea20a91715c361b24554ec4e8f51aa578568afc230799fe1a19 \ + --hash=sha256:4e66cbe84437497d951b799d3e1551291b6fb3d643820a7014b3655d57a59162 \ + --hash=sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c \ + --hash=sha256:4eafbaff06b9563f8b1c850621ce51605de05208e09d4d71ce490bc972b7b9e8 \ + --hash=sha256:52e8db01f603f5da0ca30987479acff98769382efc8e142fa3962395dcf3ffdb \ + --hash=sha256:5500eeae393a184d14e1f35645962c27129d20c81afa4069e6ef826ebc2b3aaa \ + --hash=sha256:55456ea87d8df21808446d03817be2f65e20391c1c615d9187440dff28cd08dc \ + --hash=sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988 \ + --hash=sha256:567b1c430f8bd560e689421468278ac5941bab4a05303b5d95b6ae10db03f451 \ + --hash=sha256:57f5eeb6ad4c180de583b1038e61fe5fbd9796bb69a8a1c1a0c7ddbec4c8c60f \ + --hash=sha256:59fe030d8ae4a57e3fb7756bf35a858de74e04066fc8555c53d0af979132af81 \ + --hash=sha256:5a6526a2a956bbb1e4ae3568c82e650fc99119c66bb011ea60715744955a2b4d \ + --hash=sha256:5a86a5a08c2235316bdb359d5dbb6ce0abfca7fac06363103e2c5af571d92f95 \ + --hash=sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605 \ + --hash=sha256:64b0c8c35aa4c4ed79896359f3e0b285cbe4e610042106500da4811c322cc108 \ + --hash=sha256:68242379c9b65a582b6e97318a1e9fbd6d445e58954f2d437991c4804ab11578 \ + --hash=sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf \ + --hash=sha256:6c013cd3a1721e69e14380ada97eaa4b7b0cdf1c6b96fa765d4ea47c875088db \ + --hash=sha256:6c0eb900c0e91f4aebe680845242e614f1864edfd44106380d0752ac29522bf8 \ + --hash=sha256:6c6624fe268625869485553dd7cc1daf30d22558215bb2a4ff16f67a9801a31a \ + --hash=sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965 \ + --hash=sha256:6d28a05796b99f749bf8794f17ba9ba1612d0076b802e9cfc62c554634e9ce3b \ + --hash=sha256:6d5225ef8801e4ea5e482fa9b5dfb891dd9ef6f6d870f1f25d449ca2c70ac218 \ + --hash=sha256:6da110e5f314c19ab8478464d02ae18808ae73d522c15260fa4918acdcd64da9 \ + --hash=sha256:6ecfc32dfb46fb7b565bcd6abf9412acf978775a998273d22888a6d7953730dd \ + --hash=sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b \ + --hash=sha256:71599e011ac880e8e45d46047d714871894c7d4ab6f25626f8d4f89da21f368d \ + --hash=sha256:7220697efaa6e5348fc3d18ee7f8563d4bfecd9872b37ffb915bfc1d08840622 \ + --hash=sha256:73b30cfa976659b3917c8f6153bdb0591c6a9ec6583599fd24a689b690622022 \ + --hash=sha256:779a6e7c894737e5983e7790a9c78c4000c30e23c9aada08081bdbea53b0fa60 \ + --hash=sha256:80811e1c42386ea95c6fb30571d3250ad43d7863f883f787f70517f441150e59 \ + --hash=sha256:814ff83a25b5fce8b9c80c4dd803153fb5c5599fc74db9e022466938368957ef \ + --hash=sha256:81a398f45b45a59200e13cd5ad1ae1d3f44334de98b148331afe2cdfee701c52 \ + --hash=sha256:823b92cf3c18ecd08afc70c42473888b41b6e8ef5046f3b82c05c154a2fa3d22 \ + --hash=sha256:82909c8f7eb9952656b65d3147afde4cf8e6d5a991eebc86418b5e65843b0ab8 \ + --hash=sha256:83380ffde38062a2e9bb55d83e74474f6614665528b98a6928720fc006dfffbb \ + --hash=sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca \ + --hash=sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc \ + --hash=sha256:856f743ae607f2c1380eccb566c0038a9fb3eabf0fc2be2704d76d9f73557239 \ + --hash=sha256:85ea21ec6730194d67156b0e0b5430ccb1d61f8b8b907e39b37f9812b74a13f0 \ + --hash=sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0 \ + --hash=sha256:88c2a17815c266e6d8180204ff62cb739ab869ada4a746d4c505331526ac58f1 \ + --hash=sha256:89cc46cfd15022e35084355478c9ac809d90b1152222706ac9a7655ec21df6fa \ + --hash=sha256:8ae493ed5f659a7761c43d42f183db514536073ded9bcf671d2d1df47e29a07e \ + --hash=sha256:8b2fdd7ead3c995c37940a790690660d0ca006c302db26cc51933f6766866fc3 \ + --hash=sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0 \ + --hash=sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915 \ + --hash=sha256:92bfed8deec93df30286b9fe9e3b1dd17329cc076a192b4ee5ec223841d54953 \ + --hash=sha256:96bb17dbe8bab3c0954fbebfc69ed395599de75b6bbc35e3270a878e15d4dd65 \ + --hash=sha256:97335f59082f9fe2ce6c2a9cc6433a0114bbb6cd4d5c09dd76c95c68b9f9a8b0 \ + --hash=sha256:a417149c0cba4d50b61e992e5a15e69eaf96746609b461cc4ed168aeef6b79dd \ + --hash=sha256:a5207ec414d1c4a2a7231b2086970dc036f94293cdf338190984958a013a42f1 \ + --hash=sha256:a54cf9e0ef47b96af580849db5471142200568ce1e02cbf416addab551369570 \ + --hash=sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c \ + --hash=sha256:a5fa8f1f916988d0bf1afea005bda37f56ac41a18016e813ccf0097a8d460ca4 \ + --hash=sha256:a6cd22c9da0d866558e46a041f1cc0c2bbb26b61b137b2347fa834c332e1d101 \ + --hash=sha256:aa1f1995789dca3698bc550aaceb09a51bd5df0a057ff84ff15296cd1975b801 \ + --hash=sha256:b0411b4066db926b80258c60dcb0e6db4c9cee312eab45b7e8866b17ddf9ada1 \ + --hash=sha256:b230acc1c3bfe2d6f2627ba2b95dc92e58aa494600e9722d0e6ccbc931e59702 \ + --hash=sha256:b30e600e8f337b9bd7f39b86d9fdfedc73cc46e3d0f745931a23a234220bb7e2 \ + --hash=sha256:b845b8d48088fad0cadc84be4b8fda63203be7e9237b71015b3925443c1f35ab \ + --hash=sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8 \ + --hash=sha256:bac89069bc496ebdf4f79ebb57bbd10d0b214c8454225deb672d91002bd17e18 \ + --hash=sha256:bc25fb356d0c7810bb49ff3df908ad1fda6995d660ab099ded69244ed7ab6053 \ + --hash=sha256:bccbd8e5b0bffb7106cf18eb1baa3d7194b1cebb3b4b1cdbd4bdb19382a6ee6c \ + --hash=sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162 \ + --hash=sha256:c434e072557ade9cbc642d052c89d031efe47d5c9614523619d0d74a02378e81 \ + --hash=sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a \ + --hash=sha256:c70bc1b602cf59917e8f0c7a2cbc8bcc6fbc14d5486136b00707a79619121d63 \ + --hash=sha256:c7eec6a42018bc1d45763b1c162d3d2bf7c3b9a1b0ed30d3e91dcba390efefcc \ + --hash=sha256:c802434092b769b1d613ed2e13fac15fbfce1934a74bd10283b03c0fae231cd1 \ + --hash=sha256:cc30523e3f1a23fb7511cc659834a0d01a1042bb9de359bc1c131cc4ec6c9656 \ + --hash=sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d \ + --hash=sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9 \ + --hash=sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad \ + --hash=sha256:d4c7bacb70930f3d0a56f4ecf1be474a1f0d941b01dd73b756f3c256d42cb879 \ + --hash=sha256:d5f51401d102c885b9ca509e62c79b1dbff286e1b9b047fde6f763780789356d \ + --hash=sha256:d8363d7accb0286ac3a0e633f396e93800dafb8150494505daf9515bbda591f3 \ + --hash=sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328 \ + --hash=sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659 \ + --hash=sha256:da7a94d6a3411f579d72aa3e3bc5fbca7ed4549f3dbd7e5de3aa567333374285 \ + --hash=sha256:db13ca398005abcbe538deda87b686d9bd08b7001cf40c4c06b444960ae10a26 \ + --hash=sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9 \ + --hash=sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1 \ + --hash=sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6 \ + --hash=sha256:e0d00c708fb2f5822b152429b1ac80a58dbbbc3f6c232c4d13a3f7fcf2ea5b4c \ + --hash=sha256:e1a49adf16a7c9d9646816c2946135527197b6fcf4347c7b8b761cf1bfbf4489 \ + --hash=sha256:e3b461183c5fa7681b48560f91515f53a953122fb30c71e07abc67d7ddf58c38 \ + --hash=sha256:e4c911f15a1652ca94ae9f1abd92e74cbb1b3597d2d92fdd556202f94e8cd455 \ + --hash=sha256:e56b6a368529bed262da40ce13f8fef590db0479819cca84f16a1f01ac356d0b \ + --hash=sha256:e87bc679f86a99aa3b26e3c78eeb821a247c9a28eae48eaafcc32c3bf4c3bb9e \ + --hash=sha256:e8c9a650a188e38bac005048cbe6342e81407782944d01934540ab75e417df21 \ + --hash=sha256:eab9208b00ca55bf75983ec99f7bf13acc746a36102e98953addaad7f7ea1e1b \ + --hash=sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374 \ + --hash=sha256:ec4b19788f835711a2072f9dbe6b03b3bf32ed1f0fb30cf399bdd59d9f0c33fa \ + --hash=sha256:f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8 \ + --hash=sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab \ + --hash=sha256:f54598964d357b1c5ab77cf5d92f21e598fe0e23cdbe9618480807f81b4eba15 \ + --hash=sha256:f56b397858a23dacf35ede366ed2212fdc03a6a57a1ad36468ad6e9dc5fac091 \ + --hash=sha256:f5de7feedc56337a088eb15cd9fafa9938367362221d8cc62c642b7f94821993 \ + --hash=sha256:f75720477ee05d509a310e856cacc8d909adc182f7b91193c207bcc26d7ee6db \ + --hash=sha256:f779070399f991400fc451719e0ea388eb7de313388bada2c127a35de05f798a \ + --hash=sha256:f9ca190fe9edc0eb08eec558a509a16d28d91c35667b8f043cba40ed5e77a959 \ + --hash=sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9 \ + --hash=sha256:fc1ed11c4ad0b91af24def2050f2840ea4567828e3dd058fbe608d982f6e5465 \ + --hash=sha256:febb1ce6cac545a54e6b769982824e955a700fdd9fbf3a08a3d82c990968b57d + # via mypy +matplotlib==3.11.2 \ + --hash=sha256:01dc8eaaab5a9fce9ff615eca82345728f289e4715b186ee10c6d85272fc26bb \ + --hash=sha256:02329432ae5c6af87cf208ee575b701d698bdf0b1a3bb28cc6d53c36e967e575 \ + --hash=sha256:07d9b9fa60cd4c393692f50d0bb03123242ddf61c99bb0e95e75feb354e7c1a8 \ + --hash=sha256:116cdb0eb0eb5644fc98eb2975d4b4dd4ad35e5c8e6b851c22e6976f371f7ab5 \ + --hash=sha256:1944895967f87c84c9b4bad29a707b31f5b36df4a3a2339ea1f8ea3ce5105539 \ + --hash=sha256:1a3040b209f3968b4e84161df7b174f07a9fad33b0f2d7e48ea3bbd3075e2863 \ + --hash=sha256:1b9a7ad579856284135e401ecc918c5f8a017ee30539298862a109f51b971710 \ + --hash=sha256:254d4ddb2fa8df3b4c689c0c306063aee10521df82cfb438185e499c75fe37c1 \ + --hash=sha256:2a8285cea8ef4d92aa041d1c33788bcae82248503300f93f1ca2136b9049452f \ + --hash=sha256:30ec15d7eefee71de16b7c689b42ba49715a4644650b76a6c7c70d79daf24e91 \ + --hash=sha256:399fef672f7046ef7d6a57572b2a6f9845f3f5afcff04e7b2df7a363a9f42190 \ + --hash=sha256:3aa4b8516fd26659e4363abbf317c703d9116496c5db2e9d0609a2866dd39dd2 \ + --hash=sha256:3da3bc0cbf7245e7db72cc6d29d12c5abef72cb73059c73b945f14e3545f3eb2 \ + --hash=sha256:3e8576f7c47e02fd4f21f44171302d2d1d58d4471d46da3d71fe8899d19539d9 \ + --hash=sha256:57b9ea60a835937c2012861923cbb91f47db8565775d326d1c42fc926aa10351 \ + --hash=sha256:5e1e923a3fc3326b99ec0a6ff1ab1338ac6c6cc62ad9d8a9c944197c7f8c6221 \ + --hash=sha256:643ff850d8e0f5b8319337f87ed3cb59506afb3df3cc48de777d85871233be7b \ + --hash=sha256:75b6d88402770e181b5d06a67dda4129c31da7d05004d63a21c310ec78d1b83c \ + --hash=sha256:79a258f58253dfa025af80a9e9bb228d75fced007f0e93ae7423fefbde81a74d \ + --hash=sha256:7d43ff8cebb50840648cb6429b2228621dbd709010f3117b3740abb20abf21c0 \ + --hash=sha256:7e5a90f8a707ebb6004a713b1cff091a40cc5df4c7c1cb165e5a505ebc11c292 \ + --hash=sha256:7ef7a53b66780e5d942923724f08577fbc5be1f7322da0f0f3f9dcaa45dd803d \ + --hash=sha256:854df8d7dfe9fdffcbaa6f39e44a6c24b409cb4d7561fbc09213b157d833f6a6 \ + --hash=sha256:894a9cbbecbe30ae6787d464df2e8fc7a8d475cfc68f87c3029f7c11152899b3 \ + --hash=sha256:8c8255de28f986d935a64c9ca71c0ec2d2f41d355691f5ea684725dc91413f71 \ + --hash=sha256:930efb28f59fda124e39265d177bab302625297bb147d2910de725a2fc2aef54 \ + --hash=sha256:a24d5fd36e4f0e742c3851dcd20810e56a95633a342e4bf6cb591c678e8fe61f \ + --hash=sha256:a6939df7567114b6bac7f4c5e06c84a67f197c1b2f2e4b234d4eecb3bfec9482 \ + --hash=sha256:a8756cc73d9af9a7fe0deb54ea2e75ef73b01d9e575877e72acad5458e660943 \ + --hash=sha256:af2661f6ac6bbd1d081996f54fdd9385715c625ace8cec0869055c0cfbf38981 \ + --hash=sha256:bbf1062991d826ed27e2144f3afa4461afbb8ff56e8f703043e191a8163b1ee9 \ + --hash=sha256:bc067c462a86f0e57bf52fc6e058d90171a5420007dac00c46e90153050c69a7 \ + --hash=sha256:bf3fe71fbfb8ec0e310e0bc8537c3405a01f38f25f9394ed2135e6202fed542b \ + --hash=sha256:c0b83f044ce10a98027b105b3931548719a6e8c7ef986b4362651e0b5367c8dc \ + --hash=sha256:c27e577ece613ea12a0790e00b4eb80d901c59a3e16cad474f31d8b1529690b9 \ + --hash=sha256:c5c1c68ee401fc98271263410f0e5ce88285abacf7627132914e8adf3d70ff43 \ + --hash=sha256:c9721f81275499da1feeb36a2cf8192ea086283b3bd16b7dc4c9d7aedb7396d6 \ + --hash=sha256:cc82dde2a0d3e3ad472edce04897ad7146b8d8bfd1df8a32992eebb81af18fdc \ + --hash=sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4 \ + --hash=sha256:cf41ecd1b0c0b6f7177ed965a54c2afbe888715c7cf6054dc12d53bc1494002c \ + --hash=sha256:d3304eb5a59442a8867f6920d484591c0fa09ffc29e9260be2feec3351e25869 \ + --hash=sha256:d480038c83691532ed52ff3147db51fa902fc78cb2d8349993a1cdb684435bff \ + --hash=sha256:df4f7784aca81a94f254c0a2767d592ee25f407e488f5fa7203e51093fb6ca27 \ + --hash=sha256:e43b188f0a5b75447bcc197728258166aa64365770ed1caa36595a5e1ca4bbba \ + --hash=sha256:e60cf3047a51edecdc4535a9196bbd6a732936b8e9f8184aabf4d16165884aaf \ + --hash=sha256:eac4b07d4e3743b172451e122ea964f72f152879d4f9adcf3f3d33e518f12ead \ + --hash=sha256:eb3712dc9b464793de0a4e42a7313d50293c751f94bddaf7332a1bc71bccdda9 \ + --hash=sha256:ecea603dd2fbf8242fd31a305a8b12a4ece2de28096870c65fdd0d1e35b8d9a6 \ + --hash=sha256:ef31985c4dedb5f1424e1aec6849a47dd37689cb7fa3c20b1b82187f26806261 \ + --hash=sha256:ef752769cd962f39ea0b6ffc82d1ea43a0012c5a6157c7a075212fa509cfcff2 \ + --hash=sha256:f25446b2981717dca9786bac841cb3fd7efb568e3e3c755dd980481c5cb9228d \ + --hash=sha256:f2ac30cf5eb5dff1b584627ae0b0e1186551a4f69ae3c75073911da497a29170 \ + --hash=sha256:fea03cf56568cc1cba08b470be6a0559e71c3a5b688d54b7179bb35ba23d0821 + # via -r requirements.in +mypy==2.3.1 \ + --hash=sha256:114dff494000f18bd10d5d95d84b8567b26da60279ecbe838131841df20e635d \ + --hash=sha256:18162b128c3f9c703cd35f5537446900b0d21a2549aa7a95d21380d2ef643fb0 \ + --hash=sha256:192abaedf75da1bc0b1cef104927e70ec49c1ef0031cc4825c7ee10a438ed24d \ + --hash=sha256:1c80fbc405ed8020f5ff3802dc18cf060197bcdd3fbdd6a26ef2fd34dfdd5226 \ + --hash=sha256:2166b29228835e1f88ff411e96639e6ca3c7fdde84b62ec211f70f86b4051167 \ + --hash=sha256:2329c0501293d4e1f33bc15d04d6304d65a1cdda967ee93a05c1e681a3923133 \ + --hash=sha256:2a0ba2e57847849fb0d1fcdabb32786d223095ed8bc121dfe322bcdb3d9c46bc \ + --hash=sha256:30c0477d4aab7b7f39c8397dc877f2c96b9fe5588ec379f372c56eb63d599f63 \ + --hash=sha256:375d7013876a8233b2d05be185bfa09f689696cd999ce8b1cfe6acac5c80e8a3 \ + --hash=sha256:3c80cd23d85368bdd9f37d5231dfd97d35bcbf5bf41af96ef3a9b078ad1957f9 \ + --hash=sha256:3f7e865dd51f235f60a2dbcd8728a1c095f5ca28f095d48a725b84cd935735c4 \ + --hash=sha256:47a8a7a0a7f6f6e63995c0ac36fa0c07b127413fdc81f0439b7f3dccafd33561 \ + --hash=sha256:47c1b1207258513a9d93495f69c8be9de73916186f0e52703e8c461b7a623419 \ + --hash=sha256:4956f34d145e145562a0a0bf367f642bbc85c04ec2baf47ae015947c3169a85d \ + --hash=sha256:4c5095a327483591c94e0c8d3ef9e50d4ab1369b541eae007c1f23bc2a41f6bb \ + --hash=sha256:5159ae60f5dbc3a498af5ba8365505808ac8031bc63f9e00304ad545d40bdd9b \ + --hash=sha256:52eaf3a155f35cf80b40220288c861eb45f14a2340c1f6cbfbdb0feff32879d1 \ + --hash=sha256:57a936373fc690c43a8cd7e7e12a35148e4ec5aa7698ad7fc0a9f918bdc5be41 \ + --hash=sha256:586b3612214cceabb3c0f588c97e7d1e535393f06a60e912e994f6b3ace97523 \ + --hash=sha256:667196b352f4cf304ded4c10f90cfc179263a1acfb3cdcfa984bdfd340d498bc \ + --hash=sha256:6941ab3619377bc3f32ca02876b07d27f216f5201604b664d3937ea0fdd23bb4 \ + --hash=sha256:6ed5c7e3419083268e5c9258bd1c1ef91af44a9e89374dbcaf37b775716e72eb \ + --hash=sha256:6f041a6de52c9217ca125e78ba0a335cb7fd98a1c0580978e49ab2b126f70b57 \ + --hash=sha256:71af9c8a894e862b58e92abb08e53b05a384a1e5e5d6dc7cda59126211a53d82 \ + --hash=sha256:77ad9529e67dca28e511f5cd5671436584ce91f6d3bac159a353158187b986ac \ + --hash=sha256:83d36c2924df7426333abe7faf4724a7e1aab0d9fd41625e81b4683034b80c13 \ + --hash=sha256:84081f538ce27375045c02e3d7f81bd11d853400621ae245d87ce7b6c420ec74 \ + --hash=sha256:851833db876e7b650f93719c74b7879a08e338979c96054fdfc3bfd90a486355 \ + --hash=sha256:858fc57d3d91fa728e33e7ad71def60fc6272694607b306cd3292db53ae39080 \ + --hash=sha256:8ad80807dc3ab8ea978b1b2b6e4a657194ace1d4ef03e0e731aff1abd517da29 \ + --hash=sha256:8e036f06b41630f4c8a1d48f9ac6aa26acc65f8be089973f5519da643318f03f \ + --hash=sha256:94f04929f1c44c35fb0061e912087edaf504acede963a4a7d00680bd089d8531 \ + --hash=sha256:9b4eacbee8a69836c06eff6d0dd4e134a07c2b047755b30c08625fe214f322c6 \ + --hash=sha256:a32bbbb940af990d3be0b8af321c7b6815bb1b3b48142fe7459b9cc5f58959ff \ + --hash=sha256:b091a455111214cb5c9d54a57b9618e9a49f9fe2a42e4e1ac86e9d104ed96ce8 \ + --hash=sha256:b9c53e395c12cad2c6d4b67d5da7c6057638a132d85c08b73646b18f802a0045 \ + --hash=sha256:bb26deed807bdb0457cf3e3f1cd7c4a1cf9d66864eaf1b4a61e06805d4c6b1f9 \ + --hash=sha256:bbfe022634a2a195406bd469e888d2eaf193b02ba7e607391cd7640374aaae3b \ + --hash=sha256:bf678dffd16efcda2c15cbd30e9ecc0081388e29ea23687a88e686ed92638dc3 \ + --hash=sha256:c8637731bb5eee3671eb2c3200827aa3564ed8a9309ecee4d1afe77e6d031bdb \ + --hash=sha256:cfb12e360242d23d91f5e978d94f58ea66acf5804c4fb6f2f794a20d4cb1b595 \ + --hash=sha256:d00d769056bde2f4e69c175071eba45cfb44fa1ed92bdfbfe64a93e0543b0cf0 \ + --hash=sha256:df12e20c9efd614738c71b390007ecd0181125afc4ccafca04d78a1d2eed2c01 \ + --hash=sha256:e099200a1b1b1223a4951f0a90cbff1b8c91b250ba599dab1f7217a628144d90 \ + --hash=sha256:e598c8c66401d26b150872154a286e6d484cf2789c3bb28a7556806298423021 \ + --hash=sha256:e5f1c50bb05b64e2026b52867e8d21106f01313c744a2c4ecc34c90d12e8d6e2 \ + --hash=sha256:e9144ac16fde007096f9563eb2041b4433c2d705c4218edeb79e7e9d01035ee6 \ + --hash=sha256:eda22fd4efa9dcd39331d1dede9b5b8b8a7fd69af07592e778433da98610d29e \ + --hash=sha256:ef0c6335cda9d807f8193d8ff6204a72bc909fa9882aacbca14f43cdb7188306 \ + --hash=sha256:f12fdb70459d0060dea40b29e52163a961b156106d68d57882a6a9f648983a53 \ + --hash=sha256:f5d716048611e85ca9eefb2e1baa5d73ede389b5820ded260ea27c757d667af8 \ + --hash=sha256:ff715e45b2231a8e85de1d163d1b42791e4d7aab8f5145f85fee1b710b735aff + # via -r requirements-dev.in +mypy-extensions==1.1.0 \ + --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ + --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 + # via mypy +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # -r requirements.in + # contourpy + # matplotlib +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # matplotlib + # pytest +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via mypy +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via matplotlib +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via pytest +pyparsing==3.3.3 \ + --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ + --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 + # via matplotlib +pytest==9.1.1 \ + --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + # via -r requirements-dev.in +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via matplotlib +ruff==0.16.9 \ + --hash=sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385 \ + --hash=sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161 \ + --hash=sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6 \ + --hash=sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee \ + --hash=sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d \ + --hash=sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129 \ + --hash=sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b \ + --hash=sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452 \ + --hash=sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120 \ + --hash=sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588 \ + --hash=sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96 \ + --hash=sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728 \ + --hash=sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e \ + --hash=sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67 \ + --hash=sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c \ + --hash=sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7 \ + --hash=sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c \ + --hash=sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284 + # via -r requirements-dev.in +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via mypy diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..97490b4 --- /dev/null +++ b/requirements.in @@ -0,0 +1,5 @@ +# Runtime dependencies of the analysis package (statistics, report and charts). +# Edit this file, then regenerate requirements.txt with: +# uv pip compile --universal --python-version 3.12 --generate-hashes requirements.in -o requirements.txt +numpy==2.5.3 +matplotlib==3.11.2 diff --git a/requirements.txt b/requirements.txt index a9006fd..018dc27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,512 @@ -matplotlib>=3.8 +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --python-version 3.12 --generate-hashes requirements.in -o requirements.txt +contourpy==1.4.0 \ + --hash=sha256:018227e134b73090b06f911e773a526c77c9af426be29065500ce096a5accd4c \ + --hash=sha256:072a702e2e178f4fcf96f04775d0c059e4c917925abf0d3208ebb6865df3c23d \ + --hash=sha256:0c7a4c2716a4e98342221954416a836cca77c996c14ddbf22b5f01d5d93ca09c \ + --hash=sha256:107ec46f7aa1664266d69b1181aadc953d58d39f7d8d648fa5b795d4a060b0de \ + --hash=sha256:181bea01bc742734ae672fa00c717d855dd35e1f029536406538c52e7b0cd73d \ + --hash=sha256:186ba929df36d61b6127da2e89cd1357e4cb79aca647381ab1a6feb0b152877b \ + --hash=sha256:196759a3e4db60e1546167e361090ebb6d199aa3aafc9e34ea17a5ac3b23e814 \ + --hash=sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1 \ + --hash=sha256:20da2f86bfaed60dbba729fe738c8241b1dd22b6cf7e8cbbf30df290bd04ba28 \ + --hash=sha256:2deb580178ca19437a84bd77e4bc2cd91a8ccad212413a83c273900868b5978c \ + --hash=sha256:2e741a39dfc96babe1722561e81351aeec104526c57dbe91c167c1db606c2d55 \ + --hash=sha256:3863ef2e2b13fe93f8c0ebb08ee400cb07153b8b0e91c5acb26e4537f283634f \ + --hash=sha256:3e0a2392533e5e2abdf277c951047516acc8d2f3c42b2dd9dc34c907c9ebabf2 \ + --hash=sha256:404dbcd9233513dfd1323b66ea593fef90e899f658f6b8a9ed8e93bd0ca669db \ + --hash=sha256:417be048f7e122cefbe2a34c51a1f2b0410f8eb35796f794d45475ddb7872d9d \ + --hash=sha256:423bd5f4b3f11d54a8c597234e513382e3454bb106b8ec7ae32ba4ed63f8d99b \ + --hash=sha256:432e89f835cf01f8d2123a130a3126b17ee6e0348e4d62c0973872b30aa873bf \ + --hash=sha256:43c3ccbb32c6294b183dcc8e8c46dacf5ecef809497e3d48a5be298eef185ad0 \ + --hash=sha256:43d80072a32299bf945de0a6dd4e034ea0162e832261de2e07a274a5adbba9c9 \ + --hash=sha256:4bffcb2e8cc5c0e837631acab59dd04265fdac376550e7c3e310523877c68cb7 \ + --hash=sha256:510f7d93d94cf6ebf4e2cd0640bea16ab8affac7547230175b6745b596ce0599 \ + --hash=sha256:5450f091ac1be0be3ad3a2a3b3f23b5e443e78c670ced4fd347d626f92a28fd2 \ + --hash=sha256:5545ff38229c15d11d822fc6f7415b932da9f4975f53e305b7b96ebfb01f889c \ + --hash=sha256:5a4c89c7f38a0d7a94356d74e3391073c4325c4d47ad2fd065c3fe7dbae16c0c \ + --hash=sha256:61624f6722480aa7e168fd746164e7bfdf48f8ccb6542f1613741200dc37e2b1 \ + --hash=sha256:618137ec5778fb76d494c9ec854b104a377fddba0d128bef7136cce2c3bf0df8 \ + --hash=sha256:630d32f06cedf37b7f1dd2b6a12e4492826bd477a4c9c7b2e2fee6ce3a2cea76 \ + --hash=sha256:64039341e2d8804f1a13bda8c69083eab935167bbd7e856cf096241f17d5bd45 \ + --hash=sha256:6507a016976a75a15aee47809a34605de6244ccc056e6164bf9fb14b27eecad4 \ + --hash=sha256:6ced1670fafee703b8277ab1645072a226e74f167166eae146fb743916119b67 \ + --hash=sha256:6e697d94e69f499ff6bebb899cae97a58d5d14f0e1fe9568b43a0248d2f9af8c \ + --hash=sha256:6f963ea9f0d68d2b6a02d861e3c0aae09ae25ba21f243fcf281e9fed468b078d \ + --hash=sha256:738c44fa71735a617f36da58e32810512407d283d5d1bb5b1cecb00d7eeba7cf \ + --hash=sha256:758e7496cec3195fc28a28945eab3ceef348c47fb95c7afc2eca2f3901209c8c \ + --hash=sha256:78ed5c5f962b3e156109f0531b174a65ba80d13cc681a70227b12524c162e184 \ + --hash=sha256:79b06c60d5e569ce12c5da8f0c51217cdc484386e2349fa717c30fafb56d2871 \ + --hash=sha256:7f861af508fca2384eef392bc1d8377cfa349b2b854cb11b45d89c75780e2561 \ + --hash=sha256:80c7fc8e7ac217777ce7ba2db3cf8478652b5a4b335283111012ff9eaac9d842 \ + --hash=sha256:86d05cec773c9507a3950122e0e40ce77c23c75ceeb2fc189514e71893cbb34b \ + --hash=sha256:875f42444c9cf48d56f724f2637e60d0f73b3b12c9041e1484580a233edf9591 \ + --hash=sha256:90feb8da006803a95ba573fddc9536357d0c3faddc3a4c9e22816ca49af52878 \ + --hash=sha256:912c6afaa106e2f74ba22b30416b77ef7eb94e3bdc5a4df51ab147845dba9b6d \ + --hash=sha256:96019f059bcb3774acc0104be3360a57b36d33eaaa2e1d7f77c800ca8e07618b \ + --hash=sha256:9c0e07c691f3b3321913ed9b8161c50ea5f77fadd006f52f5e755359b0dcbfc5 \ + --hash=sha256:a03b32c2e7eda8b17757c022162c13c86f5c3d6f937dddf9ba3c3ff7b513953b \ + --hash=sha256:a1c8a74744fa746eeaa12f0f9ed7f8b4add9d8f1b14d95e3b5e133675eac888f \ + --hash=sha256:a3e67bc1a6a4618a1dac7c3053a9ffece5ddfa2046b2670b342cf430bb0b87d1 \ + --hash=sha256:a58b5f130afec61a093d743cdda435cc4047748b6791627e34a68f41ec3a9b07 \ + --hash=sha256:a5e2bb871b90cd7a52bdee63bf80ef7acb46398d27efc66d7679240016efc5aa \ + --hash=sha256:abd0f7e51ecc52bf8c95713bd80b1c17c516a71fa391c54475308a4377903c2b \ + --hash=sha256:b7794ea07cab575633daad8d6e963b662053391022294d3aacc1d9ba9ef54114 \ + --hash=sha256:b9dc493e7924e5aa32d89d0b8e50986280f3d4b284968a792787bd7bcffd3cad \ + --hash=sha256:be85d160e7c795113c2a93254dde2fc7e86b375322570eee78b420969426973c \ + --hash=sha256:bee6ff96653e0807cd0f8eaef00174a031e3a2effa5c20f49a5a8d574b05af23 \ + --hash=sha256:c76f3a5318164db7d9401132fc1b5364b784c613c93fa506e3b5e6d1bf353ec8 \ + --hash=sha256:c8be3392b84cf43373a488874dc1ebf46f60784a70977f92586e18019d71c7ed \ + --hash=sha256:c927ec747633e68c9920bcaad48b0bcefe648812b822f18c34e2d0e2270a3d2a \ + --hash=sha256:cc87f2ffa84a49a77d76cb792f28a48ff3b05a222aa0b2bddbd839ba78e1d5ca \ + --hash=sha256:dd59df9e2fb0aff8bd7fd1adb9f349b7c835245a9d0f18c2f2deeb537190f2b1 \ + --hash=sha256:ddf5a2596d716fd3793434844caf31abc7a40b1e8718431420c89858268fb909 \ + --hash=sha256:de503609fdb71f597634ca64fb4bd2f13d27c97f140e912b1e8ed93544840df0 \ + --hash=sha256:e21d1d4a9db5b3a793649c7ab0ddb4697f2818929eec871a7b9c45fce7b2a1ea \ + --hash=sha256:e439ab450c93455feb0218532ded3e946ec7a9b5f459069f122cfa44b89229f6 \ + --hash=sha256:ea09dc704029930cbd097f75cb0cc1db9852adbcf1a151fac7e9559f7bcbfd19 \ + --hash=sha256:ef47fef9a912c77e3d84b014f701e5a9340f25703e9ed3bcebe37f91fb69dc49 \ + --hash=sha256:ef9440f6f8506246269a82734f5ff9e2e4c5e775b3996fc883cc491c5257eca6 \ + --hash=sha256:f1219a8898523cba821085f2da8a1b962cc696325763f09d7f93538ba43b2d70 \ + --hash=sha256:f863c6100bf926cf47d13f3cd75f9bb8ebd98aaab230eeb4468b91f98f39a6b3 \ + --hash=sha256:fa1b787362a3856e63dd2b89449f6c384d55ee1beb8f94f4bcc871b40f02462c \ + --hash=sha256:fc9feef8f1f001c5b87decadc67c4a5d1eebb62ca39c4763d1237ff62cf2b707 + # via matplotlib +cycler==0.12.1 \ + --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ + --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c + # via matplotlib +fonttools==4.66.0 \ + --hash=sha256:02117d05dddb51e39b5c5a6eed0702ed8e0e1c341cb138101b2446eb479858f8 \ + --hash=sha256:03922992966a1830b94a750961d1ccfe1a7375d792ca61077a5afb719811788a \ + --hash=sha256:056641fcedeaad24b92e343e35e9fd1337200c102450678894b17928482d010d \ + --hash=sha256:0640e69b00e089d6b82e0a54e3a03af84b775f1f0b2cbecc1b0d7fe3980a194a \ + --hash=sha256:0cf33d1041364ebb7a9958db757e213c6d2a238eb41755fbf935db028c2db20f \ + --hash=sha256:107aae7c38f05561fef5103baa98a44400b5f4c69d87839c9a5f8125405e7d9a \ + --hash=sha256:135ce8738a6341fbefeedc061e20c2889350e84405abea7e61b1c9d87e1a1f2f \ + --hash=sha256:15315302c620e6ce8582934e07932dcf442fc3f59ab208d30564200b9a2f7ea4 \ + --hash=sha256:1d120ea0f5260b04e9b5ac0d9239efde4c23d990347d5d5889cdd37221726fec \ + --hash=sha256:221defad3b1949c1fdde2558d1fe780d42f926b683a71a46a4495a0e6a6614bd \ + --hash=sha256:2b278e2abe596872b3c9fe611f956ded85588537974e48e2adbe2d6ee159e099 \ + --hash=sha256:35684b562df7154d7a0ebfa512db9199c7fb0a93b5abfb59dac9d022dbee7aaa \ + --hash=sha256:379b94fe10651d26caf398eb646b3ecde886a9c504d3cce5e46164d58a78e32d \ + --hash=sha256:3889fdbe63e85dbf2a0d9d7c9d90746827ce60af2cdd24f0bb4738380a9540eb \ + --hash=sha256:398bfc5ce9055e0b91a6189f8cf462d4230b38f522b2ceaecb4f34cbf4fb5480 \ + --hash=sha256:3ea59aaec135c96c4cff20d79fdf1d10623993712a77990585a42cfd6a42bc5c \ + --hash=sha256:4513753259f06316bdc2039c9bf3129e74923c1cfad0722b4728a9ce19beeeb2 \ + --hash=sha256:4bdaf80590fcccb1b9d8e3b8ca72225defb14d03b79788f59781ce1bb66049b9 \ + --hash=sha256:5dda4087a7f8f0b985995489b3c0115bd117f18c0fc236013a0e78e837bb9ac2 \ + --hash=sha256:6371e7ed43cc9e23550fe4a1b72af3205275f28dcecac8c09b80a4582e326802 \ + --hash=sha256:6627f48130cdbe04c7e1189bff4192886ecda757f575626ee8468f6580eaecf3 \ + --hash=sha256:67ea5af3ca60e1e5c9b2841b1f6dba5ef16aab581bf463abb68515447154ab61 \ + --hash=sha256:6b11180c4166a4fc353bb02b147322b7c454927bba4f5040ccdef109a73f9051 \ + --hash=sha256:700c448f190cc7bbd9f059bd96131f4770e90938c9692550b142153bea73b14c \ + --hash=sha256:8009e736e1491c569392b2e9782c30dcb6ac879def102c8c2e06add8d2adc10f \ + --hash=sha256:850d50b347f31667d3a277a6e66a4219236db9d98afadb2993b8addf9bd4f593 \ + --hash=sha256:85e2960a29e6882567b981ba408b5a5ca2d8865b8f67240e1b420348997a13be \ + --hash=sha256:8739d77810aeb6d2ecd55cc2e4864d98a00c317fb6131a6d22ca58b3f50cf22a \ + --hash=sha256:886f3029ea10362c592d2e1a5326a44645b85a2a0be6fdb7c9edbdb60cedb0ab \ + --hash=sha256:8af85dfc2ad564f95b3e650024578bb2268bc4d827d418aab9650bbde4b6926a \ + --hash=sha256:8ea6265535b25f8254868f5a212a1c9dbd88c2c4050f3df18d9174949df0d6d9 \ + --hash=sha256:9b0198962bbba64a88bf15dc5d95e4bab8c5208dede22a10fb06280406ed9874 \ + --hash=sha256:a5a0624aaefb0a29e4ed7500a98732a1b0ec2d3040fe86cc8837f82e3fe4f446 \ + --hash=sha256:a9a45a23b020e667499fb2dbce0f1373b1b1b8cf88e47f3031e69ef3ea21f3ee \ + --hash=sha256:ada87643b20a8fa5763e6dd4fa0f5e2901cbdbe0833deb6d5c5a9c3186782b94 \ + --hash=sha256:aea0cc5609a5f2a2500f91bd6e1989fdd22da0f225be00dc1c4cec775838a7d0 \ + --hash=sha256:b02686b2e47115b2378ab6156abd984c776c6faa4f56898fbcc1c30ee387937d \ + --hash=sha256:b432b6b54a0f3be118c6243312c826aa086aa9bcb150160be4f0bc8a21ea3aaa \ + --hash=sha256:b46500e4d6f5708a9127ea12902ba54cb0ec594fc6bec649afc53154a1125b2b \ + --hash=sha256:ba65eb3e2c46ad0922d84ef78e287c2f48b7a8cae4420728bf3b8e583282d6d7 \ + --hash=sha256:bc7b7ddc1a1f46c363354304e9a8dd93722e4a6a24f785015650898dacf40df9 \ + --hash=sha256:c144e68572af1dfcb05fe065827d0920d4118b265504928e1d2262e3701a25a1 \ + --hash=sha256:c3a66c749b69e9abd92e519ccbdea7210522b0fffdba3f6492a8fab461d4708a \ + --hash=sha256:c519433e8632284dbe64ee4c81b609a17e205bc1afff285553723964880d1c89 \ + --hash=sha256:ce818581070527883e3678b92b1ac7ed3569e48c80f51b89b264f97d84be2608 \ + --hash=sha256:cf0033f343cb1592fb24ae3eaf8b58156f208884ac16991094293ee94c585654 \ + --hash=sha256:cffa168522a0e057d3a5628c8ebd43ddf66a34613f42e116bffb85b0b32ce5a0 \ + --hash=sha256:d2d10f89c4bf7cd42b84695da617f9130fa594647ee74673bf8c8d001fa04ef9 \ + --hash=sha256:dbee639a23c4e067aedfbcf84d2e466a71a1a1156ad04d6d09023791ed167495 \ + --hash=sha256:e4677025ee40b3b1420387ca85f55545b8095f7596c3d5d59552dbee19c91f07 \ + --hash=sha256:ee4f85d1341af630d787514eef1a0e1ad883f9d6c11a4c1465faeb9cd0bb5ced \ + --hash=sha256:ee7b9f6c835ea1a9beb5a35c8eb0c62b6a3290f8105d3d0c29a0a18c90f9b8ba \ + --hash=sha256:ef0610dfe7bb5bf574d9bdad6f597403ebc9807d124ac6f148d7604b2609be98 \ + --hash=sha256:f014bbf05f30731bf8b9f5c7d2b4c0e4f28f375926f95b187f85e0bbceb16029 \ + --hash=sha256:f2032ac005e14c56e774fd19f2bc1c22c796c0c3b39d91741c0e3f9575944d38 \ + --hash=sha256:fa995d96ced196388d8ed7a67ee001b181459d230e24bb68b193ae859f5609da \ + --hash=sha256:fd3dcb69d65c05a2fffa5d9b743b77e0cdc13832665ba435065493dad2b57f83 \ + --hash=sha256:fd9297a5f670f3e59ae697289ddad5902d3c4a86188e83e7823bde06334a560c + # via matplotlib +kiwisolver==1.5.1 \ + --hash=sha256:007a5553dfc4f4e8d184f588a0200e2cd4b63a59cc8796df3c39909e679dc7a0 \ + --hash=sha256:0324cd2567259b7a095f6cf18a52b0ffc6f3de9e69528ff1bc0e7a37bd43ff1a \ + --hash=sha256:0627b9bceb9c3cdcf12b8a18655eedfed2692b038df27423383c120d0b7dc2d6 \ + --hash=sha256:06a6917674de9e0fe3f66f5430787f59a9f2ddb64af9b714eaec547e29ef5c19 \ + --hash=sha256:072bdb15a3c19a5b5dbc8f8fb1f4e1884bf4f3507eeb4cc6334401274d37a5c0 \ + --hash=sha256:0a4faea5c6db201c6a21391d2ac926ea97acf7dacdbc3c417189e1adb1a00837 \ + --hash=sha256:0ba9527afc80ae3d7814ed98b6572d02bf85eaf48065678342c5f0c6dab7a8c7 \ + --hash=sha256:0d8924877ce22e17326a99a418c3c82037da078df3c6a260b13eca677444e6e7 \ + --hash=sha256:0ebdef3eae5336568147c39a55be6a2036ffde53faa9ca2d978989ae7c2da12c \ + --hash=sha256:1209042a623ddfda5497e4066c7b77651dde8e1d3a9dd97599dc7e97f3b9b78c \ + --hash=sha256:16895f553ee6620a827d2da56b871f835fb70b9216cca5d188e885caf6e3bd23 \ + --hash=sha256:17851e5dad4484be0cbccbde3b15331deae036de9aebd45eed964487802b172f \ + --hash=sha256:1798e83840c3f627246104c4d8a9639c60fa068adf9ce92b61791781fa8a68c1 \ + --hash=sha256:18170a77ddfecf40ec60d0928268dc95880c881864e015a8f34094ed18b9b9ad \ + --hash=sha256:186884a58486651e3c217b6acea0a53eaa9498fdd472057c46f2f0fb5c25aad5 \ + --hash=sha256:18a0cfb124546a4c2e6087c5f3029c7f44b37c85b142e0ced71f73a7599ac208 \ + --hash=sha256:1983f0974a750a6f6556f368ba11105d1d8369c735b944747c9f12ae5aea7aae \ + --hash=sha256:1a7587dc335f2c0f5bd577fd0540bd16c66006bdb60f759a1059f025e6c4f071 \ + --hash=sha256:1acc7e5b7ef05e9da8bb70cd6c7c4513090213d2e1ad9720f599f0bf6c52aec5 \ + --hash=sha256:1d852545c4d0e35a72728d072cbaa59e2fa7dd84bdf01e068d670dd0ceb58eb6 \ + --hash=sha256:1ed0f5e49d0ceff8b72190824d9e59c062fbbc02c231b853112c78474b3f5ec2 \ + --hash=sha256:1fff05e239575b1481b6ed1a782f6fad616efbf1f0b1f44e6e85c4dfe426e483 \ + --hash=sha256:21e46b23a2da695c364124817bc01d970effd5483147f8d66a6a7167e3f6b851 \ + --hash=sha256:22d5e5aaad6be121f2515765e3b1c444352cb8eb4c86510801db8f2e50757316 \ + --hash=sha256:2551cf9917af48ee7c4b29cc82320489508cf96fd26a51f6fc124de661cd44c7 \ + --hash=sha256:255605693a483db7bd5c79f60437f7bf658f7f520d61aa42722e32257c941951 \ + --hash=sha256:26e8268480be5061d509e29669d59103c067a26377a56491630ece11762e3858 \ + --hash=sha256:27add358abe374ebaa3b8763ef380bc99051b5a4b18d94878366a9e4f59efef0 \ + --hash=sha256:2ae70bc59790d2af72a3f76f24b272403e135070340281108b447cb77ea70819 \ + --hash=sha256:2e10ae1bba1899188b33557c10d73affcc12033edd18adddb57d209039976a4c \ + --hash=sha256:3221f78211074f561c44ca42eac0619828171bec15a2c4cf6f7747d07df76e8e \ + --hash=sha256:34633ecf50d16187ab8e5528b7a2530f2feb4e23f300db4672538b51cfc5cd38 \ + --hash=sha256:34ec467940442c9943016fb2d4c81d1ba84351eeca2f1a78f8bc87f1ba0d414c \ + --hash=sha256:37f801b5d7cc0e5a548921308e059fd2b057bb42972b591cfa3049f95423c4ed \ + --hash=sha256:38f6e0deb4d0a4615efe0c4efc5990b06ae450ab50a0b321c0b078b6d238c083 \ + --hash=sha256:3c24cd69455e1b00ddf770c13b6e2c33e07d6dc3f2d34add0bf9277c5c6bbd46 \ + --hash=sha256:3cc210010fd2f438a3ed430b45f1b501fd13a8618bf984dc2c5ce5b69b78752e \ + --hash=sha256:3fa5855898f6d3d01b72ccd48a2d65cbdee301251603fefe34e2025bddba219c \ + --hash=sha256:416ba7ff9f233b7036689bb5a3783537e838ad483f63558d2a800f75afe738b1 \ + --hash=sha256:431dc224a1a92a5c8f582d96e505196a3b5997a7271076678da2dfde67b77e9a \ + --hash=sha256:43844c1a7ad6d723d5b5b4c4fc7f5bd399c40e288120d16257c7c9e8765c6e85 \ + --hash=sha256:44b8faef94f1857e77fa0238f3390ff1ac51d2ea20a487e2e452a59fd2b5f5ca \ + --hash=sha256:470d420f98d368d6f010633a20659b544c5fdfa5329e6b70219f2ef08fd4a7ef \ + --hash=sha256:482676e5bd48d70ac99d9fc78863469845421e01184fa83f1f9366dc49f7e974 \ + --hash=sha256:4d4ca09bf13cff792b1884f64b98ee6c2467930d632233be25c56b442d99f10e \ + --hash=sha256:5025e36fb4fb275cef0a4e30dbb11cb4ae61d1c83deb90189cb5d7e4cafd6b55 \ + --hash=sha256:509735237ae0d849e8a843551d423d2500d2e0a9ac1611a145658b29c0fb9f85 \ + --hash=sha256:534f02c1abb31ed6dbd3515545285c330b2f12d00fdb1fdb71658b9ca5a13a6a \ + --hash=sha256:5978c3340f16a35c30f8ab2fa7bcf559973c55f1a5ef6970e1f621acf3c4db13 \ + --hash=sha256:5b973887ff782cfd6b67c9904ad8ca542e0bc5e4961503408b423b5a688b4d38 \ + --hash=sha256:5c490db2168a508088f59140dd392556a54b8bd1048fc6383c8baff13c359673 \ + --hash=sha256:5d142e352eb13facc7dd047489aebdff6ba78576c239f1ea04931979caaf0567 \ + --hash=sha256:5daa1f19e097050b9c4d9a78fcc9263cb96c9dfae08037ddc1b7c4ad1889f2a2 \ + --hash=sha256:61e9a64c7635095a6bfe483e2ff055d437c59bd45f3617a228b37277f0185d62 \ + --hash=sha256:63fb7294b768f444eb4b068965f2662f28c2fd4161e23bd60fcf3ff27b74c046 \ + --hash=sha256:685929988b208a911f1285e2f8ed54210b0d681a3dc0f03e00d599d291986e7e \ + --hash=sha256:6a797a1cefc8b9c93170db580337e1fe3d011ad18b1299943231279406342048 \ + --hash=sha256:6b92f60017dda7d877fdc546438b5e28f31c523264f49cf5a48c1d0ce1a0dfbc \ + --hash=sha256:70ed9a45c7484d2b30cdacf60d220f494a1763b9fec1ad03285c6553fa0889f2 \ + --hash=sha256:719a35fa1156db3640555f95ebb94f60a444e64d1c69626b0edef5df78eba225 \ + --hash=sha256:74ad5c3dad54a4641b4c28cd15ded70899d04459c6c7aeacafea716be97cce6d \ + --hash=sha256:74ea337e0ec3f6f342a36a4f1b5cd94dd9affddcd28ba9aae2905af932ee8c6b \ + --hash=sha256:75d9b1cf8258462dbdc1eeda718c96ea7f079324c09067f6daabfcf37712b7fe \ + --hash=sha256:77a4c8187a5948d7f8795adb765a3c7b553d07d86d88e43038fc32fc1fb9a3f3 \ + --hash=sha256:7824b5e8bdbf0bccb4ccd37bbb115849a1dc45437fb4de8351385ed07c437ee0 \ + --hash=sha256:7d38b0c279c3032e8c9cc013b405c6df8e1668dbf15465779aa7f15f61201812 \ + --hash=sha256:7e9c01d3dd7ceba4d1d436cc021d40d592466e40b9bc7f5d83dc4e98a5c9cd8c \ + --hash=sha256:7fd82debf43c6acd0a94359d232f6bb516ee13f269a7993736a9ac9f988bb5d9 \ + --hash=sha256:824c3d763a05ea9e9003610145186b0e9848c7584a5575c79bac5a8e7cd80bad \ + --hash=sha256:828f75af2b0080c8a972e75f649ab46af008e92c6104a57a759157200b835b75 \ + --hash=sha256:83f78128fa28705fa85d01c59771c72fe81c11bd0e6155edbb9f818983a7d761 \ + --hash=sha256:876bbfd276473d3daffe30e8c975df4ed9429967b41a6cb362dbb5155b6f13ad \ + --hash=sha256:886fc26012f0e8b5f69d1cfe6d711f6b11f194621539bf8e6bb1c25c5dc82724 \ + --hash=sha256:8a34616dc2521cc8dc1d7d081734da63539f021ac0450ce950908340c6e7aa2f \ + --hash=sha256:8a708a47ade1fe19e8371d5da076bac0dd4b0a5a7985ad6c637f7f7e361b6baa \ + --hash=sha256:8af9b142ad719ae3a911ebf616bc4b78b32bbab84d6a40d3ad2f129670509957 \ + --hash=sha256:8bf4df63592c2a66b4f8edc5df2544998c288aa02f96ce0acd880cd1de8c8127 \ + --hash=sha256:8de6f2a4ce7e7bd27d23dd94abf0ccafe0e0e5cc9c764b0577191f2c25f08f26 \ + --hash=sha256:8f8fddb8e323bd6eee4e54e69a39243beab22689070f4c66b472c4cc88bb89d8 \ + --hash=sha256:8fca690b00c4c48f6c2a547b0160ed511357093a4e4c9b47e0fadf3128066d89 \ + --hash=sha256:9506e892bcc3b409831d363c6f53e5985e1c8d1f6f6b0256d00358684ff85378 \ + --hash=sha256:958254518717542d02d0688d0d20cbf771da5e415e6f49543f92481c850a4540 \ + --hash=sha256:95a02752aa032eef4aed01cda6d9b687c669bd0396bf4519eef8bba22a286720 \ + --hash=sha256:96c30002424670b5e1e46495c2b8cbffef39cf77c1d79e76462029d50339785b \ + --hash=sha256:98b208a7cc42c803445ef551d6753cc42a5ea13e9cab1ee66cd8b9cb70195330 \ + --hash=sha256:9b3092d8992a1d69b7a59c3e39f35e1b9be327a17f68a7c35fc17329e337d6f2 \ + --hash=sha256:9e51c119992ea8820706871c30a4642ec76de20ae82f9b50b9a45517d8e9f810 \ + --hash=sha256:a5716a33bfabb2c6ce27b6cf03253467b3804f83e215f4d202685cf93c6c9874 \ + --hash=sha256:a5a00665d1a0e26763a7338d7e911d4598fbc1d50dd0d6b7919b7dc6c5d6569f \ + --hash=sha256:a5ca5aebae78a0bc13c1943af4af615d4966c5b650b05d5aa83b50e427196fee \ + --hash=sha256:a7b85b2cc6ea45e5f7e8c9a30bc9fabd47cda09106cbb4b967335c3e6c43b69d \ + --hash=sha256:a83ee7107df13abe42a54a6654670eef9bb39425cf2e27f65e0007465e1286ab \ + --hash=sha256:aa7d00b1700966d2917e54d278aba86897890ca9276dd8b76cf6446b6c181b92 \ + --hash=sha256:ab620eb663952455271ac37f9aaad86b73c969c02f11f53cea405b38e96a4300 \ + --hash=sha256:ad8b9671348d7c8716715652ae11f85ed0eb99e265a2df2ca490577d69860b2c \ + --hash=sha256:aefe930d113798330e9462f7874542977869c0613cba3262e2de3a8d5dee8f3a \ + --hash=sha256:b03af77d77e50edba2030fd5f7c352ff209314b09030a3cba7c14edf9a09a444 \ + --hash=sha256:b390aec180a7c054919c04898835e1c77bced23ea8383eb2c570213bf25d1a86 \ + --hash=sha256:b3d78f7bb2b9d9a30345be1474b9aaa8685430b54afb51ba3639b5c6c11e9ed6 \ + --hash=sha256:b5664603a253efd3a75716d793d1d3a6a82723b61dc6db767b2460bbbeec4c0f \ + --hash=sha256:b69602970994a2ed8bbfa78c2f0394a7435226c6040489702d9f0a0ad0c07052 \ + --hash=sha256:b6ae6a0328f0bc035741820fdeecdcd67bf4694eee03972e843663107122f450 \ + --hash=sha256:bad20d4c69c851c982a1e3606f4c293edfd5a87885786c50082412240c4b1ffd \ + --hash=sha256:bb7c99f0673c03017a3ee01e54a5c2617a05468b11eabe513b0080e063ed95b1 \ + --hash=sha256:bebb89489b279b2f5661bbbb2abcc87bcd4a46607bb4a5c966f04f1db6b8df9a \ + --hash=sha256:bfd1de989b3330420e29de39352f5c049905c9e3ee67233a50d550e3d652c148 \ + --hash=sha256:c2306e8bb53601979fcb3fa09cc65e031876d9ae01eff2fcbcd7a84ef94d5bc1 \ + --hash=sha256:c3a4e41e3096bf1f0f1b76e2ffd6d828d6547f574f702d59bdbef7acfa59db9c \ + --hash=sha256:c6834b92dd2428e2dd85ef3d85f723d3c12f20aaf43a2ddd4f944ca25d833408 \ + --hash=sha256:c90d3022d8a94778939cda8638c6c8da8fa757b8958dad7ec868ce29c87681b8 \ + --hash=sha256:ca307d6c259e5c98d3cb9ade55342b47a6839762caf2536f3d7b46ee660cc82e \ + --hash=sha256:ca7f6fe0f37ca978a1e5eb7a3a68e6413f417e78e838324947ffd420202b198b \ + --hash=sha256:cb6fae641357ed2f6e533c0d3c6504a4a5703621a50c89459e46051d56b61140 \ + --hash=sha256:cdaeeb6c350106df6bf9d873395973e5f066a9713200b72cd64f55d0a3eafab6 \ + --hash=sha256:cea20da04494e662b83c872683bf4ff2345206043d036315ed0e924b652e7294 \ + --hash=sha256:cea90547bfd93807e0013a004dc76552be44fad3bc1cc2b38610a9e889ed098f \ + --hash=sha256:d09037ca068d784ebc4aec290ef952ca27ac15dd9c0b5801a88c6e1096b83e6b \ + --hash=sha256:d27c2123977cb9269c30a49ba45f03a4323017ef693e19db4ec9dbe1299a3002 \ + --hash=sha256:d50de98e8d807dc31822fff96f50293163a62418eb65487a21b42713d72ed0b7 \ + --hash=sha256:d66a64dd5dec136040ec2ae94aa026a912ee60fdd45bc28d3db30037fd809e88 \ + --hash=sha256:d79308fa689fac89cbcfbd4dbfc80b5f95c54c5a7fd4d194be221f9d33d026e6 \ + --hash=sha256:da3275833be0edbaf4830fae08bae3dc7219f40ce0c37eaa6c25825957e06612 \ + --hash=sha256:dc1a26b8e53395a01c2c611e58602fa47461f136fba7cd5542e6db6d64be1839 \ + --hash=sha256:dc23390afe9f4ef9ac3bcc72a03a56eebbde03f4c571a32cb38f859cff9a6524 \ + --hash=sha256:e05c2f7925f1d88778e53cb44f14e0223204a3bdd09a41664750363acfb1f2ef \ + --hash=sha256:e12dfea7f5fc2a34a9080efbf79c4c44eb380ec5b9c6fea09407e08f0d1e941d \ + --hash=sha256:e4e4523d6f336708d732516e6cfca7796cf3d96c9474eb5aecf6165f2f1fefc3 \ + --hash=sha256:e4e49f7e1a4e7191bdf9dc67a974db714501b1fc52c24324103d06a86abd5c08 \ + --hash=sha256:e68e151428b5384f766cd25739bf77c7e4a3dc93b5ded7a12118d9fbfdf78ab6 \ + --hash=sha256:e8e4d953faaded9ec7ede36824e9814082d22d4c7b1eafbfa079ecba8cd0d076 \ + --hash=sha256:ee9df1f0d77b9c6e94f4ac0fec533fbddd5ea3a327807f18d7b069ae019ded80 \ + --hash=sha256:f0a887b6565bbfe80efde2b7f6e8890d7d9bbdb11bdb17028a3690c32fe0621f \ + --hash=sha256:f0f4a42db92d6ec7677ab9d12830a2a8ec145a9c6d15db2b593466bc875c78d7 \ + --hash=sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a \ + --hash=sha256:f1d56ec54d257d05e0b50f5780d967540cd07beeaf9e5f645b26d50cce79f4d8 \ + --hash=sha256:f4167e87b397f273dc2356fcf1eaf50a6bac51e6105f45103ef7129c8efb0255 \ + --hash=sha256:f76fc85bd054c806960f917ec0f329e24e436f1712267d90588e4c39890caa63 \ + --hash=sha256:f942903fde7363d1d879057ec5de01310efda2597161784d752fa9953a01a71a \ + --hash=sha256:f9b1c4900736e489a812c529100de4b8fb617d4db075e931e213c57424b83d9b \ + --hash=sha256:fc271a6f0a2126958f4090e5507b9da5848927dae331f8f763bd4aa642b3d2cd \ + --hash=sha256:febcce10f2bcdbb80b4ea919238a6a4ac13dbc4c7cadbe8d5d75c3682f8b5404 + # via matplotlib +matplotlib==3.11.2 \ + --hash=sha256:01dc8eaaab5a9fce9ff615eca82345728f289e4715b186ee10c6d85272fc26bb \ + --hash=sha256:02329432ae5c6af87cf208ee575b701d698bdf0b1a3bb28cc6d53c36e967e575 \ + --hash=sha256:07d9b9fa60cd4c393692f50d0bb03123242ddf61c99bb0e95e75feb354e7c1a8 \ + --hash=sha256:116cdb0eb0eb5644fc98eb2975d4b4dd4ad35e5c8e6b851c22e6976f371f7ab5 \ + --hash=sha256:1944895967f87c84c9b4bad29a707b31f5b36df4a3a2339ea1f8ea3ce5105539 \ + --hash=sha256:1a3040b209f3968b4e84161df7b174f07a9fad33b0f2d7e48ea3bbd3075e2863 \ + --hash=sha256:1b9a7ad579856284135e401ecc918c5f8a017ee30539298862a109f51b971710 \ + --hash=sha256:254d4ddb2fa8df3b4c689c0c306063aee10521df82cfb438185e499c75fe37c1 \ + --hash=sha256:2a8285cea8ef4d92aa041d1c33788bcae82248503300f93f1ca2136b9049452f \ + --hash=sha256:30ec15d7eefee71de16b7c689b42ba49715a4644650b76a6c7c70d79daf24e91 \ + --hash=sha256:399fef672f7046ef7d6a57572b2a6f9845f3f5afcff04e7b2df7a363a9f42190 \ + --hash=sha256:3aa4b8516fd26659e4363abbf317c703d9116496c5db2e9d0609a2866dd39dd2 \ + --hash=sha256:3da3bc0cbf7245e7db72cc6d29d12c5abef72cb73059c73b945f14e3545f3eb2 \ + --hash=sha256:3e8576f7c47e02fd4f21f44171302d2d1d58d4471d46da3d71fe8899d19539d9 \ + --hash=sha256:57b9ea60a835937c2012861923cbb91f47db8565775d326d1c42fc926aa10351 \ + --hash=sha256:5e1e923a3fc3326b99ec0a6ff1ab1338ac6c6cc62ad9d8a9c944197c7f8c6221 \ + --hash=sha256:643ff850d8e0f5b8319337f87ed3cb59506afb3df3cc48de777d85871233be7b \ + --hash=sha256:75b6d88402770e181b5d06a67dda4129c31da7d05004d63a21c310ec78d1b83c \ + --hash=sha256:79a258f58253dfa025af80a9e9bb228d75fced007f0e93ae7423fefbde81a74d \ + --hash=sha256:7d43ff8cebb50840648cb6429b2228621dbd709010f3117b3740abb20abf21c0 \ + --hash=sha256:7e5a90f8a707ebb6004a713b1cff091a40cc5df4c7c1cb165e5a505ebc11c292 \ + --hash=sha256:7ef7a53b66780e5d942923724f08577fbc5be1f7322da0f0f3f9dcaa45dd803d \ + --hash=sha256:854df8d7dfe9fdffcbaa6f39e44a6c24b409cb4d7561fbc09213b157d833f6a6 \ + --hash=sha256:894a9cbbecbe30ae6787d464df2e8fc7a8d475cfc68f87c3029f7c11152899b3 \ + --hash=sha256:8c8255de28f986d935a64c9ca71c0ec2d2f41d355691f5ea684725dc91413f71 \ + --hash=sha256:930efb28f59fda124e39265d177bab302625297bb147d2910de725a2fc2aef54 \ + --hash=sha256:a24d5fd36e4f0e742c3851dcd20810e56a95633a342e4bf6cb591c678e8fe61f \ + --hash=sha256:a6939df7567114b6bac7f4c5e06c84a67f197c1b2f2e4b234d4eecb3bfec9482 \ + --hash=sha256:a8756cc73d9af9a7fe0deb54ea2e75ef73b01d9e575877e72acad5458e660943 \ + --hash=sha256:af2661f6ac6bbd1d081996f54fdd9385715c625ace8cec0869055c0cfbf38981 \ + --hash=sha256:bbf1062991d826ed27e2144f3afa4461afbb8ff56e8f703043e191a8163b1ee9 \ + --hash=sha256:bc067c462a86f0e57bf52fc6e058d90171a5420007dac00c46e90153050c69a7 \ + --hash=sha256:bf3fe71fbfb8ec0e310e0bc8537c3405a01f38f25f9394ed2135e6202fed542b \ + --hash=sha256:c0b83f044ce10a98027b105b3931548719a6e8c7ef986b4362651e0b5367c8dc \ + --hash=sha256:c27e577ece613ea12a0790e00b4eb80d901c59a3e16cad474f31d8b1529690b9 \ + --hash=sha256:c5c1c68ee401fc98271263410f0e5ce88285abacf7627132914e8adf3d70ff43 \ + --hash=sha256:c9721f81275499da1feeb36a2cf8192ea086283b3bd16b7dc4c9d7aedb7396d6 \ + --hash=sha256:cc82dde2a0d3e3ad472edce04897ad7146b8d8bfd1df8a32992eebb81af18fdc \ + --hash=sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4 \ + --hash=sha256:cf41ecd1b0c0b6f7177ed965a54c2afbe888715c7cf6054dc12d53bc1494002c \ + --hash=sha256:d3304eb5a59442a8867f6920d484591c0fa09ffc29e9260be2feec3351e25869 \ + --hash=sha256:d480038c83691532ed52ff3147db51fa902fc78cb2d8349993a1cdb684435bff \ + --hash=sha256:df4f7784aca81a94f254c0a2767d592ee25f407e488f5fa7203e51093fb6ca27 \ + --hash=sha256:e43b188f0a5b75447bcc197728258166aa64365770ed1caa36595a5e1ca4bbba \ + --hash=sha256:e60cf3047a51edecdc4535a9196bbd6a732936b8e9f8184aabf4d16165884aaf \ + --hash=sha256:eac4b07d4e3743b172451e122ea964f72f152879d4f9adcf3f3d33e518f12ead \ + --hash=sha256:eb3712dc9b464793de0a4e42a7313d50293c751f94bddaf7332a1bc71bccdda9 \ + --hash=sha256:ecea603dd2fbf8242fd31a305a8b12a4ece2de28096870c65fdd0d1e35b8d9a6 \ + --hash=sha256:ef31985c4dedb5f1424e1aec6849a47dd37689cb7fa3c20b1b82187f26806261 \ + --hash=sha256:ef752769cd962f39ea0b6ffc82d1ea43a0012c5a6157c7a075212fa509cfcff2 \ + --hash=sha256:f25446b2981717dca9786bac841cb3fd7efb568e3e3c755dd980481c5cb9228d \ + --hash=sha256:f2ac30cf5eb5dff1b584627ae0b0e1186551a4f69ae3c75073911da497a29170 \ + --hash=sha256:fea03cf56568cc1cba08b470be6a0559e71c3a5b688d54b7179bb35ba23d0821 + # via -r requirements.in +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # -r requirements.in + # contourpy + # matplotlib +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via matplotlib +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via matplotlib +pyparsing==3.3.3 \ + --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ + --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 + # via matplotlib +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via matplotlib +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil diff --git a/workers/python/greenbench_worker.py b/workers/python/greenbench_worker.py new file mode 100644 index 0000000..ff415fc --- /dev/null +++ b/workers/python/greenbench_worker.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""greenbench-worker in Python: the same commands and output as the C worker (src/worker.c). + + python3 greenbench_worker.py run --algorithm NAME --order NAME --n N [--seed S] [--reps K] + python3 greenbench_worker.py sequence --count C [--seed S] + python3 greenbench_worker.py input --order NAME --n N [--seed S] + +The sorts are line-for-line ports of src/sorts.c, including where comparisons and writes are +counted, so all three languages report identical counts for identical inputs. They use plain +lists and loops, never the built-in sort. Only the standard library is used, so the worker +runs on any Python 3.10 or later without installing anything. +""" + +from __future__ import annotations + +import sys +import time +from collections.abc import Callable + +DEFAULT_SEED = 2463534242 +MAX_N = 100_000 +MAX_REPS = 100_000_000 +MAX_COUNT = 1_000_000 +UINT32 = 0xFFFFFFFF +ORDERS = ("random", "sorted", "reversed", "nearly_sorted") +EXIT_USAGE = 2 + +Metrics = tuple[int, int] # (comparisons, writes) for one sort +SortFn = Callable[[list[int]], Metrics] + + +class UsageError(Exception): + """A problem with the command line; reported with exit status 2.""" + + +def xorshift32(state: int) -> int: + """One step of xorshift32 (shifts 13, 17, 5) on a 32-bit state.""" + state ^= (state << 13) & UINT32 + state ^= state >> 17 + state ^= (state << 5) & UINT32 + return state + + +def sequence(seed: int, count: int) -> list[int]: + """The first ``count`` outputs of xorshift32 starting from ``seed``.""" + values = [] + state = seed + for _ in range(count): + state = xorshift32(state) + values.append(state) + return values + + +def make_input(n: int, order: str, seed: int) -> list[int]: + """A permutation of 0..n-1 in the given order, exactly as src/inputs.c builds it.""" + arr = list(range(n - 1, -1, -1)) if order == "reversed" else list(range(n)) + state = seed + if order == "random": + for i in range(n, 1, -1): # Fisher-Yates shuffle + state = xorshift32(state) + j = state % i + arr[i - 1], arr[j] = arr[j], arr[i - 1] + elif order == "nearly_sorted": + for _ in range(n // 20): # swap 5% of the elements with random partners + state = xorshift32(state) + a = state % n + state = xorshift32(state) + b = state % n + arr[a], arr[b] = arr[b], arr[a] + return arr + + +def input_hash(arr: list[int]) -> int: + """32-bit FNV-1a over each value as four little-endian bytes (two's complement).""" + h = 2166136261 + for value in arr: + v = value & UINT32 + for shift in (0, 8, 16, 24): + h ^= (v >> shift) & 0xFF + h = (h * 16777619) & UINT32 + return h + + +def bubble_sort(arr: list[int]) -> Metrics: + comparisons = 0 + writes = 0 + n = len(arr) + if n < 2: + return 0, 0 + for i in range(n - 1): + for j in range(n - i - 1): + comparisons += 1 + if arr[j] > arr[j + 1]: + arr[j], arr[j + 1] = arr[j + 1], arr[j] + writes += 2 + return comparisons, writes + + +def insertion_sort(arr: list[int]) -> Metrics: + comparisons = 0 + writes = 0 + for i in range(1, len(arr)): + key = arr[i] + j = i + while j > 0: + comparisons += 1 # counted even when it ends the loop + if arr[j - 1] <= key: + break + arr[j] = arr[j - 1] + writes += 1 + j -= 1 + if j != i: + arr[j] = key + writes += 1 + return comparisons, writes + + +def _partition(arr: list[int], low: int, high: int, counts: list[int]) -> int: + """Lomuto partition around the last element; counts is [comparisons, writes].""" + pivot = arr[high] + i = low + for j in range(low, high): + counts[0] += 1 + if arr[j] <= pivot: + if i != j: + arr[i], arr[j] = arr[j], arr[i] + counts[1] += 2 + i += 1 + if i != high: + arr[i], arr[high] = arr[high], arr[i] + counts[1] += 2 + return i + + +def _quick_sort_range(arr: list[int], low: int, high: int, counts: list[int]) -> None: + """Recurses into the smaller side and loops on the larger, as src/sorts.c does.""" + while low < high: + p = _partition(arr, low, high, counts) + if p - low < high - p: + if p > low: + _quick_sort_range(arr, low, p - 1, counts) + low = p + 1 + else: + if p < high: + _quick_sort_range(arr, p + 1, high, counts) + if p == 0: + return + high = p - 1 + + +def quick_sort(arr: list[int]) -> Metrics: + counts = [0, 0] + if len(arr) > 1: + _quick_sort_range(arr, 0, len(arr) - 1, counts) + return counts[0], counts[1] + + +SORTS: dict[str, SortFn] = {"bubble": bubble_sort, "insertion": insertion_sort, "quick": quick_sort} + + +def parse_number(text: str | None, low: int, high: int) -> int | None: + """Decimal digits only, within [low, high]; None otherwise.""" + if text is None or text == "" or not all("0" <= c <= "9" for c in text): + return None + value = int(text) + return value if low <= value <= high else None + + +def parse_args(args: list[str]) -> dict[str, str]: + known = {"--algorithm", "--order", "--n", "--seed", "--reps", "--count"} + parsed: dict[str, str] = {} + for i in range(0, len(args), 2): + name = args[i] + if name not in known: + raise UsageError(f"unknown argument {name}") + if i + 1 >= len(args): + raise UsageError(f"missing value for {name}") + if name in parsed: + raise UsageError(f"repeated argument {name}") + parsed[name] = args[i + 1] + return parsed + + +def parse_seed(args: dict[str, str]) -> int: + if "--seed" not in args: + return DEFAULT_SEED + seed = parse_number(args["--seed"], 1, UINT32) + if seed is None: + raise UsageError(f"--seed must be a whole number from 1 to 4294967295, got {args['--seed']}") + return seed + + +def parse_input_args(args: dict[str, str]) -> tuple[str, int, int]: + order = args.get("--order") + if order not in ORDERS: + raise UsageError(f"--order must be random, sorted, reversed or nearly_sorted, got {order or 'nothing'}") + n = parse_number(args.get("--n"), 1, MAX_N) + if n is None: + raise UsageError(f"--n must be a whole number from 1 to 100000, got {args.get('--n') or 'nothing'}") + return order, n, parse_seed(args) + + +def command_sequence(args: dict[str, str]) -> int: + if {"--algorithm", "--order", "--n", "--reps"} & args.keys(): + raise UsageError("sequence takes only --count and --seed") + count = parse_number(args.get("--count"), 1, MAX_COUNT) + if count is None: + raise UsageError(f"--count must be a whole number from 1 to 1000000, got {args.get('--count') or 'nothing'}") + sys.stdout.write("".join(f"{v}\n" for v in sequence(parse_seed(args), count))) + return 0 + + +def command_input(args: dict[str, str]) -> int: + if {"--algorithm", "--reps", "--count"} & args.keys(): + raise UsageError("input takes only --order, --n and --seed") + order, n, seed = parse_input_args(args) + sys.stdout.write("".join(f"{v}\n" for v in make_input(n, order, seed))) + return 0 + + +def command_run(args: dict[str, str]) -> int: + if "--count" in args: + raise UsageError("run does not take --count") + algorithm = args.get("--algorithm") + if algorithm is None or (algorithm != "none" and algorithm not in SORTS): + raise UsageError(f"--algorithm must be bubble, insertion, quick or none, got {algorithm or 'nothing'}") + order, n, seed = parse_input_args(args) + is_none = algorithm == "none" + reps = 1 + if "--reps" in args: + parsed = parse_number(args["--reps"], 0 if is_none else 1, MAX_REPS) + if parsed is None: + low = 0 if is_none else 1 + raise UsageError(f"--reps must be a whole number from {low} to 100000000, got {args['--reps']}") + reps = parsed + + data = make_input(n, order, seed) + sort = None if is_none else SORTS[algorithm] + first: Metrics = (0, 0) + consistent = True + work = data + start = time.perf_counter_ns() + for r in range(reps): + work = data.copy() + metrics = sort(work) if sort is not None else (0, 0) + if r == 0: + first = metrics + elif metrics != first: + consistent = False + loop_ns = time.perf_counter_ns() - start + + if not consistent: + print("greenbench-worker: operation counts changed between repetitions", file=sys.stderr) + return 1 + # The input is a permutation of 0..n-1, so a correct sort leaves work[i] == i. + if sort is not None and reps > 0 and any(value != i for i, value in enumerate(work)): + print( + f"greenbench-worker: {algorithm} sort left the {order} input of {n} elements unsorted", + file=sys.stderr, + ) + return 1 + sys.stdout.write( + f"result language=python algorithm={algorithm} order={order} n={n} seed={seed} reps={reps} " + f"comparisons={first[0]} writes={first[1]} loop_ns={loop_ns} input_hash={input_hash(data):08x}\n" + ) + return 0 + + +USAGE = ( + "usage: greenbench-worker run --algorithm NAME --order NAME --n N [--seed S] [--reps K]\n" + " greenbench-worker sequence --count C [--seed S]\n" + " greenbench-worker input --order NAME --n N [--seed S]\n" +) + + +def main(argv: list[str]) -> int: + commands: dict[str, Callable[[dict[str, str]], int]] = { + "run": command_run, + "sequence": command_sequence, + "input": command_input, + } + try: + if len(argv) < 2: + raise UsageError("missing command") + args = parse_args(argv[2:]) + if argv[1] not in commands: + raise UsageError(f"unknown command {argv[1]}") + return commands[argv[1]](args) + except UsageError as error: + sys.stderr.write(f"greenbench-worker: {error}\n{USAGE}") + return EXIT_USAGE + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/workers/python/tests/test_python_worker.py b/workers/python/tests/test_python_worker.py new file mode 100644 index 0000000..43f2a59 --- /dev/null +++ b/workers/python/tests/test_python_worker.py @@ -0,0 +1,131 @@ +"""Unit tests for the Python worker. The cross-language checks are in tests/integration.""" + +from __future__ import annotations + +import random + +import pytest + +import greenbench_worker as worker + +DEFAULT_SEED = worker.DEFAULT_SEED + + +def test_xorshift_reference_values() -> None: + # Shared with tests/test_inputs.c and the Node.js tests. + assert worker.sequence(DEFAULT_SEED, 5) == [723471715, 2497366906, 2064144800, 2008045182, 3532304609] + assert worker.sequence(1, 3) == [270369, 67634689, 2647435461] + + +def test_xorshift_stays_within_32_bits() -> None: + state = 0xFFFFFFFF + for _ in range(1000): + state = worker.xorshift32(state) + assert 0 <= state <= 0xFFFFFFFF + + +def test_orders() -> None: + assert worker.make_input(5, "sorted", DEFAULT_SEED) == [0, 1, 2, 3, 4] + assert worker.make_input(5, "reversed", DEFAULT_SEED) == [4, 3, 2, 1, 0] + shuffled = worker.make_input(1000, "random", DEFAULT_SEED) + assert sorted(shuffled) == list(range(1000)) + assert worker.make_input(1000, "random", DEFAULT_SEED) == shuffled + assert worker.make_input(1000, "random", 7) != shuffled + nearly = worker.make_input(1000, "nearly_sorted", DEFAULT_SEED) + displaced = sum(value != i for i, value in enumerate(nearly)) + assert sorted(nearly) == list(range(1000)) + assert 0 < displaced <= 100 + + +def test_hash_reference_values() -> None: + assert worker.input_hash([]) == 0x811C9DC5 + assert worker.input_hash([0]) == 0x4B95F515 + assert worker.input_hash([1, 2, 3]) == 0x794671B5 + assert worker.input_hash(list(range(10))) == 0x3A210ED4 + assert worker.input_hash([-1]) == 0xE3160FB1 + + +@pytest.mark.parametrize("name", ["bubble", "insertion", "quick"]) +def test_sorts_match_the_builtin_sort(name: str) -> None: + sort = worker.SORTS[name] + rng = random.Random(12345) + for n in (0, 1, 2, 3, 7, 64, 500): + for _ in range(10): + arr = [rng.randint(-50, 50) for _ in range(n)] + expected = sorted(arr) + sort(arr) + assert arr == expected + + +def test_known_counts() -> None: + def hundred(reversed_: bool) -> list[int]: + return list(range(99, -1, -1)) if reversed_ else list(range(100)) + + assert worker.bubble_sort(hundred(False)) == (4950, 0) + assert worker.bubble_sort(hundred(True)) == (4950, 9900) + assert worker.insertion_sort(hundred(False)) == (99, 0) + assert worker.insertion_sort(hundred(True)) == (4950, 4950 + 99) + assert worker.quick_sort(hundred(False)) == (4950, 0) + assert worker.quick_sort(hundred(True))[0] == 4950 + + +def test_counts_published_by_algorithm_energy_c() -> None: + data = worker.make_input(4000, "random", DEFAULT_SEED) + assert worker.bubble_sort(data.copy()) == (7998000, 8003836) + assert worker.insertion_sort(data.copy()) == (4005907, 4005905) + assert worker.quick_sort(data.copy()) == (55119, 56984) + + +def test_run_prints_the_protocol_line(capsys: pytest.CaptureFixture[str]) -> None: + status = worker.main(["worker", "run", "--algorithm", "quick", "--order", "random", "--n", "4000", "--reps", "2"]) + out = capsys.readouterr().out + assert status == 0 + fields = dict(item.split("=") for item in out.split()[1:]) + assert out.startswith("result ") + assert fields["language"] == "python" + assert fields["comparisons"] == "55119" + assert fields["writes"] == "56984" + assert fields["reps"] == "2" + assert len(fields["input_hash"]) == 8 + + +def test_none_algorithm_does_no_work(capsys: pytest.CaptureFixture[str]) -> None: + status = worker.main(["worker", "run", "--algorithm", "none", "--order", "sorted", "--n", "10", "--reps", "0"]) + assert status == 0 + assert " reps=0 comparisons=0 writes=0 " in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("argv", "message"), + [ + ([], "missing command"), + (["jump"], "unknown command jump"), + (["run", "--algorithm", "merge", "--order", "random", "--n", "5"], "--algorithm must be"), + (["run", "--algorithm", "quick", "--order", "random", "--n", "0"], "--n must be"), + (["run", "--algorithm", "quick", "--order", "random", "--n", "100001"], "--n must be"), + (["run", "--algorithm", "quick", "--order", "random", "--n", "5", "--reps", "0"], "--reps must be"), + (["run", "--algorithm", "quick", "--order", "upside", "--n", "5"], "--order must be"), + (["run", "--algorithm", "quick", "--order", "random", "--n", "5", "--seed", "0"], "--seed must be"), + (["run", "--algorithm", "quick", "--order", "random", "--n", "+5"], "--n must be"), + (["sequence", "--count", "1", "--n", "3"], "takes only --count and --seed"), + (["run", "--algorithm"], "missing value for --algorithm"), + (["run", "--n", "1", "--n", "2"], "repeated argument --n"), + ], +) +def test_usage_errors(argv: list[str], message: str, capsys: pytest.CaptureFixture[str]) -> None: + status = worker.main(["worker", *argv]) + captured = capsys.readouterr() + assert status == worker.EXIT_USAGE + assert captured.out == "" + assert message in captured.err + assert "usage: greenbench-worker" in captured.err + + +def test_parse_number() -> None: + assert worker.parse_number("42", 1, 100) == 42 + assert worker.parse_number("0", 0, 1) == 0 + assert worker.parse_number("٣", 0, 10) is None # a non-ASCII digit + assert worker.parse_number("1e3", 0, 10_000) is None + assert worker.parse_number("", 0, 10) is None + assert worker.parse_number(None, 0, 10) is None + assert worker.parse_number("11", 0, 10) is None From 020b6c6e8bbff037348475984dbc587e91e65c9d Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:47 +0400 Subject: [PATCH 07/51] feat: add the Node.js worker Plain JavaScript with JSDoc types, checked by the TypeScript compiler in strict mode and linted with ESLint. It has no runtime dependencies, so it runs without a build step on Node.js 18 or later. Co-Authored-By: Claude Opus 5.5 --- workers/node/eslint.config.js | 20 + workers/node/lib/cli.js | 207 +++++ workers/node/lib/inputs.js | 99 +++ workers/node/lib/sorts.js | 116 +++ workers/node/package-lock.json | 1405 ++++++++++++++++++++++++++++++ workers/node/package.json | 23 + workers/node/test/cli.test.js | 64 ++ workers/node/test/inputs.test.js | 50 ++ workers/node/test/sorts.test.js | 50 ++ workers/node/tsconfig.json | 19 + workers/node/worker.js | 15 + 11 files changed, 2068 insertions(+) create mode 100644 workers/node/eslint.config.js create mode 100644 workers/node/lib/cli.js create mode 100644 workers/node/lib/inputs.js create mode 100644 workers/node/lib/sorts.js create mode 100644 workers/node/package-lock.json create mode 100644 workers/node/package.json create mode 100644 workers/node/test/cli.test.js create mode 100644 workers/node/test/inputs.test.js create mode 100644 workers/node/test/sorts.test.js create mode 100644 workers/node/tsconfig.json create mode 100644 workers/node/worker.js diff --git a/workers/node/eslint.config.js b/workers/node/eslint.config.js new file mode 100644 index 0000000..acd16c4 --- /dev/null +++ b/workers/node/eslint.config.js @@ -0,0 +1,20 @@ +import js from "@eslint/js"; +import globals from "globals"; + +export default [ + js.configs.recommended, + { + files: ["**/*.js"], + languageOptions: { + ecmaVersion: 2022, + sourceType: "module", + globals: globals.node, + }, + rules: { + eqeqeq: "error", + "no-var": "error", + "prefer-const": "error", + curly: "error", + }, + }, +]; diff --git a/workers/node/lib/cli.js b/workers/node/lib/cli.js new file mode 100644 index 0000000..d602d26 --- /dev/null +++ b/workers/node/lib/cli.js @@ -0,0 +1,207 @@ +// The worker's command line, shared by worker.js and the tests. Same commands, same output +// and same exit statuses as the C worker (src/worker.c) and the Python worker. + +import { DEFAULT_SEED, MAX_N, inputHash, isOrder, makeInput, sequence } from "./inputs.js"; +import { SORTS } from "./sorts.js"; + +const UINT32_MAX = 0xffffffff; +const MAX_REPS = 100000000; +const MAX_COUNT = 1000000; +export const EXIT_USAGE = 2; + +export const USAGE = + "usage: greenbench-worker run --algorithm NAME --order NAME --n N [--seed S] [--reps K]\n" + + " greenbench-worker sequence --count C [--seed S]\n" + + " greenbench-worker input --order NAME --n N [--seed S]\n"; + +/** A problem with the command line; reported with exit status 2. */ +export class UsageError extends Error {} + +/** @typedef {{ stdout: string, stderr: string, status: number }} Outcome */ +/** @typedef {Map} Args */ + +const KNOWN = new Set(["--algorithm", "--order", "--n", "--seed", "--reps", "--count"]); + +/** + * Decimal digits only, within [low, high]; null otherwise. + * @param {string | undefined} text + * @param {number} low + * @param {number} high + * @returns {number | null} + */ +export function parseNumber(text, low, high) { + if (text === undefined || !/^[0-9]+$/.test(text)) { + return null; + } + const value = Number(text); + return Number.isSafeInteger(value) && value >= low && value <= high ? value : null; +} + +/** + * @param {readonly string[]} args + * @returns {Args} + */ +function parseArgs(args) { + /** @type {Args} */ + const parsed = new Map(); + for (let i = 0; i < args.length; i += 2) { + const name = args[i]; + if (!KNOWN.has(name)) { + throw new UsageError(`unknown argument ${name}`); + } + if (i + 1 >= args.length) { + throw new UsageError(`missing value for ${name}`); + } + if (parsed.has(name)) { + throw new UsageError(`repeated argument ${name}`); + } + parsed.set(name, args[i + 1]); + } + return parsed; +} + +/** @param {Args} args @returns {number} */ +function parseSeed(args) { + const text = args.get("--seed"); + if (text === undefined) { + return DEFAULT_SEED; + } + const seed = parseNumber(text, 1, UINT32_MAX); + if (seed === null) { + throw new UsageError(`--seed must be a whole number from 1 to 4294967295, got ${text}`); + } + return seed; +} + +/** @param {Args} args */ +function parseInputArgs(args) { + const order = args.get("--order"); + if (order === undefined || !isOrder(order)) { + throw new UsageError( + `--order must be random, sorted, reversed or nearly_sorted, got ${order || "nothing"}`, + ); + } + const n = parseNumber(args.get("--n"), 1, MAX_N); + if (n === null) { + throw new UsageError(`--n must be a whole number from 1 to 100000, got ${args.get("--n") || "nothing"}`); + } + return { order, n, seed: parseSeed(args) }; +} + +/** + * @param {Args} args + * @param {readonly string[]} names + */ +function hasAny(args, names) { + return names.some((name) => args.has(name)); +} + +/** @param {Args} args @returns {Outcome} */ +function commandSequence(args) { + if (hasAny(args, ["--algorithm", "--order", "--n", "--reps"])) { + throw new UsageError("sequence takes only --count and --seed"); + } + const count = parseNumber(args.get("--count"), 1, MAX_COUNT); + if (count === null) { + throw new UsageError( + `--count must be a whole number from 1 to 1000000, got ${args.get("--count") || "nothing"}`, + ); + } + const values = sequence(parseSeed(args), count); + return { stdout: values.map((v) => `${v}\n`).join(""), stderr: "", status: 0 }; +} + +/** @param {Args} args @returns {Outcome} */ +function commandInput(args) { + if (hasAny(args, ["--algorithm", "--reps", "--count"])) { + throw new UsageError("input takes only --order, --n and --seed"); + } + const { order, n, seed } = parseInputArgs(args); + return { stdout: makeInput(n, order, seed).map((v) => `${v}\n`).join(""), stderr: "", status: 0 }; +} + +/** @param {Args} args @returns {Outcome} */ +function commandRun(args) { + if (args.has("--count")) { + throw new UsageError("run does not take --count"); + } + const algorithm = args.get("--algorithm"); + if (algorithm === undefined || (algorithm !== "none" && !Object.hasOwn(SORTS, algorithm))) { + throw new UsageError(`--algorithm must be bubble, insertion, quick or none, got ${algorithm || "nothing"}`); + } + const { order, n, seed } = parseInputArgs(args); + const isNone = algorithm === "none"; + let reps = 1; + const repsText = args.get("--reps"); + if (repsText !== undefined) { + const low = isNone ? 0 : 1; + const parsed = parseNumber(repsText, low, MAX_REPS); + if (parsed === null) { + throw new UsageError(`--reps must be a whole number from ${low} to 100000000, got ${repsText}`); + } + reps = parsed; + } + + const data = makeInput(n, order, seed); + const sort = isNone ? null : SORTS[algorithm]; + let first = { comparisons: 0, writes: 0 }; + let consistent = true; + let work = data; + const start = process.hrtime.bigint(); + for (let r = 0; r < reps; r++) { + work = data.slice(); + const metrics = sort !== null ? sort(work) : { comparisons: 0, writes: 0 }; + if (r === 0) { + first = metrics; + } else if (metrics.comparisons !== first.comparisons || metrics.writes !== first.writes) { + consistent = false; + } + } + const loopNs = process.hrtime.bigint() - start; + + if (!consistent) { + return { stdout: "", stderr: "greenbench-worker: operation counts changed between repetitions\n", status: 1 }; + } + // The input is a permutation of 0..n-1, so a correct sort leaves work[i] === i. + if (sort !== null && reps > 0 && work.some((value, i) => value !== i)) { + return { + stdout: "", + stderr: `greenbench-worker: ${algorithm} sort left the ${order} input of ${n} elements unsorted\n`, + status: 1, + }; + } + const hash = inputHash(data).toString(16).padStart(8, "0"); + return { + stdout: + `result language=node algorithm=${algorithm} order=${order} n=${n} seed=${seed} reps=${reps} ` + + `comparisons=${first.comparisons} writes=${first.writes} loop_ns=${loopNs} input_hash=${hash}\n`, + stderr: "", + status: 0, + }; +} + +/** + * Runs one worker command. argv excludes the node executable and the script path. + * @param {readonly string[]} argv + * @returns {Outcome} + */ +export function runCli(argv) { + /** @type {Record Outcome>} */ + const commands = { run: commandRun, sequence: commandSequence, input: commandInput }; + try { + if (argv.length < 1) { + throw new UsageError("missing command"); + } + const args = parseArgs(argv.slice(1)); + const command = commands[argv[0]]; + if (!Object.hasOwn(commands, argv[0]) || command === undefined) { + throw new UsageError(`unknown command ${argv[0]}`); + } + return command(args); + } catch (error) { + if (error instanceof UsageError) { + return { stdout: "", stderr: `greenbench-worker: ${error.message}\n${USAGE}`, status: EXIT_USAGE }; + } + throw error; + } +} diff --git a/workers/node/lib/inputs.js b/workers/node/lib/inputs.js new file mode 100644 index 0000000..f8db4a0 --- /dev/null +++ b/workers/node/lib/inputs.js @@ -0,0 +1,99 @@ +// The xorshift32 input generator and the FNV-1a input hash, ported from src/inputs.c. +// JavaScript numbers are doubles, so every step is brought back to an unsigned 32-bit +// integer with ">>> 0"; the parity tests check the results against the C and Python workers. + +export const DEFAULT_SEED = 2463534242; +export const MAX_N = 100000; +export const ORDERS = /** @type {const} */ (["random", "sorted", "reversed", "nearly_sorted"]); + +/** @typedef {typeof ORDERS[number]} Order */ + +/** + * One step of xorshift32 (shifts 13, 17, 5). + * @param {number} state an unsigned 32-bit integer, not 0 + * @returns {number} the next state, which is also the output + */ +export function xorshift32(state) { + let x = state >>> 0; + x = (x ^ (x << 13)) >>> 0; + x = (x ^ (x >>> 17)) >>> 0; + x = (x ^ (x << 5)) >>> 0; + return x; +} + +/** + * The first `count` outputs of xorshift32 starting from `seed`. + * @param {number} seed + * @param {number} count + * @returns {number[]} + */ +export function sequence(seed, count) { + const values = []; + let state = seed; + for (let i = 0; i < count; i++) { + state = xorshift32(state); + values.push(state); + } + return values; +} + +/** + * @param {string} name + * @returns {name is Order} + */ +export function isOrder(name) { + return /** @type {readonly string[]} */ (ORDERS).includes(name); +} + +/** + * A permutation of 0..n-1 in the given order, exactly as src/inputs.c builds it. + * @param {number} n + * @param {Order} order + * @param {number} seed + * @returns {number[]} + */ +export function makeInput(n, order, seed) { + const arr = new Array(n); + for (let i = 0; i < n; i++) { + arr[i] = order === "reversed" ? n - 1 - i : i; + } + let state = seed; + if (order === "random") { + for (let i = n; i > 1; i--) { + // Fisher-Yates shuffle + state = xorshift32(state); + const j = state % i; + const tmp = arr[i - 1]; + arr[i - 1] = arr[j]; + arr[j] = tmp; + } + } else if (order === "nearly_sorted") { + for (let k = 0; k < Math.floor(n / 20); k++) { + // swap 5% of the elements with random partners + state = xorshift32(state); + const a = state % n; + state = xorshift32(state); + const b = state % n; + const tmp = arr[a]; + arr[a] = arr[b]; + arr[b] = tmp; + } + } + return arr; +} + +/** + * 32-bit FNV-1a over each value as four little-endian bytes (two's complement). + * @param {readonly number[]} arr + * @returns {number} + */ +export function inputHash(arr) { + let h = 2166136261; + for (const value of arr) { + const v = value >>> 0; + for (let shift = 0; shift < 32; shift += 8) { + h = Math.imul(h ^ ((v >>> shift) & 0xff), 16777619) >>> 0; + } + } + return h >>> 0; +} diff --git a/workers/node/lib/sorts.js b/workers/node/lib/sorts.js new file mode 100644 index 0000000..8353352 --- /dev/null +++ b/workers/node/lib/sorts.js @@ -0,0 +1,116 @@ +// Line-for-line ports of src/sorts.c, counting comparisons and writes in the same places, +// so all three languages report identical counts for identical inputs. Plain arrays and +// loops only; Array.prototype.sort is never used. + +/** @typedef {{ comparisons: number, writes: number }} Metrics */ +/** @typedef {(arr: number[]) => Metrics} SortFn */ + +/** @type {SortFn} */ +export function bubbleSort(arr) { + let comparisons = 0; + let writes = 0; + const n = arr.length; + if (n < 2) { + return { comparisons, writes }; + } + for (let i = 0; i < n - 1; i++) { + for (let j = 0; j < n - i - 1; j++) { + comparisons++; + if (arr[j] > arr[j + 1]) { + const tmp = arr[j]; + arr[j] = arr[j + 1]; + arr[j + 1] = tmp; + writes += 2; + } + } + } + return { comparisons, writes }; +} + +/** @type {SortFn} */ +export function insertionSort(arr) { + let comparisons = 0; + let writes = 0; + for (let i = 1; i < arr.length; i++) { + const key = arr[i]; + let j = i; + while (j > 0) { + comparisons++; // counted even when it ends the loop + if (arr[j - 1] <= key) { + break; + } + arr[j] = arr[j - 1]; + writes++; + j--; + } + if (j !== i) { + arr[j] = key; + writes++; + } + } + return { comparisons, writes }; +} + +/** @type {SortFn} */ +export function quickSort(arr) { + let comparisons = 0; + let writes = 0; + + /** @param {number} a @param {number} b */ + function swap(a, b) { + const tmp = arr[a]; + arr[a] = arr[b]; + arr[b] = tmp; + writes += 2; + } + + /** Lomuto partition around the last element; returns the pivot's final index. + * @param {number} low @param {number} high @returns {number} */ + function partition(low, high) { + const pivot = arr[high]; + let i = low; + for (let j = low; j < high; j++) { + comparisons++; + if (arr[j] <= pivot) { + if (i !== j) { + swap(i, j); + } + i++; + } + } + if (i !== high) { + swap(i, high); + } + return i; + } + + /** Recurses into the smaller side and loops on the larger, as src/sorts.c does. + * @param {number} low @param {number} high */ + function sortRange(low, high) { + while (low < high) { + const p = partition(low, high); + if (p - low < high - p) { + if (p > low) { + sortRange(low, p - 1); + } + low = p + 1; + } else { + if (p < high) { + sortRange(p + 1, high); + } + if (p === 0) { + return; + } + high = p - 1; + } + } + } + + if (arr.length > 1) { + sortRange(0, arr.length - 1); + } + return { comparisons, writes }; +} + +/** @type {Readonly>} */ +export const SORTS = Object.freeze({ bubble: bubbleSort, insertion: insertionSort, quick: quickSort }); diff --git a/workers/node/package-lock.json b/workers/node/package-lock.json new file mode 100644 index 0000000..5749a2c --- /dev/null +++ b/workers/node/package-lock.json @@ -0,0 +1,1405 @@ +{ + "name": "greenbench-node-worker", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "greenbench-node-worker", + "version": "2.0.0", + "license": "MIT", + "devDependencies": { + "@eslint/js": "10.0.1", + "@types/node": "24.13.6", + "eslint": "10.11.0", + "globals": "17.12.0", + "typescript": "7.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", + "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.5.0", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz", + "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", + "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz", + "integrity": "sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.13.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.6.tgz", + "integrity": "sha512-SGrw/h3KPFshy3OE6ZL53LMBG5vGQQ8/gIpiqz/kRZhPJ7HgwCEs8LBuNtWLa8dvGZVpSF7+Bf+c11HUrCb/yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/cacheable": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", + "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.2.0", + "@cacheable/utils": "^2.5.0", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.11.0.tgz", + "integrity": "sha512-P7a6UEEqb9G95MYAtqkmsTbVXIYyzIfl6NGOIJk162PaahFxFyeGcrlXYFSiagECg4sEm8IseJdZBKR3rx6MsQ==", + "dev": true, + "license": "MIT", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.7.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.3", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "11.1.5 || >11.1.6 <12", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/file-entry-cache": { + "version": "11.1.5", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz", + "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^6.1.23" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "6.1.23", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz", + "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cacheable": "^2.5.0", + "flatted": "^3.4.2", + "hookified": "^1.15.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "17.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.12.0.tgz", + "integrity": "sha512-cezEd/DTyyht9cvSSURyygXPfy04GtWO/5e6ZPvH7fCtjKz9PYOmuawphw1Ctd1f6C+5JypXfGD7ahNMXvevBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "dev": true, + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/workers/node/package.json b/workers/node/package.json new file mode 100644 index 0000000..09abff1 --- /dev/null +++ b/workers/node/package.json @@ -0,0 +1,23 @@ +{ + "name": "greenbench-node-worker", + "version": "2.0.0", + "private": true, + "description": "The Node.js worker for greenbench: three instrumented sorts, the xorshift32 input generator and the worker protocol.", + "license": "MIT", + "type": "module", + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "node --test", + "lint": "eslint .", + "typecheck": "tsc -p tsconfig.json" + }, + "devDependencies": { + "@eslint/js": "10.0.1", + "@types/node": "24.13.6", + "eslint": "10.11.0", + "globals": "17.12.0", + "typescript": "7.0.2" + } +} diff --git a/workers/node/test/cli.test.js b/workers/node/test/cli.test.js new file mode 100644 index 0000000..9a811c3 --- /dev/null +++ b/workers/node/test/cli.test.js @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { EXIT_USAGE, parseNumber, runCli } from "../lib/cli.js"; + +test("run prints one result line in the shared protocol", () => { + const outcome = runCli(["run", "--algorithm", "quick", "--order", "random", "--n", "4000", "--reps", "2"]); + assert.equal(outcome.status, 0); + assert.equal(outcome.stderr, ""); + assert.match( + outcome.stdout, + /^result language=node algorithm=quick order=random n=4000 seed=2463534242 reps=2 comparisons=55119 writes=56984 loop_ns=\d+ input_hash=[0-9a-f]{8}\n$/, + ); +}); + +test("the none algorithm accepts zero repetitions and does no work", () => { + const outcome = runCli(["run", "--algorithm", "none", "--order", "sorted", "--n", "10", "--reps", "0"]); + assert.equal(outcome.status, 0); + assert.match(outcome.stdout, / reps=0 comparisons=0 writes=0 /); +}); + +test("sequence and input print one number per line", () => { + assert.equal(runCli(["sequence", "--count", "2", "--seed", "1"]).stdout, "270369\n67634689\n"); + assert.equal(runCli(["input", "--order", "reversed", "--n", "3"]).stdout, "2\n1\n0\n"); +}); + +/** @type {Array<[string[], string]>} */ +const USAGE_ERRORS = [ + [[], "missing command"], + [["jump"], "unknown command jump"], + [["run", "--algorithm", "merge", "--order", "random", "--n", "5"], "--algorithm must be"], + [["run", "--algorithm", "quick", "--order", "random", "--n", "0"], "--n must be"], + [["run", "--algorithm", "quick", "--order", "random", "--n", "100001"], "--n must be"], + [["run", "--algorithm", "quick", "--order", "random", "--n", "5", "--reps", "0"], "--reps must be"], + [["run", "--algorithm", "quick", "--order", "upside", "--n", "5"], "--order must be"], + [["run", "--algorithm", "quick", "--order", "random", "--n", "5", "--seed", "0"], "--seed must be"], + [["run", "--algorithm", "quick", "--order", "random", "--n", "5", "--count", "1"], "does not take --count"], + [["sequence", "--count", "1", "--n", "3"], "takes only --count and --seed"], + [["input", "--order", "random"], "--n must be"], + [["run", "--algorithm"], "missing value for --algorithm"], + [["run", "--n", "1", "--n", "2"], "repeated argument --n"], + [["run", "--colour", "blue"], "unknown argument --colour"], +]; + +for (const [args, message] of USAGE_ERRORS) { + test(`usage error: ${message}`, () => { + const outcome = runCli(args); + assert.equal(outcome.status, EXIT_USAGE); + assert.equal(outcome.stdout, ""); + assert.ok(outcome.stderr.includes(message), outcome.stderr); + assert.ok(outcome.stderr.includes("usage: greenbench-worker")); + }); +} + +test("parseNumber accepts digits only, within range", () => { + assert.equal(parseNumber("42", 1, 100), 42); + assert.equal(parseNumber("0", 0, 1), 0); + assert.equal(parseNumber("+1", 0, 10), null); + assert.equal(parseNumber("1e3", 0, 10000), null); + assert.equal(parseNumber(" 5", 0, 10), null); + assert.equal(parseNumber("", 0, 10), null); + assert.equal(parseNumber(undefined, 0, 10), null); + assert.equal(parseNumber("11", 0, 10), null); +}); diff --git a/workers/node/test/inputs.test.js b/workers/node/test/inputs.test.js new file mode 100644 index 0000000..958f67c --- /dev/null +++ b/workers/node/test/inputs.test.js @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { DEFAULT_SEED, inputHash, makeInput, sequence, xorshift32 } from "../lib/inputs.js"; + +// Reference values shared with tests/test_inputs.c and the Python tests. +test("xorshift32 matches the C generator from the default seed", () => { + assert.deepEqual(sequence(DEFAULT_SEED, 5), [723471715, 2497366906, 2064144800, 2008045182, 3532304609]); +}); + +test("xorshift32 from seed 1", () => { + assert.deepEqual(sequence(1, 3), [270369, 67634689, 2647435461]); +}); + +test("xorshift32 stays within 32 bits for the largest state", () => { + const next = xorshift32(0xffffffff); + assert.ok(Number.isInteger(next) && next >= 0 && next <= 0xffffffff); +}); + +/** @param {number[]} arr */ +function isPermutation(arr) { + return [...arr].sort((a, b) => a - b).every((value, i) => value === i); +} + +test("sorted and reversed inputs", () => { + assert.deepEqual(makeInput(5, "sorted", DEFAULT_SEED), [0, 1, 2, 3, 4]); + assert.deepEqual(makeInput(5, "reversed", DEFAULT_SEED), [4, 3, 2, 1, 0]); +}); + +test("random input is a reproducible permutation", () => { + const a = makeInput(1000, "random", DEFAULT_SEED); + assert.ok(isPermutation(a)); + assert.deepEqual(makeInput(1000, "random", DEFAULT_SEED), a); + assert.notDeepEqual(makeInput(1000, "random", 7), a); +}); + +test("nearly sorted input moves at most 10% of the elements", () => { + const a = makeInput(1000, "nearly_sorted", DEFAULT_SEED); + const displaced = a.filter((value, i) => value !== i).length; + assert.ok(isPermutation(a)); + assert.ok(displaced > 0 && displaced <= 100); +}); + +test("FNV-1a reference values", () => { + assert.equal(inputHash([]), 0x811c9dc5); + assert.equal(inputHash([0]), 0x4b95f515); + assert.equal(inputHash([1, 2, 3]), 0x794671b5); + assert.equal(inputHash([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), 0x3a210ed4); + assert.equal(inputHash([-1]), 0xe3160fb1); +}); diff --git a/workers/node/test/sorts.test.js b/workers/node/test/sorts.test.js new file mode 100644 index 0000000..cbae75d --- /dev/null +++ b/workers/node/test/sorts.test.js @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { DEFAULT_SEED, makeInput } from "../lib/inputs.js"; +import { SORTS, bubbleSort, insertionSort, quickSort } from "../lib/sorts.js"; + +/** A small linear congruential generator, so the random cases are reproducible. */ +function* numbers() { + let state = 12345; + for (;;) { + state = (Math.imul(state, 1103515245) + 12345) >>> 0; + yield ((state >>> 16) % 101) - 50; + } +} + +for (const [name, sort] of Object.entries(SORTS)) { + test(`${name} sort agrees with Array.prototype.sort on arrays with duplicates`, () => { + const source = numbers(); + for (const n of [0, 1, 2, 3, 7, 64, 500]) { + for (let trial = 0; trial < 10; trial++) { + const arr = Array.from({ length: n }, () => /** @type {number} */ (source.next().value)); + const expected = [...arr].sort((a, b) => a - b); + sort(arr); + assert.deepEqual(arr, expected); + } + } + }); +} + +/** @param {boolean} reversed */ +function hundred(reversed) { + return Array.from({ length: 100 }, (_, i) => (reversed ? 99 - i : i)); +} + +// For 100 elements, n(n-1)/2 = 4950. The same checks as tests/test_sorts.c. +test("known counts on sorted and reversed input", () => { + assert.deepEqual(bubbleSort(hundred(false)), { comparisons: 4950, writes: 0 }); + assert.deepEqual(bubbleSort(hundred(true)), { comparisons: 4950, writes: 9900 }); + assert.deepEqual(insertionSort(hundred(false)), { comparisons: 99, writes: 0 }); + assert.deepEqual(insertionSort(hundred(true)), { comparisons: 4950, writes: 4950 + 99 }); + assert.deepEqual(quickSort(hundred(false)), { comparisons: 4950, writes: 0 }); + assert.equal(quickSort(hundred(true)).comparisons, 4950); +}); + +test("the counts Algorithm-Energy-C published for 4,000 random integers", () => { + const input = makeInput(4000, "random", DEFAULT_SEED); + assert.deepEqual(bubbleSort([...input]), { comparisons: 7998000, writes: 8003836 }); + assert.deepEqual(insertionSort([...input]), { comparisons: 4005907, writes: 4005905 }); + assert.deepEqual(quickSort([...input]), { comparisons: 55119, writes: 56984 }); +}); diff --git a/workers/node/tsconfig.json b/workers/node/tsconfig.json new file mode 100644 index 0000000..2f10ca8 --- /dev/null +++ b/workers/node/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "noEmit": true, + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noUncheckedIndexedAccess": false, + "exactOptionalPropertyTypes": true, + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2022", + "lib": ["es2022"], + "types": ["node"] + }, + "include": ["worker.js", "lib/**/*.js", "test/**/*.js", "eslint.config.js"] +} diff --git a/workers/node/worker.js b/workers/node/worker.js new file mode 100644 index 0000000..426a931 --- /dev/null +++ b/workers/node/worker.js @@ -0,0 +1,15 @@ +#!/usr/bin/env node +// greenbench-worker in Node.js: the same commands and output as the C worker (src/worker.c). +// +// node worker.js run --algorithm NAME --order NAME --n N [--seed S] [--reps K] +// node worker.js sequence --count C [--seed S] +// node worker.js input --order NAME --n N [--seed S] +// +// No runtime dependencies, so it runs on the Node.js that a Linux distribution ships. + +import { runCli } from "./lib/cli.js"; + +const outcome = runCli(process.argv.slice(2)); +process.stdout.write(outcome.stdout); +process.stderr.write(outcome.stderr); +process.exitCode = outcome.status; From a73319cadac55cb3b0d9f93b612eec625e0c85db Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:47 +0400 Subject: [PATCH 08/51] test: check cross-language parity and the harness end to end The parity tests compare xorshift32 sequences, generated inputs up to 100,000 elements and the operation counts of every algorithm and order in C, Python and Node.js. The harness tests run the real binary with energy off, with fake powercap trees, and with failing or disagreeing workers. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_harness.py | 243 ++++++++++++++++++++++++++++++ tests/integration/test_parity.py | 88 +++++++++++ tests/integration/workers.py | 65 ++++++++ 3 files changed, 396 insertions(+) create mode 100644 tests/integration/test_harness.py create mode 100644 tests/integration/test_parity.py create mode 100644 tests/integration/workers.py diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py new file mode 100644 index 0000000..09bd2c5 --- /dev/null +++ b/tests/integration/test_harness.py @@ -0,0 +1,243 @@ +"""End-to-end tests of the harness binary: real workers, real processes, fake powercap trees.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +import pytest + +from powercap import build as build_powercap +from workers import REPOSITORY, interpreter_options, require_build, run_harness + +FIXTURES = REPOSITORY / "tests" / "fixtures" / "powercap" +SMALL = ["--profile", "smoke", "--sizes", "20,50", "--orders", "random,sorted", "--runs", "3", "--quiet"] + + +@pytest.fixture(scope="module", autouse=True) +def built() -> None: + require_build() + + +def read_runs(out: Path) -> list[dict[str, str]]: + with (out / "runs.csv").open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def read_meta(out: Path) -> dict[str, Any]: + meta: dict[str, Any] = json.loads((out / "meta.json").read_text(encoding="utf-8")) + return meta + + +def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: + out = tmp_path / "run" + completed = run_harness("run", *SMALL, "--energy", "off", "--out", str(out), *interpreter_options()) + assert completed.returncode == 0, completed.stderr + rows = read_runs(out) + # 3 languages x (3 algorithms + startup) x 2 orders x 2 sizes x 3 runs + assert len(rows) == 3 * 4 * 2 * 2 * 3 + assert [int(r["sequence"]) for r in rows] == list(range(1, len(rows) + 1)) + for row in rows: + assert row["energy_status"] == "off" + for column in ("pkg_uj", "dram_uj", "idle_ns", "idle_pkg_uj", "idle_dram_uj", "net_pkg_uj", "net_dram_uj"): + assert row[column] == "", f"{column} must be empty, never zero, without energy" + assert int(row["wall_ns"]) > 0 + if row["algorithm"] == "none": + assert row["inner_reps"] == "0" + else: + assert int(row["inner_reps"]) >= 1 + + # Every configuration has runs 1..3, and the run order is shuffled rather than grouped. + configurations = {(r["language"], r["algorithm"], r["order"], r["n"]) for r in rows} + assert len(configurations) == 48 + for key in configurations: + reps = sorted(int(r["repetition"]) for r in rows if (r["language"], r["algorithm"], r["order"], r["n"]) == key) + assert reps == [1, 2, 3] + first_twelve = {(r["language"], r["algorithm"]) for r in rows[:12]} + assert len(first_twelve) > 3 + + meta = read_meta(out) + assert meta["status"] == "complete" + assert meta["measured_runs"] == len(rows) + assert meta["energy"]["status"] == "off" + assert meta["energy"]["idle_baseline"] is False + assert meta["clock"] == "CLOCK_MONOTONIC_RAW" + assert meta["settings"]["runs"] == 3 + assert meta["command"].startswith("greenbench run --profile smoke") + assert len(meta["calibration"]) == 48 + assert set(meta["workers"]) == {"c", "python", "node"} + + +def test_counts_match_across_languages_in_the_csv(tmp_path: Path) -> None: + out = tmp_path / "run" + completed = run_harness("run", *SMALL, "--energy", "off", "--out", str(out), *interpreter_options()) + assert completed.returncode == 0, completed.stderr + counts: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} + for row in read_runs(out): + key = (row["algorithm"], row["order"], row["n"]) + counts.setdefault(key, set()).add((row["comparisons"], row["writes"], row["input_hash"])) + assert all(len(values) == 1 for values in counts.values()) + + +def test_fake_powercap_tree_gives_energy_columns(tmp_path: Path) -> None: + root = build_powercap(FIXTURES / "server-two-socket.txt", tmp_path / "powercap") + out = tmp_path / "run" + completed = run_harness( + "run", + *SMALL, + "--languages", + "c", + "--energy", + "require", + "--powercap-root", + str(root), + "--out", + str(out), + *interpreter_options(), + ) + assert completed.returncode == 0, completed.stderr + rows = read_runs(out) + for row in rows: + # The fake counters never move, so every energy is exactly zero; what matters is that the + # columns are filled and the idle baseline ran for the same duration. + assert row["energy_status"] == "ok" + assert row["pkg_uj"] == "0" + assert row["dram_uj"] == "0" + assert row["idle_pkg_uj"] == "0" + assert row["net_pkg_uj"] == "0.0" + assert int(row["idle_ns"]) >= int(row["wall_ns"]) + meta = read_meta(out) + assert meta["energy"]["status"] == "ok" + assert meta["energy"]["idle_baseline"] is True + assert [d["zone"] for d in meta["energy"]["domains"]] == [ + "intel-rapl:0", + "intel-rapl:0:0", + "intel-rapl:1", + "intel-rapl:1:0", + ] + + +def test_energy_required_but_unavailable_exits_3(tmp_path: Path) -> None: + root = build_powercap(FIXTURES / "no-rapl.txt", tmp_path / "powercap") + completed = run_harness( + "run", *SMALL, "--energy", "require", "--powercap-root", str(root), "--out", str(tmp_path / "run") + ) + assert completed.returncode == 3 + assert "energy unavailable: no RAPL package zone" in completed.stderr + assert not (tmp_path / "run" / "runs.csv").exists() + + +def test_energy_auto_falls_back_to_time_only(tmp_path: Path) -> None: + out = tmp_path / "run" + completed = run_harness( + "run", + "--profile", + "smoke", + "--languages", + "c", + "--sizes", + "20", + "--runs", + "2", + "--powercap-root", + str(tmp_path / "does-not-exist"), + "--out", + str(out), + ) + assert completed.returncode == 0, completed.stderr + assert "energy unavailable:" in completed.stderr + assert "does not exist" in completed.stderr + assert {row["energy_status"] for row in read_runs(out)} == {"unavailable"} + assert "does not exist" in read_meta(out)["energy"]["reason"] + + +def test_probe(tmp_path: Path) -> None: + root = build_powercap(FIXTURES / "laptop-intel.txt", tmp_path / "powercap") + completed = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "50") + assert completed.returncode == 0 + assert "energy: available" in completed.stdout + assert "intel-rapl:0:2" in completed.stdout + assert "psys" not in completed.stdout + assert "did not change" in completed.stdout # the fake counters are frozen + + missing = run_harness("probe", "--powercap-root", str(tmp_path / "nothing"), "--sample-ms", "0") + assert missing.returncode == 3 + assert "energy: unavailable" in missing.stdout + + +def test_dry_run_prints_the_plan() -> None: + completed = run_harness("run", "--dry-run") + assert completed.returncode == 0 + assert "profile study: 192 trials (3 languages x 3 algorithms + startup x 4 orders x 4 sizes)" in completed.stdout + assert "3 warm-up and 30 measured runs per trial" in completed.stdout + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["run", "--runs", "0"], "--runs"), + (["run", "--languages", "ruby"], "unknown language 'ruby'"), + (["run", "--bogus"], "unknown option --bogus"), + (["walk"], "unknown command 'walk'"), + (["probe", "--sample-ms", "lots"], "unexpected or invalid argument"), + ], +) +def test_usage_errors_exit_2(args: list[str], message: str) -> None: + completed = run_harness(*args) + assert completed.returncode == 2 + assert message in completed.stderr + + +def test_missing_interpreter_is_reported(tmp_path: Path) -> None: + completed = run_harness( + "run", *SMALL, "--languages", "python", "--python", "python-that-does-not-exist", "--out", str(tmp_path) + ) + assert completed.returncode == 2 + assert "cannot run 'python-that-does-not-exist --version'" in completed.stderr + + +def test_failing_worker_stops_the_run(tmp_path: Path) -> None: + broken = tmp_path / "broken_worker.py" + broken.write_text("import sys\nsys.exit('greenbench-worker: simulated crash')\n", encoding="utf-8") + completed = run_harness( + "run", + *SMALL, + "--languages", + "python", + "--python-worker", + str(broken), + "--energy", + "off", + "--out", + str(tmp_path / "run"), + *interpreter_options(), + ) + assert completed.returncode == 1 + assert "simulated crash" in completed.stderr + assert read_meta(tmp_path / "run")["status"] == "failed" + + +def test_disagreeing_worker_is_caught(tmp_path: Path) -> None: + # A worker that reports one extra comparison must be rejected, not averaged in. + source = (REPOSITORY / "workers" / "python" / "greenbench_worker.py").read_text(encoding="utf-8") + lying = source.replace("comparisons={first[0]}", "comparisons={first[0] + 1}") + assert lying != source + fake = tmp_path / "lying_worker.py" + fake.write_text(lying, encoding="utf-8") + completed = run_harness( + "run", + *SMALL, + "--languages", + "c,python", + "--python-worker", + str(fake), + "--energy", + "off", + "--out", + str(tmp_path / "run"), + *interpreter_options(), + ) + assert completed.returncode == 1 + assert "the implementations disagree" in completed.stderr diff --git a/tests/integration/test_parity.py b/tests/integration/test_parity.py new file mode 100644 index 0000000..5b6a5ce --- /dev/null +++ b/tests/integration/test_parity.py @@ -0,0 +1,88 @@ +"""Cross-language parity: C, Python and Node.js must generate identical inputs and do +identical work, or the energy comparison would compare different things.""" + +from __future__ import annotations + +import pytest + +from workers import LANGUAGES, require_build, run_worker + +SEEDS = ("2463534242", "1", "4294967295", "123456789") +ORDERS = ("random", "sorted", "reversed", "nearly_sorted") +SIZES = ("1", "2", "3", "20", "21", "257", "1000", "100000") + + +@pytest.fixture(scope="module", autouse=True) +def built() -> None: + require_build() + + +def outputs(*args: str) -> dict[str, str]: + result = {} + for language in LANGUAGES: + completed = run_worker(language, *args) + assert completed.returncode == 0, f"{language}: {completed.stderr}" + result[language] = completed.stdout + return result + + +@pytest.mark.parametrize("seed", SEEDS) +def test_xorshift_sequences_are_identical(seed: str) -> None: + results = outputs("sequence", "--count", "10000", "--seed", seed) + assert results["python"] == results["c"] + assert results["node"] == results["c"] + assert len(results["c"].splitlines()) == 10000 + + +def test_sequence_starts_with_the_reference_values() -> None: + first = outputs("sequence", "--count", "3")["c"].split() + assert first == ["723471715", "2497366906", "2064144800"] + + +@pytest.mark.parametrize("order", ORDERS) +@pytest.mark.parametrize("n", SIZES) +def test_inputs_are_identical(order: str, n: str) -> None: + results = outputs("input", "--order", order, "--n", n, "--seed", "2463534242") + assert results["python"] == results["c"] + assert results["node"] == results["c"] + assert sorted(int(v) for v in results["c"].split()) == list(range(int(n))) + + +def result_fields(line: str) -> dict[str, str]: + assert line.startswith("result ") + assert line.endswith("\n") + return dict(item.split("=", 1) for item in line.split()[1:]) + + +@pytest.mark.parametrize("algorithm", ["bubble", "insertion", "quick", "none"]) +@pytest.mark.parametrize("order", ORDERS) +@pytest.mark.parametrize("n", ["1", "7", "300"]) +def test_operation_counts_and_input_hashes_are_identical(algorithm: str, order: str, n: str) -> None: + reps = "0" if algorithm == "none" else "2" + lines = outputs("run", "--algorithm", algorithm, "--order", order, "--n", n, "--seed", "99", "--reps", reps) + fields = {language: result_fields(line) for language, line in lines.items()} + for language, values in fields.items(): + assert values["language"] == language + values.pop("language") + values.pop("loop_ns") # each language times itself; everything else must match + assert fields["python"] == fields["c"] + assert fields["node"] == fields["c"] + + +@pytest.mark.parametrize( + "args", + [ + ("run", "--algorithm", "quick", "--order", "random", "--n", "0"), + ("run", "--algorithm", "quick", "--order", "random", "--n", "5", "--seed", "0"), + ("run", "--algorithm", "heap", "--order", "random", "--n", "5"), + ("sequence", "--count", "0"), + ("input", "--order", "random"), + ("fly",), + ], +) +def test_usage_errors_exit_2_everywhere(args: tuple[str, ...]) -> None: + for language in LANGUAGES: + completed = run_worker(language, *args) + assert completed.returncode == 2, f"{language} {args}: {completed.returncode}" + assert completed.stdout == "" + assert "usage: greenbench-worker" in completed.stderr diff --git a/tests/integration/workers.py b/tests/integration/workers.py new file mode 100644 index 0000000..69bdb0e --- /dev/null +++ b/tests/integration/workers.py @@ -0,0 +1,65 @@ +"""How the integration tests find and start the three workers and the harness. + +The binaries come from `make` (build/ by default; set GREENBENCH_BUILD to test another +build, such as the sanitizer one). Node.js and Python are found on PATH. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +REPOSITORY = Path(__file__).resolve().parents[2] +BUILD = Path(os.environ.get("GREENBENCH_BUILD", REPOSITORY / "build")) +HARNESS = BUILD / "greenbench" +C_WORKER = BUILD / "greenbench-worker" +PYTHON_WORKER = REPOSITORY / "workers" / "python" / "greenbench_worker.py" +NODE_WORKER = REPOSITORY / "workers" / "node" / "worker.js" +LANGUAGES = ("c", "python", "node") + + +def require_build() -> None: + """Fail with a clear message, rather than a confusing error, when make has not run.""" + missing = [str(p) for p in (HARNESS, C_WORKER) if not p.exists()] + if missing: + raise RuntimeError(f"missing {', '.join(missing)}; run 'make' before the integration tests") + + +def node() -> str: + found = shutil.which("node") + if found is None: + raise RuntimeError("node is not on PATH; the parity tests need Node.js 18 or later") + return found + + +def worker_command(language: str) -> list[str]: + if language == "c": + return [str(C_WORKER)] + if language == "python": + return [sys.executable, str(PYTHON_WORKER)] + if language == "node": + return [node(), str(NODE_WORKER)] + raise ValueError(language) + + +def run_worker(language: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([*worker_command(language), *args], capture_output=True, text=True, check=False, timeout=300) + + +def run_harness(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + """Run build/greenbench with this Python and Node.js as the interpreters.""" + return subprocess.run( + [str(HARNESS), *args], + capture_output=True, + text=True, + check=False, + timeout=600, + cwd=cwd, + ) + + +def interpreter_options() -> list[str]: + return ["--python", sys.executable, "--node", node()] From d08f42117a317ca3c3df3c81cdcb3ca9ec1db691 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:47 +0400 Subject: [PATCH 09/51] feat: add grid carbon intensity for the UAE and the UK Ember's 2024 lifecycle figures as published by Our World in Data (CC BY 4.0), with the source, basis and retrieval date recorded next to each value. Co-Authored-By: Claude Opus 5.5 --- data/README.md | 29 +++++++++++++++++++++++++++++ data/grid_intensity.csv | 3 +++ 2 files changed, 32 insertions(+) create mode 100644 data/README.md create mode 100644 data/grid_intensity.csv diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..15f7e8a --- /dev/null +++ b/data/README.md @@ -0,0 +1,29 @@ +# Data + +## grid_intensity.csv + +Carbon intensity of electricity generation, used by the report to turn measured energy into +grams of CO2-equivalent. Add a row to use another region, or pass +`--intensity NAME=GRAMS_PER_KWH` to the report. + +| Column | Meaning | +| :--- | :--- | +| `region_code` | ISO 3166-1 alpha-3 code, used with `--regions` | +| `year` | the year the figure describes | +| `g_co2e_per_kwh` | grams of CO2-equivalent per kilowatt-hour generated | +| `basis` | what the figure covers | +| `source`, `source_url`, `licence`, `retrieved` | where it came from and when | + +The two rows come from Ember's yearly electricity data, as published by Our World in Data in +the chart "Lifecycle carbon intensity of electricity" (indicator 1295504, last updated +2026-06-30), read on 2026-09-26. 2024 is the latest year with a value for both countries. + +- Ember, *Yearly Electricity Data* and *Yearly Electricity Data Europe* (2026 release, + published 2026-04-21), . + Licensed under [CC BY 4.0](https://ember-energy.org/creative-commons/). +- Our World in Data, , + CC BY 4.0. + +These are lifecycle figures: they include emissions from building power stations and +supplying fuel, and cover all greenhouse gases as CO2-equivalent. They are annual averages +for the whole grid, not the marginal emissions of extra demand at a given hour. diff --git a/data/grid_intensity.csv b/data/grid_intensity.csv new file mode 100644 index 0000000..b05c6d7 --- /dev/null +++ b/data/grid_intensity.csv @@ -0,0 +1,3 @@ +region_code,region,year,g_co2e_per_kwh,basis,source,source_url,licence,retrieved +ARE,United Arab Emirates,2024,467.51,"lifecycle, whole grid, annual average","Ember (2026), yearly electricity data, via Our World in Data indicator 1295504",https://ourworldindata.org/grapher/carbon-intensity-electricity,CC BY 4.0,2026-09-26 +GBR,United Kingdom,2024,216.50,"lifecycle, whole grid, annual average","Ember (2026), yearly electricity data, via Our World in Data indicator 1295504",https://ourworldindata.org/grapher/carbon-intensity-electricity,CC BY 4.0,2026-09-26 From da4c52dc2a29bf3c8cd79b5bdab9cfd4780cfc2c Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:48 +0400 Subject: [PATCH 10/51] feat: add the statistics and report package greenbench_analysis loads runs.csv and meta.json with strict validation, computes medians with percentile-bootstrap confidence intervals and Spearman's rank correlation between time and energy, converts energy to CO2-equivalent, and writes docs/results.md with its charts. With --withhold-timings it marks time, energy and carbon figures as pending. It replaces visualize.py. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/__init__.py | 3 + analysis/greenbench_analysis/__main__.py | 5 + analysis/greenbench_analysis/carbon.py | 116 ++++++ analysis/greenbench_analysis/charts.py | 205 ++++++++++ analysis/greenbench_analysis/cli.py | 98 +++++ analysis/greenbench_analysis/load.py | 181 +++++++++ analysis/greenbench_analysis/report.py | 475 +++++++++++++++++++++++ analysis/greenbench_analysis/stats.py | 137 +++++++ analysis/greenbench_analysis/summary.py | 263 +++++++++++++ analysis/tests/synthetic.py | 155 ++++++++ analysis/tests/test_carbon.py | 81 ++++ analysis/tests/test_load.py | 81 ++++ analysis/tests/test_report.py | 137 +++++++ analysis/tests/test_stats.py | 106 +++++ analysis/tests/test_summary.py | 100 +++++ tests/integration/test_pipeline.py | 80 ++++ visualize.py | 99 ----- 17 files changed, 2223 insertions(+), 99 deletions(-) create mode 100644 analysis/greenbench_analysis/__init__.py create mode 100644 analysis/greenbench_analysis/__main__.py create mode 100644 analysis/greenbench_analysis/carbon.py create mode 100644 analysis/greenbench_analysis/charts.py create mode 100644 analysis/greenbench_analysis/cli.py create mode 100644 analysis/greenbench_analysis/load.py create mode 100644 analysis/greenbench_analysis/report.py create mode 100644 analysis/greenbench_analysis/stats.py create mode 100644 analysis/greenbench_analysis/summary.py create mode 100644 analysis/tests/synthetic.py create mode 100644 analysis/tests/test_carbon.py create mode 100644 analysis/tests/test_load.py create mode 100644 analysis/tests/test_report.py create mode 100644 analysis/tests/test_stats.py create mode 100644 analysis/tests/test_summary.py create mode 100644 tests/integration/test_pipeline.py delete mode 100644 visualize.py diff --git a/analysis/greenbench_analysis/__init__.py b/analysis/greenbench_analysis/__init__.py new file mode 100644 index 0000000..9310e2d --- /dev/null +++ b/analysis/greenbench_analysis/__init__.py @@ -0,0 +1,3 @@ +"""Statistics, carbon estimates and the results report for greenbench runs.""" + +__version__ = "2.0.0" diff --git a/analysis/greenbench_analysis/__main__.py b/analysis/greenbench_analysis/__main__.py new file mode 100644 index 0000000..dd8a8c9 --- /dev/null +++ b/analysis/greenbench_analysis/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/analysis/greenbench_analysis/carbon.py b/analysis/greenbench_analysis/carbon.py new file mode 100644 index 0000000..aa00173 --- /dev/null +++ b/analysis/greenbench_analysis/carbon.py @@ -0,0 +1,116 @@ +"""Convert energy into grams of CO2-equivalent with a grid carbon intensity. + +The intensities are annual averages for a whole grid (location-based), so the result is +an estimate of the emissions attributable to the electricity, not a measurement. +""" + +from __future__ import annotations + +import csv +import re +from dataclasses import dataclass +from pathlib import Path + +from .load import DataError + +JOULES_PER_KWH = 3_600_000.0 +GRID_COLUMNS = ( + "region_code", + "region", + "year", + "g_co2e_per_kwh", + "basis", + "source", + "source_url", + "licence", + "retrieved", +) + + +@dataclass(frozen=True) +class GridIntensity: + """Grams of CO2-equivalent per kilowatt-hour of electricity, and where the figure came from.""" + + code: str + region: str + year: int | None + g_per_kwh: float + basis: str + source: str + source_url: str + licence: str + retrieved: str + + +def grams_co2e(energy_uj: float, g_per_kwh: float) -> float: + """Grams of CO2-equivalent for energy_uj microjoules at g_per_kwh.""" + return energy_uj / 1e6 / JOULES_PER_KWH * g_per_kwh + + +def load_grid(path: Path) -> list[GridIntensity]: + """Load the grid intensity table (data/grid_intensity.csv).""" + try: + with path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + header = tuple(rows[0].keys()) if rows else () + except FileNotFoundError: + raise DataError(f"{path}: file not found") from None + if header != GRID_COLUMNS: + raise DataError(f"{path}: expected the columns {', '.join(GRID_COLUMNS)}") + grid = [] + for number, row in enumerate(rows, start=2): + try: + value = float(row["g_co2e_per_kwh"]) + year = int(row["year"]) + except ValueError: + raise DataError(f"{path}:{number}: year and g_co2e_per_kwh must be numbers") from None + if value < 0: + raise DataError(f"{path}:{number}: a carbon intensity cannot be negative") + grid.append( + GridIntensity( + code=row["region_code"], + region=row["region"], + year=year, + g_per_kwh=value, + basis=row["basis"], + source=row["source"], + source_url=row["source_url"], + licence=row["licence"], + retrieved=row["retrieved"], + ) + ) + return grid + + +def select_regions(grid: list[GridIntensity], codes: list[str]) -> list[GridIntensity]: + """The latest year for each requested region code, in the order requested.""" + chosen = [] + for code in codes: + matches = [g for g in grid if g.code == code] + if not matches: + known = ", ".join(sorted({g.code for g in grid})) + raise DataError(f"no grid intensity for region {code!r} (known: {known}); add it with --intensity") + chosen.append(max(matches, key=lambda g: g.year or 0)) + return chosen + + +_CUSTOM = re.compile(r"^(?P[^=]+)=(?P[0-9]+(\.[0-9]+)?)$") + + +def parse_custom(spec: str) -> GridIntensity: + """A user-supplied intensity, written NAME=GRAMS_PER_KWH, for example "Solar farm=45".""" + match = _CUSTOM.match(spec.strip()) + if match is None: + raise DataError(f"--intensity expects NAME=GRAMS_PER_KWH, got {spec!r}") + name = match["name"].strip() + return GridIntensity( + code=name, + region=name, + year=None, + g_per_kwh=float(match["value"]), + basis="supplied on the command line", + source="--intensity", + source_url="", + licence="", + retrieved="", + ) diff --git a/analysis/greenbench_analysis/charts.py b/analysis/greenbench_analysis/charts.py new file mode 100644 index 0000000..317a57e --- /dev/null +++ b/analysis/greenbench_analysis/charts.py @@ -0,0 +1,205 @@ +"""Charts for the results report, drawn with matplotlib into PNG files. + +The charts carry their own light background, so they read the same on light and dark +pages. Colours follow a fixed categorical order (blue, orange, aqua) that stays +distinguishable with the common colour-vision deficiencies; every chart has a legend, and +the report prints the same numbers in tables. +""" + +from __future__ import annotations + +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") # files only, no window + +import matplotlib.pyplot as plt +from matplotlib.axes import Axes +from matplotlib.figure import Figure + +from .load import ORDERS +from .summary import Configuration, OperationCounts + +SURFACE = "#fcfcfb" +INK = "#0b0b0b" +INK_SECONDARY = "#52514e" +MUTED = "#898781" +GRID = "#e1e0d9" +AXIS = "#c3c2b7" +SERIES = ("#2a78d6", "#eb6834", "#1baf7a") # categorical slots 1-3, never cycled +ALGORITHM_ORDER = ("bubble", "insertion", "quick") +LANGUAGE_ORDER = ("c", "python", "node") +LANGUAGE_NAMES = {"c": "C", "python": "Python", "node": "Node.js"} +DPI = 144 + + +def _style(ax: Axes) -> None: + ax.set_facecolor(SURFACE) + ax.grid(True, which="major", color=GRID, linewidth=0.8) + ax.set_axisbelow(True) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + for side in ("left", "bottom"): + ax.spines[side].set_color(AXIS) + ax.tick_params(colors=MUTED, labelcolor=INK_SECONDARY, labelsize=8) + ax.xaxis.label.set_color(INK_SECONDARY) + ax.yaxis.label.set_color(INK_SECONDARY) + ax.title.set_color(INK) + + +def _figure(width: float, height: float, columns: int = 1) -> tuple[Figure, list[Axes]]: + fig, axes = plt.subplots(1, columns, figsize=(width, height), squeeze=False, sharey=True) + fig.patch.set_facecolor(SURFACE) + flat = list(axes[0]) + for ax in flat: + _style(ax) + return fig, flat + + +def _save(fig: Figure, path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(path, dpi=DPI, facecolor=SURFACE, bbox_inches="tight") + plt.close(fig) + return path + + +def operations_chart(operations: list[OperationCounts], path: Path) -> Path | None: + """Comparisons against input size, one panel per input order, one line per algorithm.""" + orders = [o for o in ORDERS if any(c.order == o for c in operations)] + sizes = sorted({c.n for c in operations}) + if not orders or len(sizes) < 2: + return None + fig, axes = _figure(2.6 * len(orders) + 0.6, 3.2, len(orders)) + for ax, order in zip(axes, orders, strict=True): + for colour, algorithm in zip(SERIES, ALGORITHM_ORDER, strict=True): + points = sorted((c.n, c.comparisons) for c in operations if c.order == order and c.algorithm == algorithm) + if not points: + continue + ax.plot( + [p[0] for p in points], + [max(p[1], 1) for p in points], + color=colour, + linewidth=2, + marker="o", + markersize=5, + markeredgecolor=SURFACE, + markeredgewidth=1.5, + label=algorithm, + ) + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_title(order.replace("_", " "), fontsize=10, loc="left") + ax.set_xlabel("Input size (elements)", fontsize=8) + axes[0].set_ylabel("Comparisons per sort", fontsize=8) + handles, labels = axes[0].get_legend_handles_labels() + fig.legend(handles, labels, loc="upper center", ncol=3, frameon=False, fontsize=8, bbox_to_anchor=(0.5, 1.08)) + return _save(fig, path) + + +def _dot_chart( + configurations: list[Configuration], + value: str, + xlabel: str, + title: str, + path: Path, +) -> Path | None: + """Median per sort with its 95% interval: one row per algorithm, one colour per language.""" + rows = [a for a in ALGORITHM_ORDER if any(c.algorithm == a for c in configurations)] + languages = [lang for lang in LANGUAGE_ORDER if any(c.language == lang for c in configurations)] + if not rows: + return None + fig, (ax,) = _figure(6.4, 0.9 + 0.75 * len(rows)) + offsets = {lang: (i - (len(languages) - 1) / 2) * 0.22 for i, lang in enumerate(languages)} + plotted = False + for colour, language in zip(SERIES, LANGUAGE_ORDER, strict=True): + if language not in languages: + continue + xs, ys, lows, highs = [], [], [], [] + for row, algorithm in enumerate(rows): + match = [c for c in configurations if c.language == language and c.algorithm == algorithm] + if not match: + continue + estimate = getattr(match[0], value) + if estimate is None or estimate.value <= 0 or estimate.low <= 0: + continue # a log axis cannot show zero or negative values; the table still does + xs.append(estimate.value) + ys.append(row + offsets[language]) + lows.append(estimate.value - estimate.low) + highs.append(estimate.high - estimate.value) + if not xs: + continue + plotted = True + ax.errorbar(xs, ys, xerr=[lows, highs], fmt="none", ecolor=colour, elinewidth=2, capsize=0) + ax.scatter( + xs, ys, s=64, color=colour, edgecolors=SURFACE, linewidths=2, zorder=3, label=LANGUAGE_NAMES[language] + ) + if not plotted: + plt.close(fig) + return None + ax.set_xscale("log") + ax.set_yticks(range(len(rows)), rows) + ax.set_ylim(len(rows) - 0.5, -0.5) + ax.set_xlabel(xlabel, fontsize=8) + ax.set_title(title, fontsize=10, loc="left") + ax.legend(frameon=False, fontsize=8, loc="lower right") + return _save(fig, path) + + +def time_per_sort_chart(configurations: list[Configuration], n: int, order: str, path: Path) -> Path | None: + chosen = [c for c in configurations if c.n == n and c.order == order] + return _dot_chart( + chosen, + "time_per_sort_ns", + "Time per sort (ns, log scale); lines show 95% confidence intervals", + f"Time per sort, {order.replace('_', ' ')} input of {n:,}", + path, + ) + + +def energy_per_sort_chart(configurations: list[Configuration], n: int, order: str, path: Path) -> Path | None: + chosen = [c for c in configurations if c.n == n and c.order == order] + return _dot_chart( + chosen, + "energy_per_sort_uj", + "Energy per sort (µJ, log scale); lines show 95% confidence intervals", + f"Energy per sort, {order.replace('_', ' ')} input of {n:,}", + path, + ) + + +def time_energy_chart( + points_by_language: dict[str, list[tuple[str, float, float]]], baseline_subtracted: bool, path: Path +) -> Path | None: + """Process time against process energy for every measured run, one panel per language. + + points_by_language maps a language to (algorithm, wall_ns, energy_uj) for each run. + """ + languages = [lang for lang in LANGUAGE_ORDER if points_by_language.get(lang)] + if not languages: + return None + fig, axes = _figure(3.0 * len(languages) + 0.6, 3.2, len(languages)) + for ax, language in zip(axes, languages, strict=True): + points = points_by_language[language] + for colour, algorithm in zip(SERIES, ALGORITHM_ORDER, strict=True): + kept = [(t / 1e6, e / 1e6) for a, t, e in points if a == algorithm and t > 0 and e > 0] + if kept: + ax.scatter( + [k[0] for k in kept], + [k[1] for k in kept], + s=16, + color=colour, + edgecolors=SURFACE, + linewidths=0.8, + label=algorithm, + ) + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_title(LANGUAGE_NAMES[language], fontsize=10, loc="left") + ax.set_xlabel("Process time (ms)", fontsize=8) + ylabel = "Process energy, net of idle (J)" if baseline_subtracted else "Process energy (J)" + axes[0].set_ylabel(ylabel, fontsize=8) + handles, labels = axes[0].get_legend_handles_labels() + if handles: + fig.legend(handles, labels, loc="upper center", ncol=3, frameon=False, fontsize=8, bbox_to_anchor=(0.5, 1.08)) + return _save(fig, path) diff --git a/analysis/greenbench_analysis/cli.py b/analysis/greenbench_analysis/cli.py new file mode 100644 index 0000000..7d3cd2c --- /dev/null +++ b/analysis/greenbench_analysis/cli.py @@ -0,0 +1,98 @@ +"""Command line: python -m greenbench_analysis report --runs results/latest/runs.csv ...""" + +from __future__ import annotations + +import argparse +import os +import shlex +import sys +from pathlib import Path + +from .carbon import load_grid, parse_custom, select_regions +from .load import DataError, load_meta, load_runs +from .report import ReportOptions, write_report, write_summary_csv +from .summary import summarise + + +def _positive(text: str) -> int: + try: + value = int(text) + except ValueError: + raise argparse.ArgumentTypeError(f"expected a whole number, got {text!r}") from None + if value < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return value + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m greenbench_analysis", + description="Statistics, carbon estimates and the results report for a greenbench run.", + ) + commands = parser.add_subparsers(dest="command", required=True) + report = commands.add_parser("report", help="write the Markdown report and its charts") + report.add_argument("--runs", type=Path, required=True, help="runs.csv written by greenbench run") + report.add_argument("--meta", type=Path, help="meta.json (default: next to runs.csv)") + report.add_argument("--out", type=Path, default=Path("docs/results.md"), help="report file (default %(default)s)") + report.add_argument("--img-dir", type=Path, default=Path("docs/img"), help="chart directory (default %(default)s)") + report.add_argument( + "--grid", type=Path, default=Path("data/grid_intensity.csv"), help="grid intensity table (default %(default)s)" + ) + report.add_argument( + "--regions", default="ARE,GBR", help="comma-separated region codes from the grid table (default %(default)s)" + ) + report.add_argument( + "--intensity", + action="append", + default=[], + metavar="NAME=G_PER_KWH", + help="an extra carbon intensity in grams of CO2e per kWh; repeatable", + ) + report.add_argument( + "--per-sorts", type=_positive, default=1_000_000, help="carbon table unit, in sorts (default %(default)s)" + ) + report.add_argument( + "--withhold-timings", + action="store_true", + help="mark time and energy results as pending (for runs on a busy machine)", + ) + report.add_argument("--resamples", type=_positive, default=10_000, help="bootstrap resamples (default %(default)s)") + report.add_argument("--seed", type=_positive, default=1, help="bootstrap seed (default %(default)s)") + report.add_argument("--summary-csv", type=Path, help="also write every configuration's estimates here") + return parser + + +def _report(args: argparse.Namespace, argv: list[str]) -> int: + runs = load_runs(args.runs) + meta = load_meta(args.meta or args.runs.with_name("meta.json")) + grid = load_grid(args.grid) + codes = [code.strip() for code in args.regions.split(",") if code.strip()] + regions = select_regions(grid, codes) + [parse_custom(spec) for spec in args.intensity] + if args.withhold_timings and args.summary_csv: + raise DataError("--summary-csv would publish the withheld timings; drop one of the two options") + + summary = summarise(runs, resamples=args.resamples, seed=args.seed) + guide = Path(os.path.relpath(Path("docs/bare-metal.md"), args.out.parent)).as_posix() + options = ReportOptions( + withhold_timings=args.withhold_timings, + regions=regions, + per_sorts=args.per_sorts, + command="python -m greenbench_analysis " + shlex.join(argv), + guide_link=guide, + ) + written = write_report(runs, meta, summary, args.out, args.img_dir, options) + if args.summary_csv: + written.append(write_summary_csv(summary, args.summary_csv)) + for path in written: + print(f"wrote {path.as_posix()}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + arguments = sys.argv[1:] if argv is None else argv + args = build_parser().parse_args(arguments) + try: + return _report(args, arguments) + except DataError as error: + print(f"greenbench_analysis: {error}", file=sys.stderr) + return 1 diff --git a/analysis/greenbench_analysis/load.py b/analysis/greenbench_analysis/load.py new file mode 100644 index 0000000..3e9c69e --- /dev/null +++ b/analysis/greenbench_analysis/load.py @@ -0,0 +1,181 @@ +"""Read the harness's runs.csv and meta.json, checking every value on the way in.""" + +from __future__ import annotations + +import csv +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +COLUMNS = ( + "sequence", + "language", + "algorithm", + "order", + "n", + "seed", + "repetition", + "inner_reps", + "wall_ns", + "loop_ns", + "comparisons", + "writes", + "input_hash", + "energy_status", + "pkg_uj", + "dram_uj", + "idle_ns", + "idle_pkg_uj", + "idle_dram_uj", + "net_pkg_uj", + "net_dram_uj", +) +LANGUAGES = ("c", "python", "node") +ALGORITHMS = ("bubble", "insertion", "quick", "none") +ORDERS = ("random", "sorted", "reversed", "nearly_sorted") +ENERGY_STATUSES = ("ok", "unavailable", "off") + + +class DataError(ValueError): + """The input files are missing, malformed or inconsistent.""" + + +@dataclass(frozen=True) +class Run: + """One measured run: one worker process, as one row of runs.csv.""" + + sequence: int + language: str + algorithm: str + order: str + n: int + seed: int + repetition: int + inner_reps: int + wall_ns: int + loop_ns: int + comparisons: int + writes: int + input_hash: str + energy_status: str + pkg_uj: float | None + dram_uj: float | None + idle_ns: int | None + idle_pkg_uj: float | None + idle_dram_uj: float | None + net_pkg_uj: float | None + net_dram_uj: float | None + + @property + def is_startup(self) -> bool: + """A startup trial: the worker generated its input and exited without sorting.""" + return self.algorithm == "none" + + @property + def baseline_subtracted(self) -> bool: + return self.net_pkg_uj is not None + + @property + def energy_uj(self) -> float | None: + """Package plus DRAM energy for the whole process, net of the idle baseline when one + was measured. None when energy was not measured.""" + if self.net_pkg_uj is not None: + return self.net_pkg_uj + (self.net_dram_uj or 0.0) + if self.pkg_uj is not None: + return self.pkg_uj + (self.dram_uj or 0.0) + return None + + +def _whole(row: dict[str, str], column: str, where: str) -> int: + text = row[column] + if not (text.isascii() and text.isdigit()): + raise DataError(f"{where}: column {column}: expected a whole number, got {text!r}") + return int(text) + + +def _optional_number(row: dict[str, str], column: str, where: str) -> float | None: + text = row[column] + if text == "": + return None + try: + return float(text) + except ValueError: + raise DataError(f"{where}: column {column}: expected a number or an empty cell, got {text!r}") from None + + +def _choice(row: dict[str, str], column: str, choices: tuple[str, ...], where: str) -> str: + text = row[column] + if text not in choices: + raise DataError(f"{where}: column {column}: expected one of {', '.join(choices)}, got {text!r}") + return text + + +def _parse_row(row: dict[str, str], where: str) -> Run: + if None in row or any(value is None for value in row.values()): + raise DataError(f"{where}: expected {len(COLUMNS)} columns") + optional_idle = _optional_number(row, "idle_ns", where) + run = Run( + sequence=_whole(row, "sequence", where), + language=_choice(row, "language", LANGUAGES, where), + algorithm=_choice(row, "algorithm", ALGORITHMS, where), + order=_choice(row, "order", ORDERS, where), + n=_whole(row, "n", where), + seed=_whole(row, "seed", where), + repetition=_whole(row, "repetition", where), + inner_reps=_whole(row, "inner_reps", where), + wall_ns=_whole(row, "wall_ns", where), + loop_ns=_whole(row, "loop_ns", where), + comparisons=_whole(row, "comparisons", where), + writes=_whole(row, "writes", where), + input_hash=row["input_hash"], + energy_status=_choice(row, "energy_status", ENERGY_STATUSES, where), + pkg_uj=_optional_number(row, "pkg_uj", where), + dram_uj=_optional_number(row, "dram_uj", where), + idle_ns=None if optional_idle is None else int(optional_idle), + idle_pkg_uj=_optional_number(row, "idle_pkg_uj", where), + idle_dram_uj=_optional_number(row, "idle_dram_uj", where), + net_pkg_uj=_optional_number(row, "net_pkg_uj", where), + net_dram_uj=_optional_number(row, "net_dram_uj", where), + ) + if run.energy_status == "ok" and run.pkg_uj is None: + raise DataError(f"{where}: energy_status is ok but pkg_uj is empty") + if run.energy_status != "ok" and run.pkg_uj is not None: + raise DataError(f"{where}: energy_status is {run.energy_status} but pkg_uj holds a value") + if run.algorithm != "none" and run.inner_reps < 1: + raise DataError(f"{where}: a sorting run must sort at least once (inner_reps is 0)") + return run + + +def load_runs(path: Path) -> list[Run]: + """Load and validate runs.csv. Raises DataError with the line and column of any problem.""" + try: + with path.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + header = tuple(reader.fieldnames or ()) + if header != COLUMNS: + missing = [c for c in COLUMNS if c not in header] + extra = [c for c in header if c not in COLUMNS] + raise DataError( + f"{path}: unexpected columns (missing: {', '.join(missing) or 'none'}; " + f"unexpected: {', '.join(extra) or 'none'}); was it written by greenbench 2?" + ) + runs = [_parse_row(row, f"{path}:{number}") for number, row in enumerate(reader, start=2)] + except FileNotFoundError: + raise DataError(f"{path}: file not found; run the harness first (see README)") from None + if not runs: + raise DataError(f"{path}: no measured runs") + return runs + + +def load_meta(path: Path) -> dict[str, Any]: + """Load meta.json, the harness's record of how a run was made.""" + try: + meta = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + raise DataError(f"{path}: file not found; it is written next to runs.csv") from None + except json.JSONDecodeError as error: + raise DataError(f"{path}: not valid JSON ({error})") from None + if not isinstance(meta, dict) or meta.get("tool") != "greenbench": + raise DataError(f"{path}: not a greenbench meta.json") + return meta diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py new file mode 100644 index 0000000..37c280c --- /dev/null +++ b/analysis/greenbench_analysis/report.py @@ -0,0 +1,475 @@ +"""Write docs/results.md and its charts from a harness run. + +Every number in the report comes from the run's runs.csv and meta.json. With +withhold_timings set, time and energy sections say "pending" instead of showing numbers: +used for runs on a busy machine, whose timings would mislead. +""" + +from __future__ import annotations + +import csv +import os +from collections import defaultdict +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import charts +from .carbon import GridIntensity, grams_co2e +from .load import ORDERS, Run +from .stats import Estimate +from .summary import Configuration, Summary + +LANGUAGE_NAMES = {"c": "C", "python": "Python", "node": "Node.js"} + + +def _pending(options: ReportOptions) -> str: + return ( + "**Pending.** The numbers for this section are withheld: this run was made to check that the " + "pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet " + f"bare-metal Linux machine will fill it in; see [bare-metal.md]({options.guide_link})." + ) + + +@dataclass(frozen=True) +class ReportOptions: + withhold_timings: bool + regions: list[GridIntensity] + per_sorts: int + command: str + guide_link: str = "bare-metal.md" # docs/bare-metal.md, relative to the report + + +def _scaled(value: float, units: list[tuple[float, str]]) -> tuple[float, str]: + magnitude = abs(value) + for factor, unit in reversed(units): + if magnitude >= factor: + return value / factor, unit + return value / units[0][0], units[0][1] + + +def _digits(value: float) -> str: + magnitude = abs(value) + if magnitude >= 100: + return f"{value:,.0f}" + if magnitude >= 10: + return f"{value:.1f}" + return f"{value:.2f}" + + +TIME_UNITS = [(1.0, "ns"), (1e3, "µs"), (1e6, "ms"), (1e9, "s")] +ENERGY_UNITS = [(1.0, "µJ"), (1e3, "mJ"), (1e6, "J"), (1e9, "kJ")] + + +def format_time(ns: float) -> str: + value, unit = _scaled(ns, TIME_UNITS) + return f"{_digits(value)} {unit}" + + +def format_energy(uj: float) -> str: + value, unit = _scaled(uj, ENERGY_UNITS) + return f"{_digits(value)} {unit}" + + +def format_grams(grams: float) -> str: + if abs(grams) >= 1: + return f"{_digits(grams)} g" + if abs(grams) >= 1e-3: + return f"{_digits(grams * 1e3)} mg" + return f"{_digits(grams * 1e6)} µg" + + +def format_estimate(estimate: Estimate, units: list[tuple[float, str]]) -> str: + """Median and interval in one unit, chosen by the median: "12.3 ms (12.1 to 12.6)".""" + factor, unit = next( + ((f, u) for f, u in reversed(units) if abs(estimate.value) >= f), + units[0], + ) + return ( + f"{_digits(estimate.value / factor)} {unit} " + f"({_digits(estimate.low / factor)} to {_digits(estimate.high / factor)})" + ) + + +def _table(header: list[str], align: list[str], rows: Iterable[list[str]]) -> list[str]: + lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(align) + " |"] + lines.extend("| " + " | ".join(row) + " |" for row in rows) + return lines + + +def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: + host = meta.get("host", {}) + settings = meta.get("settings", {}) + energy = meta.get("energy", {}) + workers = meta.get("workers", {}) + cpu = host.get("cpu_model") or "not recorded (no model name in /proc/cpuinfo)" + warmup = settings.get("warmup", "?") + versions = [] + for language, worker in workers.items(): + version = str(worker.get("version", "?")) + if version.startswith("Python "): + version = version[len("Python ") :] + versions.append(f"{LANGUAGE_NAMES.get(language, language)} {version}") + lines = [ + f"- Harness: greenbench {meta.get('version', '?')}, profile `{settings.get('profile', '?')}`, " + f"{settings.get('runs', '?')} measured runs per configuration after {warmup} " + f"warm-up run{'' if warmup == 1 else 's'}, in a random order (order seed {settings.get('order_seed', '?')}).", + f"- Inputs: sizes {', '.join(str(s) for s in settings.get('sizes', []))}; orders " + f"{', '.join(o.replace('_', ' ') for o in settings.get('orders', []))}; xorshift32 seed " + f"{settings.get('seed', '?')}. Each process sorts for at least " + f"{settings.get('min_loop_ns', 0) / 1e6:g} ms.", + f"- Machine: {cpu}; {host.get('sysname', '?')} {host.get('release', '?')} {host.get('machine', '')}; " + f"{host.get('online_cpus', '?')} CPUs online; scaling governor " + f"{host.get('scaling_governor') or 'not reported'}; " + f"CPU pinning {'to CPU ' + str(settings['cpu']) if settings.get('cpu', -1) >= 0 else 'off'}.", + "- Workers: " + "; ".join(versions) + ".", + ] + if energy.get("status") == "ok": + domains = ", ".join(f"{d['zone']} ({d['name']})" for d in energy.get("domains", [])) + baseline = "an idle window of the same length after every run" if energy.get("idle_baseline") else "none" + lines.append(f"- Energy: RAPL zones {domains}; idle baseline: {baseline}.") + else: + lines.append(f"- Energy: {energy.get('status', 'unknown')}. {energy.get('reason') or ''}".rstrip()) + lines.append( + f"- Run: {summary.run_count:,} measured worker processes, started {meta.get('started_utc', '?')}, " + f"finished {meta.get('finished_utc') or 'not recorded'}" + + (f"; label: {meta['label']}" if meta.get("label") else "") + + "." + ) + if meta.get("command"): + lines.append(f"- Harness command: `{meta['command']}`") + return lines + + +def _operations_section(summary: Summary, image: str | None) -> list[str]: + lines = ["## Operation counts", ""] + if not summary.operations: + return [*lines, "No sorting runs in this data.", ""] + languages = sorted({lang for op in summary.operations for lang in op.languages}, key=list(LANGUAGE_NAMES).index) + names = ", ".join(LANGUAGE_NAMES[lang] for lang in languages) + lines += [ + "Comparisons between elements and writes into the array, for one sort. These are exact counts, " + "not measurements, so machine load does not affect them. " + + ( + f"{names} reported identical counts and identical input hashes for every configuration; " + "the harness checks this during the run and the report checks it again." + if len(languages) > 1 + else f"Only {names} was run." + ), + "", + ] + if image: + lines += [f"![Comparisons per sort against input size, by input order]({image})", ""] + lines += _table( + ["Algorithm", "Order", "Size", "Comparisons", "Writes"], + [":---", ":---", "---:", "---:", "---:"], + ( + [op.algorithm, op.order.replace("_", " "), f"{op.n:,}", f"{op.comparisons:,}", f"{op.writes:,}"] + for op in summary.operations + ), + ) + return [*lines, ""] + + +def _per_sort_rows(configurations: list[Configuration], energy: bool) -> list[list[str]]: + rows = [] + for c in configurations: + rows.append( + [ + LANGUAGE_NAMES[c.language], + c.algorithm, + c.order.replace("_", " "), + f"{c.n:,}", + f"{c.inner_reps:,}", + format_estimate(c.time_per_sort_ns, TIME_UNITS), + format_estimate(c.energy_per_sort_uj, ENERGY_UNITS) + if energy and c.energy_per_sort_uj is not None + else "energy unavailable", + ] + ) + return rows + + +def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str, str | None]) -> list[str]: + lines = ["## Time and energy per sort", ""] + if options.withhold_timings: + return [*lines, _pending(options), ""] + largest = max((c.n for c in summary.configurations), default=0) + subtract = ( + "minus the median startup trial for the same language, order and size, divided by the number of sorts " + "in the process" + if summary.startup_subtracted + else "divided by the number of sorts in the process (no startup trials were run, so interpreter " + "start-up is included)" + ) + lines += [ + f"Median over the measured runs of each configuration, with a 95% bootstrap confidence interval in " + f"brackets. Per-sort figures are the whole-process measurement {subtract}.", + "", + ] + for key in ("time_per_sort", "energy_per_sort"): + if images.get(key): + lines += [f"![{key.replace('_', ' ').capitalize()}, largest input, random order]({images[key]})", ""] + header = ["Language", "Algorithm", "Order", "Size", "Sorts per process", "Time per sort", "Energy per sort"] + align = [":---", ":---", ":---", "---:", "---:", "---:", "---:"] + lines += [f"Inputs of {largest:,} elements:", ""] + lines += _table( + header, align, _per_sort_rows([c for c in summary.configurations if c.n == largest], summary.energy_measured) + ) + lines += ["", "
Every size", ""] + lines += _table(header, align, _per_sort_rows(summary.configurations, summary.energy_measured)) + return [*lines, "", "
", ""] + + +def _correlation_section(summary: Summary, options: ReportOptions, image: str | None, reason: str) -> list[str]: + lines = ["## Is run time a good stand-in for energy?", ""] + if options.withhold_timings: + return [*lines, _pending(options), ""] + if not summary.energy_measured: + return [*lines, f"Energy was not measured in this run ({reason}), so this question stays open.", ""] + lines += [ + "Spearman's rank correlation between process time and process energy. *Pooled* uses every run of " + "every order and size, with a 95% bootstrap interval; it is dominated by input size. *Within* is the " + "median of the correlations computed inside each configuration, where only run-to-run variation is " + "left.", + "", + ] + if image: + lines += [f"![Process time against process energy for every run]({image})", ""] + rows = [] + for c in summary.correlations: + pooled = ( + "undefined" if c.pooled is None else f"{c.pooled.value:.3f} ({c.pooled.low:.3f} to {c.pooled.high:.3f})" + ) + within = "undefined" if c.within is None else f"{c.within:.3f} over {c.within_configurations} configurations" + rows.append([LANGUAGE_NAMES[c.language], c.algorithm, f"{c.pairs:,}", pooled, within]) + lines += _table( + ["Language", "Algorithm", "Runs", "Pooled rho", "Within-configuration rho (median)"], + [":---", ":---", "---:", "---:", "---:"], + rows, + ) + return [*lines, ""] + + +def _startup_section(summary: Summary, options: ReportOptions) -> list[str]: + lines = ["## Startup cost per language", ""] + if not summary.startups: + return [*lines, "No startup trials in this run (the harness ran with --no-startup).", ""] + if options.withhold_timings: + return [*lines, _pending(options), ""] + lines += [ + "A startup trial starts the worker, generates the input and exits without sorting. Its median is " + "subtracted from the sorting runs above. Random order shown.", + "", + ] + rows = [ + [ + LANGUAGE_NAMES[s.language], + f"{s.n:,}", + format_estimate(s.time_ns, TIME_UNITS), + "energy unavailable" if s.energy_uj is None else format_estimate(s.energy_uj, ENERGY_UNITS), + ] + for s in summary.startups + if s.order == "random" + ] + lines += _table(["Language", "Size", "Process time", "Process energy"], [":---", "---:", "---:", "---:"], rows) + return [*lines, ""] + + +def _carbon_section(summary: Summary, options: ReportOptions, reason: str) -> list[str]: + lines = ["## Carbon estimates", ""] + if options.withhold_timings: + return [*lines, _pending(options), ""] + if not summary.energy_measured or summary.total_energy_uj is None: + return [*lines, f"Energy was not measured in this run ({reason}), so there is nothing to convert.", ""] + regions = options.regions + largest = max((c.n for c in summary.configurations), default=0) + order = "random" if any(c.order == "random" for c in summary.configurations) else ORDERS[0] + lines += [ + f"Energy converted with annual grid averages (see the table below). Per {options.per_sorts:,} sorts of " + f"a {order.replace('_', ' ')} input of {largest:,} elements:", + "", + ] + rows = [] + for c in summary.configurations: + if c.n != largest or c.order != order or c.energy_per_sort_uj is None: + continue + energy = c.energy_per_sort_uj.value * options.per_sorts + rows.append( + [LANGUAGE_NAMES[c.language], c.algorithm, format_energy(energy)] + + [format_grams(grams_co2e(energy, r.g_per_kwh)) for r in regions] + ) + lines += _table( + ["Language", "Algorithm", "Energy"] + [f"CO2e, {r.region}" for r in regions], + [":---", ":---", "---:"] + ["---:"] * len(regions), + rows, + ) + total = summary.total_energy_uj + lines += [ + "", + f"The measured runs themselves used {format_energy(total)} (package and DRAM, before subtracting idle), " + + "; ".join(f"{format_grams(grams_co2e(total, r.g_per_kwh))} CO2e on the {r.region} grid" for r in regions) + + ".", + "", + ] + return lines + + +def _grid_section(options: ReportOptions) -> list[str]: + lines = ["### Grid intensity", ""] + rows = [ + [ + r.region, + str(r.year) if r.year is not None else "", + f"{r.g_per_kwh:g}", + r.basis, + f"[{r.source}]({r.source_url})" if r.source_url else r.source, + r.licence, + ] + for r in options.regions + ] + lines += _table( + ["Region", "Year", "g CO2e per kWh", "Basis", "Source", "Licence"], + [":---", "---:", "---:", ":---", ":---", ":---"], + rows, + ) + return [*lines, ""] + + +def _relative(path: Path, start: Path) -> str: + return Path(os.path.relpath(path, start)).as_posix() + + +def _points_by_language(runs: list[Run]) -> dict[str, list[tuple[str, float, float]]]: + points: dict[str, list[tuple[str, float, float]]] = defaultdict(list) + for run in runs: + if not run.is_startup and run.energy_uj is not None: + points[run.language].append((run.algorithm, float(run.wall_ns), run.energy_uj)) + return dict(points) + + +def write_report( + runs: list[Run], + meta: dict[str, Any], + summary: Summary, + out: Path, + img_dir: Path, + options: ReportOptions, +) -> list[Path]: + """Write the Markdown report and its charts. Returns every file written.""" + written: list[Path] = [] + images: dict[str, str | None] = {} + + def draw(key: str, make: Callable[[Path], Path | None]) -> None: + target = img_dir / f"{key}.png" + result = make(target) + if result is None: + if target.exists(): + target.unlink() # a stale chart from an earlier run must not linger + images[key] = None + return + written.append(result) + images[key] = _relative(result, out.parent) + + draw("operations", lambda p: charts.operations_chart(summary.operations, p)) + largest = max((c.n for c in summary.configurations), default=0) + publish = not options.withhold_timings + draw( + "time_per_sort", + lambda p: charts.time_per_sort_chart(summary.configurations, largest, "random", p) if publish else None, + ) + draw( + "energy_per_sort", + lambda p: ( + charts.energy_per_sort_chart(summary.configurations, largest, "random", p) + if publish and summary.energy_measured + else None + ), + ) + draw( + "time_vs_energy", + lambda p: ( + charts.time_energy_chart(_points_by_language(runs), summary.baseline_subtracted, p) + if publish and summary.energy_measured + else None + ), + ) + + energy = meta.get("energy", {}) + reason = energy.get("reason") or energy.get("status", "unknown") + lines = [ + "# Results", + "", + "", + "", + ] + if options.withhold_timings: + lines += [ + "> **Status: time and energy results are pending.** This file was generated from a run that " + "checked the pipeline end to end. Operation counts are exact and shown below; time, energy, " + "correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux " + f"machine ([how]({options.guide_link})).", + "", + ] + elif not summary.energy_measured: + lines += [f"> **Energy was not measured in this run:** {reason}", ""] + lines += ["## How these results were produced", "", *_environment(meta, summary)] + lines += [f"- Report command: `{options.command}`", ""] + lines += _operations_section(summary, images.get("operations")) + lines += _per_sort_section(summary, options, images) + lines += _correlation_section(summary, options, images.get("time_vs_energy"), reason) + lines += _startup_section(summary, options) + lines += _carbon_section(summary, options, reason) + lines += _grid_section(options) + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + written.insert(0, out) + return written + + +def write_summary_csv(summary: Summary, path: Path) -> Path: + """Every configuration's estimates as a CSV, for further analysis.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow( + [ + "language", + "algorithm", + "order", + "n", + "runs", + "inner_reps", + "comparisons", + "writes", + "time_per_sort_ns", + "time_per_sort_ns_low", + "time_per_sort_ns_high", + "energy_per_sort_uj", + "energy_per_sort_uj_low", + "energy_per_sort_uj_high", + ] + ) + for c in summary.configurations: + e = c.energy_per_sort_uj + writer.writerow( + [ + c.language, + c.algorithm, + c.order, + c.n, + c.runs, + c.inner_reps, + c.comparisons, + c.writes, + f"{c.time_per_sort_ns.value:.1f}", + f"{c.time_per_sort_ns.low:.1f}", + f"{c.time_per_sort_ns.high:.1f}", + "" if e is None else f"{e.value:.3f}", + "" if e is None else f"{e.low:.3f}", + "" if e is None else f"{e.high:.3f}", + ] + ) + return path diff --git a/analysis/greenbench_analysis/stats.py b/analysis/greenbench_analysis/stats.py new file mode 100644 index 0000000..fc2cabd --- /dev/null +++ b/analysis/greenbench_analysis/stats.py @@ -0,0 +1,137 @@ +"""Medians, bootstrap confidence intervals and Spearman rank correlation. + +Everything here is small enough to write out with numpy and test against hand-worked +examples, which keeps SciPy out of the dependencies. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np +import numpy.typing as npt + +FloatArray = npt.NDArray[np.float64] + +# Resampling a 30-run sample 10,000 times at once needs 300,000 indices; chunks keep the +# memory bounded for larger samples. +_CHUNK = 2_000 + + +@dataclass(frozen=True) +class Estimate: + """A point estimate with a confidence interval.""" + + value: float + low: float + high: float + + +def _as_array(values: Sequence[float] | FloatArray) -> FloatArray: + array = np.asarray(values, dtype=np.float64) + if array.ndim != 1 or array.size == 0: + raise ValueError("expected a non-empty one-dimensional sample") + if not np.all(np.isfinite(array)): + raise ValueError("the sample contains a missing or infinite value") + return array + + +def median(values: Sequence[float] | FloatArray) -> float: + return float(np.median(_as_array(values))) + + +def bootstrap_median( + values: Sequence[float] | FloatArray, + *, + rng: np.random.Generator, + confidence: float = 0.95, + resamples: int = 10_000, +) -> Estimate: + """The sample median with a percentile-bootstrap confidence interval. + + Resamples the values with replacement, takes the median of each resample, and returns + the (1 - confidence) / 2 and (1 + confidence) / 2 quantiles of those medians. With one + value the interval collapses to that value. + """ + if not 0 < confidence < 1: + raise ValueError("confidence must be between 0 and 1") + if resamples < 1: + raise ValueError("resamples must be at least 1") + sample = _as_array(values) + point = float(np.median(sample)) + if sample.size == 1: + return Estimate(point, point, point) + medians = np.empty(resamples, dtype=np.float64) + for start in range(0, resamples, _CHUNK): + stop = min(start + _CHUNK, resamples) + indices = rng.integers(0, sample.size, size=(stop - start, sample.size)) + medians[start:stop] = np.median(sample[indices], axis=1) + tail = (1 - confidence) / 2 + low, high = np.quantile(medians, [tail, 1 - tail]) + return Estimate(point, float(low), float(high)) + + +def rank_average(values: Sequence[float] | FloatArray) -> FloatArray: + """Ranks starting at 1, with tied values sharing the average of their ranks.""" + sample = _as_array(values) + order = np.argsort(sample, kind="mergesort") + _, first, counts = np.unique(sample[order], return_index=True, return_counts=True) + # A group of equal values occupying sorted positions first .. first + count - 1 holds + # ranks first + 1 .. first + count, whose mean is first + (count + 1) / 2. + group_rank = first + (counts + 1) / 2 + ranks = np.empty(sample.size, dtype=np.float64) + ranks[order] = np.repeat(group_rank, counts) + return ranks + + +def spearman(x: Sequence[float] | FloatArray, y: Sequence[float] | FloatArray) -> float | None: + """Spearman's rank correlation: Pearson's correlation of the average ranks. + + Returns None when it is undefined: fewer than three pairs, or either variable constant. + """ + xs = _as_array(x) + ys = _as_array(y) + if xs.size != ys.size: + raise ValueError("x and y must have the same length") + if xs.size < 3: + return None + rx = rank_average(xs) + ry = rank_average(ys) + rx -= rx.mean() + ry -= ry.mean() + denominator = float(np.sqrt(np.sum(rx * rx) * np.sum(ry * ry))) + if denominator == 0.0: + return None + return float(np.sum(rx * ry) / denominator) + + +def bootstrap_spearman( + x: Sequence[float] | FloatArray, + y: Sequence[float] | FloatArray, + *, + rng: np.random.Generator, + confidence: float = 0.95, + resamples: int = 2_000, +) -> Estimate | None: + """Spearman's rho with a percentile-bootstrap interval from resampling pairs. + + Resamples in which rho is undefined (for example, every pair drawn the same) are + skipped. Returns None when rho itself is undefined. + """ + point = spearman(x, y) + if point is None: + return None + xs = _as_array(x) + ys = _as_array(y) + values = [] + for _ in range(resamples): + indices = rng.integers(0, xs.size, size=xs.size) + rho = spearman(xs[indices], ys[indices]) + if rho is not None: + values.append(rho) + if not values: + return Estimate(point, point, point) + tail = (1 - confidence) / 2 + low, high = np.quantile(np.asarray(values), [tail, 1 - tail]) + return Estimate(point, float(low), float(high)) diff --git a/analysis/greenbench_analysis/summary.py b/analysis/greenbench_analysis/summary.py new file mode 100644 index 0000000..6a224d3 --- /dev/null +++ b/analysis/greenbench_analysis/summary.py @@ -0,0 +1,263 @@ +"""Per-configuration summaries, startup costs and time-energy correlations. + +A configuration is one language running one algorithm on one input (order and size). Every +measured run is one worker process that sorted the input inner_reps times, so: + +- process time and energy cover the whole process, as the harness measured them; +- per-sort time and energy subtract the median startup trial for the same language, order + and size (a process that generated the input and exited), then divide by inner_reps. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Callable, Iterable +from dataclasses import dataclass + +import numpy as np + +from .load import ALGORITHMS, LANGUAGES, ORDERS, DataError, Run +from .stats import Estimate, bootstrap_median, bootstrap_spearman, median, spearman + +Key = tuple[str, str, str, int] # language, algorithm, order, n + + +@dataclass(frozen=True) +class OperationCounts: + algorithm: str + order: str + n: int + comparisons: int + writes: int + languages: tuple[str, ...] + + +@dataclass(frozen=True) +class Configuration: + language: str + algorithm: str + order: str + n: int + runs: int + inner_reps: int + comparisons: int + writes: int + process_time_ns: Estimate + process_energy_uj: Estimate | None + time_per_sort_ns: Estimate + energy_per_sort_uj: Estimate | None + + +@dataclass(frozen=True) +class Startup: + language: str + order: str + n: int + runs: int + time_ns: Estimate + energy_uj: Estimate | None + + +@dataclass(frozen=True) +class Correlation: + """Spearman's rho between process time and process energy, for one language and algorithm. + + ``pooled`` uses every run of every order and size; it is dominated by input size, since + bigger inputs take longer and use more energy. ``within`` is the median of rho computed + separately inside each configuration, where only run-to-run variation is left. + """ + + language: str + algorithm: str + pairs: int + pooled: Estimate | None + within: float | None + within_configurations: int + + +@dataclass(frozen=True) +class Summary: + configurations: list[Configuration] + startups: list[Startup] + operations: list[OperationCounts] + correlations: list[Correlation] + run_count: int + languages: list[str] + energy_measured: bool + baseline_subtracted: bool + startup_subtracted: bool + total_wall_ns: int + total_energy_uj: float | None + + +def _sort_key(language: str, algorithm: str, order: str, n: int) -> tuple[int, int, int, int]: + return (LANGUAGES.index(language), ALGORITHMS.index(algorithm), ORDERS.index(order), n) + + +def _group(runs: Iterable[Run]) -> dict[Key, list[Run]]: + groups: dict[Key, list[Run]] = defaultdict(list) + for run in runs: + groups[(run.language, run.algorithm, run.order, run.n)].append(run) + return dict(sorted(groups.items(), key=lambda item: _sort_key(*item[0]))) + + +def _constant(runs: list[Run], field: Callable[[Run], int], what: str, key: Key) -> int: + values = {field(run) for run in runs} + if len(values) != 1: + raise DataError(f"{' '.join(map(str, key))}: {what} differs between runs: {sorted(values)}") + return values.pop() + + +def _check_inputs_agree(runs: list[Run]) -> None: + hashes: dict[tuple[str, int, int], tuple[str, str]] = {} + for run in runs: + key = (run.order, run.n, run.seed) + seen = hashes.setdefault(key, (run.input_hash, run.language)) + if seen[0] != run.input_hash: + raise DataError( + f"{run.order} input of {run.n}: {run.language} reports hash {run.input_hash} but " + f"{seen[1]} reports {seen[0]}; the languages did not sort the same input" + ) + + +def _operation_counts(groups: dict[Key, list[Run]]) -> list[OperationCounts]: + by_input: dict[tuple[str, str, int], dict[str, tuple[int, int]]] = defaultdict(dict) + for key, runs in groups.items(): + language, algorithm, order, n = key + if algorithm == "none": + continue + comparisons = _constant(runs, lambda r: r.comparisons, "comparisons", key) + writes = _constant(runs, lambda r: r.writes, "writes", key) + by_input[(algorithm, order, n)][language] = (comparisons, writes) + counts = [] + for (algorithm, order, n), per_language in by_input.items(): + distinct = set(per_language.values()) + if len(distinct) != 1: + detail = ", ".join(f"{lang}: {c} comparisons, {w} writes" for lang, (c, w) in per_language.items()) + raise DataError(f"{algorithm} on {order} input of {n}: the languages disagree ({detail})") + comparisons, writes = distinct.pop() + languages = tuple(sorted(per_language, key=LANGUAGES.index)) + counts.append(OperationCounts(algorithm, order, n, comparisons, writes, languages)) + counts.sort(key=lambda c: (ALGORITHMS.index(c.algorithm), ORDERS.index(c.order), c.n)) + return counts + + +def _energies(runs: list[Run]) -> list[float] | None: + values = [run.energy_uj for run in runs] + if any(v is None for v in values): + return None + return [float(v) for v in values if v is not None] + + +def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = 2_000, seed: int = 1) -> Summary: + """Summarise measured runs. The bootstrap is seeded, so the same data gives the same report.""" + rng = np.random.default_rng(seed) + _check_inputs_agree(runs) + groups = _group(runs) + operations = _operation_counts(groups) + energy_measured = all(run.energy_uj is not None for run in runs) + + startups: list[Startup] = [] + startup_time: dict[tuple[str, str, int], float] = {} + startup_energy: dict[tuple[str, str, int], float] = {} + for (language, algorithm, order, n), group in groups.items(): + if algorithm != "none": + continue + times = [float(r.wall_ns) for r in group] + energies = _energies(group) + startup_time[(language, order, n)] = median(times) + if energies is not None: + startup_energy[(language, order, n)] = median(energies) + startups.append( + Startup( + language=language, + order=order, + n=n, + runs=len(group), + time_ns=bootstrap_median(times, rng=rng, resamples=resamples), + energy_uj=None if energies is None else bootstrap_median(energies, rng=rng, resamples=resamples), + ) + ) + + configurations: list[Configuration] = [] + startup_subtracted = True + for key, group in groups.items(): + language, algorithm, order, n = key + if algorithm == "none": + continue + reps = _constant(group, lambda r: r.inner_reps, "inner_reps", key) + times = [float(r.wall_ns) for r in group] + energies = _energies(group) + base_key = (language, order, n) + if base_key not in startup_time: + startup_subtracted = False + base_time = startup_time.get(base_key, 0.0) + base_energy = startup_energy.get(base_key, 0.0) + per_sort_time = [(t - base_time) / reps for t in times] + per_sort_energy = None if energies is None else [(e - base_energy) / reps for e in energies] + configurations.append( + Configuration( + language=language, + algorithm=algorithm, + order=order, + n=n, + runs=len(group), + inner_reps=reps, + comparisons=group[0].comparisons, + writes=group[0].writes, + process_time_ns=bootstrap_median(times, rng=rng, resamples=resamples), + process_energy_uj=None + if energies is None + else bootstrap_median(energies, rng=rng, resamples=resamples), + time_per_sort_ns=bootstrap_median(per_sort_time, rng=rng, resamples=resamples), + energy_per_sort_uj=None + if per_sort_energy is None + else bootstrap_median(per_sort_energy, rng=rng, resamples=resamples), + ) + ) + + correlations: list[Correlation] = [] + if energy_measured: + pairs: dict[tuple[str, str], list[tuple[float, float]]] = defaultdict(list) + within: dict[tuple[str, str], list[float]] = defaultdict(list) + for (language, algorithm, _order, _n), group in groups.items(): + if algorithm == "none": + continue + xs = [float(r.wall_ns) for r in group] + ys = [float(r.energy_uj or 0.0) for r in group] + pairs[(language, algorithm)].extend(zip(xs, ys, strict=True)) + rho = spearman(xs, ys) if len(group) >= 3 else None + if rho is not None: + within[(language, algorithm)].append(rho) + for language, algorithm in sorted(pairs, key=lambda k: (LANGUAGES.index(k[0]), ALGORITHMS.index(k[1]))): + points = pairs[(language, algorithm)] + rhos = within.get((language, algorithm), []) + correlations.append( + Correlation( + language=language, + algorithm=algorithm, + pairs=len(points), + pooled=bootstrap_spearman( + [p[0] for p in points], [p[1] for p in points], rng=rng, resamples=rho_resamples + ), + within=median(rhos) if rhos else None, + within_configurations=len(rhos), + ) + ) + + total_energy = None + if energy_measured: + total_energy = sum((r.pkg_uj or 0.0) + (r.dram_uj or 0.0) for r in runs) + return Summary( + configurations=configurations, + startups=startups, + operations=operations, + correlations=correlations, + run_count=len(runs), + languages=sorted({r.language for r in runs}, key=LANGUAGES.index), + energy_measured=energy_measured, + baseline_subtracted=energy_measured and all(r.baseline_subtracted for r in runs), + startup_subtracted=startup_subtracted and bool(startups), + total_wall_ns=sum(r.wall_ns for r in runs), + total_energy_uj=total_energy, + ) diff --git a/analysis/tests/synthetic.py b/analysis/tests/synthetic.py new file mode 100644 index 0000000..f82617b --- /dev/null +++ b/analysis/tests/synthetic.py @@ -0,0 +1,155 @@ +"""Synthetic runs with known values, for testing the summaries and the report.""" + +from __future__ import annotations + +import csv +import json +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from greenbench_analysis.load import COLUMNS, Run + +# Exact counts for n = 100 (see tests/test_sorts.c), and made-up counts for random input. +COUNTS = { + ("bubble", "sorted"): (4950, 0), + ("bubble", "random"): (4950, 5000), + ("quick", "sorted"): (4950, 0), + ("quick", "random"): (700, 800), +} +HASHES = {"sorted": "11111111", "random": "22222222"} + + +def make_run( + *, + language: str = "c", + algorithm: str = "quick", + order: str = "random", + n: int = 100, + repetition: int = 1, + inner_reps: int = 10, + wall_ns: int = 1_000_000, + energy_uj: float | None = None, + idle_uj: float | None = None, + sequence: int = 1, +) -> Run: + comparisons, writes = (0, 0) if algorithm == "none" else COUNTS[(algorithm, order)] + measured = energy_uj is not None + baseline = measured and idle_uj is not None + return Run( + sequence=sequence, + language=language, + algorithm=algorithm, + order=order, + n=n, + seed=2463534242, + repetition=repetition, + inner_reps=0 if algorithm == "none" else inner_reps, + wall_ns=wall_ns, + loop_ns=wall_ns // 2, + comparisons=comparisons, + writes=writes, + input_hash=HASHES[order], + energy_status="ok" if measured else "unavailable", + pkg_uj=energy_uj, + dram_uj=None, + idle_ns=wall_ns if baseline else None, + idle_pkg_uj=idle_uj if baseline else None, + idle_dram_uj=None, + net_pkg_uj=(energy_uj - idle_uj) if baseline and energy_uj is not None and idle_uj is not None else None, + net_dram_uj=None, + ) + + +def study( + *, + energy: bool, + runs: int = 5, + languages: Iterable[str] = ("c", "python"), + startup: bool = True, + sizes: Iterable[int] = (100,), +) -> list[Run]: + """A small, fully known data set: per-sort cost 1 ms and 2 mJ for C, ten times that for Python, + plus a startup cost of 5 ms and 10 mJ per process for C and 50 ms and 100 mJ for Python.""" + result: list[Run] = [] + sequence = 0 + for language in languages: + scale = 1 if language == "c" else 10 + for order, n in ((o, n) for o in ("sorted", "random") for n in sizes): + algorithms = ["bubble", "quick", *(["none"] if startup else [])] + for algorithm in algorithms: + for repetition in range(1, runs + 1): + sequence += 1 + jitter = repetition - (runs + 1) // 2 # symmetric around the middle run + reps = 0 if algorithm == "none" else 10 + wall = scale * (5_000_000 + reps * 1_000_000) + jitter * 1000 + gross = scale * (10_000.0 + reps * 2_000.0) + jitter * 10.0 + result.append( + make_run( + language=language, + algorithm=algorithm, + order=order, + n=n, + repetition=repetition, + inner_reps=reps, + wall_ns=wall, + energy_uj=gross + 500.0 if energy else None, + idle_uj=500.0 if energy else None, + sequence=sequence, + ) + ) + return result + + +def write_csv(runs: list[Run], path: Path) -> Path: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(COLUMNS) + for run in runs: + row: list[Any] = [] + for column in COLUMNS: + value = getattr(run, column) + row.append("" if value is None else value) + writer.writerow(row) + return path + + +def meta(*, energy: bool) -> dict[str, Any]: + return { + "tool": "greenbench", + "version": "2.0.0", + "status": "complete", + "started_utc": "2026-09-26T10:00:00Z", + "finished_utc": "2026-09-26T10:30:00Z", + "command": "greenbench run --profile smoke", + "host": { + "sysname": "Linux", + "release": "6.8.0", + "machine": "x86_64", + "cpu_model": "Test CPU", + "online_cpus": 8, + }, + "energy": { + "status": "ok" if energy else "unavailable", + "reason": None if energy else "no RAPL package zone under /sys/class/powercap", + "idle_baseline": energy, + "domains": [{"zone": "intel-rapl:0", "name": "package-0", "kind": "package"}] if energy else [], + }, + "settings": { + "profile": "smoke", + "sizes": [100], + "orders": ["sorted", "random"], + "runs": 5, + "warmup": 1, + "seed": 2463534242, + "order_seed": 1234567, + "min_loop_ns": 2000000, + "cpu": -1, + }, + "workers": {"c": {"version": "gcc 14"}, "python": {"version": "Python 3.12.0"}}, + } + + +def write_meta(data: dict[str, Any], path: Path) -> Path: + path.write_text(json.dumps(data), encoding="utf-8") + return path diff --git a/analysis/tests/test_carbon.py b/analysis/tests/test_carbon.py new file mode 100644 index 0000000..873910c --- /dev/null +++ b/analysis/tests/test_carbon.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from greenbench_analysis.carbon import grams_co2e, load_grid, parse_custom, select_regions +from greenbench_analysis.load import DataError + +REPOSITORY = Path(__file__).resolve().parents[2] +GRID = REPOSITORY / "data" / "grid_intensity.csv" + + +def test_one_kilowatt_hour_gives_the_intensity() -> None: + one_kwh_in_uj = 3.6e6 * 1e6 + assert grams_co2e(one_kwh_in_uj, 467.51) == pytest.approx(467.51) + assert grams_co2e(0.0, 467.51) == 0.0 + + +def test_one_joule() -> None: + # 1 J = 1/3.6e6 kWh; at 360 g/kWh that is 1e-4 g. + assert grams_co2e(1e6, 360.0) == pytest.approx(1e-4) + + +def test_repository_grid_table_has_cited_uae_and_uk_rows() -> None: + grid = load_grid(GRID) + uae, uk = select_regions(grid, ["ARE", "GBR"]) + assert uae.region == "United Arab Emirates" + assert uk.region == "United Kingdom" + for row in (uae, uk): + assert row.year == 2024 + assert row.g_per_kwh > 0 + assert "Ember" in row.source + assert row.licence == "CC BY 4.0" + assert row.source_url.startswith("https://") + + +def test_latest_year_is_chosen(tmp_path: Path) -> None: + table = tmp_path / "grid.csv" + table.write_text( + "region_code,region,year,g_co2e_per_kwh,basis,source,source_url,licence,retrieved\n" + "XXX,Example,2022,300,b,s,u,l,r\n" + "XXX,Example,2024,250,b,s,u,l,r\n" + "XXX,Example,2023,280,b,s,u,l,r\n", + encoding="utf-8", + ) + (chosen,) = select_regions(load_grid(table), ["XXX"]) + assert chosen.year == 2024 + assert chosen.g_per_kwh == 250 + + +def test_unknown_region_is_explained() -> None: + with pytest.raises(DataError, match=r"no grid intensity for region 'FRA' \(known: ARE, GBR\)"): + select_regions(load_grid(GRID), ["FRA"]) + + +def test_bad_tables_are_rejected(tmp_path: Path) -> None: + wrong_columns = tmp_path / "a.csv" + wrong_columns.write_text("code,value\nARE,1\n", encoding="utf-8") + with pytest.raises(DataError, match="expected the columns"): + load_grid(wrong_columns) + negative = tmp_path / "b.csv" + negative.write_text( + "region_code,region,year,g_co2e_per_kwh,basis,source,source_url,licence,retrieved\n" + "XXX,Example,2024,-5,b,s,u,l,r\n", + encoding="utf-8", + ) + with pytest.raises(DataError, match="cannot be negative"): + load_grid(negative) + with pytest.raises(DataError, match="not found"): + load_grid(tmp_path / "missing.csv") + + +def test_custom_intensity() -> None: + custom = parse_custom("Solar farm=45.5") + assert custom.region == "Solar farm" + assert custom.g_per_kwh == 45.5 + assert custom.year is None + for bad in ("45", "=45", "Grid=", "Grid=-3", "Grid=abc"): + with pytest.raises(DataError, match="NAME=GRAMS_PER_KWH"): + parse_custom(bad) diff --git a/analysis/tests/test_load.py b/analysis/tests/test_load.py new file mode 100644 index 0000000..429e6a1 --- /dev/null +++ b/analysis/tests/test_load.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +import synthetic +from greenbench_analysis.load import COLUMNS, DataError, load_meta, load_runs + +HEADER = ",".join(COLUMNS) +GOOD_ROW = "1,c,quick,random,100,2463534242,1,10,1000000,900000,700,800,22222222,unavailable,,,,,,," + + +def write(tmp_path: Path, *lines: str) -> Path: + path = tmp_path / "runs.csv" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def test_round_trip_through_csv(tmp_path: Path) -> None: + runs = synthetic.study(energy=True) + loaded = load_runs(synthetic.write_csv(runs, tmp_path / "runs.csv")) + assert loaded == runs + + +def test_energy_cells_left_empty_are_none(tmp_path: Path) -> None: + (run,) = load_runs(write(tmp_path, HEADER, GOOD_ROW)) + assert run.energy_status == "unavailable" + assert run.pkg_uj is None + assert run.energy_uj is None + assert not run.baseline_subtracted + + +def test_energy_prefers_the_net_values() -> None: + run = synthetic.make_run(energy_uj=1000.0, idle_uj=300.0) + assert run.energy_uj == 700.0 + gross_only = synthetic.make_run(energy_uj=1000.0) + assert gross_only.energy_uj == 1000.0 + + +@pytest.mark.parametrize( + ("row", "message"), + [ + (GOOD_ROW.replace(",c,", ",rust,"), "column language"), + (GOOD_ROW.replace(",quick,", ",merge,"), "column algorithm"), + (GOOD_ROW.replace("1000000", "12ms"), "column wall_ns: expected a whole number"), + (GOOD_ROW.replace(",unavailable,", ",ok,"), "energy_status is ok but pkg_uj is empty"), + (GOOD_ROW.replace("unavailable,,", "unavailable,5,"), "but pkg_uj holds a value"), + (GOOD_ROW.replace(",10,1000000", ",0,1000000"), "must sort at least once"), + (GOOD_ROW + "x", "column net_dram_uj"), + (GOOD_ROW[:-2], "expected 21 columns"), + (GOOD_ROW + ",", "expected 21 columns"), + ], +) +def test_bad_rows_name_the_line_and_column(tmp_path: Path, row: str, message: str) -> None: + with pytest.raises(DataError, match=message) as error: + load_runs(write(tmp_path, HEADER, row)) + assert "runs.csv:2" in str(error.value) + + +def test_wrong_header(tmp_path: Path) -> None: + with pytest.raises(DataError, match=r"missing: median_seconds|unexpected: median_seconds"): + load_runs(write(tmp_path, "algorithm,order,n,median_seconds,comparisons,writes", "quick,random,100,0.1,1,1")) + + +def test_missing_and_empty_files(tmp_path: Path) -> None: + with pytest.raises(DataError, match="file not found"): + load_runs(tmp_path / "nothing.csv") + with pytest.raises(DataError, match="no measured runs"): + load_runs(write(tmp_path, HEADER)) + + +def test_meta(tmp_path: Path) -> None: + path = synthetic.write_meta(synthetic.meta(energy=False), tmp_path / "meta.json") + assert load_meta(path)["energy"]["status"] == "unavailable" + (tmp_path / "bad.json").write_text("{not json", encoding="utf-8") + with pytest.raises(DataError, match="not valid JSON"): + load_meta(tmp_path / "bad.json") + (tmp_path / "other.json").write_text('{"tool": "something else"}', encoding="utf-8") + with pytest.raises(DataError, match=r"not a greenbench meta\.json"): + load_meta(tmp_path / "other.json") diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py new file mode 100644 index 0000000..9d6c7e3 --- /dev/null +++ b/analysis/tests/test_report.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +import synthetic +from greenbench_analysis.carbon import GridIntensity +from greenbench_analysis.cli import main +from greenbench_analysis.report import ( + ENERGY_UNITS, + TIME_UNITS, + ReportOptions, + format_energy, + format_estimate, + format_grams, + format_time, + write_report, +) +from greenbench_analysis.stats import Estimate +from greenbench_analysis.summary import summarise + +REPOSITORY = Path(__file__).resolve().parents[2] +GRID = REPOSITORY / "data" / "grid_intensity.csv" +UAE = GridIntensity("ARE", "United Arab Emirates", 2024, 467.51, "lifecycle", "Ember", "https://example.org", "CC", "") + + +def options(*, withhold: bool = False) -> ReportOptions: + return ReportOptions(withhold_timings=withhold, regions=[UAE], per_sorts=1_000_000, command="test") + + +def test_unit_formatting() -> None: + assert format_time(950) == "950 ns" + assert format_time(1_500) == "1.50 µs" + assert format_time(12_345_678) == "12.3 ms" + assert format_time(2.5e9) == "2.50 s" + assert format_energy(2_000) == "2.00 mJ" + assert format_energy(3.2e6) == "3.20 J" + assert format_energy(-1_500) == "-1.50 mJ" + assert format_grams(0.25) == "250 mg" + assert format_grams(2e-6) == "2.00 µg" + assert format_estimate(Estimate(12_300_000, 12_100_000, 12_600_000), TIME_UNITS) == "12.3 ms (12.1 to 12.6)" + assert format_estimate(Estimate(2_000, 1_900, 2_100), ENERGY_UNITS) == "2.00 mJ (1.90 to 2.10)" + + +def render(tmp_path: Path, *, energy: bool, withhold: bool) -> str: + runs = synthetic.study(energy=energy, sizes=(100, 200)) + summary = summarise(runs, resamples=200) + out = tmp_path / "docs" / "results.md" + write_report( + runs, synthetic.meta(energy=energy), summary, out, tmp_path / "docs" / "img", options(withhold=withhold) + ) + return out.read_text(encoding="utf-8") + + +def test_report_with_energy(tmp_path: Path) -> None: + text = render(tmp_path, energy=True, withhold=False) + assert "Pending" not in text + assert "1.00 ms (" in text # C quick sort: 1 ms per sort + assert "2.00 mJ (" in text # and 2 mJ per sort + assert "## Is run time a good stand-in for energy?" in text + assert "1.000 over 4 configurations" in text + # 2 mJ per sort x 1e6 sorts = 2 kJ = 0.000556 kWh; at 467.51 g/kWh that is 0.260 g. + assert "| C | quick | 2.00 kJ | 260 mg |" in text + for chart in ("operations", "time_per_sort", "energy_per_sort", "time_vs_energy"): + assert (tmp_path / "docs" / "img" / f"{chart}.png").stat().st_size > 1000 + assert f"](img/{chart}.png)" in text + + +def test_report_without_energy_says_so(tmp_path: Path) -> None: + text = render(tmp_path, energy=False, withhold=False) + assert "Energy was not measured in this run" in text + assert "no RAPL package zone" in text + assert "energy unavailable" in text + assert not (tmp_path / "docs" / "img" / "energy_per_sort.png").exists() + + +def test_withheld_report_shows_counts_but_no_timings(tmp_path: Path) -> None: + text = render(tmp_path, energy=True, withhold=True) + assert "Status: time and energy results are pending" in text + assert text.count("**Pending.**") == 4 + assert "| quick | random | 100 | 700 | 800 |" in text + assert " ms (" not in text + assert " mJ (" not in text + assert (tmp_path / "docs" / "img" / "operations.png").exists() # exact counts are always drawn + assert not (tmp_path / "docs" / "img" / "time_per_sort.png").exists() + + +def test_stale_charts_are_removed(tmp_path: Path) -> None: + render(tmp_path, energy=True, withhold=False) + assert (tmp_path / "docs" / "img" / "time_vs_energy.png").exists() + render(tmp_path, energy=True, withhold=True) + assert not (tmp_path / "docs" / "img" / "time_vs_energy.png").exists() + + +def test_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + runs_path = synthetic.write_csv(synthetic.study(energy=True), tmp_path / "runs.csv") + synthetic.write_meta(synthetic.meta(energy=True), tmp_path / "meta.json") + out = tmp_path / "report" / "results.md" + status = main( + [ + "report", + "--runs", + str(runs_path), + "--out", + str(out), + "--img-dir", + str(tmp_path / "report" / "img"), + "--grid", + str(GRID), + "--intensity", + "Solar farm=45", + "--resamples", + "200", + "--summary-csv", + str(tmp_path / "summary.csv"), + ] + ) + assert status == 0 + text = out.read_text(encoding="utf-8") + assert "CO2e, United Kingdom" in text + assert "CO2e, Solar farm" in text + assert "python -m greenbench_analysis report --runs" in text + assert (tmp_path / "summary.csv").read_text(encoding="utf-8").startswith("language,algorithm,order,n,") + assert "wrote" in capsys.readouterr().out + + +def test_command_line_errors(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert main(["report", "--runs", str(tmp_path / "missing.csv")]) == 1 + assert "file not found" in capsys.readouterr().err + runs_path = synthetic.write_csv(synthetic.study(energy=False), tmp_path / "runs.csv") + synthetic.write_meta(synthetic.meta(energy=False), tmp_path / "meta.json") + status = main( + ["report", "--runs", str(runs_path), "--grid", str(GRID), "--withhold-timings", "--summary-csv", "x.csv"] + ) + assert status == 1 + assert "would publish the withheld timings" in capsys.readouterr().err diff --git a/analysis/tests/test_stats.py b/analysis/tests/test_stats.py new file mode 100644 index 0000000..370e992 --- /dev/null +++ b/analysis/tests/test_stats.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from greenbench_analysis.stats import ( + Estimate, + bootstrap_median, + bootstrap_spearman, + median, + rank_average, + spearman, +) + + +def rng() -> np.random.Generator: + return np.random.default_rng(1) + + +def test_median_odd_and_even() -> None: + assert median([3, 1, 2]) == 2 + assert median([4, 1, 3, 2]) == 2.5 + + +def test_median_rejects_empty_and_missing_values() -> None: + with pytest.raises(ValueError, match="non-empty"): + median([]) + with pytest.raises(ValueError, match="missing or infinite"): + median([1.0, float("nan")]) + + +def test_bootstrap_interval_contains_the_median_and_is_reproducible() -> None: + values = np.random.default_rng(7).normal(100.0, 5.0, size=30) + first = bootstrap_median(values, rng=rng(), resamples=4000) + second = bootstrap_median(values, rng=rng(), resamples=4000) + assert first == second + assert first.low <= first.value <= first.high + assert first.value == pytest.approx(float(np.median(values))) + + +def test_bootstrap_interval_narrows_with_more_data() -> None: + source = np.random.default_rng(3) + small = bootstrap_median(source.normal(0, 1, 10), rng=rng(), resamples=4000) + large = bootstrap_median(source.normal(0, 1, 1000), rng=rng(), resamples=4000) + assert (large.high - large.low) < (small.high - small.low) + + +def test_bootstrap_covers_the_true_median_about_95_percent_of_the_time() -> None: + # 200 samples of 30 from a skewed distribution whose median is ln(2) (exponential, rate 1). + source = np.random.default_rng(11) + true_median = float(np.log(2)) + covered = 0 + for _ in range(200): + estimate = bootstrap_median(source.exponential(1.0, 30), rng=source, resamples=1000) + covered += estimate.low <= true_median <= estimate.high + assert 0.88 <= covered / 200 <= 0.995 + + +def test_bootstrap_edge_cases() -> None: + assert bootstrap_median([5.0], rng=rng()) == Estimate(5.0, 5.0, 5.0) + assert bootstrap_median([2.0, 2.0, 2.0], rng=rng(), resamples=100) == Estimate(2.0, 2.0, 2.0) + with pytest.raises(ValueError, match="confidence"): + bootstrap_median([1.0, 2.0], rng=rng(), confidence=1.0) + with pytest.raises(ValueError, match="resamples"): + bootstrap_median([1.0, 2.0], rng=rng(), resamples=0) + + +def test_rank_average_with_ties() -> None: + # 10 is smallest (rank 1); the two 20s share ranks 2 and 3; 30s share 4, 5 and 6. + assert rank_average([20, 10, 30, 20, 30, 30]).tolist() == [2.5, 1.0, 5.0, 2.5, 5.0, 5.0] + + +def test_spearman_hand_worked_example() -> None: + # Ranks x: 1 2 3 4 5; ranks y: 2 1 4 3 5. Sum d^2 = 4, rho = 1 - 6*4/(5*24) = 0.8. + assert spearman([1, 2, 3, 4, 5], [20, 10, 40, 30, 50]) == pytest.approx(0.8) + + +def test_spearman_is_rank_based() -> None: + x = [1, 2, 3, 4, 5, 6] + assert spearman(x, [v**3 for v in x]) == pytest.approx(1.0) + assert spearman(x, [-np.exp(v) for v in x]) == pytest.approx(-1.0) + + +def test_spearman_with_ties_matches_pearson_on_ranks() -> None: + x = [1, 2, 2, 3, 4, 4, 5] + y = [2, 1, 3, 3, 5, 4, 6] + expected = float(np.corrcoef(rank_average(x), rank_average(y))[0, 1]) + assert spearman(x, y) == pytest.approx(expected) + + +def test_spearman_undefined_cases() -> None: + assert spearman([1, 2], [3, 4]) is None + assert spearman([1, 1, 1], [1, 2, 3]) is None + with pytest.raises(ValueError, match="same length"): + spearman([1, 2, 3], [1, 2]) + + +def test_bootstrap_spearman() -> None: + source = np.random.default_rng(5) + x = source.uniform(0, 1, 200) + y = x + source.normal(0, 0.1, 200) + estimate = bootstrap_spearman(x, y, rng=rng(), resamples=500) + assert estimate is not None + assert 0.8 < estimate.value < 1.0 + assert estimate.low <= estimate.value <= estimate.high + assert bootstrap_spearman([1, 1, 1], [1, 2, 3], rng=rng()) is None diff --git a/analysis/tests/test_summary.py b/analysis/tests/test_summary.py new file mode 100644 index 0000000..01d00b7 --- /dev/null +++ b/analysis/tests/test_summary.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import dataclasses + +import pytest + +import synthetic +from greenbench_analysis.load import DataError +from greenbench_analysis.summary import summarise + + +def test_per_sort_values_subtract_startup_and_divide_by_reps() -> None: + summary = summarise(synthetic.study(energy=True), resamples=500) + c_quick = next(c for c in summary.configurations if (c.language, c.algorithm, c.order) == ("c", "quick", "random")) + py_quick = next( + c for c in summary.configurations if (c.language, c.algorithm, c.order) == ("python", "quick", "random") + ) + assert c_quick.inner_reps == 10 + assert c_quick.time_per_sort_ns.value == pytest.approx(1_000_000) + assert c_quick.energy_per_sort_uj is not None + assert c_quick.energy_per_sort_uj.value == pytest.approx(2_000) + assert py_quick.time_per_sort_ns.value == pytest.approx(10_000_000) + assert c_quick.process_time_ns.value == pytest.approx(15_000_000) + assert summary.startup_subtracted + assert summary.baseline_subtracted + for c in summary.configurations: + assert c.time_per_sort_ns.low <= c.time_per_sort_ns.value <= c.time_per_sort_ns.high + + +def test_startups_are_reported_separately() -> None: + summary = summarise(synthetic.study(energy=True), resamples=200) + startup = next(s for s in summary.startups if (s.language, s.order) == ("python", "random")) + assert startup.time_ns.value == pytest.approx(50_000_000) + assert startup.energy_uj is not None + assert startup.energy_uj.value == pytest.approx(100_000) + assert all(c.algorithm != "none" for c in summary.configurations) + + +def test_without_startup_trials_per_sort_includes_startup() -> None: + summary = summarise(synthetic.study(energy=False, startup=False), resamples=200) + c_quick = next(c for c in summary.configurations if (c.language, c.algorithm, c.order) == ("c", "quick", "random")) + assert not summary.startup_subtracted + assert c_quick.time_per_sort_ns.value == pytest.approx(1_500_000) + + +def test_without_energy_there_are_no_energy_figures() -> None: + summary = summarise(synthetic.study(energy=False), resamples=200) + assert not summary.energy_measured + assert summary.total_energy_uj is None + assert summary.correlations == [] + assert all(c.energy_per_sort_uj is None and c.process_energy_uj is None for c in summary.configurations) + + +def test_correlations_within_and_pooled() -> None: + summary = summarise(synthetic.study(energy=True, runs=7), resamples=200) + assert [(c.language, c.algorithm) for c in summary.correlations] == [ + ("c", "bubble"), + ("c", "quick"), + ("python", "bubble"), + ("python", "quick"), + ] + for correlation in summary.correlations: + assert correlation.pairs == 14 # 7 runs x 2 orders + assert correlation.within == pytest.approx(1.0) # time and energy rise together in the synthetic data + assert correlation.within_configurations == 2 + assert correlation.pooled is not None + + +def test_operation_counts_are_checked_across_languages() -> None: + summary = summarise(synthetic.study(energy=False), resamples=100) + quick_random = next(o for o in summary.operations if (o.algorithm, o.order) == ("quick", "random")) + assert (quick_random.comparisons, quick_random.writes) == (700, 800) + assert quick_random.languages == ("c", "python") + + runs = synthetic.study(energy=False) + broken = [ + dataclasses.replace(r, writes=801) if r.language == "python" and r.algorithm == "quick" else r for r in runs + ] + with pytest.raises(DataError, match="the languages disagree"): + summarise(broken, resamples=100) + + +def test_counts_must_be_constant_within_a_configuration() -> None: + runs = synthetic.study(energy=False) + runs[0] = dataclasses.replace(runs[0], comparisons=runs[0].comparisons + 1) + with pytest.raises(DataError, match="comparisons differs between runs"): + summarise(runs, resamples=100) + + +def test_inputs_must_agree_across_languages() -> None: + runs = synthetic.study(energy=False) + runs[-1] = dataclasses.replace(runs[-1], input_hash="deadbeef") + with pytest.raises(DataError, match="did not sort the same input"): + summarise(runs, resamples=100) + + +def test_results_are_reproducible_with_the_same_seed() -> None: + first = summarise(synthetic.study(energy=True), resamples=300, seed=3) + second = summarise(synthetic.study(energy=True), resamples=300, seed=3) + assert first == second diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py new file mode 100644 index 0000000..6759bae --- /dev/null +++ b/tests/integration/test_pipeline.py @@ -0,0 +1,80 @@ +"""The whole pipeline: harness run, then the report, as `make smoke` does it.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +from workers import REPOSITORY, interpreter_options, require_build, run_harness + + +def report(*args: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ, PYTHONPATH=str(REPOSITORY / "analysis")) + return subprocess.run( + [sys.executable, "-m", "greenbench_analysis", "report", *args], + capture_output=True, + text=True, + check=False, + env=env, + cwd=REPOSITORY, + timeout=600, + ) + + +def test_harness_then_report(tmp_path: Path) -> None: + require_build() + out = tmp_path / "run" + completed = run_harness( + "run", + "--profile", + "smoke", + "--sizes", + "30,90", + "--runs", + "3", + "--energy", + "off", + "--quiet", + "--out", + str(out), + *interpreter_options(), + ) + assert completed.returncode == 0, completed.stderr + + results = tmp_path / "results.md" + reported = report( + "--runs", + str(out / "runs.csv"), + "--out", + str(results), + "--img-dir", + str(tmp_path / "img"), + "--resamples", + "200", + ) + assert reported.returncode == 0, reported.stderr + text = results.read_text(encoding="utf-8") + assert "Energy was not measured in this run" in text + assert "C, Python, Node.js reported identical counts and identical input hashes" in text + assert "| quick | sorted | 90 | 4,005 | 0 |" in text # n(n-1)/2 comparisons: sorted input is the worst case + assert "energy unavailable" in text + assert (tmp_path / "img" / "operations.png").exists() + assert (tmp_path / "img" / "time_per_sort.png").exists() + + withheld = report( + "--runs", + str(out / "runs.csv"), + "--out", + str(results), + "--img-dir", + str(tmp_path / "img"), + "--resamples", + "200", + "--withhold-timings", + ) + assert withheld.returncode == 0, withheld.stderr + text = results.read_text(encoding="utf-8") + assert "time and energy results are pending" in text + assert not (tmp_path / "img" / "time_per_sort.png").exists() diff --git a/visualize.py b/visualize.py deleted file mode 100644 index 573c6cc..0000000 --- a/visualize.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -"""Charts and a summary table from energy_explorer's CSV output. - - ./energy_explorer > data.csv - python3 visualize.py data.csv - -Writes comparisons.png, time.png and orders.png, and prints a Markdown table (also -appended to the GitHub Actions job summary when run in CI). -""" - -import csv -import os -import sys - -import matplotlib - -matplotlib.use("Agg") # write files only; no window -import matplotlib.pyplot as plt # noqa: E402 - -ALGORITHMS = ["bubble", "insertion", "quick"] -ORDERS = ["random", "sorted", "reversed", "nearly_sorted"] - - -def load(path): - with open(path, newline="", encoding="utf-8") as f: - return [ - {**row, "n": int(row["n"]), "median_seconds": float(row["median_seconds"]), - "comparisons": int(row["comparisons"]), "writes": int(row["writes"])} - for row in csv.DictReader(f) - ] - - -def series(rows, algorithm, order, field): - points = sorted((r["n"], r[field]) for r in rows if r["algorithm"] == algorithm and r["order"] == order) - return [p[0] for p in points], [p[1] for p in points] - - -def line_chart(rows, field, ylabel, title, path): - plt.figure(figsize=(7, 4.5)) - for algorithm in ALGORITHMS: - xs, ys = series(rows, algorithm, "random", field) - plt.plot(xs, ys, marker="o", label=algorithm) - plt.xscale("log") - plt.yscale("log") - plt.xlabel("Number of elements (random order)") - plt.ylabel(ylabel) - plt.title(title) - plt.legend() - plt.grid(True, which="both", alpha=0.3) - plt.tight_layout() - plt.savefig(path, dpi=120) - plt.close() - - -def orders_chart(rows, n, path): - width = 0.25 - plt.figure(figsize=(8, 4.5)) - for k, algorithm in enumerate(ALGORITHMS): - values = [next(r["comparisons"] for r in rows - if r["algorithm"] == algorithm and r["order"] == order and r["n"] == n) - for order in ORDERS] - plt.bar([i + (k - 1) * width for i in range(len(ORDERS))], values, width, label=algorithm) - plt.xticks(range(len(ORDERS)), [o.replace("_", " ") for o in ORDERS]) - plt.yscale("log") - plt.ylabel("Comparisons") - plt.title(f"Comparisons by input order, {n} elements") - plt.legend() - plt.tight_layout() - plt.savefig(path, dpi=120) - plt.close() - - -def table(rows, n): - lines = [f"Input of {n} elements (median of 5 timed runs):", "", - "| Order | Algorithm | Comparisons | Array writes | Median time |", - "| :--- | :--- | ---: | ---: | ---: |"] - for order in ORDERS: - for algorithm in ALGORITHMS: - r = next(r for r in rows if r["algorithm"] == algorithm and r["order"] == order and r["n"] == n) - lines.append(f"| {order.replace('_', ' ')} | {algorithm} | {r['comparisons']:,} | " - f"{r['writes']:,} | {r['median_seconds'] * 1000:.3f} ms |") - return "\n".join(lines) - - -def main(): - rows = load(sys.argv[1] if len(sys.argv) > 1 else "data.csv") - largest = max(r["n"] for r in rows) - line_chart(rows, "comparisons", "Comparisons", "Comparisons as input grows", "comparisons.png") - line_chart(rows, "median_seconds", "Median time (seconds)", "Time as input grows", "time.png") - orders_chart(rows, largest, "orders.png") - summary = table(rows, largest) - print(summary) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: - f.write("### Benchmark results\n\n" + summary + "\n") - - -if __name__ == "__main__": - main() From a62547930930ea2fe45e263d1a950e9002b9bcbd Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:48 +0400 Subject: [PATCH 11/51] feat: run the whole study on bare-metal Linux with one command scripts/run-study.sh refuses WSL and containers, checks the tools, offers to make the RAPL counters readable until the next reboot, builds and tests, records the environment, runs the study profile with energy required and writes the report. --check and --rehearse test a machine first. The Dockerfile gives Windows and macOS users the same tools with energy off. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 22 +++++ scripts/run-study.sh | 227 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 Dockerfile create mode 100644 scripts/run-study.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2eb4f90 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +# A Linux environment with everything greenbench needs: GCC, Clang, make, Python 3.12, +# Node.js 24 and ShellCheck. RAPL counters are not visible inside a container, so this image +# runs the harness with energy off. It is for trying the project on Windows or macOS and for +# running the same checks as CI; the energy study itself needs bare-metal Linux. +# REGISTRY can point at a mirror of the Docker official images, for example +# --build-arg REGISTRY=public.ecr.aws/docker/library when Docker Hub rate-limits pulls. +ARG REGISTRY=docker.io/library +FROM ${REGISTRY}/node:24-trixie-slim AS node + +FROM ${REGISTRY}/python:3.12-slim-trixie +RUN apt-get update \ + && apt-get install -y --no-install-recommends clang gcc libc6-dev make shellcheck \ + && rm -rf /var/lib/apt/lists/* +COPY --from=node /usr/local/bin/node /usr/local/bin/node +COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ + && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx +COPY requirements-dev.txt /tmp/requirements-dev.txt +RUN pip install --no-cache-dir --require-hashes -r /tmp/requirements-dev.txt +WORKDIR /work +COPY . . +CMD ["make", "smoke"] diff --git a/scripts/run-study.sh b/scripts/run-study.sh new file mode 100644 index 0000000..2a15955 --- /dev/null +++ b/scripts/run-study.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +# Runs the greenbench energy study on bare-metal Linux, from checks to report: +# +# ./scripts/run-study.sh the full study; writes docs/results.md and docs/img/ +# ./scripts/run-study.sh --check only check that this machine can run it +# ./scripts/run-study.sh --rehearse a short run with energy, written under results/ +# +# See docs/bare-metal.md for preparing a machine (for example Ubuntu booted from a USB stick). +set -euo pipefail + +cd "$(dirname "$0")/.." + +MODE=study +CPU=auto +ASSUME_YES=0 +POWERCAP=/sys/class/powercap +FAKE_POWERCAP=0 + +usage() { + cat <<'EOF' +Usage: scripts/run-study.sh [--check | --rehearse] [--cpu N | --no-pin] [--yes] [--powercap-root DIR] + + --check check the machine (Linux on real hardware, tools, readable RAPL counters) and stop + --rehearse run the short smoke profile with energy, to test the machine; results go to results/ + --cpu N pin the harness and workers to CPU N (default: CPU 2 when there are 4 or more) + --no-pin do not pin to a CPU + --yes do not ask before making the RAPL counters readable + --powercap-root DIR + read RAPL zones from DIR instead of /sys/class/powercap. Only for testing this + script with a fake tree: it skips the bare-metal checks, and the fake counters + never move, so every energy value is zero +EOF +} + +say() { printf '%s\n' "$*"; } +warn() { printf 'run-study: warning: %s\n' "$*" >&2; } +fail() { + printf 'run-study: %s\n' "$*" >&2 + exit 1 +} + +while (($# > 0)); do + case "$1" in + --check) MODE=check ;; + --rehearse) MODE=rehearse ;; + --cpu) + [[ $# -ge 2 && "$2" =~ ^[0-9]+$ ]] || fail "--cpu needs a CPU number" + CPU="$2" + shift + ;; + --no-pin) CPU=none ;; + --yes) ASSUME_YES=1 ;; + --powercap-root) + [[ $# -ge 2 && -d "$2" ]] || fail "--powercap-root needs an existing directory" + POWERCAP="$2" + FAKE_POWERCAP=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + usage >&2 + exit 2 + ;; + esac + shift +done + +check_platform() { + [[ "$(uname -s)" == Linux ]] || fail "RAPL counters need Linux on real hardware; see docs/bare-metal.md" + if [[ -f /.dockerenv ]] || grep -qE 'docker|containerd|kubepods' /proc/1/cgroup 2>/dev/null; then + fail "this is a container, which hides RAPL counters; run on the host (docs/bare-metal.md)" + fi + if grep -qi microsoft /proc/version 2>/dev/null; then + fail "this is WSL, which does not expose RAPL counters; boot Linux directly (docs/bare-metal.md)" + fi + local virt + virt="$(systemd-detect-virt 2>/dev/null || true)" + if [[ -n "$virt" && "$virt" != none ]]; then + warn "systemd-detect-virt reports '$virt': virtual machines rarely expose RAPL, and never accurately" + fi +} + +check_tools() { + local tool + for tool in cc make python3 node; do + command -v "$tool" >/dev/null || + fail "$tool not found; install the tools with: sudo apt-get install -y git build-essential python3-venv nodejs" + done + python3 -c 'import sys; sys.exit(sys.version_info < (3, 12))' || + fail "Python 3.12 or later is needed for the analysis (found $(python3 --version))" + python3 -m venv --help >/dev/null 2>&1 || fail "python3-venv is missing; sudo apt-get install -y python3-venv" + local node_major + node_major="$(node -p 'process.versions.node.split(".")[0]')" + ((node_major >= 18)) || fail "Node.js 18 or later is needed (found $(node --version))" +} + +check_rapl() { + shopt -s nullglob + local zones=("$POWERCAP"/intel-rapl:[0-9]*) + ((${#zones[@]} > 0)) || + fail "no RAPL zones under $POWERCAP: this CPU, its firmware or this kernel does not expose them" + local unreadable=() file + for file in "$POWERCAP"/intel-rapl:*/energy_uj; do + [[ -r "$file" ]] || unreadable+=("$file") + done + shopt -u nullglob + ((${#unreadable[@]} == 0)) && return 0 + + say "Since the fix for CVE-2020-8694 (late 2020), only root can read the RAPL energy counters." + say "The next step runs: sudo chmod o+r on ${#unreadable[@]} energy_uj file(s)." + say "The change lasts until the next reboot. Do this only on a machine you control." + if ((ASSUME_YES == 0)); then + local answer + read -r -p "Make the counters readable? [y/N] " answer + [[ "$answer" == [yY]* ]] || fail "stopped; nothing was changed" + fi + sudo chmod o+r "${unreadable[@]}" +} + +pin_options() { + if [[ "$CPU" == none ]]; then + return 0 + fi + if [[ "$CPU" == auto ]]; then + if (($(nproc) >= 4)); then printf '%s\n' --cpu 2; fi + return 0 + fi + printf '%s\n' --cpu "$CPU" +} + +mains_power() { + local supply + for supply in /sys/class/power_supply/*; do + if [[ -r "$supply/type" && "$(cat "$supply/type")" == Mains && -r "$supply/online" ]]; then + if [[ "$(cat "$supply/online")" == 1 ]]; then echo yes; else echo no; fi + return 0 + fi + done + echo "no mains adapter reported (a desktop, or unknown)" +} + +record_environment() { + local file="$1" f clean=unknown + if command -v git >/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then + if git diff --quiet HEAD; then clean=yes; else clean=no; fi + fi + { + say "recorded_utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + say "git_commit: $(git rev-parse HEAD 2>/dev/null || echo unknown)" + say "git_worktree_clean: $clean" + say "kernel: $(uname -srmo)" + say "virtualisation: $(systemd-detect-virt 2>/dev/null || echo none)" + say "cc: $(cc --version | head -n 1)" + say "python3: $(python3 --version 2>&1)" + say "node: $(node --version)" + say "on_mains_power: $(mains_power)" + say + say "## lscpu" + lscpu 2>/dev/null || say "lscpu not available" + say + say "## cpufreq" + for f in /sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_governor \ + /sys/devices/system/cpu/intel_pstate/no_turbo /sys/devices/system/cpu/cpufreq/boost; do + if [[ -r "$f" ]]; then say "$f: $(cat "$f")"; fi + done + } >"$file" +} + +if ((FAKE_POWERCAP)); then + warn "reading a fake powercap tree at $POWERCAP; the energy values will all be zero" +else + check_platform +fi +check_tools +check_rapl + +say "== Building and running the unit tests" +make -s clean +make -s all test + +say "== RAPL counters" +build/greenbench probe --powercap-root "$POWERCAP" +if [[ "$MODE" == check ]]; then + say "This machine can run the study: ./scripts/run-study.sh" + exit 0 +fi + +say "== Python environment for the analysis (.venv)" +python3 -m venv .venv +.venv/bin/python -m pip install --quiet --require-hashes -r requirements.txt + +stamp="$(date -u +%Y%m%dT%H%M%SZ)" +if [[ "$MODE" == study ]] && ((FAKE_POWERCAP)); then + fail "a fake powercap tree can only be used with --check or --rehearse" +fi +if [[ "$MODE" == study ]]; then + out="data/runs/$stamp" + profile=study + report="docs/results.md" + images="docs/img" +else + out="results/rehearsal-$stamp" + profile=smoke + report="$out/results.md" + images="$out/img" +fi +mkdir -p "$out" +record_environment "$out/environment.txt" +mapfile -t pin < <(pin_options) + +say "== Measuring (profile $profile); close other programs and keep the machine on mains power" +build/greenbench run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ + --label "run-study.sh $MODE $stamp" "${pin[@]}" + +say "== Report" +PYTHONPATH=analysis .venv/bin/python -m greenbench_analysis report \ + --runs "$out/runs.csv" --out "$report" --img-dir "$images" --summary-csv "$out/summary.csv" + +say +say "Done. Raw data: $out/ Report: $report" +if [[ "$MODE" == study ]]; then + say "Next: read $report, then commit it with the data:" + say " git add $report $images $out && git commit -m 'docs: add measured study results'" +fi From 243ebd20b54cabb033bad9b32e2c2d9f8123a37a Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:25:49 +0400 Subject: [PATCH 12/51] ci: test every part of the study on every push GCC and Clang builds, sanitizers with a fake powercap tree, cross-language parity, harness and pipeline tests, a short harness run with energy off, statistics tests on Python 3.12 and 3.14, the Node.js worker, ShellCheck and actionlint. Dependabot covers actions, pip, npm and Docker. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 27 +++++++++++ .github/workflows/ci.yml | 101 ++++++++++++++++++++++++++++++++------- 2 files changed, 110 insertions(+), 18 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..26f682d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + groups: + python: + patterns: ["*"] + + - package-ecosystem: npm + directory: /workers/node + schedule: + interval: weekly + groups: + node-dev-tools: + patterns: ["*"] + + - package-ecosystem: docker + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 656b737..9c6039a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,8 @@ permissions: contents: read jobs: - test: - name: Build and test (${{ matrix.compiler }}) + c: + name: C build and unit tests (${{ matrix.compiler }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -22,7 +22,7 @@ jobs: - name: Build with warnings as errors run: make CC=${{ matrix.compiler }} - - name: Unit tests + - name: Unit tests (sorts, inputs, RAPL parsing and wraparound, options, plan, protocol, output, spawn) run: make CC=${{ matrix.compiler }} test sanitizers: @@ -35,31 +35,96 @@ jobs: - name: Reduce ASLR entropy for the sanitizer runtime run: sudo sysctl vm.mmap_rnd_bits=28 - - name: Tests and a short benchmark under sanitizers + - name: Unit tests and a short harness run with a fake powercap tree, under sanitizers run: make sanitize - benchmark: - name: Benchmark and charts + workers: + name: Cross-language parity, harness run and pipeline runs-on: ubuntu-latest - needs: test steps: - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements-dev.txt + + - uses: actions/setup-node@v7 + with: + node-version: "24" + + - name: Install the Python tools + run: python -m pip install --require-hashes -r requirements-dev.txt + - name: Build run: make - - name: Run the benchmark - run: ./energy_explorer | tee data.csv + - name: xorshift32, inputs and operation counts identical in C, Python and Node.js + run: make parity PYTHON=python - - name: Draw the charts - run: | - python3 -m venv .venv - .venv/bin/pip install --quiet -r requirements.txt - .venv/bin/python visualize.py data.csv + - name: Harness and pipeline integration tests + run: make integration PYTHON=python + + - name: Short harness run with energy off, then the report + run: make smoke PYTHON=python - uses: actions/upload-artifact@v7 with: - name: benchmark-results - path: | - data.csv - *.png + name: smoke-run + path: results/smoke/ + + statistics: + name: Statistics and Python worker (Python ${{ matrix.python }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.12", "3.14"] + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Install the Python tools + run: python -m pip install --require-hashes -r requirements-dev.txt + + - name: Lint, type-check and test + run: make check-python PYTHON=python + + node: + name: Node.js worker (lint, type-check, tests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: "24" + cache: npm + cache-dependency-path: workers/node/package-lock.json + + - name: Lint, type-check and test + run: make check-node + + shell: + name: ShellCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Lint the study script + run: make lint-shell + + actionlint: + name: Workflow lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: actionlint + run: docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:1.7.12 -color From 811cbb51b61f55ef06a7efc08c3afab163137c3b Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:26:09 +0400 Subject: [PATCH 13/51] chore: mark the study script and the workers executable Co-Authored-By: Claude Opus 5.5 --- scripts/run-study.sh | 0 workers/node/worker.js | 0 workers/python/greenbench_worker.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/run-study.sh mode change 100644 => 100755 workers/node/worker.js mode change 100644 => 100755 workers/python/greenbench_worker.py diff --git a/scripts/run-study.sh b/scripts/run-study.sh old mode 100644 new mode 100755 diff --git a/workers/node/worker.js b/workers/node/worker.js old mode 100644 new mode 100755 diff --git a/workers/python/greenbench_worker.py b/workers/python/greenbench_worker.py old mode 100644 new mode 100755 From feb8969809930c9053ec026d836ea7f656ce3db1 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:34:04 +0400 Subject: [PATCH 14/51] fix: keep coinciding lines and unplottable values visible in the charts All three sorts make the same number of comparisons on reversed input, so each algorithm now has its own line width, dash pattern and marker. Values that a log axis cannot show are listed under the chart instead of silently dropped, and a time-energy chart with nothing to plot is not drawn. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/charts.py | 57 +++++++++++++++++++++----- analysis/tests/test_charts.py | 32 +++++++++++++++ 2 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 analysis/tests/test_charts.py diff --git a/analysis/greenbench_analysis/charts.py b/analysis/greenbench_analysis/charts.py index 317a57e..8d9226c 100644 --- a/analysis/greenbench_analysis/charts.py +++ b/analysis/greenbench_analysis/charts.py @@ -9,6 +9,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any import matplotlib @@ -17,6 +18,7 @@ import matplotlib.pyplot as plt from matplotlib.axes import Axes from matplotlib.figure import Figure +from matplotlib.ticker import NullLocator from .load import ORDERS from .summary import Configuration, OperationCounts @@ -28,6 +30,13 @@ GRID = "#e1e0d9" AXIS = "#c3c2b7" SERIES = ("#2a78d6", "#eb6834", "#1baf7a") # categorical slots 1-3, never cycled +# Lines can coincide exactly (all three sorts make n(n-1)/2 comparisons on reversed input), so +# each algorithm also gets its own width, dash pattern and marker, drawn widest first. +LINE_STYLES: tuple[dict[str, Any], ...] = ( + {"linewidth": 4.5, "linestyle": "-", "marker": "o", "markersize": 8}, + {"linewidth": 2.5, "linestyle": (0, (5, 2)), "marker": "s", "markersize": 6}, + {"linewidth": 1.5, "linestyle": (0, (1, 1.5)), "marker": "^", "markersize": 5}, +) ALGORITHM_ORDER = ("bubble", "insertion", "quick") LANGUAGE_ORDER = ("c", "python", "node") LANGUAGE_NAMES = {"c": "C", "python": "Python", "node": "Node.js"} @@ -72,7 +81,7 @@ def operations_chart(operations: list[OperationCounts], path: Path) -> Path | No return None fig, axes = _figure(2.6 * len(orders) + 0.6, 3.2, len(orders)) for ax, order in zip(axes, orders, strict=True): - for colour, algorithm in zip(SERIES, ALGORITHM_ORDER, strict=True): + for colour, style, algorithm in zip(SERIES, LINE_STYLES, ALGORITHM_ORDER, strict=True): points = sorted((c.n, c.comparisons) for c in operations if c.order == order and c.algorithm == algorithm) if not points: continue @@ -80,15 +89,15 @@ def operations_chart(operations: list[OperationCounts], path: Path) -> Path | No [p[0] for p in points], [max(p[1], 1) for p in points], color=colour, - linewidth=2, - marker="o", - markersize=5, markeredgecolor=SURFACE, - markeredgewidth=1.5, + markeredgewidth=1, label=algorithm, + **style, ) ax.set_xscale("log") ax.set_yscale("log") + ax.set_xticks(sizes, [f"{size:,}" for size in sizes]) + ax.xaxis.set_minor_locator(NullLocator()) ax.set_title(order.replace("_", " "), fontsize=10, loc="left") ax.set_xlabel("Input size (elements)", fontsize=8) axes[0].set_ylabel("Comparisons per sort", fontsize=8) @@ -112,6 +121,7 @@ def _dot_chart( fig, (ax,) = _figure(6.4, 0.9 + 0.75 * len(rows)) offsets = {lang: (i - (len(languages) - 1) / 2) * 0.22 for i, lang in enumerate(languages)} plotted = False + hidden: list[str] = [] for colour, language in zip(SERIES, LANGUAGE_ORDER, strict=True): if language not in languages: continue @@ -121,8 +131,12 @@ def _dot_chart( if not match: continue estimate = getattr(match[0], value) - if estimate is None or estimate.value <= 0 or estimate.low <= 0: - continue # a log axis cannot show zero or negative values; the table still does + if estimate is None: + continue + if estimate.value <= 0 or estimate.low <= 0: + # A log axis cannot show zero or negative values; say so under the chart. + hidden.append(f"{LANGUAGE_NAMES[language]} {algorithm}") + continue xs.append(estimate.value) ys.append(row + offsets[language]) lows.append(estimate.value - estimate.low) @@ -143,6 +157,15 @@ def _dot_chart( ax.set_xlabel(xlabel, fontsize=8) ax.set_title(title, fontsize=10, loc="left") ax.legend(frameon=False, fontsize=8, loc="lower right") + if hidden: + fig.text( + 0.01, + -0.02, + "Not shown, because the median or its interval reaches zero or below (see the table): " + ", ".join(hidden), + fontsize=7, + color=INK_SECONDARY, + va="top", + ) return _save(fig, path) @@ -175,14 +198,19 @@ def time_energy_chart( points_by_language maps a language to (algorithm, wall_ns, energy_uj) for each run. """ - languages = [lang for lang in LANGUAGE_ORDER if points_by_language.get(lang)] + # Log axes cannot show zero or negative energy (a run that used no more than idle power). + positive = { + lang: [(a, t, e) for a, t, e in points if t > 0 and e > 0] for lang, points in points_by_language.items() + } + dropped = sum(len(points_by_language[lang]) - len(positive[lang]) for lang in positive) + languages = [lang for lang in LANGUAGE_ORDER if positive.get(lang)] if not languages: return None fig, axes = _figure(3.0 * len(languages) + 0.6, 3.2, len(languages)) for ax, language in zip(axes, languages, strict=True): - points = points_by_language[language] + points = positive[language] for colour, algorithm in zip(SERIES, ALGORITHM_ORDER, strict=True): - kept = [(t / 1e6, e / 1e6) for a, t, e in points if a == algorithm and t > 0 and e > 0] + kept = [(t / 1e6, e / 1e6) for a, t, e in points if a == algorithm] if kept: ax.scatter( [k[0] for k in kept], @@ -202,4 +230,13 @@ def time_energy_chart( handles, labels = axes[0].get_legend_handles_labels() if handles: fig.legend(handles, labels, loc="upper center", ncol=3, frameon=False, fontsize=8, bbox_to_anchor=(0.5, 1.08)) + if dropped: + fig.text( + 0.01, + -0.02, + f"Not shown: {dropped:,} runs whose net energy was zero or below.", + fontsize=7, + color=INK_SECONDARY, + va="top", + ) return _save(fig, path) diff --git a/analysis/tests/test_charts.py b/analysis/tests/test_charts.py new file mode 100644 index 0000000..607fe9b --- /dev/null +++ b/analysis/tests/test_charts.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from pathlib import Path + +import synthetic +from greenbench_analysis import charts +from greenbench_analysis.summary import OperationCounts, summarise + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def test_operations_chart_needs_two_sizes(tmp_path: Path) -> None: + one_size = [OperationCounts("quick", "random", 100, 700, 800, ("c",))] + assert charts.operations_chart(one_size, tmp_path / "a.png") is None + two_sizes = [*one_size, OperationCounts("quick", "random", 200, 1600, 1700, ("c",))] + drawn = charts.operations_chart(two_sizes, tmp_path / "b.png") + assert drawn is not None + assert drawn.read_bytes().startswith(PNG_SIGNATURE) + + +def test_time_energy_chart_skips_runs_it_cannot_draw(tmp_path: Path) -> None: + nothing_positive = {"c": [("quick", 1e6, 0.0), ("quick", 2e6, -5.0)]} + assert charts.time_energy_chart(nothing_positive, True, tmp_path / "a.png") is None + some_positive = {"c": [("quick", 1e6, 10.0), ("quick", 2e6, -5.0)], "python": [("bubble", 5e6, 40.0)]} + assert charts.time_energy_chart(some_positive, True, tmp_path / "b.png") is not None + + +def test_dot_charts_from_a_summary(tmp_path: Path) -> None: + summary = summarise(synthetic.study(energy=True), resamples=100) + assert charts.time_per_sort_chart(summary.configurations, 100, "random", tmp_path / "t.png") is not None + assert charts.energy_per_sort_chart(summary.configurations, 100, "random", tmp_path / "e.png") is not None + assert charts.time_per_sort_chart(summary.configurations, 999, "random", tmp_path / "none.png") is None From 974f50a64c50b399ed488162364529bb8d66bd31 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:37:08 +0400 Subject: [PATCH 15/51] feat: retry a worker process that fails, once by default A study run in Docker Desktop stopped after 2,227 of 5,760 runs when one Python worker could not read its own directory (ENOMEM from the shared file system). A failed process is now rerun up to --retries times (default 1) and the retries are counted in meta.json. Disagreeing results are never retried: they stop the run as before. Co-Authored-By: Claude Opus 5.5 --- src/greenbench.c | 32 +++++++++++++++++++------ src/options.c | 9 ++++++- src/options.h | 1 + tests/integration/test_harness.py | 39 ++++++++++++++++++++++++++++++- tests/test_options.c | 6 ++++- 5 files changed, 77 insertions(+), 10 deletions(-) diff --git a/src/greenbench.c b/src/greenbench.c index 4eff565..1596246 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -77,6 +77,7 @@ typedef struct s_context { t_consistency hashes[ORDER_COUNT][OPTIONS_MAX_SIZES]; t_consistency counts[ALGO_COUNT][ORDER_COUNT][OPTIONS_MAX_SIZES]; t_spawn_result spawn; + unsigned retries_used; char started_utc[32]; char meta_path[PATH_MAX]; char command[2048]; /* the harness's own command line, for meta.json */ @@ -134,6 +135,7 @@ static void print_usage(FILE *out) { " --min-loop-ms N sort for at least N ms per process, well above the ~1 ms RAPL update\n" " --max-reps N never sort more than N times in one process\n" " --no-startup skip the startup trials (processes that exit without sorting)\n" + " --retries N rerun a worker process that fails up to N times (default 1)\n" " --seed N input seed, identical for every language (default %u)\n" " --order-seed N seed for the randomised run order\n" " --energy MODE auto (measure when possible), require (fail without RAPL) or off\n" @@ -390,7 +392,7 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure json_key_string(f, " ", "finished_utc", finished, 1); json_key_string(f, " ", "label", o->label, 1); json_key_string(f, " ", "command", ctx->command, 1); - fprintf(f, " \"measured_runs\": %" PRIu64 ",\n", measured_runs); + fprintf(f, " \"measured_runs\": %" PRIu64 ",\n \"worker_retries\": %u,\n", measured_runs, ctx->retries_used); fputs(" \"clock\": \"CLOCK_MONOTONIC_RAW\",\n", f); fputs(" \"host\": {\n", f); @@ -449,9 +451,9 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure fprintf(f, " \"runs\": %u,\n \"warmup\": %u,\n \"seed\": %" PRIu32 ",\n \"order_seed\": %" PRIu32 ",\n \"min_loop_ns\": %" PRIu64 ",\n \"max_reps\": %" PRIu64 ",\n \"startup_trials\": %s,\n" - " \"cpu\": %d\n", + " \"retries\": %u,\n \"cpu\": %d\n", o->runs, o->warmup, o->seed, o->order_seed, o->min_loop_ns, o->max_reps, - o->startup_trials ? "true" : "false", o->cpu); + o->startup_trials ? "true" : "false", o->retries, o->cpu); fputs(" },\n", f); fputs(" \"workers\": {\n", f); @@ -500,7 +502,10 @@ static void describe_trial(const t_trial *t, char *out, size_t size) { order_name(t->order), t->n); } -static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measurement *m) { +/* Runs one worker and measures it. Returns 0; -2 if the process could not start, exited + with an error or was killed (worth retrying: a busy machine can make a start fail); or -1 + for anything else, including results that disagree, which a retry would not change. */ +static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measurement *m, int will_retry) { const t_options *o = &ctx->options; char n_text[24], seed_text[24], reps_text[24], what[TEXT_LEN], err[ERROR_LEN]; char *argv[16]; @@ -549,7 +554,8 @@ static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measure return -1; /* the worker got the same Ctrl-C; command_run reports it */ if (rc != 0) { spawn_describe_failure(&ctx->spawn, err, sizeof err); - return error_message("worker for %s (%s): %s", what, argv[0], err), -1; + error_message("worker for %s (%s): %s%s", what, argv[0], err, will_retry ? "; trying again" : ""); + return -2; } if (protocol_parse(ctx->spawn.out, &m->result, err, sizeof err) != 0) return error_message("worker for %s printed an unexpected result: %s", what, err), -1; @@ -566,6 +572,17 @@ static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measure return 0; } +/* run_worker, repeated up to --retries times when the worker process itself fails. */ +static int run_measured(t_context *ctx, const t_trial *t, uint64_t reps, t_measurement *m) { + for (unsigned attempt = 0;; attempt++) { + int will_retry = attempt < ctx->options.retries; + int rc = run_worker(ctx, t, reps, m, will_retry); + if (rc != -2 || !will_retry || g_interrupted) + return rc == 0 ? 0 : -1; + ctx->retries_used++; + } +} + /* Every language must see the same input and do the same work on it. */ static int check_consistency(t_context *ctx, const t_trial *t, const t_worker_result *r) { int s = size_index(&ctx->options, t->n); @@ -646,7 +663,7 @@ static int warm_up(t_context *ctx) { t_measurement m; for (unsigned w = 0; w < o->warmup; w++) { - if (interrupted() || run_worker(ctx, t, reps, &m) != 0 || check_consistency(ctx, t, &m.result) != 0) { + if (interrupted() || run_measured(ctx, t, reps, &m) != 0 || check_consistency(ctx, t, &m.result) != 0) { free(order); return -1; } @@ -695,7 +712,8 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { t_measurement m; t_run_record record; - if (interrupted() || run_worker(ctx, t, t->inner_reps, &m) != 0 || check_consistency(ctx, t, &m.result) != 0) { + if (interrupted() || run_measured(ctx, t, t->inner_reps, &m) != 0 || + check_consistency(ctx, t, &m.result) != 0) { free(items); return -1; } diff --git a/src/options.c b/src/options.c index 82116e0..28cfe43 100644 --- a/src/options.c +++ b/src/options.c @@ -68,6 +68,7 @@ void options_defaults(t_options *o) { o->seed = INPUT_DEFAULT_SEED; o->order_seed = DEFAULT_ORDER_SEED; o->startup_trials = 1; + o->retries = 1; o->energy = ENERGY_AUTO; o->baseline = BASELINE_AUTO; o->powercap_root = RAPL_DEFAULT_ROOT; @@ -252,7 +253,7 @@ static const char *const VALUED[] = {"profile", "languages", "algorithms", "runs", "warmup", "seed", "order-seed", "min-loop-ms", "max-reps", "cpu", "energy", "idle-baseline", "powercap-root", "out", "python", "node", "c-worker", - "python-worker", "node-worker", "label"}; + "python-worker", "node-worker", "label", "retries"}; static int is_valued(const char *name) { for (size_t i = 0; i < sizeof VALUED / sizeof *VALUED; i++) @@ -338,6 +339,12 @@ static int apply_option(const char *name, const char *value, t_options *o, char o->max_reps = number; return 0; } + if (strcmp(name, "retries") == 0) { + if (parse_number(name, value, 0, 10, &number, err, err_size) != 0) + return -1; + o->retries = (unsigned)number; + return 0; + } if (strcmp(name, "cpu") == 0) { if (parse_number(name, value, 0, 4095, &number, err, err_size) != 0) return -1; diff --git a/src/options.h b/src/options.h index a186e45..b414617 100644 --- a/src/options.h +++ b/src/options.h @@ -30,6 +30,7 @@ typedef struct s_options { uint64_t min_loop_ns; /* each measured sort loop should last at least this long */ uint64_t max_reps; /* upper bound on sorts per process */ int startup_trials; /* also measure processes that exit without sorting */ + unsigned retries; /* times to rerun a worker process that fails, before giving up */ t_energy_mode energy; t_baseline_mode baseline; const char *powercap_root; diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 09bd2c5..7a5ab28 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -215,10 +215,47 @@ def test_failing_worker_stops_the_run(tmp_path: Path) -> None: *interpreter_options(), ) assert completed.returncode == 1 - assert "simulated crash" in completed.stderr + assert "simulated crash; trying again" in completed.stderr # one retry by default, then it stops assert read_meta(tmp_path / "run")["status"] == "failed" +def test_a_worker_that_fails_once_is_retried(tmp_path: Path) -> None: + # Fails on its first start only, as a worker can when the machine is briefly out of memory. + real = REPOSITORY / "workers" / "python" / "greenbench_worker.py" + marker = tmp_path / "failed-once" + flaky = tmp_path / "flaky_worker.py" + lines = [ + "import pathlib, runpy, sys", + f"marker = pathlib.Path({str(marker)!r})", + "if not marker.exists():", + " marker.touch()", + " sys.exit('greenbench-worker: transient failure')", + f"sys.argv[0] = {str(real)!r}", + f"runpy.run_path({str(real)!r}, run_name='__main__')", + ] + flaky.write_text("\n".join(lines) + "\n", encoding="utf-8") + out = tmp_path / "run" + completed = run_harness( + "run", + *SMALL, + "--languages", + "python", + "--python-worker", + str(flaky), + "--energy", + "off", + "--out", + str(out), + *interpreter_options(), + ) + assert completed.returncode == 0, completed.stderr + assert "transient failure; trying again" in completed.stderr + meta = read_meta(out) + assert meta["status"] == "complete" + assert meta["worker_retries"] == 1 + assert len(read_runs(out)) == 4 * 2 * 2 * 3 + + def test_disagreeing_worker_is_caught(tmp_path: Path) -> None: # A worker that reports one extra comparison must be rejected, not averaged in. source = (REPOSITORY / "workers" / "python" / "greenbench_worker.py").read_text(encoding="utf-8") diff --git a/tests/test_options.c b/tests/test_options.c index ee6b2d4..2857194 100644 --- a/tests/test_options.c +++ b/tests/test_options.c @@ -24,6 +24,7 @@ static void test_defaults_are_the_study(void) { CHECK(o.energy == ENERGY_AUTO && o.baseline == BASELINE_AUTO, "energy and baseline automatic"); CHECK(o.cpu == -1, "not pinned"); CHECK(o.startup_trials == 1, "startup trials on"); + CHECK(o.retries == 1, "one retry for a failed worker process"); } static void test_profile_and_overrides(void) { @@ -57,7 +58,7 @@ static void test_values(void) { "--order-seed", "5", "--max-reps", "10", "--cpu", "3", "--energy", "require", "--idle-baseline", "off", "--out", "x/y", "--python", "python3.12", "--node", "/usr/bin/node", "--label", "laptop test", - "--no-startup", "--dry-run", "--quiet", "--powercap-root", "/tmp/pc"}; + "--no-startup", "--dry-run", "--quiet", "--powercap-root", "/tmp/pc", "--retries", "0"}; int argc = (int)(sizeof all / sizeof *all); CHECK(parse(&o, argc, all) == 0, "every option parses"); @@ -68,6 +69,7 @@ static void test_values(void) { strcmp(o.powercap_root, "/tmp/pc") == 0, "text"); CHECK(o.startup_trials == 0 && o.dry_run == 1 && o.quiet == 1, "flags"); + CHECK(o.retries == 0, "retries"); } static void expect_error(int argc, char *const argv[], const char *needle, const char *name) { @@ -96,6 +98,7 @@ static void test_errors(void) { char *const q[] = {"--out", ""}; char *const r[] = {"--orders", "sideways"}; char *const s[] = {"--algorithms", "none"}; + char *const t[] = {"--retries", "11"}; expect_error(2, a, "--runs", "zero runs"); expect_error(2, b, "not a whole number", "runs not a number"); @@ -115,6 +118,7 @@ static void test_errors(void) { expect_error(2, q, "must not be empty", "empty text value"); expect_error(2, r, "unknown order", "unknown order"); expect_error(2, s, "unknown algorithm 'none'", "none is internal"); + expect_error(2, t, "from 0 to 10", "too many retries"); } static void test_parse_u64(void) { From 9839cba75c3122b2eb43e8016a09a86b6e6c1c23 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:39:14 +0400 Subject: [PATCH 16/51] chore: show timings in the local smoke report Co-Authored-By: Claude Opus 5.5 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 73faa14..2a699ae 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ sanitize: smoke: all $(HARNESS) run --profile smoke --energy off --out results/smoke $(PYRUN) -m greenbench_analysis report --runs results/smoke/runs.csv --out results/smoke/results.md \ - --img-dir results/smoke/img --withhold-timings + --img-dir results/smoke/img # The three workers must generate identical sequences and inputs and do identical work. parity: all From 982ec1cc77dc2ce6b7c4fca9faa7ef1496fd7d41 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:39:15 +0400 Subject: [PATCH 17/51] docs: rewrite the README for greenbench and document the study The README explains that greenbench grew out of Algorithm-Energy-C and says plainly that energy results are pending until the study runs on bare-metal Linux. The old README's timings, measured on a CI runner, are not carried over; timings will come from the measured study run. docs/study.md sets out the question, method and threats to validity, docs/bare-metal.md gives the steps for an Ubuntu USB stick, and docs/decisions.md records the design choices. The licence year is 2026. Co-Authored-By: Claude Opus 5.5 --- LICENSE | 2 +- README.md | 279 +++++++++++++++++++++++++++++++++------------ docs/bare-metal.md | 103 +++++++++++++++++ docs/decisions.md | 201 ++++++++++++++++++++++++++++++++ docs/study.md | 182 +++++++++++++++++++++++++++++ 5 files changed, 693 insertions(+), 74 deletions(-) create mode 100644 docs/bare-metal.md create mode 100644 docs/decisions.md create mode 100644 docs/study.md diff --git a/LICENSE b/LICENSE index aa2132b..a5cdd91 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025-2026 Farah Sharif +Copyright (c) 2026 Farah Sharif Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index d4b507e..afdd68e 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,219 @@ -# Algorithm-Energy-C +# greenbench + +Measures the energy, time and work of three sorting algorithms in C, Python and Node.js on +identical inputs, to test whether run time is a good stand-in for energy. [![CI](https://github.com/fasharif/Algorithm-Energy-C/actions/workflows/ci.yml/badge.svg)](https://github.com/fasharif/Algorithm-Energy-C/actions/workflows/ci.yml) -How much work do simple sorting algorithms do, and how long does it take? This benchmark runs -bubble sort, insertion sort and quicksort in C on identical inputs of several sizes and orders. It -counts the comparisons and array writes each one makes, and times it. - -Work and time are stand-ins for the energy a computation uses. Measuring energy directly is the -next step (see [Next steps](#next-steps)). - -## Results - -4,000 integers, median of 5 timed runs on a GitHub Actions runner. Counts are exact; timings vary -a little between runs. - -| Input order | Algorithm | Comparisons | Array writes | Time | -| :--- | :--- | ---: | ---: | ---: | -| Random | bubble | 7,998,000 | 8,003,836 | 33.7 ms | -| Random | insertion | 4,005,907 | 4,005,905 | 2.5 ms | -| Random | quick | 55,119 | 56,984 | 0.17 ms | -| Sorted | bubble | 7,998,000 | 0 | 7.5 ms | -| Sorted | insertion | 3,999 | 0 | 0.005 ms | -| Sorted | quick | 7,998,000 | 0 | 5.0 ms | -| Reversed | bubble | 7,998,000 | 15,996,000 | 62.6 ms | -| Reversed | insertion | 7,998,000 | 8,001,999 | 5.0 ms | -| Reversed | quick | 7,998,000 | 4,000 | 5.5 ms | -| Nearly sorted | bubble | 7,998,000 | 1,059,064 | 11.2 ms | -| Nearly sorted | insertion | 533,531 | 533,520 | 0.37 ms | -| Nearly sorted | quick | 146,424 | 23,412 | 0.14 ms | - -What the numbers show: - -- **The algorithm matters.** On random input, quicksort makes about 145 times fewer comparisons - than bubble sort and finishes about 200 times faster. -- **So does the input.** Insertion sort handles sorted input in 3,999 comparisons. This quicksort - always picks the last element as its pivot, so sorted input is its worst case: 7,998,000 - comparisons, as many as bubble sort. -- **Counting operations does not predict time.** Bubble sort makes the same 7,998,000 comparisons - on sorted and on random input, but takes 4.5 times longer on random input (33.7 ms against - 7.5 ms), because of the writes and the branches the processor cannot predict. The same caution - applies to energy, which is why it needs to be measured rather than inferred. - -## How it works - -- Sizes 500, 1,000, 2,000 and 4,000 by default, or your own (`./energy_explorer 100 10000`). -- Four input orders: - - random: shuffled with a fixed seed, so every run sees the same input - - sorted - - reversed - - nearly sorted: 5% of elements swapped -- Every algorithm sorts an identical copy of each input. Each sort is timed five times with a - monotonic clock, the median is reported, and every result is checked to be sorted. -- *Comparisons* counts each comparison between two elements. *Array writes* counts each store into - the array; a swap counts as two. -- Quicksort uses the last element as its pivot (Lomuto partition) and recurses into the smaller - half, so its stack stays small even in its worst case. - -## Build and run +greenbench grew out of [Algorithm-Energy-C](https://github.com/fasharif/Algorithm-Energy-C), +a C benchmark that counted the comparisons and array writes of bubble sort, insertion sort +and quicksort and timed them. Its README named measuring energy directly with Intel's RAPL +counters as the next step. This is that step, done as a small study: an energy-measuring +harness in C, the same algorithms in three languages, statistics, carbon estimates and a +written method. + +> **Status: energy results are pending.** Everything is built and tested, but the energy +> study needs Linux running directly on hardware, and this version was developed in Docker +> Desktop, where the energy counters do not exist. [docs/results.md](docs/results.md) shows +> the exact operation counts and marks every time, energy and carbon figure as pending until +> the study runs on bare-metal Linux with one command ([how](docs/bare-metal.md)). + +## Sample output + +In Docker Desktop, `greenbench probe` reports honestly that there is nothing to measure: + +```text +$ build/greenbench probe +powercap root: /sys/class/powercap +energy: unavailable +reason: no RAPL package zone (intel-rapl:N named package-N) under /sys/class/powercap; this CPU, kernel or virtual machine does not expose RAPL +``` + +The harness then measures time and operation counts only, and leaves every energy cell +empty rather than writing zeros. Operation counts are exact, so they are already in +[docs/results.md](docs/results.md), generated from a full run of the study profile. They are +identical in all three languages, which the harness checks on every run: + +![Comparisons per sort against input size for bubble, insertion and quick sort, one panel per input order](docs/img/operations.png) + +## The problem + +Developers measure time and assume that faster code uses less energy. That assumption is +rarely tested, because measuring energy properly is fiddly: the counters are coarse, the +processor never stops drawing power, and comparing languages fairly means feeding them +exactly the same work. greenbench does the fiddly parts so that the question can be answered +with data: **is run time a good stand-in for energy, and how do C, Python and Node.js +compare?** The research question, method and threats to validity are in +[docs/study.md](docs/study.md). + +## Features + +- **Energy from RAPL in C.** Reads package and DRAM zones from `/sys/class/powercap`, handles + counter wraparound with `max_energy_range_uj`, and reports "energy unavailable" with a + reason when the counters are missing or unreadable, never a fake value. +- **Identical inputs everywhere.** xorshift32 and the four input orders are implemented in C, + Python and JavaScript; tests check that all three produce identical sequences, inputs and + operation counts, and the harness stops if a worker ever disagrees. +- **A careful harness.** One process per measured run for every language, time from + `CLOCK_MONOTONIC_RAW`, repetitions calibrated so each process sorts for at least 200 ms + (well above the ~1 ms RAPL update interval), an idle baseline of the same length after every + run, 30 measured runs per configuration after warm-up, randomised run order, optional CPU + pinning, startup trials that measure each interpreter's fixed cost, and tidy CSV output. +- **Statistics.** Medians with 95% bootstrap confidence intervals, and Spearman's rank + correlation between time and energy per language and algorithm, in a Python package with + its own tests. +- **Carbon estimates.** Energy converted to grams of CO2-equivalent for the UAE and UK grids, + from a cited table (Ember data, CC BY 4.0), with any other region or value on request. +- **A generated report.** `docs/results.md` and its charts come from a script, never by hand. +- **Runs anywhere, measures where it can.** The same harness runs on WSL2, in Docker and in CI + with energy off; `scripts/run-study.sh` runs the full study on bare-metal Linux. + +## Architecture + +```mermaid +flowchart LR + subgraph harness["build/greenbench (C)"] + plan["plan: configurations x 30 runs,
shuffled from a seed"] --> run["run one worker process"] + rapl["RAPL reader
/sys/class/powercap"] --> run + run --> files[("runs.csv
meta.json")] + end + run -- "argv: algorithm, order, n, seed, reps" --> workers + subgraph workers["workers: same commands, same output"] + c["C"] + py["Python"] + node["Node.js"] + end + workers -- "result line: counts, loop time, input hash" --> run + files --> analysis["greenbench_analysis (Python):
medians, bootstrap CIs, Spearman, carbon"] + grid[("data/grid_intensity.csv")] --> analysis + analysis --> report["docs/results.md
docs/img/*.png"] +``` + +For each measured run the harness reads the energy counters and the clock, starts a worker, +waits for it to exit, and reads them again. It then sleeps for the same duration and reads +the counters once more to measure idle energy. The worker prints one line; the harness checks +that its input hash and counts match the other languages, and writes a CSV row. + +## Tech stack and why + +| Part | Choice | Why | +| :--- | :--- | :--- | +| Harness and C worker | C11, POSIX (`posix_spawnp`, `pread`, `clock_gettime`) | Small, predictable overhead around each measured process, and direct access to sysfs. | +| Energy | Linux powercap sysfs | No kernel modules, no per-model unit decoding; the kernel reports microjoules ([ADR 1](docs/decisions.md#1-read-energy-from-the-powercap-sysfs-files)). | +| Python worker | Python standard library | Runs on the Python a Linux distribution ships, with nothing to install. | +| Node.js worker | JavaScript with JSDoc types, checked by `tsc --strict` | Type-checked like TypeScript, but no build step between the source and what is measured ([ADR 9](docs/decisions.md#9-no-runtime-dependencies-in-the-workers)). | +| Statistics and report | Python 3.12+, numpy, matplotlib | The bootstrap and Spearman's rho are short enough to write out and test; no SciPy needed. | +| Quality | GCC and Clang with `-Werror`, ASan and UBSan, pytest, mypy `--strict`, ruff, ESLint, ShellCheck, actionlint | Every language in the repository is linted, type-checked where possible, and tested in CI. | +| Portability | Dockerfile, GitHub Actions | Try it on Windows or macOS; the energy study itself needs bare-metal Linux. | + +## Quick start + +On Linux (including WSL2) with a C compiler, make, Python 3.12+ (with `venv`) and Node.js 18+: ```bash -make -./energy_explorer > data.csv # algorithm, order, n, median_seconds, comparisons, writes -python3 -m pip install -r requirements.txt -python3 visualize.py data.csv # comparisons.png, time.png, orders.png and a summary table +git clone https://github.com/fasharif/Algorithm-Energy-C.git greenbench && cd greenbench +make && make test # build the harness and workers, run the C unit tests +python3 -m venv .venv && .venv/bin/pip install --require-hashes -r requirements.txt +make smoke # all three languages, energy off; report in results/smoke/results.md +./scripts/run-study.sh # the energy study, on bare-metal Linux only ``` -`make results` runs both steps. In CI, the CSV and the charts are saved as artifacts of each run. +The Makefile uses `.venv/bin/python` when it exists, and `python3` otherwise. + +On Windows or macOS, the Docker image runs the same smoke test and writes the report to +`results/smoke/` on your machine (in PowerShell, write `${PWD}` instead of `$PWD`): + +```bash +docker build -t greenbench . +docker run --rm -v "$PWD/results:/work/results" greenbench +``` + +## Configuration + +`build/greenbench run` takes these options; the defaults are the study profile. +`build/greenbench --help` lists them all. + +| Option | Default | Meaning | +| :--- | :--- | :--- | +| `--profile study\|smoke` | `study` | Study: sizes 250, 500, 1,000, 2,000; 30 runs; 3 warm-up runs; 200 ms minimum sort loop. Smoke: sizes 64, 256; 3 runs; 1 warm-up; 2 ms. | +| `--languages`, `--algorithms`, `--orders`, `--sizes` | all | Comma-separated subsets, for example `--languages c,node --sizes 100,1000`. | +| `--runs N`, `--warmup N` | 30, 3 | Measured and warm-up runs per configuration. | +| `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Minimum sort loop per process, and the cap on sorts per process. | +| `--energy auto\|require\|off` | `auto` | `require` fails with exit status 3 if RAPL cannot be read. | +| `--idle-baseline auto\|on\|off` | `auto` | On whenever energy is measured. | +| `--cpu N` | off | Pin the harness and every worker to CPU N. | +| `--seed N`, `--order-seed N` | 2463534242, 1234567 | Input generator seed, and run-order seed. | +| `--no-startup` | | Skip the startup trials. | +| `--retries N` | 1 | Rerun a worker process that fails (cannot start, crashes) up to N times; disagreeing results are never retried. | +| `--out DIR` | `results/latest` | Where `runs.csv` and `meta.json` go. | +| `--python CMD`, `--node CMD` | `python3`, `node` | Interpreters to use. | +| `--powercap-root DIR` | `/sys/class/powercap` | Read RAPL zones from elsewhere, for tests. | +| `--dry-run` | | Print the plan and exit. | + +`build/greenbench probe` checks the counters and prints idle power over one second. + +The report is `python -m greenbench_analysis report --runs DIR/runs.csv` with +`PYTHONPATH=analysis`. It takes `--out`, `--img-dir`, `--regions ARE,GBR` (codes from +[data/grid_intensity.csv](data/grid_intensity.csv)), `--intensity NAME=G_PER_KWH` for any +other grid, `--withhold-timings` for runs on a busy machine, and `--summary-csv`. + +## Running the tests + +```bash +make test # C unit tests: sorts, inputs, RAPL parsing and wraparound, options, plan, protocol, CSV, spawn +make sanitize # the C tests and a harness run under AddressSanitizer and UBSan +.venv/bin/pip install --require-hashes -r requirements-dev.txt +make check-python # ruff, mypy --strict, and the statistics and Python worker tests +make check-node # ESLint, tsc --strict and the Node.js worker tests +make integration # cross-language parity, the harness end to end, and the report pipeline +make lint-shell # ShellCheck on the study script +``` + +The RAPL tests build fake powercap trees from the manifests in +[tests/fixtures/powercap](tests/fixtures/powercap), including a two-socket server, an AMD +processor without DRAM, a missing tree, corrupt values and an unreadable counter, and rewrite +counters between readings to test wraparound. CI runs all of the above and lints the workflow +with actionlint; the C build and tests run with both GCC and Clang, and the statistics tests on +Python 3.12 and 3.14. + +## Folder structure + +```text +src/ C: harness (greenbench.c), C worker (worker.c), RAPL reader, sorts, + input generator, options, run plan, worker protocol, CSV output +workers/python/ Python worker (standard library only) and its tests +workers/node/ Node.js worker (no runtime dependencies), tests, ESLint and tsc config +analysis/ greenbench_analysis: loading, statistics, carbon, charts, report; tests +tests/ C unit tests, fake powercap manifests, integration tests (pytest) +scripts/run-study.sh the one-command study for bare-metal Linux +data/ grid carbon intensity with sources; data/runs/ will hold study runs +docs/ study.md, bare-metal.md, decisions.md, results.md (generated), img/ +``` + +## Design decisions + +Recorded in [docs/decisions.md](docs/decisions.md): why powercap rather than MSRs, why every +language is measured as a whole process, how repetitions are calibrated, why the idle +baseline follows every run, why missing energy is never written as zero, and more. -## Testing +## Limitations and roadmap -`make test` checks each algorithm against the C library's `qsort` on random arrays with many -duplicates. It also checks exact comparison and write counts on sorted and reversed input. GitHub -Actions builds with GCC and Clang with warnings treated as errors, repeats the tests under -AddressSanitizer and UndefinedBehaviorSanitizer, and runs the benchmark. +- **No energy data yet.** The study has to run on bare-metal Linux; until then the results + are pending, and the README makes no claims about energy. +- **RAPL is not a power meter.** It covers the processor package and DRAM, not the whole + machine, and its accuracy varies between processors. See + [threats to validity](docs/study.md#threats-to-validity). +- **A small workload.** Three single-threaded sorts on integer arrays of up to 2,000 elements + in the study profile (100,000 at most). -## Next steps +Next: -- Measure energy directly with Intel RAPL counters (`/sys/class/powercap`, or - `perf stat -e power/energy-pkg/`). This needs Linux running on real hardware; CI runners and - WSL2 do not expose the counters. -- Add merge sort and a median-of-three pivot, to show how quicksort's worst case can be avoided. +1. Run the study on bare-metal Linux and publish `docs/results.md` with the raw data. +2. Check RAPL against a plug-in power meter on the same machine. +3. Add merge sort and a median-of-three quicksort, to show a worst case being avoided. +4. Add hardware counters (instructions, cache misses) as explanations for energy differences. -## License +## Licence -[MIT](LICENSE) +[MIT](LICENSE). Grid intensity data: Ember, via Our World in Data, CC BY 4.0 (see +[data/README.md](data/README.md)). diff --git a/docs/bare-metal.md b/docs/bare-metal.md new file mode 100644 index 0000000..5497317 --- /dev/null +++ b/docs/bare-metal.md @@ -0,0 +1,103 @@ +# Running the study on bare-metal Linux + +The energy half of greenbench needs Linux running directly on a computer. Energy comes from +the processor's RAPL counters, which Linux exposes under `/sys/class/powercap`. WSL2, Docker +Desktop, most virtual machines and CI runners do not expose them; where a virtual machine does +show something, it is not the host's real energy. Everything else in the project runs anywhere. + +The simplest route is Ubuntu booted from a USB stick. Nothing is installed on the computer's +disk, and the study leaves the machine as it found it after a reboot. + +## What you need + +- A computer with an Intel processor from 2011 (Sandy Bridge) or later, or a recent AMD + processor. Intel processors report package energy and, on many models, DRAM energy; AMD + processors report package energy only. `--check` (below) shows what your machine reports. +- A USB stick of 8 GB or more with Ubuntu 24.04 LTS (desktop image), written with + [balenaEtcher](https://etcher.balena.io/) or [Rufus](https://rufus.ie/). +- Mains power and a network connection. +- A second USB stick, or a way to push to GitHub, to take the results away: a live session + forgets everything when it shuts down. + +## Steps + +1. Boot from the USB stick (usually F12, F10 or Esc at power-on) and choose **Try Ubuntu**. +2. Connect to the network and open a terminal (Ctrl+Alt+T). +3. Install the tools. `nodejs` is in the universe repository, which the first line enables: + + ```bash + sudo add-apt-repository -y universe + sudo apt-get update + sudo apt-get install -y git build-essential python3-venv nodejs + ``` + +4. Get the code and check the machine: + + ```bash + git clone https://github.com/fasharif/Algorithm-Energy-C.git greenbench + cd greenbench + ./scripts/run-study.sh --check + ``` + + `--check` confirms that this is Linux on real hardware, that the tools are present and that + the RAPL counters can be read. Since the fix for CVE-2020-8694 in late 2020, only root can + read them, so it asks before running `sudo chmod o+r` on the counter files. That change + lasts until the next reboot. It then builds the project, runs the unit tests and prints the RAPL zones with + the idle power over one second. + +5. Run the study. This is the one command: + + ```bash + ./scripts/run-study.sh + ``` + + Leave the machine alone while it runs: no browser, no typing, lid open, on mains power. + After the warm-up the harness prints an estimate of the time left and updates it as it goes. + The study profile makes 5,760 measured runs. The 4,320 that sort each sort for at least + 200 ms and are followed by an idle window of the same length, so those alone take at least + 29 minutes (4,320 x 2 x 200 ms); warm-up, process start-up and the slow configurations add + to that. + +6. Take the results away. The script prints where they are: + + - `data/runs//`: `runs.csv` (every run), `meta.json` (settings, RAPL zones, + versions), `environment.txt` (kernel, `lscpu`, governor, power source) and `summary.csv`. + - `docs/results.md` and `docs/img/`: the report, now with measured time and energy. + + Copy the repository folder to the second USB stick, or commit and push: + + ```bash + git add docs/results.md docs/img data/runs + git commit -m "docs: add measured study results" + ``` + +## Before you start: reducing noise + +- Close every other application. A live session may start update checks in the background; + wait a few minutes after booting before running the study. +- Keep the machine on mains power; laptops on battery often cap the processor. +- The script pins the harness and every worker to CPU 2 when there are four or more CPUs. + Choose another with `--cpu N`, or turn pinning off with `--no-pin`. +- Frequency scaling and turbo stay as they are, and are recorded. For steadier numbers you + can fix them for the session, for example on Intel with + `echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo`. Mention it when you + report the results. + +## A shorter rehearsal + +`./scripts/run-study.sh --rehearse` runs the smoke profile (sizes 64 and 256, three runs per +configuration) with energy, and writes everything under `results/` instead of `docs/`. Use it +to see that the machine works before committing to the full run. + +## If something goes wrong + +| Message | Meaning | +| :--- | :--- | +| `no RAPL zones under /sys/class/powercap` | The processor, firmware or kernel does not expose RAPL. Check that you booted the USB stick and not a virtual machine. | +| `this is WSL` or `this is a container` | The script refuses environments without real counters. | +| `permission denied reading .../energy_uj` from the harness | Run `--check` again and accept the `chmod`, or reboot and start again. | +| `the package counter did not change` from `probe` | The counters are frozen or updated rarely on this machine; its energy readings cannot be trusted. | +| `the implementations disagree` | A worker produced different counts from the others. This is a bug; please open an issue with the message. | + +The harness stops cleanly on Ctrl+C: the runs measured so far stay in `runs.csv`, and +`meta.json` records that the run was interrupted. diff --git a/docs/decisions.md b/docs/decisions.md new file mode 100644 index 0000000..84b265b --- /dev/null +++ b/docs/decisions.md @@ -0,0 +1,201 @@ +# Design decisions + +Short records of the choices that shaped greenbench: the situation, what was decided, and +what follows from it. + +## 1. Read energy from the powercap sysfs files + +**Context.** RAPL energy counters can be read three ways on Linux: the model-specific +registers through `/dev/cpu/*/msr`, the `perf` power events, or the powercap files under +`/sys/class/powercap`. MSRs need the `msr` module, root and per-model unit decoding; `perf` +adds a dependency and its own event names. + +**Decision.** Read `energy_uj` and `max_energy_range_uj` from powercap, keep each counter's +file open and re-read it with `pread`. Use only package zones (`intel-rapl:N` named +`package-N`) and DRAM zones, summed over sockets. Skip `core` and `uncore` (inside the +package), `psys` (more than the processor) and `intel-rapl-mmio` (a duplicate of the package). + +**Consequences.** No kernel modules and no unit decoding: the kernel reports microjoules. +The same code reads AMD processors, whose package energy appears in the same place on kernels +whose `intel_rapl` driver supports them. Since the fix for CVE-2020-8694 the files are +readable by root only, so the study script offers to `chmod o+r` them until the next reboot, +and the harness explains the error if it cannot read them. + +## 2. Say "energy unavailable", never zero + +**Context.** RAPL is missing in WSL2, Docker Desktop, most virtual machines and CI runners. +A benchmark that writes 0 when it cannot measure produces data that looks real. + +**Decision.** If the zones are missing, unreadable, or hold values that do not parse, the +harness records `energy_status=unavailable` and a reason, and leaves every energy cell empty. +`--energy require` turns this into exit status 3. The report prints "energy unavailable" and +keeps the energy sections empty. + +**Consequences.** The same harness runs everywhere with time and operation counts, and no +file can be mistaken for an energy measurement. The unit tests check that a corrupt counter +is an error rather than a zero. + +## 3. Handle one counter wrap; keep intervals short + +**Context.** `energy_uj` wraps to zero at `max_energy_range_uj`. At high power a package +counter can wrap in well under an hour. + +**Decision.** `rapl_delta` computes `end - start` or, when `end < start`, `(max - start) + +end`. Readings above the maximum are treated as corrupt. Every measured interval is one worker +process, seconds long at most, so at most one wrap can occur. + +**Consequences.** Correct deltas across a wrap, tested with fixture trees whose counters are +rewritten between readings. The formula loses the single step from the maximum back to zero, +one energy unit (about 15 to 61 microjoules), which is far below the noise. + +## 4. Measure whole worker processes, for every language + +**Context.** Comparing C, Python and Node.js needs the same measurement in each. RAPL +counts energy for the whole package, so there is no per-process energy; what matters is the +interval being measured. Inside one process, C could be timed around a function call, but +Python and Node.js would then be measured differently from C. + +**Decision.** The harness starts one worker process per measured run, for C as well, and +reads the counters and `CLOCK_MONOTONIC_RAW` just before starting it and just after it exits. +A startup trial for each language, order and size (a worker that generates its input and +exits) measures the fixed cost of starting, which the report subtracts before dividing by the +number of sorts. + +**Consequences.** One code path measures all three languages, as the research question +needs. Interpreter start-up is measured explicitly instead of being hidden. The cost is some +extra overhead per run, and per-sort figures depend on subtracting a median startup cost. + +## 5. Calibrate repetitions so each process sorts for at least 200 ms + +**Context.** RAPL counters update about once a millisecond. A sort that takes 50 +microseconds cannot be measured on its own. + +**Decision.** During warm-up the harness asks each worker for one sort, reads the worker's +own loop time, and sets the number of sorts per process so that the loop lasts at least +`--min-loop-ms` (200 ms in the study profile, about 200 counter updates). It refines the +number on each further warm-up run, then keeps it fixed for every measured run of that +configuration. + +**Consequences.** Every measured interval spans hundreds of counter updates, so the +update granularity is well under 1% of the measurement. Fast configurations sort thousands of +times per process, slow ones once. The number is recorded in the CSV and in `meta.json`. + +## 6. Subtract an idle baseline of the same length after every run + +**Context.** The package draws power when idle, and that power drifts with temperature and +background activity. + +**Decision.** After each measured run, the harness sleeps for the run's duration and reads +the counters again. Net energy is the run's energy minus the idle energy scaled to the run's +exact length. Negative results are kept, not clamped. + +**Consequences.** Idle power is measured close in time to each run, so drift largely +cancels. The study takes about twice as long. Clamping would bias the medians upwards, so +negative values stay visible as noise. With energy unavailable the idle windows are skipped +(`--idle-baseline auto`), so runs without RAPL are not slowed down. + +## 7. Randomise the run order + +**Context.** Running all repetitions of one configuration together lets slow drifts (heat, +turbo budget, background jobs) look like differences between configurations. + +**Decision.** Every measured run of every configuration goes into one list, shuffled with a +Fisher-Yates shuffle driven by xorshift32 from `--order-seed`. The warm-up order is shuffled +too. + +**Consequences.** Drift spreads across configurations instead of landing on one. The order +is reproducible from the seed, which is recorded in `meta.json`. + +## 8. Identical inputs, checked at run time + +**Context.** Algorithm-Energy-C already fed every algorithm the same input from a seeded +xorshift32 generator. A three-language comparison is only fair if all three generate exactly +the same arrays. + +**Decision.** Port xorshift32, the four input orders and the sorts line for line to Python +and JavaScript, counting comparisons and writes in the same places. Every worker reports an +FNV-1a hash of its input and its operation counts; the harness stops if two languages +disagree, and the report checks again. The parity tests compare 10,000-number sequences, whole +inputs up to 100,000 elements, and counts for every algorithm and order. + +**Consequences.** A bug in one port stops the run instead of skewing the comparison. The +C sorts still produce the exact counts Algorithm-Energy-C published, which the tests check. + +## 9. No runtime dependencies in the workers + +**Context.** The study has to run on a freshly booted Linux, where Python and Node.js come +from the distribution (Ubuntu 24.04 ships Python 3.12 and Node.js 18). + +**Decision.** The Python worker uses only the standard library and runs on Python 3.10 or +later. The Node.js worker is plain JavaScript with JSDoc types, checked by the TypeScript +compiler in strict mode (`checkJs`), and needs no build step or packages at run time. + +**Consequences.** Nothing to install for the workers, and no build step between source and +what is measured. Type checking still covers the JavaScript. The analysis package needs +numpy and matplotlib, installed from hashed, pinned requirements into a virtual environment. + +## 10. Statistics written out rather than imported + +**Context.** The report needs medians, bootstrap confidence intervals and Spearman's rank +correlation. SciPy provides them but is a large dependency for three functions. + +**Decision.** Implement the percentile bootstrap and Spearman's rho (Pearson's correlation +of average ranks) with numpy, seeded so the same data gives the same report, and test them +against hand-worked examples and a coverage check. + +**Consequences.** The analysis depends on numpy and matplotlib only. The functions are +short enough to read, and the tests pin their behaviour, including ties and undefined cases. + +## 11. Carbon from annual, location-based grid averages + +**Context.** Energy becomes emissions only through a grid carbon intensity, and published +figures differ in scope (direct or lifecycle), year and source. + +**Decision.** Keep intensities in `data/grid_intensity.csv` with their source, year, basis +and licence. Use Ember's lifecycle figures for the UAE and the UK for 2024, the latest year +with both, as published by Our World in Data (CC BY 4.0). Let the report take other regions or +a custom value. + +**Consequences.** The estimates are transparent and easy to change, but they are annual +averages: they do not describe the marginal emissions of running a benchmark at a given hour. +The report says so. + +## 12. Results stay pending until a quiet bare-metal run + +**Context.** This version was built on a machine running many other jobs at the same time, +inside Docker Desktop, where RAPL is not available. + +**Decision.** `docs/results.md` is generated by the report script and, until the study runs +on bare-metal Linux, is generated with `--withhold-timings`: it shows the exact operation +counts and marks every time, energy and carbon figure as pending. + +**Consequences.** No number in the repository comes from a machine that could not measure +it properly. One command (`./scripts/run-study.sh`) replaces the pending sections with +measured ones. + +## 13. Keep the last-element pivot + +**Context.** The quicksort always pivots on the last element, so sorted and reversed input +are its worst case. A median-of-three pivot would avoid that. + +**Decision.** Keep it, with the recursion into the smaller side that bounds the stack. + +**Consequences.** The data includes a case where one algorithm's time depends heavily on +its input order, which is useful for asking whether time and energy move together. It also +keeps the counts comparable with Algorithm-Energy-C. + +## 14. Retry a failed worker process once + +**Context.** The first full run of the study profile in Docker Desktop stopped after 2,227 +of 5,760 measured runs, because one Python worker could not read its own directory (the +shared file system returned "cannot allocate memory"). Nothing was wrong with the +measurement, but half an hour of work was lost. + +**Decision.** When a worker process cannot start, exits with an error or is killed, the +harness runs it again, up to `--retries` times (default 1), and counts the retries in +`meta.json`. A failed attempt's measurement is discarded. A worker whose result disagrees with +the other languages is never retried: that is a bug, and the run stops. + +**Consequences.** A transient failure no longer ends a long run, and the count shows how +often it happened. A worker that fails every time still stops the run, with its error +message, after one extra attempt. diff --git a/docs/study.md b/docs/study.md new file mode 100644 index 0000000..8887f7f --- /dev/null +++ b/docs/study.md @@ -0,0 +1,182 @@ +# The study + +**Is run time a good stand-in for energy, and how do C, Python and Node.js compare?** + +Developers rarely measure energy. They measure time and assume that faster code uses less +energy. That is plausible for one program on one processor, but it is an assumption. This +study tests it on a small, fully controlled workload: three sorting algorithms, implemented +identically in three languages, sorting identical inputs. + +> **Status.** The harness, the workers, the statistics and the report are built and tested. +> The energy measurements have not been made yet: they need Linux on real hardware, and this +> version was developed in Docker Desktop, where RAPL is unavailable. Until the study runs on +> bare-metal Linux ([how](bare-metal.md)), [results.md](results.md) shows exact operation +> counts and marks every time, energy and carbon figure as pending. + +## Questions + +1. **Time as a proxy.** For each language and algorithm, does energy rank runs in the same + order as time does? Measured by Spearman's rank correlation between process time and + process energy, both over all input sizes and orders together and within each + configuration. +2. **Languages.** How much energy does one sort take in C, Python and Node.js, for the same + algorithm and input? Is the energy ratio between two languages the same as their time + ratio? +3. **Input order.** Bubble sort does the same number of comparisons on sorted and random + input, and this quicksort does far more on sorted input than on random input. Does energy + follow those differences in the same way that time does? + +### What would count as an answer + +If time is a good proxy, the within-configuration correlation should be clearly positive in +every language, and energy ratios between languages should match time ratios within their +confidence intervals. A language or algorithm whose energy ratio differs from its time ratio +would mean it draws a different average power, and that time alone would mislead. + +## Method + +### Workload + +- **Algorithms.** Bubble sort, insertion sort and quicksort (Lomuto partition, last-element + pivot, recursing into the smaller side), carried over unchanged from Algorithm-Energy-C. +- **Inputs.** Permutations of 0..n-1 in four orders: random (Fisher-Yates), sorted, + reversed, and nearly sorted (5% of positions swapped with random partners). Sizes 250, 500, + 1,000 and 2,000 in the study profile. +- **Generator.** xorshift32 (shifts 13, 17, 5) from seed 2463534242, reimplemented in each + language. Every run of every language sorts exactly the same array. +- **Languages.** C (compiled with `-O2`), Python and Node.js, each as a worker program that + generates the input, sorts a fresh copy of it a set number of times, checks the result and + prints one line with the counts, its own loop time and a hash of its input. + +The three implementations are line-for-line ports. They count comparisons and array writes +in the same places, and each uses its language's everyday array type (C `int` arrays, Python +lists, JavaScript arrays). None uses a built-in sort. + +### Checks that the comparison is fair + +- The parity tests compare the first 10,000 xorshift32 outputs for four seeds, whole inputs + of every order at eight sizes up to 100,000, and the comparison and write counts of every + algorithm on every order. +- During a run, the harness compares every worker's input hash and counts with those of the + other languages for the same input, and stops if they differ. The report checks again. +- The C sorts still produce the exact counts that Algorithm-Energy-C published (for example + 55,119 comparisons and 56,984 writes for quicksort on 4,000 random integers), which the unit + tests check in all three languages. + +### Measurement + +- **One process per measured run.** The harness (`build/greenbench`, in C) reads the RAPL + counters and `CLOCK_MONOTONIC_RAW` immediately before starting a worker and immediately + after it exits. C is measured the same way as Python and Node.js. +- **Energy.** Package and DRAM zones from `/sys/class/powercap/intel-rapl:*`, summed over + sockets, with one counter wrap handled using `max_energy_range_uj`. If the counters are + missing or unreadable the harness records "energy unavailable", never a zero. +- **Enough work per process.** RAPL counters update about once a millisecond (Hähnel et al., + 2012). During warm-up the harness sets the number of sorts per process so that each worker's + sort loop lasts at least 200 ms. +- **Idle baseline.** After every run the harness sleeps for the same duration and measures + the idle energy, which is subtracted, scaled to the run's exact length. +- **Startup cost.** For each language, order and size, a startup trial runs a worker that + generates the input and exits without sorting. Its median time and energy are subtracted + before dividing by the number of sorts, giving time and energy per sort. +- **Repetitions and order.** Three warm-up runs per configuration, then 30 measured runs. All + measured runs of all configurations are shuffled into one random order from a recorded seed. +- **Pinning.** The study script pins the harness and all workers to one CPU (CPU 2 by + default), so the measured process does not migrate between cores. +- **Records.** `runs.csv` holds one row per measured run (tidy data: one variable per column). + `meta.json` holds the settings, the RAPL zones and their wrap values, the CPU model, the + kernel, the scaling governor, the turbo setting, the language versions and the exact + command. `environment.txt` adds `lscpu` output and the git commit. + +### Analysis + +- **Central value.** The median of the 30 runs of each configuration. +- **Uncertainty.** A 95% percentile-bootstrap confidence interval for each median (10,000 + resamples, seeded, so the same data always gives the same report). +- **Time against energy.** Spearman's rho between process time and process energy for each + language and algorithm, pooled over all sizes and orders (with a 95% bootstrap interval from + 2,000 resamples of the pairs), and the median of the rho values computed within each + configuration. The pooled figure is dominated by input size; the within-configuration figure + shows whether run-to-run variation in time tracks variation in energy. +- **Carbon.** Energy converted to grams of CO2-equivalent with annual grid intensities for + the UAE and the UK (Ember, via Our World in Data, CC BY 4.0; see + [data/README.md](../data/README.md)). Other regions or a custom intensity can be passed to + the report. + +## Threats to validity + +### Measurement + +- **RAPL is the processor's own estimate.** Studies of recent Intel processors found package + readings that track external power meters closely (Khan et al., 2018), but accuracy varies + between processor generations and vendors. This study does not calibrate against an + external meter. +- **Package, not system.** The counters cover the processor package and, where present, DRAM. + They miss the disk, screen, fans and power-supply losses, so the energy at the wall is higher. +- **Everything on the package counts.** RAPL cannot separate processes. Background activity + during a run is counted as the worker's. The idle baseline removes steady background power + but not bursts; randomising the run order spreads bursts across configurations rather than + removing them. +- **Idle after work is not idle before work.** The idle window follows each run, when the + processor is still warm and may be leaving a turbo state, so idle power can be slightly + higher than true idle, and net energy slightly low. +- **Counter security filtering.** After the PLATYPUS power side-channel attack (Lipp et al., + 2021), some Intel systems add noise to RAPL readings. The 200 ms minimum per process and the + 30 repetitions reduce, but do not remove, its effect. +- **Frequency scaling and heat.** Turbo and the scaling governor are left as the system sets + them and recorded. Long runs heat the processor, which raises its power at the same work; + the random order keeps this from favouring one configuration. + +### Workload and implementations + +- **Instrumentation.** Counting comparisons and writes adds work in every language, and + proportionally more in Python than in C. +- **Just-in-time compilation and garbage collection.** Node.js compiles hot code while it + runs and both interpreters collect garbage. A process that sorts once measures more + unoptimised code than one that sorts thousands of times. This is what running the program + costs, but it makes per-sort figures depend on the number of sorts per process. +- **Startup subtraction.** Per-sort figures subtract a median startup cost measured in + separate processes. Noise in that median shifts every per-sort value of the configuration. +- **Language versions.** Results describe the recorded compiler and interpreter versions + (Ubuntu 24.04 ships Python 3.12 and Node.js 18), not the languages in general. + +### Scope + +- One machine at a time, single-threaded, small integer arrays, no input or output. The + results say nothing about parallel code, large memory footprints or I/O-bound programs. +- Carbon estimates use annual, location-based averages that include lifecycle emissions. + They do not describe the marginal emissions of running the benchmark at a particular hour. + +## Reproducing + +```bash +make && make test # build and unit tests, on any Linux +make smoke # a short run of all three languages with energy off +./scripts/run-study.sh --check # on bare-metal Linux: can this machine measure energy? +./scripts/run-study.sh # the study itself; writes docs/results.md +``` + +## Related work + +This study asks a narrower question than Pereira et al. (2017), who compared energy, time and +memory across 27 languages on benchmark programs. Here the workload is small enough to check +line by line, the inputs are provably identical across languages, and each language is +measured by the same external harness with an idle baseline and confidence intervals. + +## References + +- Hähnel, M., Döbel, B., Völp, M. and Härtig, H. (2012). Measuring energy consumption for + short code paths using RAPL. *ACM SIGMETRICS Performance Evaluation Review*, 40(3). +- Khan, K. N., Hirki, M., Niemi, T., Nurminen, J. K. and Ou, Z. (2018). RAPL in action: + experiences in using RAPL for power measurements. *ACM Transactions on Modeling and + Performance Evaluation of Computing Systems*, 3(2). +- Lipp, M., Kogler, A., Oswald, D., Schwarz, M., Easdon, C., Canella, C. and Gruss, D. (2021). + PLATYPUS: software-based power side-channel attacks on x86. *IEEE Symposium on Security and + Privacy*. +- Marsaglia, G. (2003). Xorshift RNGs. *Journal of Statistical Software*, 8(14). +- Pereira, R., Couto, M., Ribeiro, F., Rua, R., Cunha, J., Fernandes, J. P. and Saraiva, J. + (2017). Energy efficiency across programming languages: how do energy, time, and memory + relate? *Proceedings of the 10th ACM SIGPLAN International Conference on Software Language + Engineering*. +- Ember (2026). Yearly electricity data. + (CC BY 4.0), via Our World in Data, . From 49258141c359bf183d3ef26d0a508361d1092876 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:04:31 +0400 Subject: [PATCH 18/51] fix: make the report's provenance lines easier to read and reproduce Sizes get thousands separators, worker versions and the energy reason read cleanly, the report command includes the PYTHONPATH it needs, and a withheld report says why its raw data is not committed. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/cli.py | 2 +- analysis/greenbench_analysis/report.py | 10 ++++++---- analysis/tests/test_report.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/analysis/greenbench_analysis/cli.py b/analysis/greenbench_analysis/cli.py index 7d3cd2c..b38cb1c 100644 --- a/analysis/greenbench_analysis/cli.py +++ b/analysis/greenbench_analysis/cli.py @@ -77,7 +77,7 @@ def _report(args: argparse.Namespace, argv: list[str]) -> int: withhold_timings=args.withhold_timings, regions=regions, per_sorts=args.per_sorts, - command="python -m greenbench_analysis " + shlex.join(argv), + command="PYTHONPATH=analysis python3 -m greenbench_analysis " + shlex.join(argv), guide_link=guide, ) written = write_report(runs, meta, summary, args.out, args.img_dir, options) diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 37c280c..d187181 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -110,12 +110,12 @@ def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: version = str(worker.get("version", "?")) if version.startswith("Python "): version = version[len("Python ") :] - versions.append(f"{LANGUAGE_NAMES.get(language, language)} {version}") + versions.append(f"{LANGUAGE_NAMES.get(language, language)}: {version}") lines = [ f"- Harness: greenbench {meta.get('version', '?')}, profile `{settings.get('profile', '?')}`, " f"{settings.get('runs', '?')} measured runs per configuration after {warmup} " f"warm-up run{'' if warmup == 1 else 's'}, in a random order (order seed {settings.get('order_seed', '?')}).", - f"- Inputs: sizes {', '.join(str(s) for s in settings.get('sizes', []))}; orders " + f"- Inputs: sizes {', '.join(f'{s:,}' for s in settings.get('sizes', []))}; orders " f"{', '.join(o.replace('_', ' ') for o in settings.get('orders', []))}; xorshift32 seed " f"{settings.get('seed', '?')}. Each process sorts for at least " f"{settings.get('min_loop_ns', 0) / 1e6:g} ms.", @@ -130,7 +130,8 @@ def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: baseline = "an idle window of the same length after every run" if energy.get("idle_baseline") else "none" lines.append(f"- Energy: RAPL zones {domains}; idle baseline: {baseline}.") else: - lines.append(f"- Energy: {energy.get('status', 'unknown')}. {energy.get('reason') or ''}".rstrip()) + reason = energy.get("reason") + lines.append(f"- Energy: {energy.get('status', 'unknown')}." + (f" Reason: {reason}." if reason else "")) lines.append( f"- Run: {summary.run_count:,} measured worker processes, started {meta.get('started_utc', '?')}, " f"finished {meta.get('finished_utc') or 'not recorded'}" @@ -409,7 +410,8 @@ def draw(key: str, make: Callable[[Path], Path | None]) -> None: "> **Status: time and energy results are pending.** This file was generated from a run that " "checked the pipeline end to end. Operation counts are exact and shown below; time, energy, " "correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux " - f"machine ([how]({options.guide_link})).", + f"machine ([how]({options.guide_link})). The run's raw data is not in the repository, because it " + "holds those timings; the harness command below reproduces the counts.", "", ] elif not summary.energy_measured: diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index 9d6c7e3..a6f4b16 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -120,7 +120,7 @@ def test_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> Non text = out.read_text(encoding="utf-8") assert "CO2e, United Kingdom" in text assert "CO2e, Solar farm" in text - assert "python -m greenbench_analysis report --runs" in text + assert "PYTHONPATH=analysis python3 -m greenbench_analysis report --runs" in text assert (tmp_path / "summary.csv").read_text(encoding="utf-8").startswith("language,algorithm,order,n,") assert "wrote" in capsys.readouterr().out From 997da75ce37d0e45c25d90cee9f87e92513d79f5 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:04:31 +0400 Subject: [PATCH 19/51] docs: generate results.md from a study-profile run, with timings pending Generated with --withhold-timings from a full run of the study profile (5,760 measured runs, all three languages) in Docker Desktop, where RAPL is unavailable. It shows the exact operation counts, identical in C, Python and Node.js, and marks time, energy, correlation and carbon as pending until the study runs on bare-metal Linux. Co-Authored-By: Claude Opus 5.5 --- docs/img/operations.png | Bin 0 -> 83206 bytes docs/results.md | 96 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 docs/img/operations.png create mode 100644 docs/results.md diff --git a/docs/img/operations.png b/docs/img/operations.png new file mode 100644 index 0000000000000000000000000000000000000000..015bb9abc65ac04481726ea174b3ac956a598e80 GIT binary patch literal 83206 zcmdqJbx@V<+ct`#U?U<8qBMfEbQmBFk94PibayJ!B_PtEQqls_DcuV}y1Q9)!}`wk zJiqsO=iU38J+o)_AK&aX!+>zFd)?Q0opBt;d0qzoB`bFO2FVQ!42;_n;v(`G7+0TS zU|ha_4GVr_|9PYvzVX?Ms@f}98QDAO*&1TV=-FGFTiKhN=s$Ecw6!y_vgBZ9V`FA# zdid7f-rA0jg~j6EE10cpjaj@yH{m82*R93X>@YA0XwYAmB;={LFfL(WNQgXFbWU8K zaB@~O9Bti3{@laJdM5srN+}}j;qWEVAMYz2drKagJs;Z4KX<@aaZ*~pQCfl#bd~g* zeOl^XB*vx4kI8Y=QE|H$Y;W2=UWxq3)sD2&s$jNVjI&E7Yv@iq9bc-Dp`nq*#Qpar ztY^7=J>cJ$qJLyO<$t|qRX+W>|L>J`-_kPvz3vE~=BcR2e_l-m^jQr5^(IPBQ3;uU z-?YsrX&UtJyL4X5EBx1+M9zZ#&-o~EJVPSCLqt1qvwJyb0=H-Td}I0uwefuwo_uB= zvO6dHYpqKuUfKnxIXyPmw@I41;yBC@H&Xq5IA=m>kF&JX>PC8Lk$64PB5yW(Ef zd7fMIkSTd0;KG%m0v+-mNmzGS>u4|DU0mFVQDY)3+Ni+js`cs$)?h>a^-tYe@zSj2 z3fH1uiLico8Q|ouWnH0}t5KF55)$G%q2mx%<+#Du*Vl*sR9^qaUzHAP?Fg2)%UPd_ z^%}AAt2X&2UDuw-@-Ey*9<&G;XOa8gA=jU%+%MP6e_TiRtuCt+2nxU`~T$^ACYT{O~8gXou5}k3)lx5Pf_}Vr7;S zN9Tvp_M77svf&DNSFc|6kBT`K8YA6JOn20@|1G+=HWdEx<43<_^nNS){JpuMsww`Z zN@i#CxVX~1bn&p@Xj;xr+lrY&UhKf&AM9UVUSm1`_$%RNakfL(zMOs5&hV6{M$;}r zSnky;tjrWozQZY$Q$u6prQu>@oVnKUrDj4@_ts2fHS+lUaH`^XG2wWt5ocvEPb2zJ zCCbLII|U_(tT}uD=AvY0j|{}4v)&(2Q8_k_-jSD=FW+t+OA)B-s^(IBojf=wb2sbM$i<*YUX2dYY_O~dZi;h#(L^bN-v@2QA z-v=8fGB&pDN!fSNF|xA&>bgwD2J zVx;h54sSSK?53c08gYdG`s=S}h-AT|v8jt!@SRa$aNCnBtCpPKaqEwQT%k$75o~b> zRk%QdhE?T#f<{vJ#15#{cZ7Xm(JZg=t&>E(TXs^D@s;dJR}e@MsWY`azM=Ud!g zzkXrNFD>nknsFShWTw#5(@+0VIAymORZd~_3j>_(Y({9E7 zphkO7&us1KMbB(=@bN;-)mUlTpr9btGD|^TUS1(&C*n}Cm%L_jV*5a4!p2Z=x2yOJ z*8gVor^(rVftTS}Y2LWQ(1iKu&$!HF*P;VCSq_`M)bVt$GGy)PgFF67b*OV>u2z*F zeD^fxA}yP6B*j2)FZ{rd5IVheezH&X^l3C)r9gHETSeQ2#!rvcW)}Z!r<3Jujw1Yb zj*1llvu4}zQVQY}S=X8#qrGZZP0V35ZCgG0ZFOf5GvPGFZok}XdTxGUs&erN zgYSdrz-i$jatdLRc{|k@ku0#&Mm4-~RXW)nZ)RpDEn{;xY^;nWS{QX!JboG#6*W@h zR(5!rN+FEs=`m^wh`4|M{ucQ1?<@tn!2%rwzKUjhK^=3hwXD=DpW_|9-=(WHb&owSTrPQC z9Ak4O-1hEC;^(lQekg=Gv$N{UVupR0vm7b%6?@?F0m3c53R!#8yI+n%*4dfI>$r13 zI$0p_4mqEqTWv}A_I|>B&(lG+)<1hTJAJJKKCh<4w~nJse}=E3qm#m8J}Zcf*Y=9p zQ2s(4%A+~q==5xtg-Fo#ryS+hY%oj4_wU!d<%EtufFHl7@H&f#pc2Dibw6UYwX>_( z?WrOX_7qTFiwLhdJxJdfF-|$@`ntEhBb&r`oSWtH(`nMh->~y@;hT1StUF}a>f?-) zpQA3$Hd@aJD|*x9TmRVF(+CTraKGZF23EB&V* z4pHG=XFf^%PDO^DMzgcC7(`s=MBJ0k2EKE3C#Db3J`_bKHwzx4@&bxgTJrbPihuxd zyx&567~aie?g^KbeD$0su|L{73zuws3ufM-CAcIjxUUHD+H!WhyJqh-Ah?;xYiCn6 zq3O2qRA`K)eERaO#%VBb_HXxe+Ze6I1kKHT_r-7PBzM>enTi*y}+wC=dx?YS1B^; zR=-2W^H*L<-Xoc*BJZ?<2Q(`=VsW*G&nt$cbfivJU#DzT?clIEO<^r=j!owc=dQYS zJ}CQUpxk_RJY_fIYfy7t#+zBIvbW4?iev-%w%8ak^Zq9D7Cb@pMQ!x{!I+gDB*N`! zADo?rn=JHy`(U^ty6pFfT)}o_&bS$reKdajT5^i;o+>xt!uIyU&^Pb|@+v2~p_Pkl zw{bx?dDevPMA*j#ZH_o;GJ#m-)MLqoFzrlMO3KSIouZD8T(4ih7C0C-FrTXR`JF0; zhl~5GErP0IHJgzbd}CTIhl7gB$MgK;cs=v`ccL%jQ)=$+?x9Pr)AbiOrcz>KF`k*4 zKDHSBnYDvh4A&kly&E%ga8Xg4^Hf1z{_Rly+0=7)cY*fyDmiD=>A?mS14EXifmPLp zqb&ah=Vldh5>F@}UAw~nu3a)8Qri@giAhw8vDqZ2j9IdMu0DO2m^f}b2L388X^Jzw zA)u+KxXiKG`!$4Qqo8t5@+!8!Zt;loTqrkdLT5^6(RIcwa)RLWhK-h)$T`Q2(L&u{ z2x>aIU8rvyPMa~Bt}BM<lHpImE zt)JxK$2Z0*d3vL*6kbhDfhyKx7G>L0s0)>7G|Jl-|0n6|JNack-uTJ>=?JvA=1v@nX&rmXMIp^g7>1e}xd+3E3_6%zX8K@Midj-r4CCik5*P5bR8{!T+$n zc>Vnv4|(ayArfI$Q1ePz`7@%B;E6f_l@Cr9vR>B4WgO+jW5m(5$pQxy3#HL{eOIiB zH*e%TWpuSt3U?FmIN8%%k)PVvC%=OY?w0OxN`?P{aXL;fPX9VcPZ3gxx;^2oJ1{( z4ZuviWRgEk;acG9OKC>@YDF~%30E$?d6Tv~rbSIv3*Lh?=N!>Q5EJ!N`!VNP?AFcRwA9{OmaY zdKc1}EkdwA+dibJNkp0!FM%r>`B`i5N!b~GRD08(ECn0BBn)>zLhdIk#^WwaNv$&P zkBd{miO9_?x+|>_f`>o!n{P}y&)votx8%kVI(lehW8u*fO3ZLC^M?Xn*|2TU8n)0cC%kZeXl&d0r0mE7A01J3 zf8+kNK!*in84d78}R(U=6w%E0AiBG>&@8T18{HT-BSg~7(g21wzuCNts2n9~=t zAk;<4mEw8Az+v1oyO?05{I>@`h?SH~JfTp5>UEv%7AAPW_&|jK z*V4mQ*Z4twzt)yAqw%aK)yNzRFw_3pP}Ikd&lDB!udJ-RF*53_uv@y3GkU90w>69$ z5>8*Kg~rbA?ntc%|3sw&6%S8&2bu*9QH`oirXZG6v7jzi~a>+}L%BPD+wdS0}NtvH9TZtK=M#m`EnCpkM-b zu4r;FY-NO3aE;sHtZl{eV0YBrC;UrAfZKnJZTz&FsvYM`8aN+GJGpV=1{!c|Z*Rj* zY#<1J7lKa@A|qd<2zhkmrB)du z9s~skuk?QPL72g!*T>4j6=Zpvjooc++11q4>^DXSs$_qF-~PG%H_MIx(syN{L(6ee zJ+`@z+%e@Tmkcd}Y6-YnZ{17rBpOapkyR21~dEa>i;Q2+z zSA7DjVuiovX*qklOtb{d-d>h``wuOidY%3ct3Imv^y&VMn>WEH%wEc&x%KM1|3Q%d zoNK}_|9SnN#GS!T=-2wd~2LzxI5a@7~lNEZm z(3tZL+2UvOMa%X12_B_6Vv+TxesfcG^`~uF;P00;K}lztSG0% z&IL*u8ea(cd-v`I1O;UeUg+!V2ZVc0Mdun~=rQ0+J(GAU^z$qw>Q^x{Lxi<~cCydmm!xe`pKY6UM{x)dEI zXuo{d?O^>84^J#G9)m*Fj!?iPkAN#a5OnSRCVF$-ksmaIh825#u7r^?L~-o`R9Uss zb@4M|0cYz94}1cGBD*EoL|Hm=KKn0VhSoDR$IWpGPfsD>=+sP1x!IY|MB?Mt*Uh=C zCZz#aLOLqfWJX^DMcw1{U_a0Yb{pWH0awCGU;52=@7@WWs+XAJ16;U8!uh?yU}0ea zy#sg2$z>p2cGh@SpJM_S4g?DU9ts~Yr2*&#H6M?JL^6h1t9q`es0bW}1?VI-HMJs9 zYL4BwJabHNo<^BI5f)$op+yuPWnX!PCbM$BrUcsVzf-ZNkJmrP#Yq6*H5terusJMr zLTZDO^mZT{A66o;s0wlt*r;Zkik|+vq}?cFR_r@uU!aoXYrDm?41WJkpTeU$xH(b9 zYdwvDt^t&kv*p4uV;hV0?=aIcGw}!s#iXUB(+#wk3CGH;T4$^k<0*Is5d+yujPOAi z2;=P{Gc8(b>fU^-DPxITa_XIVK}T_*#|r>0YQyKP@NxBB&6$b1k9Kn=FEYNoM&KEmd^~ z8~oNMD^!myl(zwW-K9vWZ>LLn0kvB~(iD-aRvgBASWp`;hDfei?)^%IiGAmrLS_%7 z`%tL8)qO4-Q*|j)%oU%?%N;??N`Y_Vo~IkIso*TY5UGG!r0djOP}9=dtfa^2m5rHM z8;PbBjujeU$7tFzak zuy~~FcuA*T%{WPGWvM2Zr8c&>hp(zHB{Mfh$1@IG>SJuI2nf0+V_aeIFxyBk)R=K1 zct7~oH<^Uez4ejci*xK=*xnw#wRKxFg9mF%Cva6~04~)fxl8gRh$dw1nwwCL-DGfZ za2&f~&|!QnI38Ls1@uL6NEYyiP|0-g-E}-#Nb~F=8O88_d^wnZRZ7czg=X^9oWBBnI=b(ohkWT2>s zi07dCh6|n0GcdGw#j>T2vpw)Sv-$i)?5)WB^%pB7ZO|V-bM){;RaXnB|9tyg zU;lD*vn<@b9YVBav0LNl;`A7F&-|JLlG?K!DWjf5HNBEW$TrumUDKUy3Y@eU81YgA z*^nF_HdQ8p3p~CTFewN&^}sb0KnQ;OmY$sN=i#PVOUMu}$b7`ymYShqr_F>qpk5E@ z5@7+42PwJ`3KB5PK}kjh(I#Os0q2i)J-jhpUD9Yv01OAbTYWHkYj$>YyB_oKGz3+L zatp(L;Eds2YBAPV>A*xx?2oP=+><=lt0sCty`ClaLY?D$gdz>k28vVKm-^lL9uNV= zBs(6|qC7o)?MMV&Z&g%OK$eVF&Qr(5$Jd%Ud-?JuB;>lS6x3OaRqe4K(A|^jTK8ky z!GfuJB^51~t{^FI2+44MWc5vND6DvdL!e^ee2k22s5zW$1wvY5U{$d|08I-E3^WJ` z3etxH#iUvO5yZz|ed$umqa|xLUipxNZ;|nQ#Aj5#OF*!gH-9A{Tv$F`ZDQB6@izAk zqzJcGe_!81y~bCs!XLa87jI)j_=wjqFxZfsWG58>FC_8)ogqVN0HM%Pb8)i95i(M2 zOxoNmI%zR7qSiw0B+Z5~2hv2gwHMUymT#hUE@i0g)>n{ZM=?TflMcF-5#FD7D-#AW zcXDoiagj(x%OU*|kvAxd0x5@3hv2d-g-nt={uA`@slL8<=91ku*)UQ8UV)lnRN+!$ zH8qLuoJ?-$xNlcnB2vy72nY?`=f%EBz{JW#5DdMt-_u@lCw(!sriW&Ye*0Ludz%xT z!*#*~RW;@1ZAWLjh!7A7+NvxdCa`o9Jq|{M42-+uO|D>Jk$7*f72K>DLssv~VxsT1 z4JqRj{{eIu%rRO(g?yYi2GWEXMhmn{ka(>{MeL#NQFJbIhARiJQ(qB#vw5C5Q*rEm zbUNKdpgKukBI{5WSi(!_ry)TM!40anX9~P(M%oaFU6&e2=po+T7<2nW#?HdV(Vuui zKYe;L7sAoQiZK_=BE${6)2~sF`&(*iXh6WV)GrUGjDFUa;Xe3RyeI!^|xv?L2MOb$(Yp#(8m@8RKjOctB; zr}#DNvsF57m@;XU{#hykuG|jJ#;#s!ZUVdMTn4r5yafP;dkZwS8i@p)e_SG}J3lb< zwc|VA>-ShxJTWX(4T+N5oGZ4NYqjGbe`eqGU5m(R2i>1Y$nV+&BXzB&TX~uW~6A(_of#Oq(@w;S6gjVI0js97p^Z~ zyl_6>AA)kgREC7VFHHaP*;{jZj5-;zn=IkZQ)Wa6>~ez0K|@EEIc^|}(VtmY&yblo zdwv1ZZ<kNa7|gdtyfHM4NKW?5?Wd#k%&^z`x;f!?G+x7%&<%SUCdE>7 ze@lRi!|TGmp`pPU`lRkHvWYwq3q&eL`s3v{ zfePIOUH?sU5I!0b2jUfg*1ZcJk`CD+dgFCK?Htci< zG>fcKo=!sAX^nqs#=DkVyxgCOvo@Hw+wv6vk}aSZ4ClFz-9~dkjI@!_(aqH^yY2_V zc8k&&`L!pjB*?m4Xotjd^boBS=P5DD2~dC(16{Sr4X6gt5E(gj`B~Yj1^mqIa1+qz zg}`oCKZt)wPaV%=L&(L-`*=?%LuIY4St;CEgCY02mT9>xM8m!Y1IORl*{L2g?n(6R zT+rW{m%zx?tmq>Ze*gY5@zYfSTU%SeTE2&Z5b$zPgI-!%G6M_LJ>FSJ%gAtLY$asT zmbaAUN(hDYI%F){YTGA4$7VY(zKBSHHlkbgn4O*7@f1o3fT}`{qS6E0-#4J2BRpa3 z2KWG4W+8{Ca>D!WlVIlT!(#Sl=H^VrM%~n$oSz`_YscsI2hJ-bl@6-`x)NfHLvVzU zI3zm>!{xKkcE}$r6kUycAY6b+^(y4M_36d=X&l#vwh>r;5sbK zrY4J@S&g6`0zJA?50aybr8}S)s#l3UU2(N$X3%va_O?R^7K|H6V5EwL8?{H$z)n|; zo4xeA#ZdM{Na$TCDeC)H$%iX)cB#+;p(r!g5jhhBz;o28ldnIRi__N8q2Cqzdk;C4 zq97+Ykg3)LwuZiWqU&l_Q%lR|F+?nz!O>9Ng_2%%7E}I%&c*Ho1|}vn2)gw^Q;0VD zu@;pj_(l1IB^J&DO+CD?yFL6dP#{#?9#HlP+#+>UuQ1|+3oxhAz6M`;do{p~)N*#> z&GKg!Vo;5zNW^@k6!Rs4wRn%6b`&Ib!@cF#bU#t@_{{@=BcnXeWfuxt9~eNV(!UNV zhMcUa;qElE9q&$%61Ga#unZY6UFhOK=tBLC-V_rdDSIGTsWLoH%aA1q`%cZm627%% zJuQ9~%e66Su(><^3v*JKMY}robn~=I4l}Vxa)^aF2e=xV5FQ(2lrMa)gWhM|*1;Ik zIY#yb8j1|`^jj`Wr>d4U`^rKWM|iKo1bhJCp&M_&GIjEQTbXn*Ca^LMfY5hxuz_wy z0TmBMu-iU8uK04YbS^$FlsO(->v4LIlvHEJg)qxX62NNEU^5dl^WwN2!T^}fWPW`_ z1g>gcaz`6vRPxN?Sy6w+)k=+%Uq$8$p$#wOzWv-0C*;Ovxs#?xUUAUUMaB6Fx{Fm&s3=%L*6wJCAI1%U9^D7$^^X3VjWXJY6nj zvMWV6<>yaMeFK9yZmS1zadBt`TVp}B|2!spc{dx9!jO*F0=T--WOczY=(%*DyT`^7 z(aI%vLH7C3ZIp9h6HchZcoOoM@`8%<=UEAbqWifDi7*Ubyt#b&GNU^MS>R2tvu$j^ zmJdWlMXy}F8UcDFG*kfspPjj*`|nUE7bd45RhXj4AcIpeGlv17j`64PyasdxI{u>2 zTtKL|wAywsWd5+3!_s!Mz!E;+qbdND3HMBNx21s2z-#8}try;p-I>Gq1oF2&crky! za1p*&#@K@=^6J&&EQZkck@5TUF**cACjGSF;$1MrvWOP~a&BM_?Vg~1(B%78tQ^9;E5Y0I( zMr+*nev6^aewqf_t>y+te#WWr=lc3*fNcq^lt6GEbKWQ+7TEuzz>_4K?0J%9N>RpH ztGuVSr#dE@w!#of5iWRu1D%w!xiGKO4JZ1?k7upCPD#0cZVM2n-y96fE=VhtM-68M55C~U!;NgV! zoBZh2=034nhystI;$jbNz*WL=!gcN{aM!0D<4cUBqMlmBU172 z-~UT~R>NcxLU6Nu?oOrsN&^%CalrH-cs%0b`i!0~0g~5aTn%j)psN_V%e@^G_`>Qd z*kfmB)wTr=o8xh5Y1B~k5QIAJuUCYil3*+Y?}8o&lTK~*_?(PmRZTxv-<$`>Ap@JLiXbUKfnTfbLd%y2&pqRDlEY5Vm>ib?oWeKXRc# zyal`(gtp_8wE{Ey@n1gJ2JCT1TaDozN(X4>2t-iP#FKIs1yooX1NLs^M$8+`DOE*R za*t+wJdb~?dbzt|GGVIYKC<@j8z^))n{pn{acbP440Nkn~m zic_YXSD_}eE;=N^oG`V)Jak^ouAGA(-2B25SW1D)VKol%tlG=vgow%(e%p{hal6{v?$rx+~S^D=+5hJO+OlULvlt& zjuu16CA+S!+?eVIB-JB$s0lRM4f=K+Qz%agttv;zxEoORivj$w*)RW<;vgU*(gBTm zQR#S204sETWk`mmR=STVDTrzp9T?}K1?5i^cqcIUG0Un@;Pjt5nX5uf`ZHiw?=CwB z&h~0fL81MMJS7#CN4zeXoP6X`&o*O0wPcPmE(m4!>c_ffe~VE70<@g(U@Vx;&dHyz z-t9?sSmR1dCx5fM_{1G>qNd0Gz_`;?4G>avG+i7cyx5Jpj?smDZ(}S5fN1qr!%gUT z6dcGU2XLAc78Z^}qplhvJcNiX5_W|HlvDMRAl<3j8ua`bu<-!Uz8a0SN`{A3UxkvO z(Gq%e20DtGcHK`)N>LqX(g~>BgH}5B+2JM~5H1Z6Yn_*poL~lGvh`7p+xUkMSJ1g? zu_tNmyc(ngIzROc^=~w6VGFVqc&$hGhxnm2AK(e&>xzU{KQ`jexx! z1J|oD<99@EX6aYM*jL5rW>sLYSt~&P>g|>Uw2Kce5fTzEg{nM%{v3FIJbG;9_&oJh zXogaba-!D-5nA>FrVSZxVZ<1Pvnu%80f$Q$*;&|*9hHw1W@J!%Iu>@@1UWtbXu+BFrXv=Wwk4TJG{_Ru#6LJ zXhA_;%18(sf4%xA3ok~?k)fbQxwdb)n|o@ydiNFDlc4+CxKsiL@VIl0tX64$Ztw9IZ!10=n)gsKm8E=d_KnO%6@PlFkyD}$~GGyARgE~>EJ(>+R zeR}Qj0BHZ7M>E*)bhn{UfaBwPUbJB@Czm^J< zq25DHCY|&=wWz%~Aq7mb@;j9u9E~4Zu@e(bx?yq0p2GP77giL&VF3m;gct!P&E6?taQ+82j#-deH;)-k$$tSw&$_%8B9Kb zJ_>6;9-GQw2Of*wAx5*VVDa-p^*ppVg+}a0uF%hbmc4F;StCf9#jg90g^uU((4Ka% zIUxq}>c-gLJk5%O+#3YVBGUn+bjk%KSU;n@JQlixcAZbW(xH(B^dsuiC&dNcOaG|f zj0Lsmp*rwq7%=byf>M^9iRlekzz?*HI8fi>VRqp!O-*P)@uSnTD52JT|N6#yF+v~o z3Q#3T&n`}pFvJXsNICFiB5q4E+Ndmb;UR-?LCGpdn=Tlc1&@R_$L!pgi%vn03pNqI zBNLEs^h}H%6c#8yK{uIsQeZ?f3@U^{A$=0{ZG@uyEN7Uo^gj)0ku(!Vi)VZAskX3n z`wgrxVB z++W`r$*$ddA#eYz!iJJk=G|-xIEVP|98SumXt)_~Sl`1M+n6M7LZd=+fAU72oyztr zKK#C{CL&ib`93T+Xp6*$q|QyjvAE%kjHj>_vlcZZJ#L-|kV^MaEy?$KYL;3tKF(Qh4~z;l3*(yf z6{PVD#2T1G;9KwO5I0}R7xf7HfwKpO_Y6YFqHvw zZX5gQ{m_%stJr{#?QE$J%6^()dQ0&P&*bF5$Tc$Z!v1zHc=rD`2+}@(D!(K3l*0{I%(ckk=O?`ab(7j zs6}nhkx1l(5fRpAYBpzh*gG*5Gp+%m??-ff_J(6x0(_?ri&d5`UpV~z2>;vQ5`Ur^ zGdmrmqHr`+Mm#?(c!`(uhml@U267i0y==rgb2=x_*p5$Xr@XNWmz`%-C7el&lj{02 zC5sge!J1x_vvOq3mKWz9STQJW-+^{to*;n*3Ac5%@$#LQo)iwga?wUv9se4|b==a2 zH^;Vot9hbJJ9;hD3+!W_cSyx`34-~9znf6C@jn6G7T8f(b#!=0?0(>u?=-e~OK#=d zMC%1>+;`>Un)Z1Y9ffqXm3XwUD44q!qf>BZ=nt+`_8mDrT*tZLcG0#WXoVf7NFV(; zPceS-P~Vb-xSe}#*P0al#KHuYJxePtD``eC&Se`t^hOqg+O$kAgqbt55fNpVTEpHM zR(<3C$B;kG`Bj_nbTSHQkJ}q9GN)mdN0&7_=$E2hLBGH||6elpf4#B_4R z#>(}pN(N$#gu|^cf%qnSTm?^XWNI?OVXK)g48WwRjm9Z=!IN*FT8jA=RE=+qbdY5v z;&rFt;^q#Uv?}@1ykIhl<)r-f*_e@1-mPeZSvxW17h#UAM=2>SA#LL<9%>UgA~8s~;CSw-b-(7;OOdpD>1(uFCsXs1c=72%8gWv}?}9Ipzi>8p zH%oa%gkKmtQ@EYCGWF<6YURc(6pC0qqAsnZ&T_&iH1i$_q|(DbuBmNib;%wShIc=Y zNLc=oE&S>e5&jo+H5(kgkBz4zG2@G5x4vJoB40#`=`$N^F`vYat)iRkv~J93M|?L< zyz1lU0iiH-8s~xmS?TkK(ODGh1)N{iI;Qg9DM7~SZYj{X;k=r|nc@%?M$@^e-;t|0 zS^brKPS)bN=7;Dq0;58E{|RFc+^BX1OJNg{58SZ{(iwaw3?d@gR$}V7a_(_G*{PO` zL`3t4iQ)Xzq~703z3zP)O<<}DPMDf#P2m1X)n;V0z+468;V9j_DtKj=Bi7@{NS1%- z`Ng7h_yvs@Yc@is&BE=jcf$eWwGmrdTIn|Rp}a)UBA=B;8P`cHWeRr|B-jhA9o}a@-tee)cOZl60w=SsG580mRd>sR<$9lS zWIFH`c?K{qi>9&J+^=#ukCC3#Q1qOj#=xV!+{dRdeBp$lIkQ4U6wB%|Hznou`V0>j zrJ$a4(QtOFXv5OnFN!g^b}_r5;9Ar5dM$>|W4DAmy6o0h7xad2u<`m_THfvDl9jzS z8*Mgs4QEFnKA(u5@rohg6I@%o^)vhL&^cxrS2wqG}4l~jEIbMI%=1QIabnqw>9r_Y$@?-)Rh9nv| zil`pNtfOnBI3b~WszbfD1q%yC2J8vfZsuI5OTlrZtfKr{ni*`HpE~cJFuwazlj&TR zz=~J7Ae~YO2*FaAKj=E9hJMMB=U+-_RMVlozPNuiuG*_S$106KVT_ETZ=y2HSXL;| zU4zmpPj#hVRlg-=H*R>=#kMYxfjJ>T5&M$!%lWCd#C@HJ#x@4V9f{(d{iphOo4|FXpHT|usAcv z%GB|?QDgVfY)ahx(XHI(kO%CE?r=(nWp0)zCH_YJC58K_I~DPoc03q@vzJ|#8&qAJ zv6WoO(0I+flssQn$F7{OhW1+{lQbz_ z634J4Vsl*oS+&bGL{sFKIee|&}R=F)9|Q6;-BI18$YDTG(?M$le)>7pI)%F4{Z<;w)95R6=vp?M$G(E72;M!H|>E2UZHs>;qn8KL4(_^s03!gq7gl%udZO4wqJ0 zM?YVi#vZMYQ}Gfr;l-WzaO%HL!qdcBYmo#8pj3afD+YmnWQ}iV**X-Ggwl$$`$e0)(u{ar7T4@OM>MU^ennrq z$!XWh`&x7GN_IaQ9oH$o1{?BaQ zSzp(lus`l<5KK)a?iU+afwB%IECYeiJ(yYDbIkWS>)w8VDaExTTIDD(Sxkq4TIq`` zzVm=OV_2c^*r9%<>stJ?FkUjSe5ij;Kc-R=Go@$43VUn?b?}g9Pg<`D8R~X4(F9$H z+biy*)xC^ug#htln}=iV#Zzg!{V7lDI2>~GYbhy1m6WpkF%{hALL{+@NyiCp<=`nu zd^vOg8&eENGDO?&yVzu~V8+xiWiY`u!ip$+h0(`QE?O z`I6GGuSTleYrOersPNb5ZSP-6?_G3r(MGGw9o?7?Wa%G{P)2)T#C=0>?!sG}Wb(lr zX3psI$vW1Rw}dp~yBYEA_w&MFm(id4I9rxmn)#N-SHwx|=LUi+8!N$C)OY{RnH4tu zg5VwD{R1ix9lVt}R>DiKpzh!(=;m{Oy?$4hNMLJ;R{7N{+ku`McQV#6xkf~zVUcoH zP67Bpl+tvTY)UcTA$S?C?!IE?6l=AKawd0Y&dGW%6P*3?pRlAWbug0^EYW!>Cm|9x zrtA|5v7QAVdB^y5|fg=HR>37Elo6! zTB8jV-?gzJ|KPsK;#+Cj z@`muLlOXZkc}LPNwWQ9yW8!%p(_w+jL;%RJTUtK;`H<6}pe1hyQz~a`6-&z_Idrch zUaMM^`XxsOD1L*aJ)l<{TdwDPg*)u~M1E~y-kh0*4ke`+D7T1ODWu|69nGP z3MFdeMjY?3T@}kyUe^e&(PlxSIcxu|<>8NCFpE6kzSi3FH1p0Ij~r!;Go8aIXPa(y zj61R4j-1*CmtW)=OfAawKv1EW#vkm?u9H}M&&=zD7Wl=DDC%LS`XKkmN5l8 zvGw%%90TXodjzlp5R{WwD$%CYV{zOaMZ|Tf%O8)fv>I?h`g>B5N34{|eUg>|BLEYX zC1(d~1?v-4F&sSt=t;~1YxC7w4-jqrq04sn!Q^}Nr0dqlN0UQ`>mx5=0GZAK`3oji zMa-p{YFrQV3n)ZIvUUFpHcf<)AgS2@H2h>M85`hUJ9yMeaRts4ndFfHmrF^lI{9pS z)!2afR_hzGV`~!YFZ`AXKJ}Or9)o{=s!5MqLjko=hBW$zCbLs_@SdE5>qvgBrP-rc zt#u~Z>HhlHny>e6XTDjoZXL|qLv+1A8$5fup0Zx#ocx7&V652DRbge7k+Qq-_rS{2 z-^EGB8+#ABbNOn1i=BE29MFR_e~Xy!6U@y-M8~Pkl*5@o=jC76M6A|#hcw;D+pC;* zp?e+i+41Ae)==z+yS7d zF*x<>v>%aQJGo&*=exZo7#@S%c=^|A1Zg2{qqMc8SIYUVRiZ8*!fXh@qnM!V{@b?_ z*~S>8HGR?_27T-{-qG%L?qSk?WkwYh>}0&9-YZgjlhjX-J&I@{OO~Qf?e7#RUo0@N zQfJQ01K{Sv)%c3JM0#ek3@8plS;zI5IQQ<8N!!@471ZubNw#*oqdcI;3`eBL;LxM} zn&Z6@US4#ACQ(SAI#0cXlA7~VZmu*m$cQOZzoE})Lr1JlT#k;9HsgWUV_b%uZxBlk z@*0RriJAdOGiPlF0LfJ7VsyCMiAxe@OA|2YP0KEgeLYYOYUm6NzEKQy^H%C9=ea|a zNZLx<-JdYC!Xz`?pmRDADR_$@O;Vjp$@uv`e zBaYm(jiGuwGUDoFdDnP`qi4A^Sn>Vzn6F>aEzF|`wmA1*{0*jxPgbfT7uYr=;qQB_ z&0q64@|(E;57*~kNiR)pj4rZ%(>=|Ba|py`V=^!Fzab8|EJpQFEgCU!=CNQ(n zW+da*OXLMJbWOq}4eOYC%n7uH?7sHI!1e;MixiFuzck0C&OJ>YHuZslKkrEgzoIf_m~#Qo}8~g*@smI>p1OKZaAQr~HSfPabhG zuUnXWh6)$pWy^EhD35-oxP+#B{O(H6B3G-weOdloT$G%?+4`#&jvw?pmuEe~+wrHX zKC@cuQQREDDa9oKp@;I~mjzK5s=s+nDsetq$Krd(2~BJqd*3T}#wR7Om7~N>E60|_<@5Ve#hG~OUS0DO}t&D zZi6x>`bynZ@uvzZQyB-RY8q^Rp*-c#I&cosZrPsL{Rd})iYK3<W2sjhH~AdBBWD5m1akY8qX=Dwmu;aRfJ8>?nq;~^{;aqVGp*BgIz zN+caW87!{$W3;*YdHqTk&BBlv5_d_Bzs8tqd37w&69*qff`i`8yGzz5*Ogfu+7!6k~f4-{F4jR zL(Irn34W&?7zSU*xud^#ah~hA5YZUXWN3$kVsX)~J-;|`)jzg`npB0xER!O9NT7`yr5{4dlKE{r8XF#e;=D85Qt1@lR%4RaCQ4X!n=Uk4MoLAk11O>S2dJg z>k(`clXu4o&D{1hmK3;;DH_HywyA$|cFU+>M{B(gKk}+y`ysBK*p*-W{`%d)&6@7i zHYR{twyhm1D^IxYfP&Pz^k4dbTjhx#i6bqqkO5C!QH>H286P{>@Wx^aDfhi)$GIM> z#>KlniH$>>sjmKP7Sg-0>R9@VK*jF&GRFdfEJ>KWPp!l!inSi%=nzSTzxpd_a`Cn;|8vxp8fZB<^do(Zn{kHlkxmUw>swqd}kE!)7nD?AHesd1sr8M>m2e5?@z`9NHoq$*TgT{E}J(GZSB^>wZDF@fq{S zskrkxeGsholwt~e&1IxX*ou;QoAbel>e~^Wt$CK3Y(|%);VSm2Sz_<4o1DA#!$d|C ziC;b#;XjQ-(J)sY-J5jwz=AQ8RJoL>_U`TluM~G^Pq}fR57U8}tz5y4*xo zl_I6#SZ7O#frzUp0=k8dngla?mI{dK!h z3Bm;0K8jsxw&i>t&y6Zd`g07VQ>?+&WPVyMDoUjaYovIb2*WYdFk=sfBlyoy;NSp( zxBmmr576(sYz-%XqLPxj?NEna&};n|7bhD2V4`BGW5(%_lB;UxexIP8Yz4wqMz;5 zUN>raFKxu!^DPZc>P4Silim=0edfwkkyp4_y6rOY<~@iewx=Vne_k7`ln9Lm@rAABkX2PJ=blj;`Si9w)>i+bs)1_46WhpW$IEPp?V+H5_~n*e>!XTt zY2|N?0$Lto`^pZk2ASaLB1X@z_t$mmZpK#wzOFab?_0suELh#c>syYB|NZ;7h?tm) z3wpp64i8&I&Oj{h9?gYgZY`3s47gHJ;(FOs=hrH|PSNt1xXQ~@>mEHzqw{_y!P{YD zQhYm-nX6BsFoU{5`_FKvNmQq6{9ZevH zsp*aTpho8TW0P)sr76$cnKd>hMS-w)khWo;6 z-+*Gw*t;>FNna-G9AM;vnIfWp<7Sa8A%yBJ2})py;!W&EnT2D+w2i&Ok({)hhZsKE zL_n3Hf;0-u(+?jmsc8bE!B{r;+Y?1-wfE=o*!ZOGi#rMX+CSc)ps0SqY3~2@8Ps`lDmXw{&G_{zd<#ow(>X7EMf6FFdr&)>FidUnh;70JlfQHmy7_g6BOe*Yk#4+7 zexq-tO@J}Rr(eGs2Mv>ur@aViRK4W_@&*l7x;w_VG`K-}6)@@ugxz6~NM{G8OGFOy zr0ZV3d>(1ln7vEez+Yqe;ig%6?$WZ3eNDkhQ=s2BVt-YgUmIHaFatAywmjI3OTKhC zBcM7nflthMTdoMFL8YF8^-mhl+Hgf0)eE19ZqXa(2HxW)eT>LqomS6}kaa&_Q6!;z z8d$j&_}I7E6@1_+skdoF-6%V9_`|4~Zk42kO;hrvZqS%T8VHdU*a7akwLBCaXY6%K zEa>5-ncEJ9nxT-9w7-NL&&I17$1kgq(*s@evXqm?IWqdsbI^$oo2}yDS<5f%Kmk?}1@3DslHthX^=>eVVvB7Z#>$~lo;jC^wA@=ll5@q?ioUmFwN z39J|kyUkN`0}FO8(j|CJ-eq|YW^zG?}Jc(LhmSEOW5a?UJ<-KF4^8%o*eC5L0E&q&;K z*!+Onr`~}N%K1P|Z!&)dGKG#9A}RU4!sc=^9quis($ujvx;qJ_4X6OL2VSuEs*7v4 zc$g$71h4in_ZO=|ZpAb+&>JHoCdyz?w$`?`Q**aZY2F^mr8F!$^ST_}N} zDk)YXRK5Ggq0$3Br^??6t-|^9Ji{J7@FE7$JBqLSm2mkE)u&|Hmvv)jRZGjp8Dw#& z7y*fbO`0im#;QF5C#vgLU*o(%6a`Q%E3n__zpVVM6;&rfN{S~X2R zkoD%rW=&Br#UTiM{ZaR>p2jy+vU81L|Y;C&>AV-7u<#pqjCwLsZ&?4*azqo9v z_7I!wO=N!R&(}9iSKGA5#JIO{KoI~cU}4P9*F1cTZ6s;0CHj>`xzkr;TdQ#2xZe~1 z=e(vE2__-A2RCo0hR?=9OH$&^OBxfHi8-z$smQq!^;P2TKl+|O*lIg1eCeTuKVf~& zYUZpZ%QZNl_gRZZqf9=U09D2yu4t490A|n!xN9*WsP-6m@$IkFmOdu#D{hjy_&R2y zs@t)b=W;p8vvGr$L!Likn}@65ujPSQm6Zs6rc2dBMb4P zseSC6Wg^3?8pa_>p;Tc+6F<8M+6vM{Ic3d*0LHjz%vsgd$tFtedQ&Wdm2UzWb~ER8 zQp04ex2*FOJQ}XRQ5v&?YwLxJZ2DzL(LbiB?U{43MEb@3KoJ!fY5vCAxTl2fuq9weHpqtO`?SxTSqZG9t?1ht5xpNhLTtpWs*M`HZigKdVBrc>=cc90o|XpLe+%!z zX3K9I^N2E^L7Ft^^twgKY1d5&N_E97swC(v57V}!Sah~Vwzjrqy&P3g`%*!=V!`?g7T=0g-1tylGGPc$#Wt@puC?;K?R z_0t&*4G8<=Kr!#`8lfz6T^KM8ZwVSafqcfw1y8ST9Ks22vHfLFnxBw4H3`B0v7&6c zN#yCi=bzqW9Z>vR7n-H0yPK5xMw9VoX)95_>Tr~j$WIiR^jZ}*NY}r&q&jUo#USe< zEYx6WN~yj6{01M!j+m)gBelo3o!X=J7M+E_!FoC{@qT2O2u{X|p5{C{N@{z1wh1Ko zExiq%!RNz|;KZ@q1}R~%o9^X8RXzIAOEpixeASQ2=^ykj){vLV1MXM3HHGa47(Fmk z{`u8{F!QBDa!_L@qW+xOijX;>XMScXCV3#Mf;;fH>7{yozwDcW04anjI_^AoEpoEC zv{s<%6H#TWzr65GWvLE^ZJjtE))2a3eMQLx5V$uU#3XQx#&xEKx#xdU#l#F);Q|Sy zGiJ|+Z|l96$eru)@qy|+-9LLc<7f*tmAPltL?U$jemO z|9tM>k5IL?r(0=5^z5HD!x+!Lbh2*;()VMpikXUZj*9GjN{R@zX?z~=_euPEwA&{P zFB0^duKN^uv)@tBR&1|bnYg`o9|K<9Vgdgn3>Knm(ddJ~sRCZpubni+A&{@g@TbCw zkLa7%eLV{;evgFpxM#`@qw-hCc?2E^KSg{El_7-3yE$zXPeBZpZAczX0x;Q#KNKUg z2>PyZpx)3S3ru$~b71jjc>OEL+0VlxLGN`w=&9<47elmVF~`KjP8DB9MDRu9H`bp9 zfvH1>%BIa#md{~EuXP3hpLJ>MNt&HH)U}3U;Mcnr;R6H-LyJY+Q)Mg^z%3aS0bZ#E zTw$e~>+`urSN2lc7ST^#Vtu)j65*)&rxUl~JTJme#-PIOK!^@p<^p3}#vW|Fmka!( z_kweCQ!fhD;f(MkocPj{2Cv>Gb{kn6vmQl1Vp!N{B)3z+M_^bmre%HUZTjK!)-xp*T9V5 z%DS#!vA?feMu*L$#e$+RNDW!-8c-JQ7-Nr^sVkXhk%>c$aM|Ys=9C`ajno9xp>Q*G z?3Jn=mzBDzZC%IxjLQKzz=vu*C81SaN#oUKQ55H78a(5t*_$MVBLaMP@nxmzMI3i8 zvR{ZR-NbgA?f6iXd46DSv0{Xgy&+UXmbw3YUuw1lga|NUpA5#CfjxnYg@py^yru9$ ztK%YKCJ$TgO4aohy5qT&C#WUU!bd61<$RK7mbBJagHyt82pt)l^ZPUP3$aUaVp6Q~ zgsJ0=B+@R_0eG8apMRRhh53BA@=q)tflq3*h66T?(by!dK%QFI`-1`3B@F%eu_^0@ zfRV}bs>p=nrS{LWq&}#ZSeWcD`=YLJQ%PJl!2oLPcm?qJyMb1wyO_tm9FIM*_u@3n zT(8hm^3~r+XGU0N_^BYO0Fmi_4GF^v=y|HeYW<$C;m^VM9&PZ~V^b(Sb_e!;%?$?F zxJwrsjG4UG$WWjd`D}lvh9vVJ;pP#6aS)GY2wZTfQ z&+)!LI&$-iisTo`;`qL@(k-rSE`(4B^uj*0e;;}WioZFKICnu=*L}DMS-&gO=5;@C zV+uW3txPaBIx4qNb?6c<3z^c%-{(cE%X!( z@K@mzdy~QqZr^^&gSaufPXrTYPKa1V5BhjM57qtP*zUY zN#GQox!#OV?rn04%rDCQ7cFakK+EVOizz5j8H-!HK-YGFS1WyC$kDrz_~2K|8aj;c zF#~*~*4ng%Auq{7-#}5PFd!EO)C~^rRv3(LfND_w?n5I&!c{h)bjYSN_tuijN7X9| zZ_*2MJZsnzTTmkOM1r1?5-q0mY!C|=%h&!3nn6<)*``;43M;vX`RH^v`y@8Fs(eFp zd?BBZM=KonoZEY_qw@*@BmoD9CB72hMf$e2tt_d-E(vlM@BetJ$Riwh)q;cCj1eI; zcnksh#+iZqHnIZ?Mzh}DmHyzlFYoc?%dz#9g~3$UujZ-?gr4ZjHCbk4Qh??pOns<2 zib0Xv^9#Ss=ZhBz1S^PXOn&N8-(3G}zK&l==F>=+?}}3|Y;t|g-<>|=TZcq%%{Tz; z9UF(QC}?8y=e$OOgEde$hDwXSLXIJI=rpQAKX63=ii^y*^HO@Eh?ewCu4`cd-#kk_ zomnHY>cuP3nzXIhDHKUw929o+^^M~V)MfbT0g$nkPXb0uk*J3BP&vO`hF3A&O79bC z{NTj`<;BM?<~Mxl3zI{9Nn-a^T71$UwR2XXB>9JEb@}$w1Gc0_nu7~Jbo1Zf;emh= zTM-={Qdiduh?;CZR;9Z3x7_^_38%x0L{kf4mm7X1am|+ePrcR$tHP6=tbj}dfZ8YE z{M$9Vi;Z26jLKoJkx01{G$zL4ylDi)8n!O_X8sKx^WkiNEhoH$-%hj+0)>!JA4T22 z`LRmCGV zn26c6QxOZ;Q&pUEqj!=P7%j!SueFUhuYOzO)r9SVV~TE>V{P!PTxT-!6U^{I;Ywy2 z5_tESK}T;F+_Cz|mjc3GaUu4f=sE83cWG-E{a6U#j!Zr{Kuh?v%nXgqyyO91Q)Qi^CkCHHfLH3*%PvbglX1 zLld%${7o!Y&IRoseGp+tGgG@81aZpKb#PdhaqrlwaI$?uA%noskJDyS+nLSvmVX~uGv8M8E1VEJR-h$I$SQLC#FX9T2L^QtcOUC7aI=aLaS!;b21=b zX(AE3^0s}j>cqy@pznzf1oE$==>r!CKq)Id7zCQgo3h9II}w%RLI6a>vZ*MR6J=-Z z>-PKb0e_@^MNMgzRXznB+zKfm;AhUf(%b&ai?`-A36j9nVB`E+4Q9=< zN2VdrybMNyk*x4ZUuBgZhn3C%;o}BK8rp#I0owRV^<$vk$#=*m>6<`A@B>1MTX!i8 zueXE_kQamoi0+%82*7!>dbZb_IB@)0j@MlQ;aLX!(PlyM%jCv69B6Vt+jy@9ZtVA& z$HzbG)Z98%7RY6e%1sp&m2xpE@ETn`U(pezeU3bK{dNJpJ|IfOR}J(Ia+7`R%X&N4b=LRO#z_r|Y6B*B$lS zX%`y{20#9wxY-uyFEaP{j-HHMI!d)2A*xac{1}L{e+i_`Jb*VwBb@ac%c(Ao?WQay zZICsSE?SwCJv2uFBqAGLr9l9yqIMP5+9pQd(+uMT+TP2f)`fLLhNkA zl%!qPKFVrBa~U3j22ccPlpHrz9e{<6OaToHG+m&aBDCkiyO0A3F?o@>9KcIO!WS}M zDkw1+lW^@BlLGVTJyxu2)Y8?eG8E&Z8H(vJ&u1=h**h&Nl5}j~@VuJEGbIPWvB0$> zpRje&WqKh%aFvE44cfD=TG)*&7h}CgyOo+g)(kt@r=s~wrv;YUFS?~i1z({5O?kU2 zJ2vPuwXF#q|LQbULs}60)Lw@Qe?S`i_>z8`yBN%jDTiR_T^ zRv5qm)S1K)ca;s^Wq5k?79JD2AmZ$TQa?3Ekkc7}B_=LJp-`u1q91JwA9y@oOZ)<4 z96!LCzzj^GGdO?{r7B(5^Iy|Cn3yRauz&KMLPc3}lqw1qu z*=Y>E%@A710w+JlDe!?7b@PhLL91iT39HMJ3!}!CV`GKoz#K?M{u?!Cu4CVzy^g4` zFD8vw#jBq?{o$ym&+GL$~-1b6E@Um_m5YW^gU#=fwD>}$-=qS0hN zce>avO1KV_D=dUaE^4WJlxVU+;t|{srOi?SbY_V|5!@Ug$2VQs1JV<80o-F#_Pt?I zq*!SZ4}vBXzIP;X(FEd`0-Dk*_y2S)V7hk~hj-HXVhHfxh4gmr|CFaVfpvMYW|lA| zAa!|a0d2$wK^ep-3&ACIeosruF7tfVx1js(-qUGQ20kZd16zR)uogQ$7BCux2Koj* zDQcj%?{9}TUcK*Gl5GRp66j$39IvG^Vm}+!AwTUytf(@D%pF|~f4NjZi7P4snMo@Q z__(TB&Z(Tjf!??Ctat&KHWlDw?8#Un#Ja_THEV{bQWj`=XgU6g?*`LAMScI(sz+r8 z@>UX09Zqc~jzm(?=sG@jh}qKrbg{_+GR{r$rG|`Q<%;|K_{VM)AuUHvJA;6|I+3x~ zWZ8SH6o3eZfKj<+^NG$_o4q;j+t^K(@Rah)7cz+kS;TTR^IbWW=yrQV?x4M0_KD$% z+d$94I=7r%W%yE@7d)vzZ|YAdxMl~`j76xL!9f@bu@eBdbhgEa?e2>qKxIV=^i7Ix zJu_Z5Lu`pAa4NA_1ok4(i z-g(OAdS3vgA=%peeSn0_=u1TNDbYex=BV!giI}J1RU~r+_98{~sPT#r!GtdnH2L5tB8X{F zNZKb3ZD4fvM*83h&p{mDTX=>a*D`6Bf z4@h*I!b>;h2eKK>EtyvCLFG;{i{>^VqYsI%c|j^s@e;qxP?0MsqhuRgYtJk4vsnnA3(@ELO7DdLrm8f!<_nD z3`QiZ{|XB13zv+zl?SN%>?0#dpJLjfv)zLz8<7=3Qy=%96hL{y{C`Un&NysO5C>zy zdv6R*Q{G4ahJ`-xvUdTYeU6(d)u0Q4;4tr zXAre$(aFibSDtgm(tCd*>F_-JhgVm6;Di4GV>fJ67w?85pggyx0SZk9$0wg4HdljH zu>%*Z!&(`BSK8kOq-er;iM^82b>Kx8fU?iRG5;1M<=xjv6)8yq?MmsSdH{tTOZ$dH z!TErh|8xG^a<+ZEJQPo&)Zjklc#|7FlexFG-I(ROeF~5aI!U#sBKqxSCF(~>Qhl@x z9neHl79^4upv?`eNIv{S$AUwjo4|^+YIuowQ<1C6qD?R#1wl`zk*E91{s$3bT$q;W zt0TojmA{(TeSo2rfwm^0H{{>;H3+t5vEZhT$6212q;j~cSVf#eapa>@BrAdMwq>cg zKSPo7(qIS2<=D{0$*SSj9r@!95m0oC0wPrLqF$_DxPJsmn z|KXOpJd z%AEz4HYPuEc56D}pcm(^hXn=K;^h($D0n|9+1+p}e^zZ2Atl6Y`UU=ZKph(li zd%;E`mg$G2#t;c3tFv#z6-xS)7i70Rr&5^*z z{dy^UCPJUFTjN=XzLUP4rYncTlO@piT-_=HndJQn1JIG0|2khq$F~$SWq;QWm{nt9 zV*?+oE|xj?$0YG}tgkF~CbWd~VLfGqF5jS0D_Z}9FO_8#TdM^82VY8m0hQ~nFs?V5 z!SOq>1^xKYhlS)5dJ+pBnd^3Y%4}d(T(er`%2(`~Q!DOdy%lmfD3li(-15K+^sWaS z04xVC+@Or^D&?%3i8Q1MH;l56e?klM&0*QArEjhYvveQc(SlNj3YyQ7@dAKHh59X7 zne0I>zzBqCg%%6SYGpBw?ao>rE#_MfrhRsGfhLFo#f5Z;3R`H{)%bif#asuYRoWa}vZm!fM3_DR zSBQ9zcY>8^TEYUpFvZ}B@fw$$_@eE>4^Z()VfozXe@9}~PWYAnX;B|seVy>Pf=A)Y z1e*Hfj_uoAQ=a$WP@qcvd{M3E8x!mq_hz>%Azt1}ODzke#r=q8pw2lWJ2T*B$2DX- z0sZB_M(6Bm>3hAUdfPJ45`8X9s8X&*25LtEO-47V{ z@#Y*WXX|%(;Pp!IzIDI#_a5$eEkc7{Ea!^-y&Wiv;qxAuC+Nx!XE-5 zkfTB)6#Pum5VdAG!j2VEW>Wl3O&JB){sW}jx zpKP4G2$^$xuk6PJZ;fUyaoFVyqsQ+G&q#I?oQ097lJpasS0U8r&RXW1A98u_VY@TI zf`6Wm54!<6LNoGEtDqGDf>8!Ol?*Bv1pu;dQd?Y`{}EIoVW$iXTLx#dtWBX=e#8fEYnV!uVR38?1$>< z$a@-Lr~j3+gez|k8zhS9x3)EL{%uTm+nc=9Yo!~Awybd62c=;&jV$X=u>B2;{R|3i z8%NUfJQWXGuWM&VTJR!#6vXGE%iai!3{;(K&kqH%c&2orkh}gZkjkE$fcILyV~DE9 z1Z7k8ZvH#{$&BN2jG(*A#jwo6d6KuzwQv1j{E}R4_yfPhZJ%4OaC2h)lxzJ2U_fYV zal{ZCYPagAYuVrx4IUjNHWe^ihpghzCxD$-q`>c;8@d|}Y^Zy!rj|6E5?^Jk0=bP@NXbu>UL1y;f(WRJtetJs}KZ(+yn1wqQGewaZ8}3%l)bZ-+ z3Hy(lM0D>iR(_iR6BrO;o@hGpOP3v+Q=$Sy7c_!R0iiM?{Q~&LgBFs&kqY4~I{M8T`*H(Pr}!8ZV=k66RE=K=f#oG@sMTfxlZE*M{#tw*!TU-d+Yvp9|9rDRM8 zWFkX&1T?SusHFCz$s1z;sUZyvyFPjko=+%q%S3V4@$XE^`vwq20TMR^#uh`s$;P2r zeVoes@uwaFDxnrpBW9rr6dzUj7m4~~Z_-z7&HtE8FwDGm6VACevmZpCXkd*ocKH?f z2>zqhF$3k>8F6FvNnck2*5%6N7j z9={9BL2<68FMi{w!PymNn}DCIHDyoSo0iftes+(N+$~m|f4K!JxUy!6hZFPLgK8)J zL0kjq!00wEX+Ne7Y;p(V@LYmpIO-{2Q7jrEN4&IK_7tS0QaqWm+% z`VFbUT?FnYHW|?9OV;}^Ph{daNVE*t0clct7 zxh*wn^`{O;f%U(=C8^k2(nKDQi-#5AEZW4)kg{lQ1eil+sgnB0I8-wZ^jRUEDSjqFBj{)K0fr3ZF;A#Y13vf~AV}&HiJ(g1!%^ikZj?z?f4f z1mt%QKpSW#MDJTz*?at$9x{Tt$(OMn@fF}U&Di~wE{Dhbyp;ti872CbQw$P=;)K6| zy7pn-Fc@%ECe`_Tf;|c+M)!eGA&*uJ{GS`~Qff|!`XM{JaT7vPaAah=MNNhNEdtcA zibltl8c*B7){;arrGDvC=s}7J^dR&6S_Ig^?Z)DAe&(mZrB|WpxbZ{eQ+&7QuX4)D zPcDPst*R=ZFqFzdfCd;}mTJgW0>4Mn$&J5-Rs<7{vJ7s#3d+OV8YQBl@tM9Auludea^>w&0cZpT+71Cmia#(?ekT4jm*t z?CCF^Oh?*amb}qPp6?;84y&#OI}saiMpu8b z;N3%-W`i{c#~yLL?ceit^Fo+R!DU(Kq64up zkap)nXNN#rj4Ic|4yPU>t{3X-!x2M%d@d6U6zQ_w-=Gyv;>BNgn# zSi@?kU(nfM5q{hN`|-doxSu)N&kkf{^+2~C@K+1kMgLYpw{yvMB+_l5sED)B)V?Gy zj>xhvr~aK*%sQ=n$2^jzLXDuqN#)Dxslj6O{-rq^+S2a|6jc|5n%5!z7QmnxQnTw% z2`SqII$iXPVRaJzZ0+L@*KQNUklO|B5dl}} zVpaOdW(y*9ylR{Vf2Qb<&TF7d>H;D%x_j?8ejV^>3W3vnbl5i>P}%f}`&UKaJ1{dM;! zT_-e+&&l-5!O?ad2?H+O0nUO97OKf?|wwIg*CKJhSiy zai>aKN>AXU|J`={H)58|;J$8jF~2IXL-QxJJdMeZ~Eq8_Y_C3FsI107?UjFvR}4N#fA1~Mv<+mrRNK5P4OuwdrbXKM602pMp{ z1gln%E4!qDm$I;ja3-)B4@40xD==Ipz_gi@pB) zdfWerqL#y5teZHYi35Tgbs&6Vdh{v}EJr&E(PA8-i}#3x5m*trQi;uRu*JBhp05Va zz11^g&ct_-No7x}|BRT>o?vu__#emg!n`gJ*DlbibaByh|%KQp+ICl6WqX_-d29CC=6(ucLd9P8V@pn@4k&Cv1Cdz@ChT~PD=cL_ZaubZAJYPX`v$xaGTqi6%Z~GO6JND z+l>MqQ5xxvi4QjE!+@NZ`+sP*A>#A_yuzId9u0#&$BO}t0%7bLD@J&NP=`@_F`$Wy7_xnY$D1}(FxgO=HUDnMGZ9YP+Ub{+>K zm64IY@>T)2*V8C7i6>!%mWEKY)W{i>fi6pJLC_VKj*CKbfvH@jv(MbatGV7rQuF2{ z8Sxj@;_d)uMiC7s79Xee=g@P88DrY#BctQa9}}-Yh09k(c1=61aJASOa8(o#i6RyJs!?s_r#!r3+5D736}o6Q5g$jSX9~ux8_{-k z3j=<<0}Mgw%b$)0@ie<)h0^QhPRES-eWJgM z6EPf+!d5>2Zu7X7;-TpXZ;Qrv(K%qbv$Zvc%AGCmYa~y8UpT8+I5*rLVx)JA!HKUd zX30K>&MW_=>TSF!_i+cE{&pqJ0zU1O2#x;Jo6Yq^Kr4SgzfQcnrH~UilFM+p$-G&t z93Gp{&kbQm@slac@Yj2#fMdQNrFhj0?LF00e%_|eM$Y&nCwXR3D3DEnEeiT&7+&W) zfZTi+0P>^@Lsv8#s)Xc0gxFy;L;)2^rjCey8+ZDI{ofu4K4+AJ-ja*l`(p`E(sL0i zaPK(QHxmuwN^dSADHGq5Zn5~Epy#i245#eW{RMIjN| z63|qV`e0({PCMa64GR6>EI%-^&cizFWOEh~GKrAQ^DLInd`ayM?o$Q*{WHLehHu$Z z?T>Ldg$8rL=l3tozf}*vc4mX@GJ@oXX*X=1+Jp1XP4x4O3gNv4E0hzRZ2Zn~ms~vu zGTabAY}Ef=>}6Qkc3ryxw+wElFiSl5Ye9U;@Q^nD*3{RW^XqE!$!9n?gTH@$fWZg$ z)E0|h(}id=wTOcFp9ciIz1g|kG@WiA#hAd&;-n!=$pO8b7Q^F(&kT}t&kRpIo(Yyp zA<8+p;u9+g7qwZAHWBUvur1Qox_O+N?d znIUwLTkJ5eF?eOZm-Bq6G-+YQ?j2>}jkP^YFA=>2~HwR-I-^LICeA&m&Qrtg3R z!;vhFjZ5OX=5*WOc)C5_f9(mJr-xQnu=vElCV5~;Dg~B7fc>NNB#9|0(CyeCF&2D* zUSm)bS04Hl1&Qk2{1j1uZY}oVn5yZ6(;MSFz$I#8f!!d2^Ar88vm)j=fn23`e$0Wy zARvaS4)tJ8hAcg`>c-X*MhyjUz%F=3z{K69XKjax`!v2wPqO9jXQb+7k`I<`0_Od1 z|1oSsC53|)iC}mEpdB7x`%day({{)KsXv^Rh8p)h(t4Eb?Ba+S>evPazWAZO6fqV$ zx0lZXhA1(H7t4L#5=w$mPr1*yTR+L1wnUw4i}qrC0YbP{m*3}1Bmzck9dY)blL4KN!@pNoR8bY&g#xyN zj*i11NGv3e{LR6z%J#QtVIcRwMB z&RtI+whs?}_ZpZk%^UO7fBtRGUixpD?FwZGhEN^KTa}bVCKj6kliCf*U;V_)I(hH`fi6Ix@|U%y?sL~e(1(vvrxJH zr1m-V3U~sqgQx_flf_Ilede;Jyt~Ty`WvHX%P{n3Wa-M>-J$ zWs&qw;Dv`(O<5XD+bnG|4q)%*A>sPbBhvb1;O|NNqiu;Cx4ZjrYlWIqC=iUZa)^P0 zW^~hp#a(5ewTH9DT+fsgrItvGI?3WreNWRoIT1`8X8;TNOkY@ju%Rw#Fykp}{%EhZ zvm9kspt))!UlWtb5i6;#bK|_n^&oee(+tZ@br@o+S*7i-w~2>iN^*xi-4UnT^F-dj zz!+u*0nr{T;5(kXFJyQ3RIlx6!?GWpvRk^4!T@e_(#i3Cz^d*8ShgD+R&hMh&H%g) zuS=K>*j)Sez)@Tft7nCcr+6{Xk9~LXs7_lAvjM~A_=g6dbvV#_iewR4aDu?}q&K@8 zEy1;$?MWjS)I{$U7dyT4G!YfRCDRIWibpBuc;~h+g6-2Y){Gh`yI;A)QgeuT z|DT?jWmYhd^6mo|tSxRV3=$j@ra|cZog}R=sW zS7#?C)@maou(tmF`}d51G%wi91y%v2#>(>bnp{f&H~qsNGVz#{FJLX=B1HK9e!B7C zH%(a;>e6t5dJN-vD3Kmt5?bEtCsZQjBCxqZ*dz23PAye~`^HLKk4LzSMrZRri6K8D z4_FYb+!|ccv5jWzeaXH^hTvmUE1qB|*kbR2mf>!);r2;a>Y&NqI_Jbb-UGc4O~qU?^kQPWvU!bm$B zen+dAb>-#IC{G3>p~&kwoZFuS1%HNHU;k1uJHrLpQEv}%pi{lU6IIW)*BqZ@?jW5& zMmpRJNxI$R{Eb#xANY$Ed7m(HwigHPcCUK#8TC!wWTaf}w0;feoSgPAC%4RXr@+KD=1HDC$r}?+;2TN5!mjk7bc_;7c{J2zxo3I3X^sI3UQwW@)T}p5C_kw zdZjt1=TJxT2`R4!{_Ce^Hs`u6Vk(orTvh{Rk)D>wxUk*zc97`?*`4H{dxt|xoxdq} z?m(tKMNfvC{21;yhwQILm3JT{zs$bbgFJJ(M|gy}I~}Aj?n(EI0DVbo;EB7rR2Z7( zVzs05&CavoAGo*^H0cyf?=sA)HSkdouyJu7!Jv@}cyDI9?@Jf^gAFhGcO--EX-OKg zF;pdjyRGg-mUP_Q^DRQ|m5eTKwz#C3gn?6sm^U)&d7bIg_pRw|2si*K@u z$sT@5{#9$e_)r+VdBw>)QZa|8fwE*%Lad*H3nD_pDHe3BJG!x|SqmF)sc@5oeR%6o z?k5{*b*59%(mR%Sh57l3VvoZP!;Ucro@P9%%nE4!>1#A=&-&pV3oP91mLNHX0#ULh zX7iL)kF)#kUf#u?!;&D|lP12?2G(5_1a zQfGxjgQ%mc?R((-zCqw66*8LgxyDm*8&((*l>{Wmw@sDWHfipaMem=~Icnis5#b?} z&N%AGID=nZK`(#pd|bQJb1~_`736i z%)-IUV{~G{jXCRH4X~#sSyQStNaZ>;FE-jnh)foo(hu z2(lj!@ny7M%W2J`MUN>Lnn6$+bF2<+cy{HB$6#Or6p>41C(AyZaO0*1pWpLDL~$py z?SE{5c)~wQ5@di!mkt-HkX6s0w!F%e1h%VBw%SALIr{Yx9wS+OuX-GRLd%ldR$ z!Xe~`CzV9|H$kzch9Y>QaYwi;@)y|!tgSvXxyCYcuM%@N)H@ot57(+bI)oGn57kj3 zU}98{FO{LAh4eXMs1<{|O< z`Z~vCYPF>wt1>Cd=&=JUoCxDR_<$jcfB>w>p5)FZ76}@DgqVLoP`{f{&MzeJUzU?j zU}YG^Wh1J1(NX(1gSEKoX1+>Qt-bWZG5@nd-?&{awFwV2u$BP!EO7I=f+G>qEhi*^ zt;W7FJnTm*E5d~1=O3Y)2{`)Vc{^G}m{dqS=iSPwocL#-1U`l;J^QrDH);QReCy4c zApZS!5ex6(%r>US{bqf>*2Y*#yI?}OjTcpFMq&^(@=$Ha!M}!2AjQp3ByzLCLi4xWgs_JOZ7~0rhen4I}nTqic_(TC&Bk z?Q$J|HJfj3LcR+y^WoQ71h$Xi@0Mv`Tk&0*1Az!BgENqR@1IvkKV~E`wD+VDH}E?< zdGO}l%@bG_6BnfYma4mL2a03TaL6VH;5}YgT;#day)5-Sx>&viKM!MlFc;wQ0{4DI zIMhK!4n{;nl_VpLG-4>)<4GX6?sxSwu}0mMQlU0g6@Q+dQoS8}pC7gR9^%NSU_KqW z7kD|Vy8MHG;iIt34EFD`CdAOCU?;-l8$$^*QUMznu!fUV$_yOdC}bIdV5>0L0b}wy zr_fXB6fEPxz{d{;OKtnynjb$U?Y_CJg@ZXhk-gZ{e%biCU3IqaD6{$tOZD>5Sgu#K zbDn|k>}Hww$0GmB{966_rrcj%DcP}EjoYv2P>$bUVP5iew`vffsjgnGzx}~zpMKSe zn})6wB@FWgF0xgz;N=Q=u$jsACGy}|nJn$+8vUDM$Ik^$`=`Qk4Zu9k46Ep>%okg) zy;f16Y+zV4vu~!4=_$vBvLIX-98-|C{Bw$TA>y=^eCa<_S?qVxX>}?*WUHm6cpDtBQdVL%i z9P9!cJ|Dq!1fu!TWI5Sp0lPnBunG!3<9bq<)vNt2kK-{FSe5JNe$Ytf_FZR!l>h|+ zgg+P&3EP^Io->@SY1|ugQw+5A$*L>-BmI{Gtc-*o+^dGuUr5C0&Hka6US#w&6qwf& ze*1>@_ofhnV<+eTdK^@>bO+s`#0X`#!GV)ivqsec31YtK?pNS zujNNgg&mSRvWr5UcxByH^Rc2mZaD8A{QoMh@+A1Yj>|oWNFo>(GI+~W>uf#W|-B*l8dn0 z@i~m{pZ0cn6H@lm7VXW}J_iW`a8qhMUxY+~{Y79GTm30PMnWK1(Nujgl_;C$FO^!a z`>D;FfZe#}f|zkV>Llb{j6aThgo~>Yvg~AKk+nVQ&gTxLQ6imjRjZhe$dttvAJ;BK z@9_~mVIw=T-{rT$1(*%B{dK_}$->>OS4E@L?<(GAh{)u_A|dVxWQDU8Us6htWGswyOIo0F}9vu5E^Wd@G$e;D^$i4lBCA`u;OMZtFPC zwrYDssV8aju(WZ#AbVYEmZD^Fc0np04e^K*?iQ|8>4bogChM zX$E@#tsri_*;~z=+qmBE=46})bG7MW6V2cxk&J;?a3Fv2;#a*4wl5hnw_GCa%})xOWZVM?WpH_W^0uS3wcV2ugPcQ}irqE#K3K>j^uvWxoBF^*j zxdm{fpYCIO=jV&(kb##$EN1XYzxBP}dZHW=;~XaJ>#WA(478TtUl|UVMn=TQ83!R7 zWT%OjZy2~R$V%utdWib;?5pA}+uU{8mz4Gy9WzYJ)f<_1X zddAv@v|Te#xm-rP_-JkfME+?F&+v?|gaVF690*j~hQtKVNMe5)k~x;^i+H0|jNN{q z@)vt0e2wuDAw1Dh>&+OeA?{?^b|d9zHJ>&nF?cAsO_Yt0<|f7d3CnI&oVE~e)5NnScc&wBjv3S;qik#= zU-zSg!J@RFsw#{muXU8Q_4Vhpv_u_2`JA3GWWS3F_! zx-|;p4HkV0+|O11o~M;ima`K|?ciJ7r=R;%2%?eq$s_W$=1T z#@Q1vF?;+8GR~Zcrxac1bgujSJo zQ|_385M-64%k}ESz^WLPRNmhB=QS3MhWv1M=)7>&cf5kRe$K$+=}X0mJ{eBzL)q2_ z<1N?V(4F2R2Lof4)tx^>@I!Iz?q(Szc=;9J%Xe%CA04!BpH9frKL)`ee0*!nO8Es` z7J(4K#2)>l0L(|LWxiJ19!L#YdCvRZ-~ifK-!K6C)m*l>TCySck1j4Z*XfvE!oqy4 z)&SQ5Z??v&tt(OMWzUvCS&mH1|3}$dM&*?)-NFwVAUFh<;O_1Y!QFy8!3plJK>`Fx zfMCJhU4tgUf(Ex>!QHuw^f|ZR?|y&o-J}0>kI_79uU)HZ)~s2z+E@mHtx|DTjSUqi zG6~U13wp4$CC%)V4`ubABf&ve-ZR)eYam})vwcZ>jQ~j?;#Wgv6a=O3mY`kNFfR1n z6VF+4B`C;VIC<(9>J{Rrz%0zKc2_$q+4eZ_#<#5++5H(hz1Et;sdm#s!_Y4)EhT2d zJ@xd-1!4ju#4fEVNLxWGBHTAx1ZdWD@PGT~Q1pq&CRX{zV^o%vlq)%wuNt^{z6IgA z_98;y1 zt>6?m+MHAKZEZg&=@`yegFfvKHF>yl+U8ukj8Md1E;k4minvX%L}ji$j4JkDoBk7Y|pF9d82R+O;bdL5C$%(#+XlF8u1v8z%GRn4Q z3kW0aMtfSN_Y}yC1%20#Ta5+VEU8FWR=1v?^{(17z~0R}n&SC$g{&?I+WyQCHMW*7 zuxP}5yL*+eM>iGiSHi}a4yd0IYqLS@!?^zPlx?ir0f8Gp2>7yzu=AgnO>S0EV3D(s z7wZIsgebGXx@&D`uzipFwrJ_-#K2vv^6AAb1nE4EJwlgDGy}y&u}UdAz`c^&35OdwfX9*b||7JA|r7y&Zho%%Q8TlD}yu0Me zJ`fJ8$QL8EwNf~BGAo#s#1t3&U)eCcj6{bi$>xk`IWu|M|JQdg%tcD6lri=Dy)c_B z=uA`0yL?q`L?OzlCfc5O8o8nCVnLC`ILDvenNw{0B`~qUD^KLIH9P~T<}R6~zw_w> zQm~2!`LG}kaQ*#=DK=@^{-FSRLG_*rXU*EP`ZXemd35e3!6UOB6}t5P9q`KbdhVNJdb}1$ZTbf3?E%;! z0l||?hyW&O&PvlrHj#hQHoS=9I3I6T#jT~NM$|;aqtL))Mm(Jh7S#Z*uDO``)FvE` z{&@R$LR0XQLl{k6TMnDrc?yu$4@B*u_Xw#Y?KIdBLFX07(#{e}JQ05k@&NjRaTCt( zXyc*}T});@P6AIa`o$I9;AyRo|>P<4gBo8GT>k0x|>)@gDF|tS_+Z%;e8R1-Gz?ICOZC z)YK?-uR3L`7hdtl=Xt<()ST#5SG#L2YvPx8a^z1SG+ibl==0>>A1Lr4DfOH!lH(ot zv>S}bNHVQIXR87^``^H+#2XxK@+u3y+qq&_O+8UbsWxAyuGW5l75pHG3N9B8>PSu=@oUl4UJp7WZ~Nkd&|2>YfqZS5tk`|Z zc%XeL9-u_M8{i&a@A3oZ0|M$ocY+UcL0@H$znya4y@sE(IQ~qtLH}f0eOQF^W5wao zpU}Bt%4OabR8(2lhk3mcE{o3J#;kZ_fR~HRMNkwCT!^?tTIx7Sm-ukBe}UB2T!5{) zRlYl~mU@QRYg8yn#$D&9b)L*P#aelT$JOb#|2kMv(y(vdY-KZV%F{uYtwy<3t5E*E z%Azfkq``ikddijSc1`%SPt2382mxD}D+bQiH>43vwK8S+tfv*a58`1ff6Z~@ZsWMr zk3J5+TT@x_eRAv3^PSkyMu!kgQIyFm1s|#o)HLYUY*TPhroPXw8~r!G^4?T(WKPCx zaA)Y|H!0*iAlcj7*TAlNG5C)csjrPW(h)M#md7Ii|FY z`?dnw$JdodKa&+5oP7NfPAFqphwvE<+ixk8b+5~H>fgd?RX)J=^-)16L&L&MdP3iU z`*4%To}Dnu_{?VM2{`@?YMTaqzt|-`>)MhgzLgic)rNqLeAGvAp`e<5a>f9HB`2e3 z7d}Tve?L#U>5vYAB>qH2#DvZ8nOKQ>a(E3jz=1SXAd}iVl`B=E2cuI>k7Ro3Z6cP% zybIGT4wc4M%^6>boPA!o;~?I_P5@!`(P1S?8+^ix#DDQ(6?45yp+^+hoFNHJDV@g3 zP_LnYm-yca4vMBl1&*n$S=vd03;b|%Z;pC#1X~|*U%huy-&);H<)8Y6^98BP$lD0I zt6mNZLX^18HT9{6l1oap_Z^F1JcSIZ*~Xx1nyW762RW*fw=UH-Q*z+?NL0bcyLZG5 zZzl-xm{8m58?cQWZNFyH!8x6#u`aOX+;1w@Y!QWd9gEM#AHJzlp4HQvvtX3mNGAXa zh7Vj2*i@MxQ~8;ztk@C;pOBN;`hWB88vx`?8Swzj))wQgkKa4xFWA`-ctC)+bWYQ1U zw&lN8Px)dgEzQr)p}YdFHx(&b zC!)qOiB|2fUW$6WZTil}A2vOb&RIc|Mf)qrtjkz!OA%8cts`0z@Pe8N$}4652HQZyRJ< z+FdnwokhQ{)bqShr8LOTYDvReMqf3?@2q3#9d}1D4YH-K$98M4f4kDKnm;T0-yGl=m9u^cz}E z`xgg9cU2Qq$V-0e?pO+{+;C3`WX{(_o-T@X9ls14`4>%SgZV5tAYBA5$h(QyMJ2hr zL|^~r%JXex4}6k{MvkRjdbaYae%%W@i9e%v4LhN}&6OzH?uo%}T=t1C3JMxOK!a1% zqpE2Id2+-l$KhAcCXF(e)P&5qJa@lYBfm2L?;&C=9d^;s=3G$C3%jWC8Z-1|f~~(( zC)SY?;6$3?G|FELCUW6}7`t@?0=z&jr?)_!Bv|Qp>jd*);-rF{LQmZOJfR&ALG1QK zVI@)F3DRf7>+*i;wLBDyDw#n7hWyzd zt?9pzAI7EdIVwsWUvU-XAvBSkXqjKUhtAZCOL!p^1ZWUIi!=p^AHiXCL~NJtYEPA3 znd5vqH=i=M@Lbu>I1VGo1+^~N?lKxlMYi?_TGq^TgdRwG|43xorw&~7_k8XAnvEnH z+2TOnNrkBB$jF*RRs5^e!_jqj67hQ~Qy*X!!sQHn2DGuI>=vp>ENk%rf0nqeVc4C- z&}lDZ+2is(+Nq@O9Lx;un*^p#+w6Cs5Y^9~mf8XHC&eQ6LS2$ZEa3dB$|e4|HK+tI zD))7%Q`MhR+Y24=J0?mP!ROQcut-1HP?<~%$r)A*d-=Ik+~yEC#9bZ17mxJAz#V3l zE5&2T9|i8X_!9GD?!=iqCG169VeKz%6cA{);lj=ra^vZf1Jh*yof`RiK>fI~diN}) zsm>Aoft?Donhe;7SLD;0+V#@6tj$fZr0lG=tyltEwpbfufBz}V)G_iU65U@&YjhP# zRo^j*?ANB)GO_kF&7>cR6HUpo1_Avx;SElC27Mh`R3vw^kVwucFnU^qOB5+ zQ|fwcU?Qq|vWcO|JDN{FB3DYJBIo|2_!`coT>i*Q4!XfzI5&Mls~FrA+NmQurLOX|%g5uY1Kogk04fB)YDC z)okzqAbl*~Ew)Ghz1v4(s|DLyA7_64D1okt? znXrp7J~;@n4uu#P2`sFQmRdUZOef5k%oX) zAr{8aD#*AsZcUyt^$JsFe_*hrZlL_|ehp1jdk*E>rxpHzzfmPut`^3zf~l|Z{72@J zK0OtyqXOP$ycKW`%J}p6;?^1=vcq$N1AZ!@)0s1!Tzk1N6Fd*N$=uH&#zs=YY7(R6 zX~J|3xJr78O^gk=-gtKmjUajAV6un(`SWV9Y5{d)3UEmxjo=KdiECVSGUPg`9%dH^ zvxSE+HOi-J6->$U4=s^vmkCAxVe4LT|?y`dK-xOsq~IbQ3k# z-%q;vL_ZMB=|()|vbJ@H%>Lp_iYg`9S*HjkhMhD2{JslGP|JL_7V!)-f7QAx&dHA# zP4v#$F0H`j^^FT1sXOoo&dK~3Rko3~Me)oYxYEj;m~m@+wkr@w{|?6S-kZS4ktt$! zPCy-LrC{8oX+76zTgVsmHT_V~;b94*xWt1sK(A9j6z!V*i2lL%8Y7{!c5275j81CG z>y%pd8$l%0l**@`%z12XA=i&@RAyaO@Xhq&){Fl_r&-@Rf>D;rawS|!4D~WXC%s{w zK~ycMX_xyCkk9;1kY^zwZ2Xl3qkmYWyFWV32uX(&R>t&qQ36{?IONqe?F009)qZI6 z&Ik&^1jF&*#w!Iv0Ic4MxLtOupnXJ`AK?n*MsHy_0L#-YA92A>V{pE1@ldhkAk)e3 ztGn`Nb}?4^Gl@2JL7gFi+eK0ARx9d9D>ikbfzS^QNQp425kf209Svv7FfiF z$RBgbzEV`kI}=Y;HP<&v9`1b7g=2HOXNc{9wNa8JPH$Z^`cH-e@uTHd4!Sl9pl*QH+HsvfuReY30jprs(9;w4?g(6S4Gj_Yd_?K(>#Mg8$P_Lb z$;{hR1BrlP%&?4NIf-_WVNe9iyWZrG;2f2v$Kf&^RcJEaTwRau$wheaMf>de0ZGY& znG`%7qdSh*L|3DL0ctPLb#QC>i6noL^?tIRN{;Iof6Q{mFXL-Epy!I#69t~2!LmbT zS2{>F>eWyj8ty{*xJt8wVNP4o`c_xw2EJwenII&!UxtDU4Ea{!smX&TqnQZ>NV>=c z<%6qdO@Nctil3cpo&cxjEynkEFSkR;tm_mn0Ef|hCTnhx~0X%LfznLCxYS7%pUIMp&-{@G_QSAzZoOAT`yTX z8}E=%pf;Es{P-9Way({{fbiV)F_Ugc3+DSAi;3<%9kio2{3kKo*82=lU7o{)VTlQ_ zERA$*)HJs6)@+}!M~H)1*FO8Y;KmS2AoM{q!R^NLFEY@7+b>dT(1RABwX(RZlvg6% zOFPdjXnjLetyWw+`baDo*_7oFS`7OPWf z+40+myTwL!LExl4W+?d7){zG<0}>+6?Br_)5;4oWCnoaPnrd$k6#!AC*DswG;7M#M z%M9r{`$`C410q?VM}OpFT%EfUdyDh_;mug@(A_pEKjZyoeNX^sD=@ zYjbC|?2pXI@D*y{<${e);6$`a5q6g#@G7bTiMRL58xP%h37E5Mqn!{Z*(}jJH#uxm zU_0#9#B-<mYNcEn{n;Fgkhh7>jr7CE4}?7NzVyifo64i`XT%?=eXA;v_2ScC zW@^V$B6LM!3xS+C+{EDQYGvAU;X&0QDafD{$7!Z-waHmSr}anr2JN@-omQzT(%H2d z1A2x~i*u_n#ny>^vzO${Z%yvz6W;W2eU*TMI6i7c2UPOnJ|?d1ArQ6U7X-tMVV{8t z2J4nHECK@U-tV`2A0>j1k1X`3QX_q{`b@s+$;_;0s^*O?>V&a0IP_g7ln3NQYP+Es z7dDVWjE{hOaR3O5oMoe8v=gAHop>XRu;VhNLaoR+S?emrMcXD0kW^IZnLHeD0rK?i zyHS_y8ptDG0~Dl+{xMr#4GZ#Fz8yLNDkyRytnO-7Bh~I>f8&xb%sP#NsEthkq>He! zGkiFm2MPit@}V@JedQ|DuVZSV?h6f)uLhLEoFdQ9p&E!dOl}de9)v02wNX*~(|KHb zVrG=NbfQH%v|j?%R+y5>K_>^*RGJC6x=Y9Z7O84RWG3qO3`-6^8{D@d$;SAINM8xR zW5+pBm-wO_ z;mZ!~ctZ%uzNro9Y4OXi9lkouI$mB+H_@hbVx`1erlwk(E3=yvjb*8yIMVFG)`3G2 zKSu(DDs}WN@vqz)V&iw=W5p3mx4WWB)G{iHr9DNnYlXP#)LUbgjX>t|gWnBFS66o= z)KW}M)rbEsf2Ie+g_Kq(t+e_6?Te?D>P+Evt4)=+ylviBJZCH{11mM`dn)H&$rpzQHTyx928aUq zW|2o|U}4h&kDI1uJxgI=q7xTA~}s-mKw}HnwdO0_OoZQ6vQBma+T~tW->gGf{E1TuZI3csvHVdK zU|^vZO>Igio>4tzKfWPESpiTl6)Jm!zf%AcJ>gZ4frD`U>2z7rDT}YM4cOiigw_;C z+xW;(#qUB`A%_PExmmE;#xnEaOZrs$9CbD8zM!d1X{&Iu{DAWEJykIs0yHDn)@IkM z?a~0KpKpsKd~*b|H&>6vosP1z<8#~< zg2p%RQd672Z5j#~0lM_K7NzI~qC|~WBehU?4@2y@N!apv?ekwRk&NNt`BqzVrI`8$+D@IWEkVTC|=L^tCiWV2O zE=#qCq>WLMFT0B+Z-yPSTqF`g`asyOJ$9TPL#O798lsmwFs$|kZy;dXnNX-|l zECFcKiXu7H^KYUDJ+~|U@xVJR3R9DxpbSR^S*`2%X?}uKQS#(9yxs|J=9;H?0F#Tg zT$W*=&$Zt`Z}oL@kd};?-cgV{PA6UlFuA#lG*W zQME4iGk8>ULT_~Cy9{+!mf&i74N6lZnF6h_O@o84Ql&!qgTtQV zlF`xO%gPQ74i1W{s@|7|#2-}1$QNc{htAzENMRBKK9GX`k?+}btO$3;n8q_vfqf-d zk!vdOEzZYwLb4}rxAfz|Yvi^5$x)jw|0($A&L;mU-V{3pfF;|MZgxC_RVkD^Eqgke z@n6{f23HbvUP{O9;W3CZ=^U|cj#g%hkck{hhIs5GQE=fU zcUtOXPU=vX_jD-ZG%yH9V?V>A4dK4N~26eEG?(=7?QXuIYot4yPEFPwE!46oYFj>pr2(asH0 zqWo*tfJXwmoj(SI*mBm?Gwt11{o`&UHp5Ym9fMotcx>4%z+ub%7n8d>8K%AxtGP1`=c&6FUe?TX15V?XZoONGjR=SN|)? z4OAvCGz3l-U{cq?oBbk=*I?JX{mF^@c@Cf1P4nA72^8aOdHq%iKWLy*&b=MlwU_M=1%QctQ=%q*Ljt;rJ*p7aF)C6S!d zAu2#pvLo;~KGHXn$M1paFxUGNFGemgvQ$>lai56HdGZ``FBt%@9nH zo&J{ahe+FMjRAmGuG&j5Qm?-3ZBgzHj$~PX+T|yFgca+GQ%I5lpErTU7?`fnmSR@G zkO%@jm#a2Q*Hmjqf5M6}r0d&VNE*&^W#2^Cdq?m^KPDyy@6R_3ITxXB*%^}poC6T` zAbq{*geq2vtt3#(vbNkd^GPlckHPehN)K;Zpg(8NQP@Vk-;>Ff!(VAQ#c-p9+ms_J zV(jW;6ViX+6&~{wEaf6xQ3RV()SgK?gZCIYXmoT>TF;|+$f@7v!55ct{qQ|WfR(9( z6ru#Pj%F@r<1PWZld9IXemCu%45`rbb?;t}piNqlR@;?i>GkJmh>2XPM#9nL;QLX* zDuv2V+A*A5uIFx(Kz5(rev9evZ#|vOPT*)f_;hpw<2f2vhlG>RbEo+lm)*zhbsF~7 z^=JGz9d?ALLnyvnP|K^i01}he`;1m8lRt{o|B57&zmMYO%dY&zVsT z%a;4y0$A6Rn)UCx@}|+hBay*|iK)y`_dTW@a|jDH6GO(hSfM@K2-o)ri6`(03x!a^ z#*Q)`q{8^DZm64_CnuLabw*220H6uj#I&@?jScgnDf-CVufOFn)2r`3X0fK18kcZ4 zT=T{y>T9%L;^G-K6F0=5u*bvu%`i}EN zS#CI3;N6RL()v*I#_pOM4h=yc6voc`(dKr?R~e_vq!tcgimpw2Gdki){LH%aToX?c zI{Q$fh+NC}WjKG;)Yk#tiiC1`qnD6U$v8=-l;4M6fEPF2pa=&((spvo-yuwG} zDx9Dg{4n|Df=m|E&S$3k?*0$hL;bnlpT0WN{)meo<^?(V$(L1LtHQJ|~#pqm6G?-R^Xp&>s&LpG(&h z6rKfno~($abp{ZNof=67s)a2xw~%^k-n-N#1|)J7e@sdWaSOPMTUgMKOpKgO(}}?s zKwWHcOFHHBA5wL~EI7pJ_P6#4@nf_VmgpLMqY^6BxIP%^``uRNq07o@)AX8+XOXJ7 zShJ+sb&q(P2ybKnH}dbZ@Tz8B<@A<^wek*(Rm31BLCtGvC^CQJOS9m&hu3pi5whQ< zRdgF;ES|Lco~ihH;IMPG{c;KJj??UF8W|GQy^5Ot_#BU1=&&T^qHm?(GFe26t79JiNQpUxlQ*T4nad#sBtOgN{NTmg@mtukmdP6a-5Kt=e@K;uiz6i}Ti7Vqkt5%;(jM1b=CH6XU}PME(M z?vP|TRQw&rJNV(AvQVgz2C*E(wCQ3}X-U`qNhLo6%ukbo`xr1$k!={j7hW^;izI$u z+Vk@?9cY{_#9`$ZM1SP-s!)U+%^fU8mK6sTfDC}thomI*;4=aJHAI(OwZ8)y+)unx zkiNx-^$C)Vh|qP$U}gLxjcvBxw(u#^J;!AhS5spLI4U6_At*POTsPivR4BY;=0NPc zFu;*Br+7Cre1)bX@Zp;efOY)wzf2@@@-Gx;l=*g;?So&gl#|suOKh1SOdMSna4ir& zEfyTYzl*hD7;*@)*vEapbpQM!i>`7W`$d?Um>BcDDVH~%gD*jN#Il8W(kDv!{DNQR z;tC21A^U7>Y^>=D0EjX${sj}A#E|z{9=UxU#h<+Or>@q911Y&@D*pZ zg@l;r8WosCwX^fp!|kDpp<&$Em`ZBHr%wb=AR>uIIpL{tX3W}UTy#&xKsS=|2#Z3+ zIYnSivjn@JKO-?_{ab`30nN*4=JKk%$o(N1ob#sEc^xagU}5a#_zn~p@%5a|_1`rv zKFE~hCu%VPk{gwHk402+HYHU@M~7F;K0IsX?!K3Q2vP?^afyV3X;6tIL#F_l;jS3w zKa!_Hb$0Y4aL8&XadK~_ku*3&;;45$;Hk^{HJU++f_t$}&8UYuDJ6pBR;MMy!xR3wX;0TD$&3UTOf9|NMapsCJDBv40s zAxA|LaEI7-dmwPN_HNW*&iiaTN`}M>nLkr0PW_r#$`$iWTLc-jAABZ3Qf>9QoR#+V z+Eq6H`0Z#tZJv#nar~FNt{n?9qI~a0lW-bI)^4{dCdSiqO{0pLpzB9KCzb$#%qJM2 zclg0+hZP-rZH;WJt3qXNJ1xqpyfofpw607;3lpA&$t#ExPx(R(36+stiK5Bs*_kLz zNKJnZ{v09JyKU*sxi8Yzu49TFFQ9OT!?M+Uyg4OFoZCuZbqm-bUGc#5WrsyIA^vzS zt%c#zf8Fp#qg^h{yIPO+`}K6^Ih&qM6CSAVOCjf@Q;uKqdt8P2~!OM;b6NhCBmi>p_mT zmbmv%P|UqhciO7)jIZv51|WQEQ**Q!nrl?-MZbqGY@cV+wB#H*@}DisNAq^^drB8J(nD zxx@f;xV@Pu*-CXYIR?ECsw@XIbW zGiU#-4iA=_^FLd6hI!*%mi0isNPfZwJZ)nS1+OV1G=%Z{a96p34F9}(F!}KxsY@~R z&~Q|-e|EsZ5{p=c;REyhfdYQ)ej%W*YZQOS9#nSjis^`xXJ_(fC4Zb`HZPu3*!dK^ z*y?UFQ;|8_V9)S#;?i_%rF-NeW6zF~iUx>xfdc|~5ZqdRH0sAdCYvZf{{x89=<1NT z=~7g`)k!YxpF)66}eYlu=C5L;d?j9b8J#kCS1qGrx-J7cd_kBxeL zKvcBhsnM{ZrlelWeaF%bWJ_oept)cHtqc$dJw3hC-mL0^%c0-fjOfU=_Xe)U3zd}= z%W8;_CVpr1o6uhV`NH0FsX*~qS?6~2FGxp|tpCLHZRk;A{aD8uchJu%(1Cb}ocVmn zL}oag)Wq0^Lbz{+=<6=0wV&-*1}r zTD|~o&F9-&RWDbXqs2c?8zgX$G;Lql;rFrtH~E;iNunhDK0d9@&CMNecCoS&$AW;l zFa|gns9MO(!=KWHNg4!(6O7EUw_W3|5B!n8TQhh@Ex#TLSCeQey*Meh*K-J z@hav|==?{P4}VSz@Z#`zki@0>DD_4M+wdB8j(!}2h3E{k-d{-z#HSU#DX*AZsIlPC zy1V0$jw1-Gs$wkzT6F2#cp@$&^#TR^{T==fxjl1y^too3f`XVQ+WdYh?=Ds8ustuY zsIfD=dVM91o%M+(zWx#>Ya!$|JVD0}F1wWb2k8~X3J)!WcODucLRjY*(+Fl}bj#`Z z_B_|>5L13vim_`?wRYOiR93})LUpTL-RHKTC~q7li<(vOB_hq5-K{a8qh+NKegVnG`M|5Hx%RUH9b3WrA8+rK^XB*- zJ-evfc3wLnK2Ng`0=bPs#xv@8vDx>WP{A5H&~nn17R#G+R%rtfYyy#>IRZQy{@O`9 z3z!Dez4|qR)%*zE`lAySv`XvlVt&BgGbt`rt2IKB zc@i1ScwjX;QqQzDNW0vIBr^9wRC|ROZ8uZ>pI}$JsYKdPen$CZ3VVr>+Yp!vc@pT@ zHT}7*osLwAzf2JpJt$KR+1VZGfF8QVv)1gVbTh1GSOg|P0F(~|Jy#%aU0nRAi<|9! zsUhRDV-LiWUY@B$3a-WG{eeILA}yKV=I9UsGJ>?FV$$$%n)=JJf4=;-Ac`WfdrnXe zBEk%8Zc%?^E&+xAkVxp8<1Ko|{g1bQ8=z~@c)W>*2N|-{{n0?jZv%4r;67-LaNqv~ za`eXU&PCqD0^Znvc*xuh4S*%roWLk%&oo}vl`!pB5u#HOyw&p zxpTAsoH*`HQeK^R>~x+_i)z!f)TB$No3$B%I8)OjM2?i2jLzGu7`< z?=k`)a4B*NAKB-cg#f3_WbpGyZyW>sJ3(rO?yM$fN_X2Cu}&kS?OD^`eoQaql8WNk z3yKq*nzG&Tavu+HXP(J*4Va{0_m=4Aj?w4?7D%s!!2y1P%XfEo-nTDsa8_6Xys(@3 z_?pgsMny)&_PIRX;DUzC^*suHqN|mVOXu7D(;O8iF|6II81Con-%j-#+MQjAi4qYa zDES$sfp4ytOaS4Yxwf zKAc#hO6}@14Pk-C@z8>74pE+447`UEHW1U)fp=f%@FT1NEfL_RLSK-+M8?KSsl5M_ zf%bLcC#&trP~7;9UZ(fEryP(yIS~LzlUE|2*}7idGYsfa`~7N5hx*jrrlOq z{=pb;DA3JWji*>jv!~uT-}O2Zi8vFXM!>YX@Anp|=3x-@*VydxD$QuDFGoz9zBv#M zwLP148VXTCKuV=ogypRJ__szvQ>blmzW*| zs56A%z<{&#VAgsPKn4U73ZQxXQZA#HIe#<^J)XLbrRGdNUNrH%GJsvW!UkWX_9@oN z69$3aueU@cnl&{wD^9Zs85w&uLc(NF^vOsx>>O;XYbhgdlFkmH<)b;sa>Anj9&3c96M4uT~@}6@~&^v@ayH9axY@(0m>#@UI!X& z4Nk@jKUzhr)$GB!sN)V7)1IS0t>P}AP zj8;}wj8pdk%rR!=ooD%w!VUd8b7X&=;%x+bI=%<-4L;2ei!pkQCf>H^!5;{$Zn@a} zoMm-z6}A^hA!Agu&@>gE1>idI^2x1|E3b5YUSg~5Lc{LeE-nGo!Z%s`Wwu-c3)fwnyK3NM49hjt{`2{x?JJ16EOVRa9U_P zc(r8+;zmoZnNCJK;m&U_9gkxeHyk^A>zgXD$7H;eMBQX=q=b#be#n}K!n-w~OA6(N zFJ3&Ey-P?W25seOSC0vR_I&}Q3m!{egnfNd=MBmq* ztePOncXsEKjc;o0Gn!K6_3MX;jCnY*E;m_;qn|lOY(gER@DViJw`ogwG5i#0k7<~t24{a4Ve?&u4saRGP4p>agTiF7NtLU zQo{XFL8s1|OgbJPCVdJiZ#)vr(O%yuPMWRp!lG9G7##%jY{BCx%VYCTEPwr2@6#=K({UEwkO`3x4XkoFbSvG`C;0=1P2GnN) z@@ukwQSuY*|0KzC1wSptyuFzFCG&Y3i$Nrp+m82$2T6<_i_q`HNjtE1lyQS|^w2Kj z?Sf?#DX04(tz(TxYuz7B0&33U zMK*ZMzm)D1`7p!Jg(^U@a4r;H;g8?w@JI^YH}!%~HpE%x8*SM~Cl-=J4%voNdI|xu zqwvDCu`j5~DRRS!iI}bj3vduHGQy<(b*6nc85tgcHr)1ek#8mnzB{+@1LFL}M8SH1 zk$(`KdLcPgJ|QE{a~zfjoW<5l(aNQE3!&xzDQfaejh>VIAZf=a9Q0_h-+NqwSw<3o z1}o;&V3aC|{acHpLXjkxf&ct`2hnx5-3M2) z+WmJ9kb$iBMy-Q#De3DK{bS$VNd^@){g_Und(IZuo#37xXh~^llWvN5ZqL1Gc06QO zJY+~<;+oyBc;$z_wW{^cnae%am6=%XdOLJ6yp{YerV7dm(fhx1#xbV)@LowiME^q) z8?Be@bLo5xK*xXKbtThD$82AV^^81^9L+D-48$fM>rhGJq&<$li-RdbK0dyyy$NMj z>my@un}XT!CoWKFq$uZK>Jke{_>h4NN#nCkk)OYQ4nYhK^lDs){#aWf?r%sEj0{gn z*<^)*JwhX1{`ukaikQQA7Nva%#Ex#*Ro3RSQN8R23LZ?)5%_*vXr{YlRS9jOHVFFmJw+*AOw&TfGWw&uLw~EOy-E?Xsb3qYNfb@9!8T{Ho z>tXj?z3tJS$YYSQ$TeQ7zBgSsGX7kN84V?6Ht0h=_s=Xx6*7AD3c>sQV80=hy6al9 z=rD`d z9hn>1+e_&|w6xBfg}_BUuMxhVU+6UDB`Y76-Nhvva=g}Ek1(^X>e9U^L~hjczYQe0 zb3gL!a&RO z>vFBK*UiwS+K<6&YnTwso}M07lOCxw;d{Al7?|JuFA}tCqTLrd?itEhgi)coiUyg1 zB#L+5iUw6YR~Q+~%$rPfC*>CLpG~Ktst2vm_AHbm}a%)Ft)R5VAvk8hac@t%H4Pfp2H zUjqp~ti`dKo?O`!kfl$2uH?fv(lw)OH85704)>@qJlru}p6)6`y^~Eb!&GGg+_G$x>}<84 zPsl!yF!V-r{I@?OY|-68(@GV)+JBLhHGLS5%LWw3$!&0DVx+FNI+B|&SG*PRSX!y%NA){A0qgfSA|sK-02iBYz5pq&1gKbrIICHI$tD1D8El zC!ovqA|B{U#H~5ccwcP5GE0n~EjoA$TR&6`M& zu1`G@%+K$Tkrcc7y}yMFJq@(r!yi&wfjKObgUD_g{w$?ol?h2*8 z=KA+T`Z7J-->$sgqZ{oT+MSqn=Kr2=FQ-p>$7v}Hvi)e|dLPVaC^Jh@jAWfgu^dSt zdq?oHWOGd&HWN=Oiv6Ei$2S_+FS3dE$pF+*``~u)*JIr>AXT}oK}*8J^388iqV$`9 zpa=fuc!Gf-uNwK`*MA*hV}EDTR6d{``@hT@j@ZA>Czx2fUHz^eAkIYVJg5`7iZL{D zaazi~D5X-IlkUt?vivBr#qHL~Ha6UZ`)zi_K|~~>6X?u#9QnV2W2Ar%>zh5`~$@CFwKGZ=$cbsJAVbM)^3R7P^>Z71M{|BNpA9UnI-HCBNQ} zo50}uW?gISckHaNKRIJ@3y77fQbt08FiZ=rqzGx5e=L>>qhva#sQ?LLTsLE^*-iFD z2-zKCVZIY4t9IaP)X{{JQhAkh4=X0RvOg+;fE=_!m0T`K9BU^#ygm@W+S>Qd`Ra&F zLsN6=O%}Ta{pRA^QvE^-k%Eb5T#JbH@o6IMhgMJeVao^<2AD(l>#y!o`bHW$!)eK^ zkX5Ll0|Ms%kbUBD#YD?ndtyP&m@LR?2Mf#y&=tUYghd1lNR2NhWZUu&&1%I3C$T=B zkc*jM$8TA($*C?w%gUPYE{bb-lxl3V3YhyN08fu0J1ygcHfPc5(5dr zvuXggcQ-h5pNA5yFQZp1a++X)A9dW50Ce%@T25e!x`G)R4_7#IvOLY_ZIY9mU?Dp` zmoNGF@JEGj^)t;aWz3(DJ^|FEu+1S;66Gxem_TEt%boJua>db)B|^^mfXmb1CpzF= zMS*7Ore)_ChCJY3gVfYpGjjP{2b!|;r)^UE^zxvC-{U023{!nXCtEFCa7}eX2knE} z1?Kf#C$~IRW!K16O-kC9hWB)w6ou88NrPX_y=W{EP^IncGK?reAMt9B6c{JKQ|y#Qn`n{Wx?5|J<0rkkwo_xLfp zMA}@-Te-hNh3`w{AoM_V6spRuvJLtrBvC-G9v`47KYaLbHB8tFW}ZoFP1BiMWQqg5 zRc;=r2T^vO0C|(=`ZoX8C5*&+=HtV1aVE1LNbS0+U=f1H9M&mzxBp$-TkSj0=}<|k z8?in`1x~cMx726(+nCUYE4$uUP)tKA|9Sr$Z9Z-<^!T@)$<1EY`)^2udo-baqEqz& zA^Tveh`HCDe!kXvf(%LDR#pIfrcwlm zA98-!h^I-%gB@iY0#v#nP}G1{l$GfUild#OeE>6z^iu^Vzfw@`Uj4gNu#W<_9}H5n z5%%c5cZ0XF(tA%2QD47VYt0Zs(6Dg=l{lCYhSzUctZ<=f_UC6kWy(qBqIkr7#uqG^ zutDbu)%@-jA&*1X#s0i8clsZDa2jnQWNp-LIRxwwB}yHTKU)D1xjSJsS( z)uz{Eu~|;*2??#4K;!62WYI74A8P(*#2M)DZK)UD zk-*P*y&kFj2@3&iV=32v812@{%P$6%D|qp-GN2Xx@eU4j`HIs!`YNJ75!jZ(5V!d5 zeV|COq0_}dl9M|?t}T8T7>W91Yc_-k6jBffj#BA5_9KYRPxU$d<WB z$!>UB4Vc?bU(IR}az}V#;VD`VrAjNH$!TuLIN^u-EY<^7{LrwK-Etz^9TP$v1zJa{-W+ zgRzfzFWSGFT%F>K!l`?0Wi)b)Nm;3ObIUEN1oLM&`A@m(P=5sB0T~F%kBlP3)F6ng z&577CDY`F)+Ets@z4hjBJ6%P%A6F7pN>5zKFV~%~sp%Mpc*fJ4pGvtMBtx2kDHsJI z-ptq{BusC78So8AkO+iDSOeU&Hgn_X$X>-%@kjGCpfw5gVxd_a4z3v;I3QXO7dG zIC?*|Y&qMd$8&m8WL7SYwIAI#WC{-2YR9SSjqe!Qy$B8r1PJC=8z4L%Vi0JZ$zqHJ^M)m^DcU`%V ztAx2Zo4Ay?sSHcgdung(FmC8riT<#oSy>H-3fbJo_0}w=h~( zK$XhCo|iF+unq0&O#XV$iHBy@#`Hs;Lo5#!2Z+B)85X_4wC(a zZ&!RiE-&p^T{gM?R2lox(_xk`WE6j%aeyL4tXN=n@z&{m($BxvQMj1G!;UdNj?FE` z69!|nK2A5{jvyc+>d|$Qk-=$a3ZY6(u(bx(l01GpT5-t*A}Vjf;FuUQq_;eAB-^wt zmmbX!@@KV2>uvYt!=VT*FglTKq;Z<6*K{9c*niIP<#X{_BM_hZGt<9evf?W%sp8|~ z+03}OxCZm}`IzLt&^M=7p4V!=L_-Xka^K)x$S|TaW)+u2ARX5-MlLQa67$77^Nt1V~W^nIV9T#Aa7sDd|jD=A~HH;omM>q{cmK74d5p&Z5^IBe0>6Z zA6PS3nH}DGSBHRqh#NOMcYnW(%mou!5k-V;BZPf!TeMqPppovBAfYmk|4cPWWr<9OFY*61gT*^S1=J?*376 zoBAM9f!P;C--V_hQ0-AX5$Dr#y^uFuO;gijd)vBlb14(^Qu-O;i)>|oZyHxLd;xnQ zJSgDQu5v1IjO#4~n;OV%JWN8ogbC2*k%8a*{8=<&0tHqKw6 zGxy{&@;qsJ$BWInuT&EXR=*Bv+gI+1gHIDgW|oH`@dv;m-g_JLp?F8%QqtUF_v3Hh zzHw(-Sj*r%l9twRx@C+W8t1Nc8N4-vRa+@z<51(#KC}1#V4lQ#Zy>K_g=@+h$%Vnw73k{8!$aKHjc!+Wj0eQ)Ns*BcHJ&e6LSR<< zMOu&!>Tf|lE1DGXrbl#VQz8kj$O4)k5`Bi69KK?UhGV`p)7A8a_YssrwS@YF2wk%& zJFwiY+M2fi;aZ)^H)v=JCb7G ziR!S2x(5*u{#-?FQRu_L3)>>HyXgQ~1(1em6vaR@z}=#&n$Ihw&f!Ijd}>#muL9r~ zfvkN|&z3X3PzzjY{;9E!m-i;@@V#&;1KG67g>wfQp5gk-LVil=2`5T8{*?$J`^}yd2KR^k#l`6Z(C1t#a8M677k@PjO+JdU%L0SeK{Z zPkP_UG*)?=yKcDwfsh!fj)IdvvKxLcL~~!~`qBOJ?n2~8`hls+NM!B&OEiTFf{zKF z%!SCDHbS>6PA$!HkT8yaZ{l{*6WZCxHqe;DjEfDkly?b(Wrisr%w2KcuRw34f@WAc z7eEJuXg70Y79D<$SU8Db~rt$15p@Y!Y=APb2 znU_?D<2gv(Sn7DD-cr0*+ukXekJDDxi|kl>)B;&=@!wMzRHL{%ZH+Rkf)#*N`Ri~eD_?#s3N zD+HfpYj&ROkfYrAw0fIHc|Z~XZEW9$3)9{OXI&Y@JG-ROgbuko z(K}zv^Kw~T7(Njh@HohH#98@Yd0mHr+;*D}|KwH=$nrC&id~{~rFHND;SOxTStq9C zlyPPcAD`>;mJyEhMU?0p5pZ|)?~0$aqQ12cVjlTgPgXjVM@eVhFyDPEf;)9Yc79Xz zKR0!VPnl~rWqUDSt&ke%nR0gT#xp)K0yGDVM@W%3i}LqdzMtEJhOUO9FMSb)qkHW`)rpMTPduUyCdXW2+o@9uZ6)tTCg$SB4n4yz$%Ha z6D-=&54_JQWI);(bp9<@e<@gSdU~FFZm#X_g*O$FabCWrwK=oQk&+EX&w!VmIu1t; zM4T47ja19#CFgpt{W8~@$dR~#xJMLTh=Xq?dx94|G z;C@B?Vt|Q*le-?ufX<@H6+Ct=MVk>Nb1ntWx)|T{f!z(b^on~MO8Gx~ktPUPtcfb*q;Pc*_*M_C{q*=7DNC+)ir)uo{7RIF7 zXKWRPg~UkT-fL(10~9ha=SIIV$*rg!udc9Lub;N4c>tJEXre+UG)3PYX+!cfW0>{{ z0UGeaO=X_B_;zif<~`$}tjdh!QX7h7Xpoqn^2LT*Q%=Zg4ttq1_t; zEV^0|R;=@}VodcVUtdW{8IQ(iOVA{Wf*xyM z9TaRI>4}~nt~6?Um*lw+c>!ERo_cn`?$Jb;PJ-R~csPeX6C%!TT`-)}jtz=d#WXq0 zc%SL($G9{HQez;XVCHDSF~X%I8B^Y?#X_ zT?v@=d*i%X1TYl_MnchXOH6M&VM}|A`m^d%C2vALn|HldWZ!5S+|$r{qpJ?o9$%f& zbx|?Vc~?z=N`6*}0q7P0xx@%nVf+W)GEQ~|%_j;nGO!SF0~q~AJb7znJ$#5W`LXKA{z9XCM zl$W9A(p`m^K(QN<=^~{rUny>0jC{?>yu=lpi6+L zeV&oiV{j)Tdi6C2MZj$elT0BqZc1W5JPxj#go*{X(LY|l=VxUc5>CFXC-enRG6C`r zu0DTDr3f7o@d$k{Up6lIcJbhVZA3m+5WqsyreSTMfr0SpJvck?1BSR{{|phK180-4 zG5*IJ^{%gl+NCERG+GTLZqgR$oo?U`2Cc&0<^QJ!3wdM3bM}2OiF5@F(2YkoI| z@5}?3rTCwxL4S+?xH4$#j|qcy^l%vMf=#*dXooJD9GKpUk*CIKjyRw!R1YvD zU4#Df=D|>lV3J!!!(REDU-`I_DuhYPDS}qOqz3>a3ZZa+0EQKN|p}O9r~J#a4(r~C7mK9M7To}^(kanlxM;@02%!g5fK42 z*M&K4t89rTS)KwOXhO@n9;0mNDgz1HzXdnYDn+V8dGhaT zEIJ2%)~f^GuQ(Nvj=qxE!Xiu^oE5+L@B2#<}~Y=oX7flb0mJAka#eRWc~_L=bp2O4k?MULI(rT_7tYmh^Arazx?J~jy> zK`2XxggD|TUw~~(pF5169&f2PA87_N2gB^i$LID;2k1o|^( zZV);3&OA>R%+UmX8-)08L&ypu*}7xfAy!d$@(sA{i4;&p2=^C#%>eq|s8ZM?>z;Lud_yY2(g79&&Rwt= z2(R=U)?K!F^{)+oZZh_w40R9iFsG-l2~aJKH;m~0nHlj-y^edJlQEvd>PUU$7r;56 zSNam&gSbEZY{uQDO4-=FpzgwKI^_bY#5s>55DHb{+ zjS~jNhxanH8{(r*Fh#$d_+FBwkN6AlK@RQ_Y>~g0{bB$rjggv*VAcc4&xKaazBzr0 zH76~vU_!#G*!3$U_kiG*u)G816!rVoeAy zW9=PXS}z+ihB2Kpb#A&69cPm9VYbJ~2rh`;N`FXThiC$uS+&`S*FBAA^v{Y#?uje+b02#vvLSTtda8r8EwOtQ+Yuw4C~= zE8b^lEB#k!#V`XskbC$YyJZ8WZ%YraF^C{zq+oDhV1k)%VFwPa{7FXS#KUH7ZavB7 z4W!Rmj_`DC5{>AkyukcV6>cEXfZt>etbdX=REP#O)CB$-96ZT!?zfVKK=l|CMJQg{ z#Q*qb3s9VkkOPwZdl*0!-S;)-u6mkNsz?m)6(Glm$HcN}8~pRdG>3D)rY2$OKJm{5 zL}po+>svairCyFA-}WzJz_hk{SM<-}F~&mfs$RN#eqZB#9=Tt|4A>>=MN!z3duO&z zYcmYI2|wC{?&E=-4f6f`<}K3ahg4ANGWDzHYkUPie?0VgpoZzXDScz6oVzTS*G9(d z@e>8%KCYr-iURN)6N$!zUj2O<=lSdyN-ATUdKZI2IbRZ`(6T~}o&m9e_N0{4!|NvV z&38S@OiKTp`dKC+>eCo(25;H{U(&5~h8b zbq_Zkzzx3}2^rt!bX0;i{};6QgoL$w?&7mGNC|u%bwCC!w&o}B5G_CW>MEcBPDTAP zYTFv+P?o+|RnbL+uKFf7Xcc5{po3a8l<#aOSCvCW2r2qDPkenM&u<3hqi{10*u(!a zTC7fWL0pX8uBkSXE4IWGqX_)$1~{y++WK*rcwsjD;^OoF8QA%n$I_#DijLV?o6#2x zyuw7`mB;^+M)VmbN3Qhj4zZ$dkyJZqG#gpZI0eVQo`(3W2-2qyR(Wi@h9P^FkMz(V z$lO5_`Mv$&&z@?LzK!(fwZ+Elh{4^1U55@`807Jho*Dd95})6m>+PCrUym4s-;h7F zalf|_fBugAt^UV*{`0rZn}31`=XQCp{8TjZ?YP0B+xSP~rJ+7c{#%X!UpgRG{+}Cx zxE+@$`0S-L4$I%vTImZ@NGN|puFc=+yj+ANN&Fx7aZ8UFgasDAhO(3rSU!kX6 z+Vk%Bi;1!-3uv%HqMo<=d}fuUPbB@s19T2eh|1bUIV#TW^|5__c_1Ae{WUk zbxR(=PD`hx|C(7p-I5VTdHlEM^NjqJR;GF$ zj{0+>|KIodpDXnA3yrYERbHwaH*UE18HkyhvJ|SDLDO#eFXA>fN-V)l=#r_BmbxFn#KCc;=ciAoo8c~r zWzQcmg+kVQ#Bbec1e}UrPC&sr0GY)Sl^N(b2(ccV?A@JRo0k<=6t}b_%*d35Vj)lM zT#K&LzP>)-MSc(`NDwnUGm;%!!pzG0z`=n$AyFT6*-{}XDQSFKT3V;A*asA;)>fP1 zqiK`OBB&~Z(s*q{CvkhUXEW1Frh6p={hmP1Ui^8+%^#s=D3mPf3%Qknp|#B1?EV{YD|2o0m{lP5xp{MJ$+QG&`E$upBxPjY zK2kbs4xiiI;<|Y8;tf_-5$9BU4=>Lm`-$)09|75E-;v$0)=8yr>vK_8R~j5IU1SJg zzv=;y?6DYd@-{Z}9G)t!xsGQ+EwzI~*-zsl2qvfV1g=liij41}X$p?kEzN`St2O*MhyX)e@bJ z#p|>Tyu9dreM${mF(^77#4<$GBu7Iu%Ql@)@`2#So_|H#tTC> z>-Fo`9FE&Bk%1Vf-{N+752M3__@lsOA|l1dkKg|Ojmf{+l4xVj+cNee>g0IqHNdYg zL;IJmF2$DCR*?zUlUGM(TZH5!zg9|IPaKi`t`}VDF!pH#94E`MKSsJ8Z}178E-;8o zNW_A zyW+IF(llamh9Tgz`&!Bixf_3(4}{m4%*$1t>}mk|gWv%nQwaIqn41tf4%uLaYxVQK zjL^Y6S86{$KLy~0bAl2WBDV5UwIj&t>37wu1c!tKgomp}1NkI?3dzkY7Y)MDDo0N$ zTjSqhY@F=*TTa%ZFfuX4XRX}rHRdv}+Ofr{tTEFW!3m+f$1grLYYQACyIyyq8CtOgpl2ftfsOj9A%;s{ov1bTy1AiZEESa_P zW%F-e!g{mP(rQiowHwo)q!Lh4dS_)lL6%;YI*hYw9}OG(_oT|t(7L&y`}#`RnB(w6 z-RRF~QlgW^BAa*tCtLj;_L|AT{KT}ZtZ#$)Gq2WN_IONmx~6I|XkFcRmQMlFduMa= zbyt;>9smB%3&@s=*|9t~+1To*8~yVgdLBZKv=&;V~Ir<_7X`+j2U3F{T?amOuO z=(GfgOO&CZAu?`LvbC~#?X=D|o7S=1#ueMBZ@K+0)c5jq$`4W~lUOyl%S8;2Bz9Et&Xe9$^>qRN({r&?pCOFwz!(4j{~Pw$QO zSdmo{mx*g|RzF3nd5M%-s@$V3^J)>tHJvll*Jl&18q8W;CSHXm1F;{+%tK`Ql)8z6 z&XgSY<1wO}tt5mj&dS$@?{`l(PDoxUkor1e_E^d?c)RvwPZD47RqXTZ{Ch_(TOBIh zMFZ_doAyV0hLS2Bk#yPWMfC^6jys`YNg8uwH?T_beng^D1SwGRZF1)dI!#(U) z=}!;o9?)^Q`D^yMG@PzrgN|snU(XG>OeXyNUN6o?nSaoBGmC#-poW}8R-Iz;^1}@B zZrG`#{Xa&k*UGnS7oi5>V@75sV@1_&RpZRrUijU!U5)4wii%UKdChAsCp%}+Bfx_# zuvl&~FZsapqWSXts?Lw&RVkyXdX=QFUwh66onpKQoku?8R2c+rop{4E?5>Tmc0}IS z;*8P+!}&!;hU0Er^UI5=G+|fxkK?v17GMxbLn=Q$er7x9%08)FT;3VD z^62cine*{f#QwAFe95zu^@{F3m5vXPtQlRXzT&S|zW0SM+3j?OeBLRhBZ~V`fq|Ue z!SQuFSfazZsN5t8S_f6$2#5Nz)l}M=FyvgC4HwoWuOr|3T8F(0|5izAk~dAIdEX}RIAFG8MkL?uz?)I4-N zDe~+X-*Y=V5=Cx_&Vj|np5*Q^yD_?|-^VqhAnB_6a}FMq&#&<%pd;ZYoD5jO=_|F; z03x>A&t`9A>D1&jHbawX7L8(@nI?H*2uPzKAVwR0fBDu5uzmoJ;*-&Ab@R)+(|Z%H zjX)3)6Pg{Ys2I-w;+E^ts>)OH0M?~>6EA%Eh9s#lx-^oYeHJZXd%tw4bfI0##Z&an zvPCK3ix)3c?Z%g%f_WgG*fqIWBk<0jt$gsHR1sRzXufVu2F|7H85v7*xR%w|gibs} z-k~Gg0~R%7W7dwE5XsYp44&c52OQCdaEqHr0U0Ep^YZsx3^a{&uj$Z7e7>VG*Ba$f@)o*5W_x~5OSY<+ujv^TRfR1gysXS+O3%^(p_ z-<>RdRqCS#yjoXc^W+(gTfN%NG=*E1V>Sa~AKjrH^xg9HU9%DBq#3lFQ!##^ZD_Af z;kdiv36nVsZ6)I2x#vnnPQF-rxOD$#tb=2Q z$s??X0Tcl+m%_vk%DmRBkaHKm*pvG$4;ZmE8x2c_&Kdmnr=_EFd!TB$J^VfqpRK8c zw)Pj~3j$c;c)5&J^VI7-1+)(;w`o4K32-xKXJ&2y<`&Dw!bRZo!BF6{nSK$b<;9ih z_GZY`R=TDi67#rV6#oRh#XF*Oiv$$iifu)kE!aLp%|SB@T?{)+igI80FD2$*b~)O$ z85+;kxrMmOAfVOM~HAjsj}D%@2yW@csINeglap}^?_VkDm;u`+QEqg z*HgfouS}HSyBwBQrvOwO%*j<}+19ZiBj8?IuWRq2zIl}G#VB<*n zCilKVCq0Gqr=78CYp&J0zbDIs%wl3HvdYWLz?J0dpyA zv%_8+qnnD>lg|y9Kq$Ni%Yk?Gbbt7x^g~D9%a^CM=7OO0vkM^X9y)p*$=T{-tb9$ULLfT zgw)*j%23ZQv21`Fm6(qKFXs5P>*@5$`jZLAaiUm37Yd+ZFkhTEL&u|dYIrJ>{P>cm zsC&Bae%O~_yejRWPOb9N^T~oOIDOnsk9=y0h0wh0*RAxYx56`)t5b9CP*~3X){Aa$ z_ZTzC?D^t4x3^Wl$A0qoiK3$7(RfPYEm&pX5COiTi(OM`J!=-ZtYtteO~n+FzGjl4 zV|L7Y#3KSiAO%`g@5t$;Jx-5wK7%6t!UJJ@|3RGPII0LE3W8Eni~d-s%W{zxS_)*a5(EEA;!&IHB2i zB{|?)6x7sw!GDp7v?kA~nR+JuKHdxt->v(h0&oM+0Z`qcVVX| z^mii>$lkTru3g+4b&B0;QgCp%-)>;Vkjpwz%@)ww+S&-6$Wmc&ghX=RjxmK0;B2V|K_6TTb0xK)|(J(M(b`Q3+a>OJY&5C+6bE_}Bt8*V6 zMT)$sHK#S3GOvplJr|tj(9aGM5)$*xmOo$F+ad*b@8{yS%^Yr!;JSb%NQjja=Q=Ok z<*8TqmQJ(;bo4}APu8Uq5`mo&8n1td9#z|QtzM{ob8-XhzLsEyrcV1gI` zHiDqG-Hn;+lT+Tbmeq7)1%^D@4hPp%S+4LO4(cEf2GVA&X70snF}d#U?%biXjs?ty z?X>_Eh#^`y+h|?~x|uTs!u*RJHlp_RoMFxHoVeZgJ;PjCtD^&dtK6GrF~tZogr6S^ z92@K_Zb!JATW2*@M|dx%S7s2WxCFZ&1zx;!r{uZe<5b+4k?mAyVd7nE)2Fr9q-fH| zCgH8VJnmO&se~q?xI50+?%}Z)RNESikixuhcyyfHy8gwfbiVAm-H)*?{+r<(R?6AT zy8Ta68DotSlahAJQ%yCkh@Dq&)z^Q~UC5it!hS+(NNPGp@hz)9CNlEv&T{|sQ7Chs z%`*4P<%M2L&BDkX%XC=@TH5!qzv(z}7A?L-#Ao_<=g6kl2@zH4Fp zfKplWF^&r=YG^Z;O%Sb6eE=oh@8u!?`|K|WKYdQ9)MTmX$bGGy!JBjckxx+b#!U0n z++62U*4;aI!O)lPDvo*n$S0v*6iMz!$&=L_$+a9_Vm$u2JQF3O3boY0QN=aS%-quD(jP=XUiIS8mUCPpuqwqz4 zx}V9k-uG5FAvZzEiDoFH{P>dxq-ZanVA_5+3gQQ+Wkds%QM~lOP}cr3yU`T zqRSS>t1#|S;h|kX*7T40Erl8OyR;et@SSn4C zW4t`&tyK}1lHwJd_^^*c!(m+MOe>ooJ+xW8&+>cR)ch^$&Dl$uH8k((#Jm!oq;_Vl zJ04)R=R2mO_F&+^HZC@c(VttYBaRGHtRqisLjqY-$u zF?|v9ku-|rTn$Q6knFEBu3Ob%JZoC1E=G4LEuDC1JRIP{z98yiVd<*L5`S#4l>^2U{7H&*pR;UJo7nTn+JbYF5#!@8Ch z%vAZ2K|RH(Jl~^k~R#sNZOE_ zPitz8T_wf}jtISkXumBZ7~AYOv%z@Ald-QoUTXK?*F@X=MpE_6hW+h3cg#POrm(QA zTyh;OB7%TR9Tb)+8uD9de^-6}Xp@JMOH$M7V|X~Yy)slQ3`nf4 zt@G@mgh4N<&&b$VR7ItvbL(-RWrviaA|bL02^c1v_r@-(s;c*w1ysLz^So3q)bnDb z)7WDYr=!n5fvfJ(gQL`x96VH1)ZfN02i^w+5c@fWkx^16#>ELQp?}?s94wu7eqy8f zM2U&`Sx*x#1RcNeOmx00D6ondAtR@h9(#g71cm8rR4Kmu#&uojq!1&BM(|r!kWjE= zdMXoJP7d(D>-NXq=H`B=r$;E>H}}ce<*ffW(Em=UCL0wM)rZK#!;&ej9E}~tC$Ykc zdUWCmUwM!4J_VS3KRsQR*Q|O?t}jI9U43#mKyIPA;hu1~e#=T5kOOJ=Z~pvVLQEYK4N(n(Sb`sZ(vlx!0#_KDgVsk9UejGOGi!BD z?3DXhBOv(2#q?HC=#$31&88}5m`q(WGgce^@87>eHUuZtPtPJF&?MuTc2)hHEalYf z?0bv(jxdvJ0+c2uFO7_(PdduV$|koQ+8xrDO6TaAIf+b^bKh21(_CR!!*X$PDJ$0R z-0-{aSyWi4tFO=8ck26q9>Txe6*G@%``jqU`^uBCM@|_^ip@s)st6hPNXW?}9yi3!%*^QN>LLpKYzUrG z4`@;eI%9%+Quj?Ngph1dVr>-*Yk%@JS40s}bWF6OHrvBz&*+)i1nFp~0*CpyNT2Ua zc-a-1_odi_G z>*^Y;G$K+j+N>Qdn`ohOgfJ=n&TKkV(*VPz=bJYu(OQ}a$u9_dr;?@)N*-0xqJek0 zBA{7TN93Z)RXsN1M3sQYQ~Sk!iQ29_7d6W86x ztWx{ybwnS^L}Pg}4oFmZXutSOwW}+nSZZe^g2I15ZUGERl!;q5k3bE}LR=l}h=9sU zP2};9!0twTRH|U0uZMtVgY(?-ymxpw8KaWg(7juCE-x-F{@z+0(zTS0QDtcYP2uC` zS3fY}j4dzE9%1=)xJCk>mY$s*e|5OTP|zd?)XQ_~C=maNm74E&`&=mjf*av|?Ed_8 zs<)ST|M1`@)n2h9sKaN^o`s(sS#Q57F6Lg2R0S>E-$D6uq8fFJ07R5*7rW4UbIeG| z#>VwCu>o~gY~<@bP1etL!fzF&;ISCj6=c43tN!FDJ}bTfjv-#>m-_*3L1*tkiR%yM zP2JId9y0@EHulF09K%WxpA(_$$Iq^=#()pY?S{{SF?lRJ;><>h@2FUqufYq4tK>px zwXK}nm2~;@osS_Qu+cj!rOXY?@t{U$w9@h&iJibbGIVsWmm?`=Aj%%tt&KJTX93|) zK7tSvaz!dqO8@yEiQ#7@n=dbNa&mf%mUjA8xSsH?tgc32sAU8`X&T`XX{7uFo1(~M zK={cMBCv17thYh6t*y~azAlrLmTiOr-bvb@l*PrxH(+p~ZDGHc`k0*$`l$L9I&MBm zdDi-Ioq)KINq%15^}#ZIzBzVPOxjpEgnN90*UZ`&pK6IM*TeTz7X@R3phw2lJb9hc ziVDsn1yBZ$!2;;Ztd;XlQhFbbMweHUj8z>U7*r-^fOlB)pJr)nu4y zmak3l+(HT~B*FO2^qb%)U^cxcZ&0s88jTUEjP>P3nZX}F`la`z&~Y+{M6dyr#%wh_ zo&uY+QzDE_Yif1{$HvBn-*KOyu5RGA>ENsW?}=jGIE`a`CHc0i6z?gxC@x-frsB0G zKtZYM`rfl8d*+K9sH5QiLYg!&%nhMFT;h=@;}gYWKG~Hh5iaP$%PE6uU$XR>C3IEm z*N5YUu&$QQRWZ~Gt#X#F` zf(<%YswW^w#)C&6jOj)t=z&Mgn_yBh5Q7Tae986fglDwEj`Rcr$0H?$X*s<3rQQWf zN~z|JASYWf4Gj#)M&7=ATO2l4w4e*t`1p7`J-P5!w#E)}BJSPiYZ&XGRv%XP!NhO& z#Gap;A0UdUu-2P)+}V_9+0iu}E~S}il68lpPU}V`3Ia$qv*bJM4)8Yt80+Jp8xkE2 z#A&b6($n*Kdq5Ao$KfE!r~?Hb)2~j*$}No?M~}e-l#Qa=!$CO?d=_whrKr! z78Z23!}Hs1k)szCTb}|Ls;9R%#&Drg18<6Yr1~qH5Sc_GDra1LT0wCT{CWoFf$z7i zqwiT>;N=~Lpe9y9K+YE$D5`v4<>dA@Fo#W3Q}fm8k`j{NsxvxwVmo&4;qlHuY)ePm z0~j#U++<{AocvaERzE&Y1F<@T>G7g+B`pZ?Sh>+%_KhI5N7N1tc`ni(?Co139fNn? zr@DvADRNP7UI18jK5+H-*0L|ah|w@~8diSPxEq8v8iQW^#P3GqNj>row0*!A0jKGq z0?fIJr5d`L(?e_!Ir(-IuC04ZOW@32;WpO{ns7TJ0TCs(aH1goP_SY8nzT-8;`F^2 zufBix_+{fwsHV;D7TDm8-?x7^1qu#s-0}_{*WM*Bmi+_1GLU zA&c5DRABfa@*@rdgZ?%ijz@{zGWro8g~sD&^l#tgLNA_65K!x6V_P5^#9+GP)dZw< zll7k1EuG&z(j)Fe!tAP*WN@Wr#{*4M7T1#<(uj$j)e(GGfom2ekRQA3jZZDE@Ds1U zQXA?-?>WugO-R&nZ|f6J?TndPr@h+Onh9nLJu4T9&}nH!)4E}M_$59kCq(U!qrEAx zxR3Lk@RvnIwawY&Vg)_-_w5%J7wZOo;6t~8;78JMf4mPahvx`S@UBU{DpB4d9&|;eVirNr0(XX&w{G8^_AYc#&#G zu6q{tK7DCyRNs(D;<$(R>A=8ZzbCcScJ-EhrSst6;IrQ!=<*5+jq_>}^MNIS0H20m z*x(`W_V)G^)V&{{pg!-EJW6 zfeJWpFEuuA*?X*gfm&3u=lT`j+crzjWnwuXyM4f8Ya5w-eC*sF&4CLQ`o`TcgUl4=<5~T!nq{m%Oa139Lm3L^|KRb>32!Q)qrqg~SvjtgMQA|1y zf)1HLWhIl9l!pY*sM96>H~f@?HEI zpYi+qK-;ij(GjkP+EJHjg1iBn)6|LXAW`)`@F56zeM7NdUxb209UX@$#|%wPaiDhw z$#@RO=I~P=ae8ra4+zL5!5WT>z^<{sLfADpq)?XhO&sExm0d0dNvV+7mYZ6oV91qi z%~W9z4v85X9J;z52Zs+XW_F(F?}fZ3RD@Ce`t{4gH1?u1&?Lv+tklwd^RiQ=bw(3uuRz1>chNYo=`}J2?M6hq8Kk1w1 z87|?RadX%^ky(X>cadM`;K*Hy`TRNMN@Znbd~R;;+vq-7aL9yC_fZ)h2EO%r8xv`P z{i01?M~C(um~(F?+!FoOH{~=+%yDzHs}epI^7He9DV6@QaMtp7EsFJ8_eI;?U04p; zfUm!JcHiN@T3aDc9)Epp)@)-w)*ES@+>&bR?8K9k>qL%r{zvRP1y!%{D%6QV!6=Fs zZ59xg1!59t>|H{6RmI~jeu>3IJq#D-*4OtYyk(uDl(CI}pz9mHl6AjifRJu3sLA z3o~A3;N%Q(7&bA|Q(Jb>fCB@sy|A=YU*CP%*f=*<1CLUWs;Q|7;l90n_xXz#tul!( zq%p|I$Zns_|C(oD6`(HX<;^Q9K=ppI_rg-$_!gj^d>W-ce?Qiaxch4;(B`<{Yi-*P zZt?ETXB6M@dQ9zw-hTdvpX?D3V?e zny!CqlWFMfeP1Srrme$?^07FWxmnCOU-doc`g)X_s!*41a&5qJoS%G)Hizb~OyB7Az!5`w zQxJe@C@U}wj*5D>vSNz#AT~ZG%e>Pf=dn1w5Y6jw`N0Fvmha!azI&FrBlejT3Gh>IbX`>-LH1V4nFz^uwKni?vcuPu5 ztlKsxYrnHZ0*`^GVG|KOIbQFP*E2C`##z)Ze919vx8M%J5NTX?*l%dDKN{9+Xg!Jp z@h(r#$N_F5-Ic(g!r0B>LWvRL~_q3!;0@SHM({5 z(f6$=oCRqY7mZ%o?bRM1U<048@5#~b+qPfy92|rU58r>om}z3JiXC1OGy8S@ioXdf zGXjpLdqDvw*!VC(M%I}sEK?)Uono(~BRdi6DKWg?bXq8*PXbd2`kL6HSW`DSS z(?b*m3k&PERe$rsPys1IYW^2eQ!S>((#3t(;3p?1SIu`Ohd`zuyy+BV@(Hjm`n-4c zTJbw*JltKysG;EHa)U@ij%GCuAD;%MrzphCWn#Fg1qEXel3vyrXlRoSK}^#|s$`Jz zXdM>3ex1_D$SAx{M>sz(iAh#+wBug*H&e*E~OqB2u` zaLqM~tkUxGdcB0ic7Akl2Q6w&vBbp0UYeL7cNH`qXaIv6MY6LQ?By)&0z8-mnaEI( z!^p@F4mCH_dCGc$Hx7zr25OCOa|DkPZyoRhLIYdCv3I(FVjK2&MSK6WK3#xQ&L z`7<{d9zTKO$@yfbq+OtQh|};lpg{E?WSn(!d@!#^Omao2S4?&nVexCU7W-V_k4u)J}LjjgGwV(+%YcjEN*PqI(GtlkPbTk12(cRAut z%QOLE#!KMNH!XDO|6bp4d$l^u<s&rO8-^K4TfeW>X|Xr0+8m znKNu8^VB9GWJ-pNnZq^@nM>v&WXup-<|*UW4sq_cwa#yyb^bnU9jiZDtF`jJ&wD@j zbKTc<-S=}DmSkq7=)UPAm%B}o-JPIzk2NZ_Vo{+6q03|!hIksEVS1xG36?EafY+3E#z$1=1j`Bc&VS4az*)KVq-6((HB7& zlsX-wrtxJB2p7L45z9tEJFs0zoN+`gKi+{I1+MAxt+FOt=F&ew5qSFT#|fxa3P z&VnvQx7vTJ3!YxDZ!3oZZjOl9LM1NlHhk%(i(U;@GFvqT1_2$c)DgTk2r`06vO!^q zmDiFTBu7Uny)JT#G{BBPcHTA)h(rbIg(f(M1O*FGzG|07u1HB;GN`l<$jUkks)Xy> zgrAG8j@fwSTHj!Q-gVh&fU=)IhY@#8eEK!bYoW?B*@?d%kG4aD{2ME%_YyqOC&tFc zV1;bs-y{N}r`>M&&4iH0F8q9@sWr>;YI8gedLHiOpTLe56(5iC+8>jOqO!IE`#3Zq z-0s_D0a>jH&@13$BhoLTxeMu$xjC}XQm#LqL2m)vjdvZmbaEaZ9@0KVd;JIH=p1$d z42?G{TbB+VFIMT>+m^!SX4W}qeg2v;JE|kmH%_oCrT6pD=dc({u$0`eoA9lwk|2>t zC{#F$R0!?}6Mll4Da}r*gV@zVy-;H2bq0&YPA@L%nmtVcN)Pr$IsOJQu+*Bj)DRBy z@1|z-$#81hH({(#g#kS)e8 zF)&MPfRlU{_hen@tsA?$w+t;SslI2I4z;^FN+a&()}8WO(&^jTCz$gA0RgItdR%G( z0=c5}U~mdJ(P5&QOH@=ZI|I-y z-~5Ty$a^9W2pGtenBW2=ek1HG`f@uy(dq}U$tlMg{H@nCzdV@6M;8ahDo0`Yo6= z3jCs&?j$536C0aO&(MLP&op2QEpQr#avD2Xek?mXn`G1T0L1SHHQrQk7^fbrYgjJ6 z1HJ}sf$;ZoTffZ1lhe8b>7RokM%l&qYGQW!xQEC4z~tmir#{eI*TyQEWRIAeAP2>B z$oTpBpioCzdM_2f{BAd3%Zq-G#}^btf#>g_t`0rGnVAOi%XRTN;O6{K(%b6foS511dFYy#!oyUJ48g@xfA8rfjByp;A+-d0TnKt!zes)=G}cJ88UVBkr_OM&W$ z-ZVNqtis9uRUnOdlQu<58~)hE#iBwdh{3oS`3AKWNCyhG{1_{3@Zn*D_P zJU9QfVmeW=ijBoDO*W2_@zC zcDVlGuo(m$ma3+C5D1 zpCHam+Pjv)2~Tv)(9rNf)nnH6`kykW4x&Cd`JxgMLIww4d#>jE*kcc^19z=m*lnAm zzYr>>y1J^S^w0sme%gJMOlog}f5DzK2MDtONdlUmAD_3>81nY^-k6D~E%6*uabBM! zw;QVrsIFE>9;Z4^hMg#8<>KPfEj7O|a1#>|@+>%*3fy*Yssm6R2Jt{&C@H;&OQ0xa zNNuA)<^Dp$!TDt^uP2BM1+@^E04ypn9R&+`LJ%eIhO<224(rdqZQg76Y2jQaq2& zX{%Z!YSkOUDP|EEz*olRu_EuX6{|JpK%c7pkyFhV0%he^CWNql`L#RuVw@-=rC$yf z?e4G7&xcd|T!&#Ws`JdzL^+8Zn>_lpB z5I`D87!3PSMKYQ>N_tBxD;oQX38!jQdzSr8Td6`QQ*mX9iOXf}o+ATEx%+SaJJakCY0i8O zEfm3QT?D$VdP%2NX~srLP!0~y6(>8E-TWV@w*>`Z>6be`NsE26iVME2vLFB}0=AAB zP`V}DH&9R_5T>W$0**uhF=Zz$*|lHvFy<&;94BKNGuhwI04|xXB)8_Su0*k150{uJ zDn9P4A_v&AIXPYS%wkm!7MGNJ<_tB2gSP+9ogGs(I#Z1)lF83Ypc1hcd|3V2~UQ!%VM^U^E-b_ zKRq}Pt&G00u}sK(!k_uw1Gy5nX1f7O*|{)O6r{UGhNt7;=t%x8oUb{TvIT4b$kuau^cArvBO)UUPV3B99IOtIwm$NxiiU!3$P0P-c zxkK#s-zy@tiZuE|%lLZ=iMk`0VxO91zfXRvg`@UViSFr#4`sW-P#89_ z2WZ@MCeQ^$XOiorH}^eF?ZDDfVF=02IyT_k^C~pphY_r7zy>-Q{>iazb5F7kjc8~J z2Z?NPe8EN7qSvDl(!^U#ut>U_AA{Q?&|<-z78aJJheB5cPDHesadV1`iXLazo5)$_ zoclL_lUE`lJwly_L~?$n%YR4Hob%bUN|U1oAF;xZUSP0${`^53iHfb1$qI$i5OD&I zr%NnO*(FBZcPo!3QTxW{z1ggPernwP)uEvc{d?USH{=pRyKUCk*saHT$I&c?hVVV! zf-Q+R8?7+^%5I79v3vCOt0WNpej2aD3Xov>tVP3#ZO|h?We?q5lTs46qamrfUp87c z=Q&gFnWCnunu(+syt>g}U(W33<^qPuKalRYU$M-m%Gm$9!=?><6mN5K+MMwhE?!ir zzyTu!hFMde*4$0{NKq*35T9nNI_ZowD;?rr0pE0A=KwYI_q+;O*mQ?pGt3T=xJeYB zDjico&s!gvG0s0=8qLeJ-e3RFhDWEsw~P~d_$eoPwvh@t0SXEPjOyGQ$9`y6 z47DG-siQ;3b*d7FGB&fNhDd)M8w)JwZC)8bH^7fSudY7PuxArK_)I1l?GgyPQ!ptm zWkHRTp+mKd7dB>dYw&9B_%Z zl)7-lV=e9B;m5A7jai}RIG94SDvMS;+nbumL7Kt)@F5@Ka#7&d2>cd+!B-579A_vY zhBT&sGxX~M#WR_Fl;eM6W4cbLBI~P|#MY2)L##&7w zruj0xN=WFNBU>A5VPLG{GRTdcZHI1Bu0h1D2KOdf#u)2oD za>|F&guU!5dn9!5dn6TvjI^u*@H4=~=PYXQ=X}mIvoS>={sZuLJrFzxe}IR_d!Tvk z=*CWwlgqomw+~|y(1(s6T9}6%FJb+vTN)W#?&-O=IJYiDy#1{#h6nf|6PP0XwZ=l^ zBpoXm)Elq=iX+VB$GMXTGOnu!X~SlOfn3IwPprQE*2CgLTmKTD+VIMHbgFPNaDbr+ z?!Eb^W9m+OB6Q&B0H~*j>i(_j)n*Wu3A3}$eR=tgXfL2o*WyUWKsez^N=kN=71f+S zRm955TE%wpl#sg6;?hcE|EE_T8O3?vK$!s&tHyQZB;Q@W+1YslX>$f{MX4P;a7Ar& z3GFw|%$coHuqg>(2THTZU%q^~(QzcxLRag0SzP=a1mi7vdGg@kV9ILNmy%8mmQ4Q9 zSsDvHRlrz4Z}}VXf*2$xMCr({Dr)}33(vaOwR^0Dkar}+mWvylJN}5 zfxba3s%4LG;W2!1W=pSH7RTQU0fvc;jSZA~@6b3>#4|w8b2lhl2KJCV3_12B?FpwM z@x7H!skIJo16PT5YH7o!O3TQg77}uZIk06ImX{~m(4l%NTlv^(x` zzeraC@3bHYz#lF;xaE#(rUcqs`sofO)Kq3JzSY&W{!RZhMSVDu11jfjTG}xH^|?N$ zj*|f?#f*Srus8G#_um1j3bI4`qave!H=!z$@c91F-P7|^XJ^RJ;Sxesk6r-1G-7e_ z9Ng}AERgmREl~RaJlIz|7~k1i5kiF;@SEqqj*cV#K;R(K@3z=1qh8-DlkxcHdEU<} z`3FhN$+NuqUqOapVnQ6_yuZdL&qATC3(1-=H%GJ0Ta>=Cy!f%}?%sSSkXS}Wlif*} zm>TCF&*1AlsC7LH_#fdhY{iUG$QVR^egVgY9+q;eFJv&Q|Bi~vDX9A29ODD!C|=y* zSWk*nXf)5goBT{>tuNmQJ8S_#_d?3e9$Laq1D|{Cx>(Dz0O~hqD=I`Q;P?Ob%o0}Q zdC_8;B<(22Tc8ly;GbQC2Kx#v_`9ln7H$AbLm@AJrK+adWTy4A3v1gk@{@$EtWYG~ z*$SI@^y5=>Jl^N$CgJ%7EE?8ke*EC5KpXn(YShcX(d80z_WrL*1NuEpeE+@n#P7?? zt*TM~bPNoH4snzD2G*eAXfVN=ckfMfPcVb52d#iPk%`gdI2;ZL&!h?l`%KvU}ZhU za5t7edbC7eYozd-vH1?Q{RCy>#`OC2)UHI!p*mn8@K1#M#3LnnijxZjMDuC7ih4mJ zqtKhyMBhN#%GZw&;mETzvlAGKnk5j}x7RS|&gsmXxNreD_ug!YlX!Z- zHt_++Hat8WFScT)t0|dkhs73Sf1OH_b)yBL;WoICuE7xY%&hFvY|iCxQfI(agH+&I zTwISG9Rr$fE9QEGG-%fNNukP>$cvEdG&D5jwo|OoMrX!kTwGRWR+ziv%#~~bm9oUE z_;!_i*h$5u7b(GYgPkm;(Gh$o2fm^gowuPjXw2-;$YdhR?E!~oGTQA9_)N5Ovg@q1 zKj&W_@8IQu=SGCcfa_itQ$fH(BACeQ&^G)?*4No5QtK&rW;7oGg$z>+pzcj85WpzP za?rGVZP$H*)3C62b`lYJlN@|wc=$}ad$jlWW@QPl-P6FoQ$iUGbp7ec$s(y$F;UTC z?_Vm+vWIlY`%ow& zV7qzs49H3+SH!WVJ&J&*;F0~wJIc?**m)CwU0GXQC9FzQqtR&HlJ8#6Cn#^}eq}_G z2gsL;lNl8G8KvAnF*I-D4w2=E_qaNzR%T;#cC;}2$rDG%sG5<2y51XoxZmv64g0x9 zZ00dj>FZ1*dfL9`v(GXz7$?5j9u;t4lS*NGm_2${#{$^IuVqT0xicnzF~%B|c*72* z1J^!(NErC4XYHz8s7nhDLDhU=4_OYiZ$XT$CbRFpz@1~x;4}FGwkM_>^<>0RRjtjz zUBm*!?av>REqgSVvI1ur`ZWuTh?r(m`dEI$N%-HK9UUE^w3fMHKX$qn4K_|J@Mp!I zLqESVmHUg0uyLr3LT~+MZ%KxPGpV<=M*Tjd*dW?7<2V(Y)%0Jv8}quk?FBCg+nDJ8-fHi3QwR7 z7{}Dq$Yi-a0KU?Hn(7@jOzT7ep7h^$FS?`dfQWajbLxZ_A2kW$^#GMdg2ZTK<1L fGXL`dVvfjjtO$Hp=#zp_@T077?{=Y_@w5K|1)2TF literal 0 HcmV?d00001 diff --git a/docs/results.md b/docs/results.md new file mode 100644 index 0000000..ebb3718 --- /dev/null +++ b/docs/results.md @@ -0,0 +1,96 @@ +# Results + + + +> **Status: time and energy results are pending.** This file was generated from a run that checked the pipeline end to end. Operation counts are exact and shown below; time, energy, correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux machine ([how](bare-metal.md)). The run's raw data is not in the repository, because it holds those timings; the harness command below reproduces the counts. + +## How these results were produced + +- Harness: greenbench 2.0.0, profile `study`, 30 measured runs per configuration after 3 warm-up runs, in a random order (order seed 1234567). +- Inputs: sizes 250, 500, 1,000, 2,000; orders random, sorted, reversed, nearly sorted; xorshift32 seed 2463534242. Each process sorts for at least 200 ms. +- Machine: AMD Ryzen 7 6800H with Radeon Graphics; Linux 6.18.33.2-microsoft-standard-WSL2 x86_64; 16 CPUs online; scaling governor not reported; CPU pinning off. +- Workers: C: gcc 14.2.0; Python: 3.12.14; Node.js: v24.21.0. +- Energy: unavailable. Reason: no RAPL package zone (intel-rapl:N named package-N) under /sys/class/powercap; this CPU, kernel or virtual machine does not expose RAPL. +- Run: 5,760 measured worker processes, started 2026-09-25T22:37:21Z, finished 2026-09-25T23:00:54Z; label: pipeline check in Docker Desktop; timings withheld. +- Harness command: `greenbench run --profile study --energy auto --label 'pipeline check in Docker Desktop; timings withheld' --out results/study-docker` +- Report command: `PYTHONPATH=analysis python3 -m greenbench_analysis report --runs results/study-docker/runs.csv --out docs/results.md --img-dir docs/img --withhold-timings` + +## Operation counts + +Comparisons between elements and writes into the array, for one sort. These are exact counts, not measurements, so machine load does not affect them. C, Python, Node.js reported identical counts and identical input hashes for every configuration; the harness checks this during the run and the report checks it again. + +![Comparisons per sort against input size, by input order](img/operations.png) + +| Algorithm | Order | Size | Comparisons | Writes | +| :--- | :--- | ---: | ---: | ---: | +| bubble | random | 250 | 31,125 | 31,196 | +| bubble | random | 500 | 124,750 | 119,688 | +| bubble | random | 1,000 | 499,500 | 496,926 | +| bubble | random | 2,000 | 1,999,000 | 1,988,792 | +| bubble | sorted | 250 | 31,125 | 0 | +| bubble | sorted | 500 | 124,750 | 0 | +| bubble | sorted | 1,000 | 499,500 | 0 | +| bubble | sorted | 2,000 | 1,999,000 | 0 | +| bubble | reversed | 250 | 31,125 | 62,250 | +| bubble | reversed | 500 | 124,750 | 249,500 | +| bubble | reversed | 1,000 | 499,500 | 999,000 | +| bubble | reversed | 2,000 | 1,999,000 | 3,998,000 | +| bubble | nearly sorted | 250 | 31,125 | 4,056 | +| bubble | nearly sorted | 500 | 124,750 | 14,926 | +| bubble | nearly sorted | 1,000 | 499,500 | 59,780 | +| bubble | nearly sorted | 2,000 | 1,999,000 | 232,556 | +| insertion | random | 250 | 15,844 | 15,838 | +| insertion | random | 500 | 60,338 | 60,339 | +| insertion | random | 1,000 | 249,457 | 249,457 | +| insertion | random | 2,000 | 996,390 | 996,389 | +| insertion | sorted | 250 | 249 | 0 | +| insertion | sorted | 500 | 499 | 0 | +| insertion | sorted | 1,000 | 999 | 0 | +| insertion | sorted | 2,000 | 1,999 | 0 | +| insertion | reversed | 250 | 31,125 | 31,374 | +| insertion | reversed | 500 | 124,750 | 125,249 | +| insertion | reversed | 1,000 | 499,500 | 500,499 | +| insertion | reversed | 2,000 | 1,999,000 | 2,000,999 | +| insertion | nearly sorted | 250 | 2,277 | 2,251 | +| insertion | nearly sorted | 500 | 7,962 | 7,926 | +| insertion | nearly sorted | 1,000 | 30,889 | 30,860 | +| insertion | nearly sorted | 2,000 | 118,277 | 118,270 | +| quick | random | 250 | 1,933 | 1,940 | +| quick | random | 500 | 4,578 | 4,552 | +| quick | random | 1,000 | 11,196 | 10,210 | +| quick | random | 2,000 | 25,197 | 24,544 | +| quick | sorted | 250 | 31,125 | 0 | +| quick | sorted | 500 | 124,750 | 0 | +| quick | sorted | 1,000 | 499,500 | 0 | +| quick | sorted | 2,000 | 1,999,000 | 0 | +| quick | reversed | 250 | 31,125 | 250 | +| quick | reversed | 500 | 124,750 | 500 | +| quick | reversed | 1,000 | 499,500 | 1,000 | +| quick | reversed | 2,000 | 1,999,000 | 2,000 | +| quick | nearly sorted | 250 | 4,346 | 1,008 | +| quick | nearly sorted | 500 | 10,192 | 1,850 | +| quick | nearly sorted | 1,000 | 28,499 | 7,892 | +| quick | nearly sorted | 2,000 | 58,002 | 11,768 | + +## Time and energy per sort + +**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). + +## Is run time a good stand-in for energy? + +**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). + +## Startup cost per language + +**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). + +## Carbon estimates + +**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). + +### Grid intensity + +| Region | Year | g CO2e per kWh | Basis | Source | Licence | +| :--- | ---: | ---: | :--- | :--- | :--- | +| United Arab Emirates | 2024 | 467.51 | lifecycle, whole grid, annual average | [Ember (2026), yearly electricity data, via Our World in Data indicator 1295504](https://ourworldindata.org/grapher/carbon-intensity-electricity) | CC BY 4.0 | +| United Kingdom | 2024 | 216.5 | lifecycle, whole grid, annual average | [Ember (2026), yearly electricity data, via Our World in Data indicator 1295504](https://ourworldindata.org/grapher/carbon-intensity-electricity) | CC BY 4.0 | From 4b92622816a89cfd221ad7cab32911088684867f Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:49:10 +0400 Subject: [PATCH 20/51] fix: refuse RAPL counters that never move instead of recording zeros Counter files that exist but never change, as some virtual machines and firmware leave them, used to pass as valid energy: the harness wrote energy_status=ok with 0 uJ and the report printed zero energy and carbon. The harness now reads the counters 200 ms apart before measuring and treats a package counter that has not moved as unavailable (exit 3 under --energy require). A measured interval or idle window of 100 ms or more with no package change stops the run. probe exits 3 on frozen counters. Tests that need energy now move the fake counters with CounterWriter (tests/integration/powercap.py), which advances them in place and wraps them at max_energy_range_uj; make sanitize runs the harness under it. Co-Authored-By: Claude Opus 5.5 --- Makefile | 7 +- docs/bare-metal.md | 2 +- docs/decisions.md | 13 ++- scripts/run-study.sh | 6 +- src/greenbench.c | 90 ++++++++++++++--- src/rapl.c | 4 + src/rapl.h | 11 ++ tests/fixtures/powercap/README.md | 6 ++ tests/integration/powercap.py | 161 ++++++++++++++++++++++++++++-- tests/integration/test_harness.py | 88 +++++++++++----- tests/test_rapl.c | 13 +++ 11 files changed, 339 insertions(+), 62 deletions(-) diff --git a/Makefile b/Makefile index 2a699ae..d463b22 100644 --- a/Makefile +++ b/Makefile @@ -47,11 +47,12 @@ test: $(TEST_BIN) @for t in $(TEST_BIN); do ./$$t || exit 1; done # Unit tests and a short C-only harness run under AddressSanitizer and UBSan. The harness -# reads a fake powercap tree, so the energy and idle-baseline code runs too. +# reads a fake powercap tree whose counters move, so the energy and idle-baseline code runs too. sanitize: $(MAKE) BUILD=build/sanitize CFLAGS="$(SANITIZE)" LDFLAGS="$(SANITIZE)" all test - $(PYTHON) tests/integration/powercap.py tests/fixtures/powercap/server-two-socket.txt build/sanitize/powercap - build/sanitize/greenbench run --profile smoke --languages c --quiet --energy require \ + $(PYTHON) tests/integration/powercap.py build tests/fixtures/powercap/server-two-socket.txt build/sanitize/powercap + $(PYTHON) tests/integration/powercap.py advance build/sanitize/powercap -- \ + build/sanitize/greenbench run --profile smoke --languages c --quiet --energy require \ --powercap-root build/sanitize/powercap --out build/sanitize/smoke # A short run of every language with energy off, then the report. Works anywhere. diff --git a/docs/bare-metal.md b/docs/bare-metal.md index 5497317..a117b41 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -96,7 +96,7 @@ to see that the machine works before committing to the full run. | `no RAPL zones under /sys/class/powercap` | The processor, firmware or kernel does not expose RAPL. Check that you booted the USB stick and not a virtual machine. | | `this is WSL` or `this is a container` | The script refuses environments without real counters. | | `permission denied reading .../energy_uj` from the harness | Run `--check` again and accept the `chmod`, or reboot and start again. | -| `the package counter did not change` from `probe` | The counters are frozen or updated rarely on this machine; its energy readings cannot be trusted. | +| `the package energy counter did not change` | The counter files exist but their values never change, as some virtual machines and firmware leave them. The probe exits with status 3, the script stops, and this machine cannot measure energy. | | `the implementations disagree` | A worker produced different counts from the others. This is a bug; please open an issue with the message. | The harness stops cleanly on Ctrl+C: the runs measured so far stay in `runs.csv`, and diff --git a/docs/decisions.md b/docs/decisions.md index 84b265b..722070f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -28,12 +28,19 @@ A benchmark that writes 0 when it cannot measure produces data that looks real. **Decision.** If the zones are missing, unreadable, or hold values that do not parse, the harness records `energy_status=unavailable` and a reason, and leaves every energy cell empty. -`--energy require` turns this into exit status 3. The report prints "energy unavailable" and -keeps the energy sections empty. +Counters that exist but never move, as some virtual machines and firmware leave them, count +as missing: before measuring, the harness reads the counters 200 ms apart, and a package +counter that has not changed makes energy unavailable. During the run, a measured interval or +idle window of 100 ms or more with no package change stops the run with an error; a live +counter moves about once a millisecond. `--energy require` turns unavailable energy into exit +status 3, and so does `greenbench probe` when the counters do not move. The report prints +"energy unavailable" and keeps the energy sections empty. **Consequences.** The same harness runs everywhere with time and operation counts, and no file can be mistaken for an energy measurement. The unit tests check that a corrupt counter -is an error rather than a zero. +is an error rather than a zero, and the integration tests check that frozen counters are +refused. Tests that need energy move the fake counters with a writer thread, as the kernel +would. ## 3. Handle one counter wrap; keep intervals short diff --git a/scripts/run-study.sh b/scripts/run-study.sh index 2a15955..f271093 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -27,8 +27,8 @@ Usage: scripts/run-study.sh [--check | --rehearse] [--cpu N | --no-pin] [--yes] --yes do not ask before making the RAPL counters readable --powercap-root DIR read RAPL zones from DIR instead of /sys/class/powercap. Only for testing this - script with a fake tree: it skips the bare-metal checks, and the fake counters - never move, so every energy value is zero + script with a fake tree whose counters move (tests/integration/powercap.py + advance); it skips the bare-metal checks, and the energy values are not real EOF } @@ -170,7 +170,7 @@ record_environment() { } if ((FAKE_POWERCAP)); then - warn "reading a fake powercap tree at $POWERCAP; the energy values will all be zero" + warn "reading a fake powercap tree at $POWERCAP; the energy values are not real" else check_platform fi diff --git a/src/greenbench.c b/src/greenbench.c index 1596246..bf718ac 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -153,7 +153,8 @@ static void print_usage(FILE *out) { "\n" "Options for probe:\n" " --powercap-root DIR as above\n" - " --sample-ms N measure idle power for N ms (default 1000; 0 skips it)\n" + " --sample-ms N measure idle power for N ms and check that the counters move\n" + " (100 to 60000, default 1000; 0 skips both)\n" "\n" "Exit status: 0 success, 1 measurement error, 2 usage error, 3 energy required or unavailable.\n", INPUT_MAX_N, OPTIONS_MIN_STUDY_RUNS, INPUT_DEFAULT_SEED, RAPL_DEFAULT_ROOT); @@ -569,6 +570,11 @@ static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measure m->energy.dram_uj = 0; if (ctx->energy_ok && rapl_energy_between(&ctx->rapl, &before, &after, &m->energy) != 0) return error_message("a RAPL counter reading was above its max_energy_range_uj"), -1; + if (ctx->energy_ok && rapl_is_frozen(&m->energy, m->wall_ns)) + return error_message("the package energy counter did not change while %s ran for %.0f ms; the RAPL " + "readings have frozen, so energy cannot be measured", + what, (double)m->wall_ns / 1e6), + -1; return 0; } @@ -744,6 +750,12 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { if (ctx->energy_ok && (rapl_read(&ctx->rapl, &after) != 0 || rapl_energy_between(&ctx->rapl, &before, &after, &idle) != 0)) return free(items), error_message("reading the RAPL counters failed during the idle window"), -1; + if (ctx->energy_ok && rapl_is_frozen(&idle, idle_end - idle_start)) + return free(items), + error_message("the package energy counter did not change during a %.0f ms idle window; the " + "RAPL readings have frozen, so energy cannot be measured", + (double)(idle_end - idle_start) / 1e6), + -1; record.has_idle = 1; record.idle_ns = idle_end - idle_start; record.idle_package_uj = idle.package_uj; @@ -785,15 +797,57 @@ static int pin_cpu(int cpu) { #endif } +/* Before measuring anything, the harness checks that the counters move. */ +#define LIVE_CHECK_NS (2u * RAPL_FROZEN_NS) + +/* Reads the counters twice, sample_ns apart. Returns 0 when the package counter moved, or -1 + with the reason when it did not or could not be read. */ +static int counters_move(const t_rapl *rapl, uint64_t sample_ns, t_rapl_energy *energy, uint64_t *elapsed, + char *reason, size_t size) { + t_rapl_sample before, after; + + if (rapl_read(rapl, &before) != 0) { + snprintf(reason, size, "reading the RAPL counters failed: %s", strerror(errno)); + return -1; + } + uint64_t start = timing_now_ns(); + timing_sleep_ns(sample_ns); + if (rapl_read(rapl, &after) != 0) { + snprintf(reason, size, "reading the RAPL counters failed: %s", strerror(errno)); + return -1; + } + uint64_t took = timing_now_ns() - start; + if (rapl_energy_between(rapl, &before, &after, energy) != 0) { + snprintf(reason, size, "a RAPL counter reading was above its max_energy_range_uj"); + return -1; + } + *elapsed = took; + if (rapl_is_frozen(energy, *elapsed)) { + snprintf(reason, size, + "the package energy counter did not change over %.0f ms: the RAPL files exist but their readings " + "are frozen, as some virtual machines and firmware leave them", + (double)*elapsed / 1e6); + return -1; + } + return 0; +} + static void setup_energy(t_context *ctx) { const t_options *o = &ctx->options; + t_rapl_energy energy; + uint64_t elapsed; if (o->energy == ENERGY_OFF) { ctx->energy_status = "off"; snprintf(ctx->energy_reason, sizeof ctx->energy_reason, "turned off with --energy off"); - } else if (rapl_open(&ctx->rapl, o->powercap_root) == 0) { + } else if (rapl_open(&ctx->rapl, o->powercap_root) == 0 && + counters_move(&ctx->rapl, LIVE_CHECK_NS, &energy, &elapsed, ctx->energy_reason, + sizeof ctx->energy_reason) == 0) { ctx->energy_ok = 1; ctx->energy_status = "ok"; + } else if (ctx->energy_reason[0] != '\0') { + rapl_close(&ctx->rapl); /* opened, but the counters do not move */ + ctx->energy_status = "unavailable"; } else { ctx->energy_status = "unavailable"; snprintf(ctx->energy_reason, sizeof ctx->energy_reason, "%s", ctx->rapl.reason); @@ -929,7 +983,8 @@ static int command_probe(int argc, char **argv) { if (strcmp(argv[i], "--powercap-root") == 0 && i + 1 < argc && argv[i + 1][0] != '\0') { root = argv[++i]; } else if (strcmp(argv[i], "--sample-ms") == 0 && i + 1 < argc && - options_parse_u64(argv[i + 1], 0, 60000, &sample_ms) == 0) { + options_parse_u64(argv[i + 1], 0, 60000, &sample_ms) == 0 && + (sample_ms == 0 || sample_ms * 1000000u >= RAPL_FROZEN_NS)) { i++; } else if (strcmp(argv[i], "--help") == 0) { print_usage(stdout); @@ -945,29 +1000,32 @@ static int command_probe(int argc, char **argv) { printf("energy: unavailable\nreason: %s\n", rapl.reason); return EXIT_ENERGY; } - printf("energy: available\n"); + printf("zones:\n"); for (size_t i = 0; i < rapl.count; i++) printf(" %-16s %-12s %-8s wraps at %" PRIu64 " uJ\n", rapl.domains[i].zone, rapl.domains[i].name, rapl.domains[i].kind == RAPL_PACKAGE ? "package" : "dram", rapl.domains[i].max_range_uj); - if (sample_ms > 0) { - t_rapl_sample before, after; - t_rapl_energy energy; - if (rapl_read(&rapl, &before) != 0) - return rapl_close(&rapl), error_message("reading the counters failed: %s", strerror(errno)), 1; - uint64_t start = timing_now_ns(); - timing_sleep_ns(sample_ms * 1000000u); - uint64_t elapsed = timing_now_ns() - start; - if (rapl_read(&rapl, &after) != 0 || rapl_energy_between(&rapl, &before, &after, &energy) != 0) - return rapl_close(&rapl), error_message("reading the counters failed"), 1; + if (sample_ms == 0) { + printf("energy: available (not sampled, so not checked for frozen readings)\n"); + rapl_close(&rapl); + return 0; + } + t_rapl_energy energy = {0, 0}; + uint64_t elapsed = 0; + char reason[RAPL_REASON_LEN] = ""; + int moved = counters_move(&rapl, sample_ms * 1000000u, &energy, &elapsed, reason, sizeof reason); + if (elapsed > 0) { double seconds = (double)elapsed / 1e9; printf("idle power over %.2f s: package %.2f W", seconds, (double)energy.package_uj / 1e6 / seconds); if (rapl.has_dram) printf(", dram %.2f W", (double)energy.dram_uj / 1e6 / seconds); putchar('\n'); - if (energy.package_uj == 0) - printf("warning: the package counter did not change; the readings may be frozen\n"); } rapl_close(&rapl); + if (moved != 0) { + printf("energy: unavailable\nreason: %s\n", reason); + return EXIT_ENERGY; + } + printf("energy: available\n"); return 0; } diff --git a/src/rapl.c b/src/rapl.c index 6d0df97..ee597ed 100644 --- a/src/rapl.c +++ b/src/rapl.c @@ -289,6 +289,10 @@ double rapl_net_uj(uint64_t gross_uj, uint64_t idle_uj, uint64_t run_ns, uint64_ return (double)gross_uj - (double)idle_uj * ((double)run_ns / (double)idle_ns); } +int rapl_is_frozen(const t_rapl_energy *energy, uint64_t interval_ns) { + return energy->package_uj == 0 && interval_ns >= RAPL_FROZEN_NS; +} + void rapl_close(t_rapl *rapl) { for (size_t i = 0; i < RAPL_MAX_DOMAINS; i++) { if (rapl->domains[i].fd >= 0) diff --git a/src/rapl.h b/src/rapl.h index 2dbbad5..f616156 100644 --- a/src/rapl.h +++ b/src/rapl.h @@ -20,6 +20,12 @@ #define RAPL_PATH_LEN 512 #define RAPL_REASON_LEN 512 +/* A live package counter moves about once a millisecond, and even an idle package draws + far more than the counter's one-unit resolution in that time. A package counter that has + not moved over an interval this long (about 100 updates) is frozen: the files exist but + the readings are not real. */ +#define RAPL_FROZEN_NS 100000000u /* 100 ms */ + typedef enum e_rapl_kind { RAPL_PACKAGE, RAPL_DRAM } t_rapl_kind; typedef struct s_rapl_domain { @@ -78,6 +84,11 @@ int rapl_energy_between(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl idle; the value is kept, not clamped, so the noise stays visible. */ double rapl_net_uj(uint64_t gross_uj, uint64_t idle_uj, uint64_t run_ns, uint64_t idle_ns); +/* 1 when the package energy over an interval of interval_ns shows frozen counters: no + change at all over at least RAPL_FROZEN_NS. Shorter intervals are never judged, because a + live counter can legitimately stand still between two updates. */ +int rapl_is_frozen(const t_rapl_energy *energy, uint64_t interval_ns); + void rapl_close(t_rapl *rapl); #endif diff --git a/tests/fixtures/powercap/README.md b/tests/fixtures/powercap/README.md index 50dcefd..80028d4 100644 --- a/tests/fixtures/powercap/README.md +++ b/tests/fixtures/powercap/README.md @@ -18,3 +18,9 @@ The values are invented but shaped like real sysfs output: counters in microjoul | `bad-max-range.txt` | A package whose `max_energy_range_uj` is not a number | | `counter-above-max.txt` | A package whose counter is above its wrap value | | `dram-only.txt` | A DRAM zone without a package zone | + +The counters in a built tree stand still. The harness treats a package counter that does not +move as frozen and refuses it, so tests that need energy run with `CounterWriter` from +`tests/integration/powercap.py`, or with `powercap.py advance ROOT -- COMMAND`. It advances +the package and DRAM counters in place, as a live system does, and wraps them at +`max_energy_range_uj`. diff --git a/tests/integration/powercap.py b/tests/integration/powercap.py index 3d5ddbe..775d63f 100644 --- a/tests/integration/powercap.py +++ b/tests/integration/powercap.py @@ -1,17 +1,28 @@ -"""Build a fake powercap tree from a manifest in tests/fixtures/powercap. +"""Fake powercap trees for testing: build one from a manifest, and make its counters move. -Each manifest line is a path relative to the powercap root, whitespace, then the file's -contents. Zone directories are named like ``intel-rapl:0``, which Windows cannot store, so -the trees are kept as manifests and built on demand. +Each manifest line in tests/fixtures/powercap is a path relative to the powercap root, +whitespace, then the file's contents. Zone directories are named like ``intel-rapl:0``, which +Windows cannot store, so the trees are kept as manifests and built on demand. - python3 tests/integration/powercap.py tests/fixtures/powercap/laptop-intel.txt /tmp/powercap +A built tree's counters stand still, which the harness rejects as frozen. CounterWriter moves +them the way live RAPL counters move, so the energy code runs end to end: + + python3 tests/integration/powercap.py build tests/fixtures/powercap/laptop-intel.txt /tmp/pc + python3 tests/integration/powercap.py advance /tmp/pc -- build/greenbench run --powercap-root /tmp/pc """ from __future__ import annotations +import argparse +import os import shutil +import subprocess import sys +import threading +import time +from dataclasses import dataclass from pathlib import Path +from types import TracebackType def build(manifest: Path, root: Path) -> Path: @@ -33,12 +44,142 @@ def build(manifest: Path, root: Path) -> Path: return root +@dataclass +class _Counter: + zone: str + kind: str # "package" or "dram" + fd: int + maximum: int + width: int + value: float + wraps: int = 0 + + +class CounterWriter: + """Moves the package and DRAM counters of a fake powercap tree, like live RAPL counters. + + Package counters gain ``idle_watts`` (or ``busy_watts`` while ``busy_marker`` exists, which + a fake worker can create while it runs), DRAM counters gain ``dram_share`` of that, and + every counter wraps to zero after its ``max_energy_range_uj``. + + The harness keeps the counter files open and re-reads them with pread, so values are + written in place, never by replacing the file. Each is one fixed-width, zero-padded write + of as many digits as the maximum has; a read that overlaps a write therefore still parses + and stays within range. + """ + + def __init__( + self, + root: Path, + *, + idle_watts: float = 10.0, + busy_watts: float | None = None, + busy_marker: Path | None = None, + dram_share: float = 0.1, + interval_s: float = 0.0005, + ) -> None: + self.root = root + self.idle_watts = idle_watts + self.busy_watts = idle_watts if busy_watts is None else busy_watts + self.busy_marker = busy_marker + self.dram_share = dram_share + self.interval_s = interval_s + self._counters: list[_Counter] = [] + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._last_ns = 0 + + @property + def package_wraps(self) -> int: + """How many times a package counter has wrapped since start().""" + return sum(c.wraps for c in self._counters if c.kind == "package") + + def _open(self) -> None: + for zone in sorted(self.root.glob("intel-rapl:*")): + name_file = zone / "name" + if not name_file.is_file(): + continue + name = name_file.read_text(encoding="utf-8").strip() + kind = "package" if name.startswith("package") else "dram" if name == "dram" else None + if kind is None: + continue + maximum = int((zone / "max_energy_range_uj").read_text(encoding="utf-8").strip()) + value = int((zone / "energy_uj").read_text(encoding="utf-8").strip()) + fd = os.open(zone / "energy_uj", os.O_WRONLY) + self._counters.append(_Counter(zone.name, kind, fd, maximum, len(str(maximum)), float(value))) + if not any(c.kind == "package" for c in self._counters): + raise ValueError(f"{self.root}: no package zone to move") + + def _write(self, counter: _Counter) -> None: + text = f"{int(counter.value):0{counter.width}d}\n".encode("ascii") + os.pwrite(counter.fd, text, 0) + + def tick(self) -> None: + """Advance every counter by the energy used since the previous tick.""" + now = time.monotonic_ns() + elapsed_s = (now - self._last_ns) / 1e9 + self._last_ns = now + busy = self.busy_marker is not None and self.busy_marker.exists() + watts = self.busy_watts if busy else self.idle_watts + for counter in self._counters: + share = 1.0 if counter.kind == "package" else self.dram_share + counter.value += watts * share * elapsed_s * 1e6 + while counter.value > counter.maximum: + counter.value -= counter.maximum + 1 + counter.wraps += 1 + self._write(counter) + + def _run(self) -> None: + while not self._stop.wait(self.interval_s): + self.tick() + + def start(self) -> CounterWriter: + """Rewrite every counter at its fixed width, then keep them moving in a thread.""" + self._open() + for counter in self._counters: + self._write(counter) + self._last_ns = time.monotonic_ns() + self._thread = threading.Thread(target=self._run, name="powercap-writer", daemon=True) + self._thread.start() + return self + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join() + for counter in self._counters: + os.close(counter.fd) + self._counters.clear() + + def __enter__(self) -> CounterWriter: + return self.start() + + def __exit__( + self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None + ) -> None: + self.stop() + + def main(argv: list[str]) -> int: - if len(argv) != 3: - print("usage: powercap.py MANIFEST OUTPUT_DIR", file=sys.stderr) - return 2 - build(Path(argv[1]), Path(argv[2])) - return 0 + parser = argparse.ArgumentParser(prog="powercap.py", description="Build fake powercap trees for testing.") + commands = parser.add_subparsers(dest="command", required=True) + make = commands.add_parser("build", help="build a tree from a manifest, replacing OUTPUT_DIR") + make.add_argument("manifest", type=Path) + make.add_argument("output_dir", type=Path) + advance = commands.add_parser("advance", help="run COMMAND while the tree's counters move") + advance.add_argument("root", type=Path) + advance.add_argument("--watts", type=float, default=10.0, help="package power (default %(default)s)") + advance.add_argument("run", nargs=argparse.REMAINDER, metavar="-- COMMAND", help="the command to run") + args = parser.parse_args(argv[1:]) + + if args.command == "build": + build(args.manifest, args.output_dir) + return 0 + command = args.run[1:] if args.run[:1] == ["--"] else args.run + if not command: + parser.error("advance needs a command after --") + with CounterWriter(args.root, idle_watts=args.watts): + return subprocess.run(command, check=False).returncode if __name__ == "__main__": diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 7a5ab28..c73e205 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -9,6 +9,7 @@ import pytest +from powercap import CounterWriter from powercap import build as build_powercap from workers import REPOSITORY, interpreter_options, require_build, run_harness @@ -81,33 +82,37 @@ def test_counts_match_across_languages_in_the_csv(tmp_path: Path) -> None: assert all(len(values) == 1 for values in counts.values()) -def test_fake_powercap_tree_gives_energy_columns(tmp_path: Path) -> None: +def test_moving_counters_fill_the_energy_columns(tmp_path: Path) -> None: root = build_powercap(FIXTURES / "server-two-socket.txt", tmp_path / "powercap") out = tmp_path / "run" - completed = run_harness( - "run", - *SMALL, - "--languages", - "c", - "--energy", - "require", - "--powercap-root", - str(root), - "--out", - str(out), - *interpreter_options(), - ) + with CounterWriter(root, idle_watts=10.0, dram_share=0.2): + completed = run_harness( + "run", + *SMALL, + "--languages", + "c", + "--energy", + "require", + "--powercap-root", + str(root), + "--out", + str(out), + *interpreter_options(), + ) assert completed.returncode == 0, completed.stderr rows = read_runs(out) for row in rows: - # The fake counters never move, so every energy is exactly zero; what matters is that the - # columns are filled and the idle baseline ran for the same duration. assert row["energy_status"] == "ok" - assert row["pkg_uj"] == "0" - assert row["dram_uj"] == "0" - assert row["idle_pkg_uj"] == "0" - assert row["net_pkg_uj"] == "0.0" + for column in ("pkg_uj", "dram_uj", "idle_pkg_uj", "idle_dram_uj"): + assert int(row[column]) >= 0, f"{column} is a whole number of microjoules" assert int(row["idle_ns"]) >= int(row["wall_ns"]) + expected = int(row["pkg_uj"]) - int(row["idle_pkg_uj"]) * int(row["wall_ns"]) / int(row["idle_ns"]) + assert float(row["net_pkg_uj"]) == pytest.approx(expected, abs=0.051) + # Both sockets' packages move at 10 W, so the package total is about 20 W times the time measured. + seconds = sum(int(r["wall_ns"]) + int(r["idle_ns"]) for r in rows) / 1e9 + joules = sum(int(r["pkg_uj"]) + int(r["idle_pkg_uj"]) for r in rows) / 1e6 + assert joules == pytest.approx(20.0 * seconds, rel=0.5) + assert sum(int(r["dram_uj"]) for r in rows) > 0 meta = read_meta(out) assert meta["energy"]["status"] == "ok" assert meta["energy"]["idle_baseline"] is True @@ -119,6 +124,26 @@ def test_fake_powercap_tree_gives_energy_columns(tmp_path: Path) -> None: ] +def test_frozen_counters_are_not_energy(tmp_path: Path) -> None: + # Files that exist but never change, as some virtual machines expose, must not become zeros. + root = build_powercap(FIXTURES / "server-two-socket.txt", tmp_path / "powercap") + required = run_harness( + "run", *SMALL, "--languages", "c", "--energy", "require", "--powercap-root", str(root), "--out", str(tmp_path) + ) + assert required.returncode == 3 + assert "energy unavailable: the package energy counter did not change" in required.stderr + assert "frozen" in required.stderr + + out = tmp_path / "auto" + automatic = run_harness( + "run", *SMALL, "--languages", "c", "--energy", "auto", "--powercap-root", str(root), "--out", str(out) + ) + assert automatic.returncode == 0, automatic.stderr + assert {row["energy_status"] for row in read_runs(out)} == {"unavailable"} + assert all(row["pkg_uj"] == "" for row in read_runs(out)) + assert "frozen" in read_meta(out)["energy"]["reason"] + + def test_energy_required_but_unavailable_exits_3(tmp_path: Path) -> None: root = build_powercap(FIXTURES / "no-rapl.txt", tmp_path / "powercap") completed = run_harness( @@ -155,12 +180,22 @@ def test_energy_auto_falls_back_to_time_only(tmp_path: Path) -> None: def test_probe(tmp_path: Path) -> None: root = build_powercap(FIXTURES / "laptop-intel.txt", tmp_path / "powercap") - completed = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "50") - assert completed.returncode == 0 - assert "energy: available" in completed.stdout - assert "intel-rapl:0:2" in completed.stdout - assert "psys" not in completed.stdout - assert "did not change" in completed.stdout # the fake counters are frozen + with CounterWriter(root, idle_watts=5.0): + live = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "100") + assert live.returncode == 0, live.stdout + live.stderr + assert "intel-rapl:0:2" in live.stdout + assert "psys" not in live.stdout + assert "idle power over" in live.stdout + assert live.stdout.rstrip().endswith("energy: available") + + frozen = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "100") + assert frozen.returncode == 3 + assert "energy: unavailable" in frozen.stdout + assert "did not change over" in frozen.stdout + + unsampled = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "0") + assert unsampled.returncode == 0 + assert "not sampled" in unsampled.stdout missing = run_harness("probe", "--powercap-root", str(tmp_path / "nothing"), "--sample-ms", "0") assert missing.returncode == 3 @@ -182,6 +217,7 @@ def test_dry_run_prints_the_plan() -> None: (["run", "--bogus"], "unknown option --bogus"), (["walk"], "unknown command 'walk'"), (["probe", "--sample-ms", "lots"], "unexpected or invalid argument"), + (["probe", "--sample-ms", "50"], "unexpected or invalid argument"), ], ) def test_usage_errors_exit_2(args: list[str], message: str) -> None: diff --git a/tests/test_rapl.c b/tests/test_rapl.c index fbe44ca..dc78e43 100644 --- a/tests/test_rapl.c +++ b/tests/test_rapl.c @@ -181,6 +181,18 @@ static void test_net(void) { CHECK(rapl_net_uj(1000, 400, 100, 0) == 1000.0, "no idle window leaves the gross value"); } +static void test_frozen(void) { + t_rapl_energy still = {0, 0}; + t_rapl_energy moved = {1, 0}; + t_rapl_energy dram_only = {0, 500}; + + CHECK(rapl_is_frozen(&still, RAPL_FROZEN_NS), "no package change over 100 ms is frozen"); + CHECK(rapl_is_frozen(&still, 250000000u), "no package change over 250 ms is frozen"); + CHECK(!rapl_is_frozen(&still, RAPL_FROZEN_NS - 1), "a shorter interval can fall between two updates"); + CHECK(!rapl_is_frozen(&moved, 250000000u), "one microjoule is movement"); + CHECK(rapl_is_frozen(&dram_only, 250000000u), "only the package counter decides"); +} + int main(void) { test_parse(); test_delta(); @@ -191,5 +203,6 @@ int main(void) { test_unavailable(); test_permission_denied(); test_net(); + test_frozen(); return check_report("rapl"); } From beb98fe132b98e796251b791062c257c089b914f Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:01:35 +0400 Subject: [PATCH 21/51] fix: measure startup cost over a batch of launches, not one short process A startup trial used to be one process that exits at once: about 1 to 3 ms for C, one to three RAPL updates, so its energy was mostly counter quantisation, and a quarter of the study's runs were affected. Each measured startup run now launches the worker back to back until the batch lasts about --min-loop-ms, calibrated during warm-up like the sorts. runs.csv gains a launches column (schema 2, harness 2.1.0), meta.json records it per trial, and the analysis divides startup time and energy by it before subtracting the cost per process. Every launch's result line is still checked against the other languages. probe now needs at least 200 ms to judge frozen counters, so a sleep that ends early on CLOCK_MONOTONIC_RAW cannot hide them. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/load.py | 12 +- analysis/greenbench_analysis/report.py | 9 +- analysis/greenbench_analysis/summary.py | 11 +- analysis/tests/synthetic.py | 2 + analysis/tests/test_load.py | 10 +- docs/decisions.md | 19 ++- docs/study.md | 7 +- src/greenbench.c | 162 ++++++++++++++---------- src/output.c | 9 +- src/output.h | 2 +- src/plan.c | 1 + src/plan.h | 6 +- tests/integration/test_harness.py | 43 ++++++- tests/test_output.c | 8 +- tests/test_plan.c | 3 +- 15 files changed, 201 insertions(+), 103 deletions(-) diff --git a/analysis/greenbench_analysis/load.py b/analysis/greenbench_analysis/load.py index 3e9c69e..8cac1c4 100644 --- a/analysis/greenbench_analysis/load.py +++ b/analysis/greenbench_analysis/load.py @@ -17,6 +17,7 @@ "seed", "repetition", "inner_reps", + "launches", "wall_ns", "loop_ns", "comparisons", @@ -43,7 +44,8 @@ class DataError(ValueError): @dataclass(frozen=True) class Run: - """One measured run: one worker process, as one row of runs.csv.""" + """One measured run, as one row of runs.csv: one worker process, or for a startup trial a + batch of ``launches`` processes started back to back.""" sequence: int language: str @@ -53,6 +55,7 @@ class Run: seed: int repetition: int inner_reps: int + launches: int wall_ns: int loop_ns: int comparisons: int @@ -124,6 +127,7 @@ def _parse_row(row: dict[str, str], where: str) -> Run: seed=_whole(row, "seed", where), repetition=_whole(row, "repetition", where), inner_reps=_whole(row, "inner_reps", where), + launches=_whole(row, "launches", where), wall_ns=_whole(row, "wall_ns", where), loop_ns=_whole(row, "loop_ns", where), comparisons=_whole(row, "comparisons", where), @@ -144,6 +148,10 @@ def _parse_row(row: dict[str, str], where: str) -> Run: raise DataError(f"{where}: energy_status is {run.energy_status} but pkg_uj holds a value") if run.algorithm != "none" and run.inner_reps < 1: raise DataError(f"{where}: a sorting run must sort at least once (inner_reps is 0)") + if run.launches < 1: + raise DataError(f"{where}: a run must start at least one process (launches is 0)") + if run.algorithm != "none" and run.launches != 1: + raise DataError(f"{where}: a sorting run is one process, but launches is {run.launches}") return run @@ -158,7 +166,7 @@ def load_runs(path: Path) -> list[Run]: extra = [c for c in header if c not in COLUMNS] raise DataError( f"{path}: unexpected columns (missing: {', '.join(missing) or 'none'}; " - f"unexpected: {', '.join(extra) or 'none'}); was it written by greenbench 2?" + f"unexpected: {', '.join(extra) or 'none'}); was it written by greenbench 2.1 or later?" ) runs = [_parse_row(row, f"{path}:{number}") for number, row in enumerate(reader, start=2)] except FileNotFoundError: diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index d187181..183559c 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -260,8 +260,9 @@ def _startup_section(summary: Summary, options: ReportOptions) -> list[str]: if options.withhold_timings: return [*lines, _pending(options), ""] lines += [ - "A startup trial starts the worker, generates the input and exits without sorting. Its median is " - "subtracted from the sorting runs above. Random order shown.", + "A startup run starts the worker, which generates the input and exits without sorting, several times " + "back to back, so that each measured batch lasts about as long as a sorting run. Figures are per " + "process; their median is subtracted from the sorting runs above. Random order shown.", "", ] rows = [ @@ -274,7 +275,9 @@ def _startup_section(summary: Summary, options: ReportOptions) -> list[str]: for s in summary.startups if s.order == "random" ] - lines += _table(["Language", "Size", "Process time", "Process energy"], [":---", "---:", "---:", "---:"], rows) + lines += _table( + ["Language", "Size", "Time per process", "Energy per process"], [":---", "---:", "---:", "---:"], rows + ) return [*lines, ""] diff --git a/analysis/greenbench_analysis/summary.py b/analysis/greenbench_analysis/summary.py index 6a224d3..4aa1394 100644 --- a/analysis/greenbench_analysis/summary.py +++ b/analysis/greenbench_analysis/summary.py @@ -4,8 +4,10 @@ measured run is one worker process that sorted the input inner_reps times, so: - process time and energy cover the whole process, as the harness measured them; -- per-sort time and energy subtract the median startup trial for the same language, order - and size (a process that generated the input and exited), then divide by inner_reps. +- per-sort time and energy subtract the median startup cost for the same language, order + and size, then divide by inner_reps. A startup run starts a batch of processes that + generate the input and exit, back to back, so its cost per process is the batch's time and + energy divided by the number of launches. """ from __future__ import annotations @@ -163,8 +165,9 @@ def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = for (language, algorithm, order, n), group in groups.items(): if algorithm != "none": continue - times = [float(r.wall_ns) for r in group] - energies = _energies(group) + times = [r.wall_ns / r.launches for r in group] + batches = _energies(group) + energies = None if batches is None else [e / r.launches for e, r in zip(batches, group, strict=True)] startup_time[(language, order, n)] = median(times) if energies is not None: startup_energy[(language, order, n)] = median(energies) diff --git a/analysis/tests/synthetic.py b/analysis/tests/synthetic.py index f82617b..bb97506 100644 --- a/analysis/tests/synthetic.py +++ b/analysis/tests/synthetic.py @@ -28,6 +28,7 @@ def make_run( n: int = 100, repetition: int = 1, inner_reps: int = 10, + launches: int = 1, wall_ns: int = 1_000_000, energy_uj: float | None = None, idle_uj: float | None = None, @@ -45,6 +46,7 @@ def make_run( seed=2463534242, repetition=repetition, inner_reps=0 if algorithm == "none" else inner_reps, + launches=launches, wall_ns=wall_ns, loop_ns=wall_ns // 2, comparisons=comparisons, diff --git a/analysis/tests/test_load.py b/analysis/tests/test_load.py index 429e6a1..17a594d 100644 --- a/analysis/tests/test_load.py +++ b/analysis/tests/test_load.py @@ -8,7 +8,7 @@ from greenbench_analysis.load import COLUMNS, DataError, load_meta, load_runs HEADER = ",".join(COLUMNS) -GOOD_ROW = "1,c,quick,random,100,2463534242,1,10,1000000,900000,700,800,22222222,unavailable,,,,,,," +GOOD_ROW = "1,c,quick,random,100,2463534242,1,10,1,1000000,900000,700,800,22222222,unavailable,,,,,,," def write(tmp_path: Path, *lines: str) -> Path: @@ -46,10 +46,12 @@ def test_energy_prefers_the_net_values() -> None: (GOOD_ROW.replace("1000000", "12ms"), "column wall_ns: expected a whole number"), (GOOD_ROW.replace(",unavailable,", ",ok,"), "energy_status is ok but pkg_uj is empty"), (GOOD_ROW.replace("unavailable,,", "unavailable,5,"), "but pkg_uj holds a value"), - (GOOD_ROW.replace(",10,1000000", ",0,1000000"), "must sort at least once"), + (GOOD_ROW.replace(",10,1,1000000", ",0,1,1000000"), "must sort at least once"), + (GOOD_ROW.replace(",10,1,1000000", ",10,0,1000000"), "must start at least one process"), + (GOOD_ROW.replace(",10,1,1000000", ",10,3,1000000"), "a sorting run is one process"), (GOOD_ROW + "x", "column net_dram_uj"), - (GOOD_ROW[:-2], "expected 21 columns"), - (GOOD_ROW + ",", "expected 21 columns"), + (GOOD_ROW[:-2], "expected 22 columns"), + (GOOD_ROW + ",", "expected 22 columns"), ], ) def test_bad_rows_name_the_line_and_column(tmp_path: Path, row: str, message: str) -> None: diff --git a/docs/decisions.md b/docs/decisions.md index 722070f..3b59bbc 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -64,9 +64,12 @@ Python and Node.js would then be measured differently from C. **Decision.** The harness starts one worker process per measured run, for C as well, and reads the counters and `CLOCK_MONOTONIC_RAW` just before starting it and just after it exits. -A startup trial for each language, order and size (a worker that generates its input and -exits) measures the fixed cost of starting, which the report subtracts before dividing by the -number of sorts. +A startup trial for each language, order and size measures the fixed cost of starting: a +worker that generates its input and exits without sorting. One C process starts in about a +millisecond, close to the RAPL update interval, so each measured startup run launches the +worker several times back to back, enough to last about `--min-loop-ms`, and records the +number of launches. The report divides by it, then subtracts the cost per process before +dividing by the number of sorts. **Consequences.** One code path measures all three languages, as the research question needs. Interpreter start-up is measured explicitly instead of being hidden. The cost is some @@ -83,9 +86,13 @@ own loop time, and sets the number of sorts per process so that the loop lasts a number on each further warm-up run, then keeps it fixed for every measured run of that configuration. -**Consequences.** Every measured interval spans hundreds of counter updates, so the -update granularity is well under 1% of the measurement. Fast configurations sort thousands of -times per process, slow ones once. The number is recorded in the CSV and in `meta.json`. +Startup trials are calibrated the same way, by the number of processes launched back to back +in one measured interval (ADR 4). + +**Consequences.** Every measured interval, sorting or startup, spans about 200 counter +updates, so the update granularity is well under 1% of the measurement. Fast configurations +sort thousands of times per process, slow ones once. The numbers of sorts and launches are +recorded in the CSV and in `meta.json`. ## 6. Subtract an idle baseline of the same length after every run diff --git a/docs/study.md b/docs/study.md index 8887f7f..712d079 100644 --- a/docs/study.md +++ b/docs/study.md @@ -77,8 +77,11 @@ lists, JavaScript arrays). None uses a built-in sort. - **Idle baseline.** After every run the harness sleeps for the same duration and measures the idle energy, which is subtracted, scaled to the run's exact length. - **Startup cost.** For each language, order and size, a startup trial runs a worker that - generates the input and exits without sorting. Its median time and energy are subtracted - before dividing by the number of sorts, giving time and energy per sort. + generates the input and exits without sorting. A single start can be as short as the RAPL + update interval, so each measured startup run launches the worker back to back until the + batch lasts about as long as a sorting run, and is divided by the number of launches. The + median cost per process is subtracted before dividing by the number of sorts, giving time + and energy per sort. - **Repetitions and order.** Three warm-up runs per configuration, then 30 measured runs. All measured runs of all configurations are shuffled into one random order from a recorded seed. - **Pinning.** The study script pins the harness and all workers to one CPU (CPU 2 by diff --git a/src/greenbench.c b/src/greenbench.c index bf718ac..8a09d65 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -33,7 +33,7 @@ #include "spawn.h" #include "timing.h" -#define GREENBENCH_VERSION "2.0.0" +#define GREENBENCH_VERSION "2.1.0" #define EXIT_USAGE 2 #define EXIT_ENERGY 3 #define TEXT_LEN 256 @@ -48,8 +48,9 @@ #endif typedef struct s_measurement { - t_worker_result result; - uint64_t wall_ns; + t_worker_result result; /* the last process's result line */ + uint64_t wall_ns; /* the whole batch of processes */ + uint64_t launches; t_rapl_energy energy; } t_measurement; @@ -154,7 +155,7 @@ static void print_usage(FILE *out) { "Options for probe:\n" " --powercap-root DIR as above\n" " --sample-ms N measure idle power for N ms and check that the counters move\n" - " (100 to 60000, default 1000; 0 skips both)\n" + " (200 to 60000, default 1000; 0 skips both)\n" "\n" "Exit status: 0 success, 1 measurement error, 2 usage error, 3 energy required or unavailable.\n", INPUT_MAX_N, OPTIONS_MIN_STUDY_RUNS, INPUT_DEFAULT_SEED, RAPL_DEFAULT_ROOT); @@ -476,9 +477,9 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure for (size_t i = 0; i < ctx->trial_count && strcmp(status, "planned") != 0; i++) { const t_trial *t = &ctx->trials[i]; fprintf(f, "%s\n {\"language\": \"%s\", \"algorithm\": \"%s\", \"order\": \"%s\", \"n\": %zu, " - "\"inner_reps\": %" PRIu64 "}", + "\"inner_reps\": %" PRIu64 ", \"launches\": %" PRIu64 "}", first ? "" : ",", language_name(t->language), algorithm_name(t->algorithm), order_name(t->order), - t->n, t->inner_reps); + t->n, t->inner_reps, t->launches); first = 0; } fputs(first ? "]\n" : "\n ]\n", f); @@ -503,10 +504,49 @@ static void describe_trial(const t_trial *t, char *out, size_t size) { order_name(t->order), t->n); } -/* Runs one worker and measures it. Returns 0; -2 if the process could not start, exited - with an error or was killed (worth retrying: a busy machine can make a start fail); or -1 - for anything else, including results that disagree, which a retry would not change. */ -static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measurement *m, int will_retry) { +/* Every language must see the same input and do the same work on it. */ +static int check_consistency(t_context *ctx, const t_trial *t, const t_worker_result *r) { + int s = size_index(&ctx->options, t->n); + t_consistency *hash = &ctx->hashes[t->order][s]; + char what[TEXT_LEN]; + + describe_trial(t, what, sizeof what); + if (!hash->seen) { + hash->seen = 1; + hash->value = r->input_hash; + hash->first = t->language; + } else if (hash->value != r->input_hash) { + return error_message("%s: input hash %08" PRIx32 " differs from %08" PRIx32 " reported by the %s worker; " + "the input generators disagree", + what, r->input_hash, hash->value, language_name(hash->first)), + -1; + } + if (t->algorithm == ALGO_NONE) + return 0; + t_consistency *count = &ctx->counts[t->algorithm][t->order][s]; + if (!count->seen) { + count->seen = 1; + count->comparisons = r->comparisons; + count->writes = r->writes; + count->first = t->language; + } else if (count->comparisons != r->comparisons || count->writes != r->writes) { + return error_message("%s: %" PRIu64 " comparisons and %" PRIu64 " writes, but the %s worker reported %" PRIu64 + " and %" PRIu64 "; the implementations disagree", + what, r->comparisons, r->writes, language_name(count->first), count->comparisons, + count->writes), + -1; + } + return 0; +} + +/* Runs the worker `launches` times back to back and measures the batch as one interval. + Sorting runs launch one process; startup trials launch enough processes to span about + --min-loop-ms, because a single start can be shorter than the RAPL update interval. + Every result line is checked. Returns 0; -2 if a process could not start, exited with an + error or was killed (worth retrying: a busy machine can make a start fail); or -1 for + anything else, including results that disagree, which a retry would not change. */ +static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, uint64_t launches, t_measurement *m, + int will_retry) { const t_options *o = &ctx->options; char n_text[24], seed_text[24], reps_text[24], what[TEXT_LEN], err[ERROR_LEN]; char *argv[16]; @@ -545,27 +585,40 @@ static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measure describe_trial(t, what, sizeof what); if (ctx->energy_ok && rapl_read(&ctx->rapl, &before) != 0) return error_message("reading the RAPL counters failed: %s", strerror(errno)), -1; + int status = 0; uint64_t start = timing_now_ns(); - int rc = spawn_capture(argv, &ctx->spawn); + for (uint64_t launch = 0; launch < launches && status == 0; launch++) { + if (spawn_capture(argv, &ctx->spawn) != 0) { + status = -2; + break; + } + const t_worker_result *r = &m->result; + if (protocol_parse(ctx->spawn.out, &m->result, err, sizeof err) != 0) + status = (error_message("worker for %s printed an unexpected result: %s", what, err), -1); + else if (strcmp(r->language, language_name(t->language)) != 0 || + strcmp(r->algorithm, algorithm_name(t->algorithm)) != 0 || + strcmp(r->order, order_name(t->order)) != 0 || r->n != t->n || r->seed != o->seed || + r->reps != reps) + status = + (error_message("worker for %s reported a different configuration: %s", what, ctx->spawn.out), -1); + else if (check_consistency(ctx, t, r) != 0) + status = -1; + } uint64_t end = timing_now_ns(); if (ctx->energy_ok && rapl_read(&ctx->rapl, &after) != 0) return error_message("reading the RAPL counters failed: %s", strerror(errno)), -1; - if (rc != 0 && g_interrupted) + if (status == -2 && g_interrupted) return -1; /* the worker got the same Ctrl-C; command_run reports it */ - if (rc != 0) { + if (status == -2) { spawn_describe_failure(&ctx->spawn, err, sizeof err); error_message("worker for %s (%s): %s%s", what, argv[0], err, will_retry ? "; trying again" : ""); return -2; } - if (protocol_parse(ctx->spawn.out, &m->result, err, sizeof err) != 0) - return error_message("worker for %s printed an unexpected result: %s", what, err), -1; - const t_worker_result *r = &m->result; - if (strcmp(r->language, language_name(t->language)) != 0 || - strcmp(r->algorithm, algorithm_name(t->algorithm)) != 0 || strcmp(r->order, order_name(t->order)) != 0 || - r->n != t->n || r->seed != o->seed || r->reps != reps) - return error_message("worker for %s reported a different configuration: %s", what, ctx->spawn.out), -1; + if (status != 0) + return -1; m->wall_ns = end - start; + m->launches = launches; m->energy.package_uj = 0; m->energy.dram_uj = 0; if (ctx->energy_ok && rapl_energy_between(&ctx->rapl, &before, &after, &m->energy) != 0) @@ -579,51 +632,16 @@ static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, t_measure } /* run_worker, repeated up to --retries times when the worker process itself fails. */ -static int run_measured(t_context *ctx, const t_trial *t, uint64_t reps, t_measurement *m) { +static int run_measured(t_context *ctx, const t_trial *t, uint64_t reps, uint64_t launches, t_measurement *m) { for (unsigned attempt = 0;; attempt++) { int will_retry = attempt < ctx->options.retries; - int rc = run_worker(ctx, t, reps, m, will_retry); + int rc = run_worker(ctx, t, reps, launches, m, will_retry); if (rc != -2 || !will_retry || g_interrupted) return rc == 0 ? 0 : -1; ctx->retries_used++; } } -/* Every language must see the same input and do the same work on it. */ -static int check_consistency(t_context *ctx, const t_trial *t, const t_worker_result *r) { - int s = size_index(&ctx->options, t->n); - t_consistency *hash = &ctx->hashes[t->order][s]; - char what[TEXT_LEN]; - - describe_trial(t, what, sizeof what); - if (!hash->seen) { - hash->seen = 1; - hash->value = r->input_hash; - hash->first = t->language; - } else if (hash->value != r->input_hash) { - return error_message("%s: input hash %08" PRIx32 " differs from %08" PRIx32 " reported by the %s worker; " - "the input generators disagree", - what, r->input_hash, hash->value, language_name(hash->first)), - -1; - } - if (t->algorithm == ALGO_NONE) - return 0; - t_consistency *count = &ctx->counts[t->algorithm][t->order][s]; - if (!count->seen) { - count->seen = 1; - count->comparisons = r->comparisons; - count->writes = r->writes; - count->first = t->language; - } else if (count->comparisons != r->comparisons || count->writes != r->writes) { - return error_message("%s: %" PRIu64 " comparisons and %" PRIu64 " writes, but the %s worker reported %" PRIu64 - " and %" PRIu64 "; the implementations disagree", - what, r->comparisons, r->writes, language_name(count->first), count->comparisons, - count->writes), - -1; - } - return 0; -} - static void format_duration(uint64_t ns, char *out, size_t size) { uint64_t seconds = ns / 1000000000u; @@ -666,23 +684,28 @@ static int warm_up(t_context *ctx) { for (size_t i = 0; i < ctx->trial_count; i++) { t_trial *t = &ctx->trials[order[i]]; uint64_t reps = t->algorithm == ALGO_NONE ? 0 : 1; + uint64_t launches = 1; t_measurement m; for (unsigned w = 0; w < o->warmup; w++) { - if (interrupted() || run_measured(ctx, t, reps, &m) != 0 || check_consistency(ctx, t, &m.result) != 0) { + if (interrupted() || run_measured(ctx, t, reps, launches, &m) != 0) { free(order); return -1; } - t->warm_wall_ns = m.wall_ns; - if (t->algorithm != ALGO_NONE) { + /* Calibrate, and estimate the next run's length for the progress report. */ + if (t->algorithm == ALGO_NONE) { + uint64_t next = plan_calibrate(m.wall_ns, launches, o->min_loop_ns, PLAN_MAX_LAUNCHES); + t->warm_wall_ns = m.wall_ns / launches * next; + launches = next; + } else { uint64_t next = plan_calibrate(m.result.loop_ns, reps, o->min_loop_ns, o->max_reps); - /* Estimate the next run's length for the progress report. */ uint64_t overhead = m.wall_ns > m.result.loop_ns ? m.wall_ns - m.result.loop_ns : 0; t->warm_wall_ns = overhead + m.result.loop_ns / reps * next; reps = next; } } t->inner_reps = reps; + t->launches = launches; if (!o->quiet && progress_due(&progress, i + 1 == ctx->trial_count)) fprintf(stderr, "%swarm-up and calibration: %zu/%zu trials%s", progress.tty ? "\r" : "", i + 1, ctx->trial_count, progress.tty ? "" : "\n"); @@ -718,8 +741,7 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { t_measurement m; t_run_record record; - if (interrupted() || run_measured(ctx, t, t->inner_reps, &m) != 0 || - check_consistency(ctx, t, &m.result) != 0) { + if (interrupted() || run_measured(ctx, t, t->inner_reps, t->launches, &m) != 0) { free(items); return -1; } @@ -797,7 +819,8 @@ static int pin_cpu(int cpu) { #endif } -/* Before measuring anything, the harness checks that the counters move. */ +/* Before measuring anything, the harness checks that the counters move over this long. It is + twice RAPL_FROZEN_NS, so a sleep that ends early on CLOCK_MONOTONIC_RAW is still judged. */ #define LIVE_CHECK_NS (2u * RAPL_FROZEN_NS) /* Reads the counters twice, sample_ns apart. Returns 0 when the package counter moved, or -1 @@ -901,12 +924,15 @@ static int command_run(int argc, char **argv, const char *argv0) { plan_trials(o, ctx.trials); if (o->dry_run) { - uint64_t processes = (uint64_t)ctx.trial_count * (o->warmup + o->runs); + uint64_t intervals = (uint64_t)ctx.trial_count * (o->warmup + o->runs); printf("profile %s: %zu trials (%zu languages x %zu algorithms%s x %zu orders x %zu sizes)\n", o->profile, ctx.trial_count, o->language_count, o->algorithm_count, o->startup_trials ? " + startup" : "", o->order_count, o->size_count); - printf("%u warm-up and %u measured runs per trial: %" PRIu64 " worker processes, %zu CSV rows\n", - o->warmup, o->runs, processes, ctx.trial_count * o->runs); + printf("%u warm-up and %u measured runs per trial: %" PRIu64 " runs in all, %zu CSV rows\n", + o->warmup, o->runs, intervals, ctx.trial_count * o->runs); + if (o->startup_trials) + printf("each startup run launches a batch of processes lasting about %" PRIu64 " ms in total\n", + o->min_loop_ns / 1000000u); cleanup(&ctx); return 0; } @@ -984,7 +1010,7 @@ static int command_probe(int argc, char **argv) { root = argv[++i]; } else if (strcmp(argv[i], "--sample-ms") == 0 && i + 1 < argc && options_parse_u64(argv[i + 1], 0, 60000, &sample_ms) == 0 && - (sample_ms == 0 || sample_ms * 1000000u >= RAPL_FROZEN_NS)) { + (sample_ms == 0 || sample_ms * 1000000u >= LIVE_CHECK_NS)) { i++; } else if (strcmp(argv[i], "--help") == 0) { print_usage(stdout); diff --git a/src/output.c b/src/output.c index ab727ab..b73f636 100644 --- a/src/output.c +++ b/src/output.c @@ -5,7 +5,7 @@ #include "rapl.h" void output_csv_header(FILE *file) { - fputs("sequence,language,algorithm,order,n,seed,repetition,inner_reps,wall_ns,loop_ns,comparisons,writes," + fputs("sequence,language,algorithm,order,n,seed,repetition,inner_reps,launches,wall_ns,loop_ns,comparisons,writes," "input_hash,energy_status,pkg_uj,dram_uj,idle_ns,idle_pkg_uj,idle_dram_uj,net_pkg_uj,net_dram_uj\n", file); } @@ -13,10 +13,11 @@ void output_csv_header(FILE *file) { void output_csv_row(FILE *file, const t_run_record *r) { const t_trial *t = r->trial; - fprintf(file, "%" PRIu64 ",%s,%s,%s,%zu,%" PRIu32 ",%u,%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 - ",%08" PRIx32 ",%s,", + fprintf(file, + "%" PRIu64 ",%s,%s,%s,%zu,%" PRIu32 ",%u,%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 + ",%" PRIu64 ",%08" PRIx32 ",%s,", r->sequence, language_name(t->language), algorithm_name(t->algorithm), order_name(t->order), t->n, - r->seed, r->repetition, t->inner_reps, r->wall_ns, r->loop_ns, r->comparisons, r->writes, + r->seed, r->repetition, t->inner_reps, t->launches, r->wall_ns, r->loop_ns, r->comparisons, r->writes, r->input_hash, r->energy_status); /* pkg_uj, dram_uj */ if (r->has_energy) diff --git a/src/output.h b/src/output.h index 7ea33a3..b53de5d 100644 --- a/src/output.h +++ b/src/output.h @@ -9,7 +9,7 @@ #include "plan.h" -#define OUTPUT_CSV_SCHEMA 1 +#define OUTPUT_CSV_SCHEMA 2 /* 2 added launches */ typedef struct s_run_record { uint64_t sequence; /* position in the randomised run order, from 1 */ diff --git a/src/plan.c b/src/plan.c index bd28e71..c437a25 100644 --- a/src/plan.c +++ b/src/plan.c @@ -23,6 +23,7 @@ void plan_trials(const t_options *o, t_trial *trials) { t->order = o->orders[r]; t->n = o->sizes[s]; t->inner_reps = t->algorithm == ALGO_NONE ? 0 : 1; + t->launches = 1; } } diff --git a/src/plan.h b/src/plan.h index 184269f..9b604ae 100644 --- a/src/plan.h +++ b/src/plan.h @@ -15,9 +15,13 @@ typedef struct s_trial { t_order order; size_t n; uint64_t inner_reps; /* sorts per process, set by calibration; 0 for startup trials */ - uint64_t warm_wall_ns; /* duration of the last warm-up run, used for the time estimate */ + uint64_t launches; /* processes per measured run: 1 for sorting, calibrated for startup */ + uint64_t warm_wall_ns; /* estimated length of one measured run, for the progress report */ } t_trial; +/* A startup batch never launches more processes than this. */ +#define PLAN_MAX_LAUNCHES 10000u + /* One measured run: the trial it belongs to and its repetition number (1-based). */ typedef struct s_run_item { size_t trial; diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index c73e205..f287ca5 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -47,8 +47,10 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert int(row["wall_ns"]) > 0 if row["algorithm"] == "none": assert row["inner_reps"] == "0" + assert int(row["launches"]) >= 1 else: assert int(row["inner_reps"]) >= 1 + assert row["launches"] == "1" # Every configuration has runs 1..3, and the run order is shuffled rather than grouped. configurations = {(r["language"], r["algorithm"], r["order"], r["n"]) for r in rows} @@ -71,6 +73,41 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert set(meta["workers"]) == {"c", "python", "node"} +def test_startup_runs_launch_a_batch_of_processes(tmp_path: Path) -> None: + # One C process starts in about a millisecond, so a 40 ms target needs a batch of them. + out = tmp_path / "run" + completed = run_harness( + "run", + "--profile", + "smoke", + "--languages", + "c", + "--algorithms", + "quick", + "--orders", + "random", + "--sizes", + "20", + "--runs", + "2", + "--min-loop-ms", + "40", + "--energy", + "off", + "--quiet", + "--out", + str(out), + ) + assert completed.returncode == 0, completed.stderr + startup = [r for r in read_runs(out) if r["algorithm"] == "none"] + assert len(startup) == 2 + for row in startup: + assert int(row["launches"]) > 1 + assert int(row["wall_ns"]) >= 20_000_000 # about 40 ms of launches, allowing for calibration error + (entry,) = [c for c in read_meta(out)["calibration"] if c["algorithm"] == "none"] + assert entry["launches"] == int(startup[0]["launches"]) + + def test_counts_match_across_languages_in_the_csv(tmp_path: Path) -> None: out = tmp_path / "run" completed = run_harness("run", *SMALL, "--energy", "off", "--out", str(out), *interpreter_options()) @@ -181,14 +218,14 @@ def test_energy_auto_falls_back_to_time_only(tmp_path: Path) -> None: def test_probe(tmp_path: Path) -> None: root = build_powercap(FIXTURES / "laptop-intel.txt", tmp_path / "powercap") with CounterWriter(root, idle_watts=5.0): - live = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "100") + live = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "200") assert live.returncode == 0, live.stdout + live.stderr assert "intel-rapl:0:2" in live.stdout assert "psys" not in live.stdout assert "idle power over" in live.stdout assert live.stdout.rstrip().endswith("energy: available") - frozen = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "100") + frozen = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "200") assert frozen.returncode == 3 assert "energy: unavailable" in frozen.stdout assert "did not change over" in frozen.stdout @@ -217,7 +254,7 @@ def test_dry_run_prints_the_plan() -> None: (["run", "--bogus"], "unknown option --bogus"), (["walk"], "unknown command 'walk'"), (["probe", "--sample-ms", "lots"], "unexpected or invalid argument"), - (["probe", "--sample-ms", "50"], "unexpected or invalid argument"), + (["probe", "--sample-ms", "150"], "unexpected or invalid argument"), ], ) def test_usage_errors_exit_2(args: list[str], message: str) -> None: diff --git a/tests/test_output.c b/tests/test_output.c index 76ef969..823a08d 100644 --- a/tests/test_output.c +++ b/tests/test_output.c @@ -24,15 +24,15 @@ static void test_header(void) { size_t n = fread(buf, 1, sizeof buf - 1, f); buf[n] = '\0'; fclose(f); - const char *start = "sequence,language,algorithm,order,n,seed,repetition,inner_reps,wall_ns,loop_ns,"; + const char *start = "sequence,language,algorithm,order,n,seed,repetition,inner_reps,launches,wall_ns,loop_ns,"; CHECK(strncmp(buf, start, strlen(start)) == 0, "header starts with the identifiers"); int commas = 0; for (const char *p = buf; *p; p++) commas += *p == ','; - CHECK(commas == 20, "21 columns"); + CHECK(commas == 21, "22 columns"); } -static t_trial trial = {LANG_PYTHON, ALGO_QUICK, ORDER_RANDOM, 1000, 40, 0}; +static t_trial trial = {LANG_PYTHON, ALGO_QUICK, ORDER_RANDOM, 1000, 40, 1, 0}; static t_run_record base_record(void) { t_run_record r; @@ -56,7 +56,7 @@ static void test_row_without_energy(void) { r.energy_status = "unavailable"; capture_row(&r, row, sizeof row); - CHECK(strcmp(row, "17,python,quick,random,1000,2463534242,3,40,250000000,200000000,11039,11846,0badf00d," + CHECK(strcmp(row, "17,python,quick,random,1000,2463534242,3,40,1,250000000,200000000,11039,11846,0badf00d," "unavailable,,,,,,,\n") == 0, "energy cells are empty, not zero"); } diff --git a/tests/test_plan.c b/tests/test_plan.c index 573bc29..311edc7 100644 --- a/tests/test_plan.c +++ b/tests/test_plan.c @@ -21,8 +21,9 @@ static void test_trials(void) { if (trials[i].algorithm == ALGO_NONE) { startup++; CHECK(trials[i].inner_reps == 0, "startup trials never sort"); + CHECK(trials[i].launches == 1, "startup batches start at one launch"); } else { - CHECK(trials[i].inner_reps == 1, "sorting trials start at one sort per process"); + CHECK(trials[i].inner_reps == 1 && trials[i].launches == 1, "sorting trials: one process, one sort"); } } CHECK(startup == 3 * 4 * 4, "one startup trial per language, order and size"); From 0c681e9847d6a62f5f95b0d9023b992dea399377 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:05:41 +0400 Subject: [PATCH 22/51] fix: pin to two CPUs on different cores so Node.js helper threads can run The study pinned the harness and every worker to one CPU. A Node.js 24 worker runs 7 threads (V8's compiler and garbage collector), a Python worker 1, so on a single CPU Node's helper threads took turns with the sorting thread and the cross-language comparison was biased against it. The harness now takes --cpus LIST (for example 2,3 or 2-3, validated against the size of a cpu_set_t, which --cpu did not do) and records the set in meta.json. run-study.sh picks two CPUs on different physical cores of one processor from sysfs, never CPU 0. ADR 15 and the threats to validity describe the choice and what has not been compared. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- analysis/greenbench_analysis/report.py | 9 +++- analysis/tests/synthetic.py | 2 +- docs/bare-metal.md | 6 ++- docs/decisions.md | 23 ++++++++++ docs/study.md | 17 ++++++-- scripts/run-study.sh | 52 +++++++++++++++------- src/greenbench.c | 26 ++++++----- src/options.c | 60 ++++++++++++++++++++++---- src/options.h | 9 +++- tests/integration/test_harness.py | 13 ++++++ tests/test_options.c | 17 ++++++-- 12 files changed, 189 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index afdd68e..b4bb55b 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ docker run --rm -v "$PWD/results:/work/results" greenbench | `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Minimum sort loop per process, and the cap on sorts per process. | | `--energy auto\|require\|off` | `auto` | `require` fails with exit status 3 if RAPL cannot be read. | | `--idle-baseline auto\|on\|off` | `auto` | On whenever energy is measured. | -| `--cpu N` | off | Pin the harness and every worker to CPU N. | +| `--cpus LIST` | off | Pin the harness and every worker to these CPUs, such as `2,3` or `2-3`. | | `--seed N`, `--order-seed N` | 2463534242, 1234567 | Input generator seed, and run-order seed. | | `--no-startup` | | Skip the startup trials. | | `--retries N` | 1 | Rerun a worker process that fails (cannot start, crashes) up to N times; disagreeing results are never retried. | diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 183559c..7a96024 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -98,6 +98,13 @@ def _table(header: list[str], align: list[str], rows: Iterable[list[str]]) -> li return lines +def _pinning(settings: dict[str, Any]) -> str: + cpus = settings.get("cpus", []) + if not cpus: + return "off" + return ("to CPU " if len(cpus) == 1 else "to CPUs ") + ", ".join(str(c) for c in cpus) + + def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: host = meta.get("host", {}) settings = meta.get("settings", {}) @@ -122,7 +129,7 @@ def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: f"- Machine: {cpu}; {host.get('sysname', '?')} {host.get('release', '?')} {host.get('machine', '')}; " f"{host.get('online_cpus', '?')} CPUs online; scaling governor " f"{host.get('scaling_governor') or 'not reported'}; " - f"CPU pinning {'to CPU ' + str(settings['cpu']) if settings.get('cpu', -1) >= 0 else 'off'}.", + f"CPU pinning {_pinning(settings)}.", "- Workers: " + "; ".join(versions) + ".", ] if energy.get("status") == "ok": diff --git a/analysis/tests/synthetic.py b/analysis/tests/synthetic.py index bb97506..4c84ee0 100644 --- a/analysis/tests/synthetic.py +++ b/analysis/tests/synthetic.py @@ -146,7 +146,7 @@ def meta(*, energy: bool) -> dict[str, Any]: "seed": 2463534242, "order_seed": 1234567, "min_loop_ns": 2000000, - "cpu": -1, + "cpus": [], }, "workers": {"c": {"version": "gcc 14"}, "python": {"version": "Python 3.12.0"}}, } diff --git a/docs/bare-metal.md b/docs/bare-metal.md index a117b41..67823c3 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -76,8 +76,10 @@ disk, and the study leaves the machine as it found it after a reboot. - Close every other application. A live session may start update checks in the background; wait a few minutes after booting before running the study. - Keep the machine on mains power; laptops on battery often cap the processor. -- The script pins the harness and every worker to CPU 2 when there are four or more CPUs. - Choose another with `--cpu N`, or turn pinning off with `--no-pin`. +- The script pins the harness and every worker to two CPUs on different physical cores of + one processor, never CPU 0, and prints which. Choose others with `--cpus 4,6`, or turn + pinning off with `--no-pin`. On a processor with performance and efficiency cores, check + with `lscpu --extended` that both CPUs are the same kind. - Frequency scaling and turbo stay as they are, and are recorded. For steadier numbers you can fix them for the session, for example on Intel with `echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo`. Mention it when you diff --git a/docs/decisions.md b/docs/decisions.md index 3b59bbc..5b22a44 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -213,3 +213,26 @@ the other languages is never retried: that is a bug, and the run stops. **Consequences.** A transient failure no longer ends a long run, and the count shows how often it happened. A worker that fails every time still stops the run, with its error message, after one extra attempt. + +## 15. Pin to two CPUs on different physical cores, not one + +**Context.** Pinning keeps the measured processes on known cores and stops them migrating. +The first version pinned the harness and every worker to a single CPU. C and Python run one +thread, but a Node.js process also runs V8's optimising compiler and concurrent garbage +collector on helper threads: a Node.js 24 worker had 7 threads while it sorted, a Python +worker 1 (counted in `/proc/PID/task` in the Docker image). On one CPU those helper threads +take turns with the sorting thread, which slows Node.js down in a way an ordinary machine with +idle cores would not, and biases the comparison between languages against it. + +**Decision.** The harness takes a CPU list, `--cpus 2,3`. The study script chooses two CPUs +on different physical cores of the same processor from `/sys/devices/system/cpu/*/topology`, +leaving out CPU 0, which handles most interrupts; `--cpus` overrides the choice and +`--no-pin` turns pinning off. Different physical cores rather than the two hardware threads of +one core, so that a helper thread does not compete with the sorting thread for the same +execution units. + +**Consequences.** Node.js's helper threads run beside the sorting thread, as on an idle +desktop. C and Python can move between the two CPUs, which are the same kind of core, so this +changes little. RAPL counts the whole package, so the helper threads' energy is included +either way. No pinned-against-unpinned comparison has been measured; the threats to validity +say so. diff --git a/docs/study.md b/docs/study.md index 712d079..bcc9b62 100644 --- a/docs/study.md +++ b/docs/study.md @@ -84,8 +84,10 @@ lists, JavaScript arrays). None uses a built-in sort. and energy per sort. - **Repetitions and order.** Three warm-up runs per configuration, then 30 measured runs. All measured runs of all configurations are shuffled into one random order from a recorded seed. -- **Pinning.** The study script pins the harness and all workers to one CPU (CPU 2 by - default), so the measured process does not migrate between cores. +- **Pinning.** The study script pins the harness and all workers to two CPUs on different + physical cores of one processor (never CPU 0), so the measured processes stay on known + cores. Two rather than one, because Node.js runs its compiler and garbage collector on + helper threads ([ADR 15](decisions.md#15-pin-to-two-cpus-on-different-physical-cores-not-one)). - **Records.** `runs.csv` holds one row per measured run (tidy data: one variable per column). `meta.json` holds the settings, the RAPL zones and their wrap values, the CPU model, the kernel, the scaling governor, the turbo setting, the language versions and the exact @@ -138,6 +140,12 @@ lists, JavaScript arrays). None uses a built-in sort. runs and both interpreters collect garbage. A process that sorts once measures more unoptimised code than one that sorts thousands of times. This is what running the program costs, but it makes per-sort figures depend on the number of sorts per process. +- **Helper threads and pinning.** C and Python run one thread; a Node.js worker runs seven, + because V8 compiles and collects garbage on helper threads. The study pins every worker to + the same two CPUs, so these threads run beside the sorting thread, and their energy counts + because RAPL measures the whole package. On a machine with more free cores, or with a + different choice of CPUs, Node.js could be somewhat faster. Pinned and unpinned runs have + not been compared. - **Startup subtraction.** Per-sort figures subtract a median startup cost measured in separate processes. Noise in that median shifts every per-sort value of the configuration. - **Language versions.** Results describe the recorded compiler and interpreter versions @@ -145,8 +153,9 @@ lists, JavaScript arrays). None uses a built-in sort. ### Scope -- One machine at a time, single-threaded, small integer arrays, no input or output. The - results say nothing about parallel code, large memory footprints or I/O-bound programs. +- One machine at a time, single-threaded sorts (Node.js adds its runtime's helper threads), + small integer arrays, no input or output. The results say nothing about parallel code, large + memory footprints or I/O-bound programs. - Carbon estimates use annual, location-based averages that include lifecycle emissions. They do not describe the marginal emissions of running the benchmark at a particular hour. diff --git a/scripts/run-study.sh b/scripts/run-study.sh index f271093..779706a 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -11,19 +11,20 @@ set -euo pipefail cd "$(dirname "$0")/.." MODE=study -CPU=auto +CPUS=auto ASSUME_YES=0 POWERCAP=/sys/class/powercap FAKE_POWERCAP=0 usage() { cat <<'EOF' -Usage: scripts/run-study.sh [--check | --rehearse] [--cpu N | --no-pin] [--yes] [--powercap-root DIR] +Usage: scripts/run-study.sh [--check | --rehearse] [--cpus LIST | --no-pin] [--yes] [--powercap-root DIR] --check check the machine (Linux on real hardware, tools, readable RAPL counters) and stop --rehearse run the short smoke profile with energy, to test the machine; results go to results/ - --cpu N pin the harness and workers to CPU N (default: CPU 2 when there are 4 or more) - --no-pin do not pin to a CPU + --cpus LIST pin the harness and workers to these CPUs, such as 2,3 (default: two CPUs on + different physical cores of one processor, never CPU 0) + --no-pin do not pin to any CPUs --yes do not ask before making the RAPL counters readable --powercap-root DIR read RAPL zones from DIR instead of /sys/class/powercap. Only for testing this @@ -43,12 +44,12 @@ while (($# > 0)); do case "$1" in --check) MODE=check ;; --rehearse) MODE=rehearse ;; - --cpu) - [[ $# -ge 2 && "$2" =~ ^[0-9]+$ ]] || fail "--cpu needs a CPU number" - CPU="$2" + --cpus) + [[ $# -ge 2 && "$2" =~ ^[0-9]+([-,][0-9]+)*$ ]] || fail "--cpus needs a CPU list, such as 2,3 or 2-3" + CPUS="$2" shift ;; - --no-pin) CPU=none ;; + --no-pin) CPUS=none ;; --yes) ASSUME_YES=1 ;; --powercap-root) [[ $# -ge 2 && -d "$2" ]] || fail "--powercap-root needs an existing directory" @@ -120,15 +121,33 @@ check_rapl() { sudo chmod o+r "${unreadable[@]}" } +# Two CPUs on different physical cores of the same processor, leaving out CPU 0, which +# handles most interrupts. Two cores, not one: Node.js runs its JIT compiler and garbage +# collector on helper threads, which must not queue behind the sorting thread. Prints nothing +# when there are no two such CPUs. +default_cpus() { + local total cpu dir package core first="" first_package="" first_core="" + total="$(nproc --all)" + for ((cpu = 1; cpu < total; cpu++)); do + dir="/sys/devices/system/cpu/cpu$cpu" + [[ -r "$dir/topology/core_id" && -r "$dir/topology/physical_package_id" ]] || continue + if [[ -r "$dir/online" && "$(cat "$dir/online")" != 1 ]]; then continue; fi + package="$(cat "$dir/topology/physical_package_id")" + core="$(cat "$dir/topology/core_id")" + if [[ -z "$first" ]]; then + first="$cpu" first_package="$package" first_core="$core" + elif [[ "$package" == "$first_package" && "$core" != "$first_core" ]]; then + printf '%s,%s\n' "$first" "$cpu" + return 0 + fi + done +} + pin_options() { - if [[ "$CPU" == none ]]; then - return 0 - fi - if [[ "$CPU" == auto ]]; then - if (($(nproc) >= 4)); then printf '%s\n' --cpu 2; fi - return 0 - fi - printf '%s\n' --cpu "$CPU" + local cpus="$CPUS" + if [[ "$cpus" == none ]]; then return 0; fi + if [[ "$cpus" == auto ]]; then cpus="$(default_cpus)"; fi + if [[ -n "$cpus" ]]; then printf '%s\n' --cpus "$cpus"; fi } mains_power() { @@ -210,6 +229,7 @@ fi mkdir -p "$out" record_environment "$out/environment.txt" mapfile -t pin < <(pin_options) +if ((${#pin[@]} > 0)); then say "== Pinning the harness and workers to CPUs ${pin[1]}"; else say "== Not pinning to CPUs"; fi say "== Measuring (profile $profile); close other programs and keep the machine on mains power" build/greenbench run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ diff --git a/src/greenbench.c b/src/greenbench.c index 8a09d65..fb4af17 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -142,7 +142,7 @@ static void print_usage(FILE *out) { " --energy MODE auto (measure when possible), require (fail without RAPL) or off\n" " --idle-baseline MODE auto (on when energy is measured), on or off\n" " --powercap-root DIR where to look for RAPL zones (default %s)\n" - " --cpu N pin the harness and every worker to CPU N (Linux)\n" + " --cpus LIST pin the harness and every worker to these CPUs, such as 2,3 or 2-3 (Linux)\n" " --out DIR write runs.csv and meta.json here (default results/latest)\n" " --python CMD Python interpreter (default python3)\n" " --node CMD Node.js executable (default node)\n" @@ -408,7 +408,7 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure fprintf(f, " \"online_cpus\": %ld,\n", sysconf(_SC_NPROCESSORS_ONLN)); char governor_path[128]; snprintf(governor_path, sizeof governor_path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", - o->cpu >= 0 ? o->cpu : 0); + o->cpu_count > 0 ? o->cpus[0] : 0); read_first_line(governor_path, text, sizeof text); json_key_string(f, " ", "scaling_governor", text, 1); read_first_line("/sys/devices/system/cpu/intel_pstate/no_turbo", text, sizeof text); @@ -453,9 +453,12 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure fprintf(f, " \"runs\": %u,\n \"warmup\": %u,\n \"seed\": %" PRIu32 ",\n \"order_seed\": %" PRIu32 ",\n \"min_loop_ns\": %" PRIu64 ",\n \"max_reps\": %" PRIu64 ",\n \"startup_trials\": %s,\n" - " \"retries\": %u,\n \"cpu\": %d\n", + " \"retries\": %u,\n \"cpus\": [", o->runs, o->warmup, o->seed, o->order_seed, o->min_loop_ns, o->max_reps, - o->startup_trials ? "true" : "false", o->retries, o->cpu); + o->startup_trials ? "true" : "false", o->retries); + for (size_t i = 0; i < o->cpu_count; i++) + fprintf(f, "%s%d", i ? ", " : "", o->cpus[i]); + fputs("]\n", f); fputs(" },\n", f); fputs(" \"workers\": {\n", f); @@ -804,18 +807,21 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { return 0; } -static int pin_cpu(int cpu) { +/* Pins the harness to the CPUs; the workers it starts inherit the set. */ +static int pin_cpus(const int *cpus, size_t count) { #ifdef __linux__ cpu_set_t set; CPU_ZERO(&set); - CPU_SET((size_t)cpu, &set); + for (size_t i = 0; i < count; i++) + CPU_SET((size_t)cpus[i], &set); if (sched_setaffinity(0, sizeof set, &set) != 0) - return error_message("cannot pin to CPU %d: %s", cpu, strerror(errno)), -1; + return error_message("cannot pin to the CPUs given with --cpus: %s", strerror(errno)), -1; return 0; #else - (void)cpu; - return error_message("--cpu is only supported on Linux"), -1; + (void)cpus; + (void)count; + return error_message("--cpus is only supported on Linux"), -1; #endif } @@ -959,7 +965,7 @@ static int command_run(int argc, char **argv, const char *argv0) { else fprintf(stderr, "energy unavailable: %s\nMeasuring time and operation counts only.\n", ctx.energy_reason); } - if (o->cpu >= 0 && pin_cpu(o->cpu) != 0) { + if (o->cpu_count > 0 && pin_cpus(o->cpus, o->cpu_count) != 0) { cleanup(&ctx); return EXIT_USAGE; } diff --git a/src/options.c b/src/options.c index 28cfe43..8384e49 100644 --- a/src/options.c +++ b/src/options.c @@ -72,7 +72,7 @@ void options_defaults(t_options *o) { o->energy = ENERGY_AUTO; o->baseline = BASELINE_AUTO; o->powercap_root = RAPL_DEFAULT_ROOT; - o->cpu = -1; + o->cpu_count = 0; o->out_dir = "results/latest"; o->python = "python3"; o->node = "node"; @@ -147,6 +147,54 @@ int options_parse_sizes(const char *text, size_t *sizes, size_t *count, char *er return 0; } +static int add_cpu(int *cpus, size_t *count, uint64_t cpu, char *err, size_t err_size) { + for (size_t i = 0; i < *count; i++) { + if (cpus[i] == (int)cpu) { + fail(err, err_size, "--cpus: CPU %llu is listed twice", (unsigned long long)cpu); + return -1; + } + } + if (*count == OPTIONS_MAX_CPUS) { + fail(err, err_size, "--cpus: at most %d CPUs", OPTIONS_MAX_CPUS); + return -1; + } + cpus[(*count)++] = (int)cpu; + return 0; +} + +int options_parse_cpus(const char *text, int *cpus, size_t *count, char *err, size_t err_size) { + char buffer[LIST_BUFFER]; + char *tokens[OPTIONS_MAX_CPUS]; + int n = split_list(text, buffer, sizeof buffer, tokens, OPTIONS_MAX_CPUS); + size_t found = 0; + + if (n < 0) { + fail(err, err_size, "--cpus: expected a comma-separated list of CPUs or ranges, such as 2,3 or 2-5"); + return -1; + } + for (int i = 0; i < n; i++) { + uint64_t low, high; + char *dash = strchr(tokens[i], '-'); + if (dash != NULL) + *dash = '\0'; + if (options_parse_u64(tokens[i], 0, OPTIONS_CPU_LIMIT, &low) != 0 || + (dash != NULL && options_parse_u64(dash + 1, low, OPTIONS_CPU_LIMIT, &high) != 0)) { + if (dash != NULL) + *dash = '-'; + fail(err, err_size, "--cpus: '%s' is not a CPU from 0 to %d or a rising range of them", tokens[i], + OPTIONS_CPU_LIMIT); + return -1; + } + if (dash == NULL) + high = low; + for (uint64_t cpu = low; cpu <= high; cpu++) + if (add_cpu(cpus, &found, cpu, err, err_size) != 0) + return -1; + } + *count = found; + return 0; +} + static int parse_languages(const char *text, t_options *o, char *err, size_t err_size) { char buffer[LIST_BUFFER]; char *tokens[LANG_COUNT]; @@ -251,7 +299,7 @@ static int parse_choice(const char *option, const char *text, const char *const static const char *const FLAGS[] = {"no-startup", "dry-run", "quiet", "help"}; static const char *const VALUED[] = {"profile", "languages", "algorithms", "orders", "sizes", "runs", "warmup", "seed", "order-seed", "min-loop-ms", - "max-reps", "cpu", "energy", "idle-baseline", + "max-reps", "cpus", "energy", "idle-baseline", "powercap-root", "out", "python", "node", "c-worker", "python-worker", "node-worker", "label", "retries"}; @@ -345,12 +393,8 @@ static int apply_option(const char *name, const char *value, t_options *o, char o->retries = (unsigned)number; return 0; } - if (strcmp(name, "cpu") == 0) { - if (parse_number(name, value, 0, 4095, &number, err, err_size) != 0) - return -1; - o->cpu = (int)number; - return 0; - } + if (strcmp(name, "cpus") == 0) + return options_parse_cpus(value, o->cpus, &o->cpu_count, err, err_size); if (strcmp(name, "energy") == 0) { if (parse_choice(name, value, ENERGY, 3, &choice, err, err_size) != 0) return -1; diff --git a/src/options.h b/src/options.h index b414617..0face33 100644 --- a/src/options.h +++ b/src/options.h @@ -8,6 +8,8 @@ #include "inputs.h" #define OPTIONS_MAX_SIZES 16 +#define OPTIONS_MAX_CPUS 64 +#define OPTIONS_CPU_LIMIT 1023 /* the highest CPU number a cpu_set_t can hold */ #define OPTIONS_MIN_STUDY_RUNS 30 typedef enum e_energy_mode { ENERGY_AUTO, ENERGY_REQUIRE, ENERGY_OFF } t_energy_mode; @@ -34,7 +36,8 @@ typedef struct s_options { t_energy_mode energy; t_baseline_mode baseline; const char *powercap_root; - int cpu; /* CPU to pin to, or -1 */ + int cpus[OPTIONS_MAX_CPUS]; /* CPUs to pin to, in the order given */ + size_t cpu_count; /* 0: not pinned */ const char *out_dir; const char *python; const char *node; @@ -60,6 +63,10 @@ int options_parse(int argc, char *const argv[], t_options *options, char *err, s /* Parses a comma-separated list of sizes (each 1 to INPUT_MAX_N, no duplicates). */ int options_parse_sizes(const char *text, size_t *sizes, size_t *count, char *err, size_t err_size); +/* Parses a CPU list such as "2,3" or "2-5" (each CPU 0 to OPTIONS_CPU_LIMIT, no duplicates, + at most OPTIONS_MAX_CPUS). Returns 0, or -1 with a message in err. */ +int options_parse_cpus(const char *text, int *cpus, size_t *count, char *err, size_t err_size); + /* Parses a decimal integer in [min, max] with nothing else in the string. */ int options_parse_u64(const char *text, uint64_t min, uint64_t max, uint64_t *out); diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index f287ca5..06e9068 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -68,6 +68,7 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert meta["energy"]["idle_baseline"] is False assert meta["clock"] == "CLOCK_MONOTONIC_RAW" assert meta["settings"]["runs"] == 3 + assert meta["settings"]["cpus"] == [] assert meta["command"].startswith("greenbench run --profile smoke") assert len(meta["calibration"]) == 48 assert set(meta["workers"]) == {"c", "python", "node"} @@ -108,6 +109,17 @@ def test_startup_runs_launch_a_batch_of_processes(tmp_path: Path) -> None: assert entry["launches"] == int(startup[0]["launches"]) +def test_pinning_to_a_cpu_set(tmp_path: Path) -> None: + out = tmp_path / "run" + completed = run_harness( + "run", *SMALL, "--languages", "c,node", "--cpus", "0-1", "--energy", "off", "--out", str(out), + *interpreter_options(), + ) + assert completed.returncode == 0, completed.stderr + assert read_meta(out)["settings"]["cpus"] == [0, 1] + assert read_meta(tmp_path / "run")["status"] == "complete" + + def test_counts_match_across_languages_in_the_csv(tmp_path: Path) -> None: out = tmp_path / "run" completed = run_harness("run", *SMALL, "--energy", "off", "--out", str(out), *interpreter_options()) @@ -252,6 +264,7 @@ def test_dry_run_prints_the_plan() -> None: (["run", "--runs", "0"], "--runs"), (["run", "--languages", "ruby"], "unknown language 'ruby'"), (["run", "--bogus"], "unknown option --bogus"), + (["run", "--cpus", "3-1"], "rising range"), (["walk"], "unknown command 'walk'"), (["probe", "--sample-ms", "lots"], "unexpected or invalid argument"), (["probe", "--sample-ms", "150"], "unexpected or invalid argument"), diff --git a/tests/test_options.c b/tests/test_options.c index 2857194..1f33c41 100644 --- a/tests/test_options.c +++ b/tests/test_options.c @@ -22,7 +22,7 @@ static void test_defaults_are_the_study(void) { CHECK(o.warmup >= 1, "warm-up runs"); CHECK(o.seed == INPUT_DEFAULT_SEED, "default seed"); CHECK(o.energy == ENERGY_AUTO && o.baseline == BASELINE_AUTO, "energy and baseline automatic"); - CHECK(o.cpu == -1, "not pinned"); + CHECK(o.cpu_count == 0, "not pinned"); CHECK(o.startup_trials == 1, "startup trials on"); CHECK(o.retries == 1, "one retry for a failed worker process"); } @@ -55,7 +55,7 @@ static void test_lists(void) { static void test_values(void) { t_options o; char *const all[] = {"--runs", "31", "--warmup", "2", "--seed", "99", - "--order-seed", "5", "--max-reps", "10", "--cpu", "3", + "--order-seed", "5", "--max-reps", "10", "--cpus", "3,5-6", "--energy", "require", "--idle-baseline", "off", "--out", "x/y", "--python", "python3.12", "--node", "/usr/bin/node", "--label", "laptop test", "--no-startup", "--dry-run", "--quiet", "--powercap-root", "/tmp/pc", "--retries", "0"}; @@ -63,7 +63,8 @@ static void test_values(void) { CHECK(parse(&o, argc, all) == 0, "every option parses"); CHECK(o.runs == 31 && o.warmup == 2 && o.seed == 99 && o.order_seed == 5 && o.max_reps == 10, "numbers"); - CHECK(o.cpu == 3 && o.energy == ENERGY_REQUIRE && o.baseline == BASELINE_OFF, "choices"); + CHECK(o.cpu_count == 3 && o.cpus[0] == 3 && o.cpus[1] == 5 && o.cpus[2] == 6, "CPU list with a range"); + CHECK(o.energy == ENERGY_REQUIRE && o.baseline == BASELINE_OFF, "choices"); CHECK(strcmp(o.out_dir, "x/y") == 0 && strcmp(o.python, "python3.12") == 0 && strcmp(o.node, "/usr/bin/node") == 0 && strcmp(o.label, "laptop test") == 0 && strcmp(o.powercap_root, "/tmp/pc") == 0, @@ -99,6 +100,11 @@ static void test_errors(void) { char *const r[] = {"--orders", "sideways"}; char *const s[] = {"--algorithms", "none"}; char *const t[] = {"--retries", "11"}; + char *const u[] = {"--cpus", "2,2"}; + char *const v[] = {"--cpus", "5-3"}; + char *const w[] = {"--cpus", "1024"}; + char *const x[] = {"--cpus", "0-64"}; + char *const y[] = {"--cpus", "two"}; expect_error(2, a, "--runs", "zero runs"); expect_error(2, b, "not a whole number", "runs not a number"); @@ -119,6 +125,11 @@ static void test_errors(void) { expect_error(2, r, "unknown order", "unknown order"); expect_error(2, s, "unknown algorithm 'none'", "none is internal"); expect_error(2, t, "from 0 to 10", "too many retries"); + expect_error(2, u, "CPU 2 is listed twice", "duplicate CPU"); + expect_error(2, v, "rising range", "falling CPU range"); + expect_error(2, w, "from 0 to 1023", "CPU beyond a cpu_set_t"); + expect_error(2, x, "at most 64 CPUs", "too many CPUs"); + expect_error(2, y, "'two' is not a CPU", "CPU not a number"); } static void test_parse_u64(void) { From 1868fab947ffdcc93ab261f76f8ec135c9326f3a Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:08:51 +0400 Subject: [PATCH 23/51] fix: record a command that pastes back into a shell, and keep inputs valid - meta.json records the harness command as invoked (build/greenbench run ...) rather than a bare "greenbench", and quotes an embedded single quote as '\'' so the line can be pasted back from the same directory. - JSON strings from the harness replace bytes that are not well-formed UTF-8 with U+FFFD, so a --label or path in another encoding can no longer produce an invalid meta.json. - The analysis rejects nan and inf energy cells with the line and column, and reports a meta.json that is not UTF-8 as a DataError instead of a traceback. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/load.py | 8 ++++- analysis/tests/test_load.py | 5 +++ src/greenbench.c | 54 +++++++++++++++++++++------- src/output.c | 42 ++++++++++++++++++++++ src/output.h | 3 +- tests/integration/test_harness.py | 29 +++++++++++++-- tests/test_output.c | 28 +++++++++++++++ 7 files changed, 152 insertions(+), 17 deletions(-) diff --git a/analysis/greenbench_analysis/load.py b/analysis/greenbench_analysis/load.py index 8cac1c4..4aabde2 100644 --- a/analysis/greenbench_analysis/load.py +++ b/analysis/greenbench_analysis/load.py @@ -4,6 +4,7 @@ import csv import json +import math from dataclasses import dataclass from pathlib import Path from typing import Any @@ -102,9 +103,12 @@ def _optional_number(row: dict[str, str], column: str, where: str) -> float | No if text == "": return None try: - return float(text) + value = float(text) except ValueError: raise DataError(f"{where}: column {column}: expected a number or an empty cell, got {text!r}") from None + if not math.isfinite(value): + raise DataError(f"{where}: column {column}: expected a finite number, got {text!r}") + return value def _choice(row: dict[str, str], column: str, choices: tuple[str, ...], where: str) -> str: @@ -182,6 +186,8 @@ def load_meta(path: Path) -> dict[str, Any]: meta = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: raise DataError(f"{path}: file not found; it is written next to runs.csv") from None + except UnicodeDecodeError as error: + raise DataError(f"{path}: not UTF-8 text ({error.reason} at byte {error.start})") from None except json.JSONDecodeError as error: raise DataError(f"{path}: not valid JSON ({error})") from None if not isinstance(meta, dict) or meta.get("tool") != "greenbench": diff --git a/analysis/tests/test_load.py b/analysis/tests/test_load.py index 17a594d..93401de 100644 --- a/analysis/tests/test_load.py +++ b/analysis/tests/test_load.py @@ -50,6 +50,8 @@ def test_energy_prefers_the_net_values() -> None: (GOOD_ROW.replace(",10,1,1000000", ",10,0,1000000"), "must start at least one process"), (GOOD_ROW.replace(",10,1,1000000", ",10,3,1000000"), "a sorting run is one process"), (GOOD_ROW + "x", "column net_dram_uj"), + (GOOD_ROW.replace(",unavailable,,,,,,,", ",ok,nan,,,,,,"), "column pkg_uj: expected a finite number"), + (GOOD_ROW.replace(",unavailable,,,,,,,", ",ok,5,inf,,,,,"), "column dram_uj: expected a finite number"), (GOOD_ROW[:-2], "expected 22 columns"), (GOOD_ROW + ",", "expected 22 columns"), ], @@ -78,6 +80,9 @@ def test_meta(tmp_path: Path) -> None: (tmp_path / "bad.json").write_text("{not json", encoding="utf-8") with pytest.raises(DataError, match="not valid JSON"): load_meta(tmp_path / "bad.json") + (tmp_path / "latin1.json").write_bytes(b'{"tool": "greenbench", "label": "caf\xe9"}') + with pytest.raises(DataError, match="not UTF-8 text"): + load_meta(tmp_path / "latin1.json") (tmp_path / "other.json").write_text('{"tool": "something else"}', encoding="utf-8") with pytest.raises(DataError, match=r"not a greenbench meta\.json"): load_meta(tmp_path / "other.json") diff --git a/src/greenbench.c b/src/greenbench.c index fb4af17..66c1614 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -884,18 +884,48 @@ static void setup_energy(t_context *ctx) { ctx->baseline = o->baseline == BASELINE_ON || (o->baseline == BASELINE_AUTO && ctx->energy_ok); } -/* "greenbench run ARGS", quoting any argument that the shell would split or expand. */ -static void record_command(t_context *ctx, int argc, char **argv) { - size_t used = (size_t)snprintf(ctx->command, sizeof ctx->command, "greenbench run"); - - for (int i = 0; i < argc && used < sizeof ctx->command; i++) { - int plain = argv[i][0] != '\0' && strspn(argv[i], "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789-_=,./:+") == strlen(argv[i]); - int written = snprintf(ctx->command + used, sizeof ctx->command - used, plain ? " %s" : " '%s'", argv[i]); - if (written < 0) - break; - used += (size_t)written; +/* Appends length bytes of text to out when they fit. Returns 0, or -1 when out is full. */ +static int append_text(char *out, size_t size, size_t *used, const char *text, size_t length) { + if (*used + length >= size) + return -1; + memcpy(out + *used, text, length); + *used += length; + out[*used] = '\0'; + return 0; +} + +/* Appends word to out, after a space unless out is empty. A word the shell would split or + expand goes in single quotes, with each single quote inside written as '\''. */ +static void append_shell_word(char *out, size_t size, size_t *used, const char *word) { + int plain = word[0] != '\0' && strspn(word, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789-_=,./:+") == strlen(word); + + if (*used > 0 && append_text(out, size, used, " ", 1) != 0) + return; + if (plain) { + append_text(out, size, used, word, strlen(word)); + return; + } + if (append_text(out, size, used, "'", 1) != 0) + return; + for (const char *p = word; *p != '\0'; p++) { + int rc = *p == '\'' ? append_text(out, size, used, "'\\''", 4) : append_text(out, size, used, p, 1); + if (rc != 0) + return; } + append_text(out, size, used, "'", 1); +} + +/* The command line as invoked, for example "build/greenbench run --profile study", quoted so + that it can be pasted back into a shell in the same directory. */ +static void record_command(t_context *ctx, const char *argv0, int argc, char **argv) { + size_t used = 0; + + ctx->command[0] = '\0'; + append_shell_word(ctx->command, sizeof ctx->command, &used, argv0); + append_shell_word(ctx->command, sizeof ctx->command, &used, "run"); + for (int i = 0; i < argc; i++) + append_shell_word(ctx->command, sizeof ctx->command, &used, argv[i]); } static void cleanup(t_context *ctx) { @@ -913,7 +943,7 @@ static int command_run(int argc, char **argv, const char *argv0) { t_options *o = &ctx.options; options_defaults(o); - record_command(&ctx, argc, argv); + record_command(&ctx, argv0, argc, argv); if (options_parse(argc, argv, o, err, sizeof err) != 0) { error_message("%s", err); fprintf(stderr, "Try 'greenbench --help'.\n"); diff --git a/src/output.c b/src/output.c index b73f636..bd3e229 100644 --- a/src/output.c +++ b/src/output.c @@ -52,9 +52,51 @@ void output_csv_row(FILE *file, const t_run_record *r) { fputc('\n', file); } +/* Length of the well-formed UTF-8 sequence starting at p (RFC 3629: no overlong forms, no + surrogates, nothing above U+10FFFF), or 0 if the bytes there are not one. */ +static size_t utf8_sequence(const unsigned char *p) { + unsigned char lo = 0x80, hi = 0xBF; + size_t length; + + if (p[0] < 0x80) + return 1; + if (p[0] >= 0xC2 && p[0] <= 0xDF) + length = 2; + else if (p[0] >= 0xE0 && p[0] <= 0xEF) { + length = 3; + if (p[0] == 0xE0) + lo = 0xA0; + else if (p[0] == 0xED) + hi = 0x9F; + } else if (p[0] >= 0xF0 && p[0] <= 0xF4) { + length = 4; + if (p[0] == 0xF0) + lo = 0x90; + else if (p[0] == 0xF4) + hi = 0x8F; + } else + return 0; + if (p[1] < lo || p[1] > hi) + return 0; + for (size_t i = 2; i < length; i++) + if (p[i] < 0x80 || p[i] > 0xBF) + return 0; + return length; +} + void output_json_string(FILE *file, const char *s) { fputc('"', file); for (const unsigned char *p = (const unsigned char *)s; *p != '\0'; p++) { + if (*p >= 0x80) { + size_t length = utf8_sequence(p); + if (length == 0) { + fputs("\\ufffd", file); /* not UTF-8: the replacement character keeps the JSON valid */ + } else { + fwrite(p, 1, length, file); + p += length - 1; + } + continue; + } switch (*p) { case '"': fputs("\\\"", file); diff --git a/src/output.h b/src/output.h index b53de5d..9b45b33 100644 --- a/src/output.h +++ b/src/output.h @@ -35,7 +35,8 @@ typedef struct s_run_record { void output_csv_header(FILE *file); void output_csv_row(FILE *file, const t_run_record *record); -/* Writes s as a JSON string literal, quotes included. */ +/* Writes s as a JSON string literal, quotes included. Bytes that are not well-formed UTF-8 + become U+FFFD, so the output is always valid JSON. */ void output_json_string(FILE *file, const char *s); #endif diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 06e9068..cb37451 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -4,6 +4,8 @@ import csv import json +import os +import shlex from pathlib import Path from typing import Any @@ -11,7 +13,7 @@ from powercap import CounterWriter from powercap import build as build_powercap -from workers import REPOSITORY, interpreter_options, require_build, run_harness +from workers import HARNESS, REPOSITORY, interpreter_options, require_build, run_harness FIXTURES = REPOSITORY / "tests" / "fixtures" / "powercap" SMALL = ["--profile", "smoke", "--sizes", "20,50", "--orders", "random,sorted", "--runs", "3", "--quiet"] @@ -69,7 +71,7 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert meta["clock"] == "CLOCK_MONOTONIC_RAW" assert meta["settings"]["runs"] == 3 assert meta["settings"]["cpus"] == [] - assert meta["command"].startswith("greenbench run --profile smoke") + assert meta["command"].startswith(f"{HARNESS} run --profile smoke") assert len(meta["calibration"]) == 48 assert set(meta["workers"]) == {"c", "python", "node"} @@ -112,7 +114,16 @@ def test_startup_runs_launch_a_batch_of_processes(tmp_path: Path) -> None: def test_pinning_to_a_cpu_set(tmp_path: Path) -> None: out = tmp_path / "run" completed = run_harness( - "run", *SMALL, "--languages", "c,node", "--cpus", "0-1", "--energy", "off", "--out", str(out), + "run", + *SMALL, + "--languages", + "c,node", + "--cpus", + "0-1", + "--energy", + "off", + "--out", + str(out), *interpreter_options(), ) assert completed.returncode == 0, completed.stderr @@ -120,6 +131,18 @@ def test_pinning_to_a_cpu_set(tmp_path: Path) -> None: assert read_meta(tmp_path / "run")["status"] == "complete" +def test_meta_json_stays_valid_and_the_command_can_be_pasted(tmp_path: Path) -> None: + out = tmp_path / "run" + label = os.fsdecode(b"it's caf\xe9") # a quote, and a byte that is not UTF-8 + args = ["--profile", "smoke", "--languages", "c", "--sizes", "20", "--runs", "1", "--label", label] + completed = run_harness("run", *args, "--energy", "off", "--quiet", "--out", str(out)) + assert completed.returncode == 0, completed.stderr + meta = read_meta(out) # json.loads would fail on a raw 0xE9 byte + assert meta["label"] == "it's caf\ufffd" + expected = [str(HARNESS), "run", *args[:-1], "it's caf\ufffd", "--energy", "off", "--quiet", "--out", str(out)] + assert shlex.split(meta["command"]) == expected + + def test_counts_match_across_languages_in_the_csv(tmp_path: Path) -> None: out = tmp_path / "run" completed = run_harness("run", *SMALL, "--energy", "off", "--out", str(out), *interpreter_options()) diff --git a/tests/test_output.c b/tests/test_output.c index 823a08d..f190349 100644 --- a/tests/test_output.c +++ b/tests/test_output.c @@ -102,11 +102,39 @@ static void test_json_string(void) { CHECK(strcmp(buf, "\"Intel(R) \\\"Core\\\" \\\\ i7\\n\\t\\u0001\"") == 0, "JSON escaping"); } +static void json_of(const char *s, char *buf, size_t size) { + FILE *f = tmpfile(); + + output_json_string(f, s); + rewind(f); + size_t n = fread(buf, 1, size - 1, f); + buf[n] = '\0'; + fclose(f); +} + +static void test_json_utf8(void) { + char buf[256]; + + json_of("caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x94\x8b", buf, sizeof buf); + CHECK(strcmp(buf, "\"caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x94\x8b\"") == 0, "well-formed UTF-8 passes through"); + json_of("caf\xe9", buf, sizeof buf); + CHECK(strcmp(buf, "\"caf\\ufffd\"") == 0, "a Latin-1 byte becomes U+FFFD"); + json_of("\xc3", buf, sizeof buf); + CHECK(strcmp(buf, "\"\\ufffd\"") == 0, "a sequence cut short at the end"); + json_of("\xc0\xaf", buf, sizeof buf); + CHECK(strcmp(buf, "\"\\ufffd\\ufffd\"") == 0, "an overlong encoding is not UTF-8"); + json_of("\xed\xa0\x80", buf, sizeof buf); + CHECK(strcmp(buf, "\"\\ufffd\\ufffd\\ufffd\"") == 0, "an encoded surrogate is not UTF-8"); + json_of("\xf4\x90\x80\x80", buf, sizeof buf); + CHECK(strcmp(buf, "\"\\ufffd\\ufffd\\ufffd\\ufffd\"") == 0, "nothing above U+10FFFF"); +} + int main(void) { test_header(); test_row_without_energy(); test_row_with_energy_and_baseline(); test_row_energy_without_dram_or_baseline(); test_json_string(); + test_json_utf8(); return check_report("output"); } From 004f3500b748aa76488b7311be880dbfbe528f71 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:13:19 +0400 Subject: [PATCH 24/51] feat: record CPU and memory limits and short sort loops in meta.json meta.json said "16 CPUs online" for a run in a container limited to two, and nothing recorded whether calibrated sort loops reached their target. - host.allowed_cpus (the affinity mask before pinning) and the cgroup v2 cpu.max and memory.max of the harness's own cgroup, when present. - short_loops and capped_loops: measured sorting runs whose loop fell short of --min-loop-ms, and how many of those --max-reps held back. The harness prints a note when there are any. - The help text describes --min-loop-ms as a calibrated target rather than a guaranteed minimum, which is what it is. Co-Authored-By: Claude Opus 5.5 --- src/greenbench.c | 60 ++++++++++++++++++++++++++++++- src/options.h | 2 +- src/plan.h | 4 +-- tests/integration/test_harness.py | 4 +++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/greenbench.c b/src/greenbench.c index 66c1614..0749278 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -79,6 +79,10 @@ typedef struct s_context { t_consistency counts[ALGO_COUNT][ORDER_COUNT][OPTIONS_MAX_SIZES]; t_spawn_result spawn; unsigned retries_used; + uint64_t sorting_runs; /* measured runs that sorted */ + uint64_t short_loops; /* of those, runs whose sort loop fell short of --min-loop-ms */ + uint64_t capped_loops; /* of those, runs held back by --max-reps */ + int allowed_cpus; /* CPUs this process could use before any pinning, or -1 */ char started_utc[32]; char meta_path[PATH_MAX]; char command[2048]; /* the harness's own command line, for meta.json */ @@ -133,7 +137,7 @@ static void print_usage(FILE *out) { " --sizes LIST input sizes, each 1 to %d\n" " --runs N measured runs per trial (the study uses at least %d)\n" " --warmup N unmeasured runs per trial before measuring; the first calibrates\n" - " --min-loop-ms N sort for at least N ms per process, well above the ~1 ms RAPL update\n" + " --min-loop-ms N calibrate each sort loop and startup batch to about N ms (RAPL: ~1 ms updates)\n" " --max-reps N never sort more than N times in one process\n" " --no-startup skip the startup trials (processes that exit without sorting)\n" " --retries N rerun a worker process that fails up to N times (default 1)\n" @@ -330,6 +334,38 @@ static void read_first_line(const char *path, char *out, size_t size) { fclose(f); } +/* The first line of a file in this process's cgroup v2 directory, such as cpu.max, or "" + when there is none. In a container the directory is the container's own. */ +static void cgroup_value(const char *name, char *out, size_t size) { + char line[512]; + char path[PATH_MAX]; + + out[0] = '\0'; + FILE *f = fopen("/proc/self/cgroup", "r"); + if (f == NULL) + return; + while (fgets(line, sizeof line, f) != NULL) { + if (strncmp(line, "0::", 3) != 0) + continue; + line[strcspn(line, "\r\n")] = '\0'; + const char *dir = strcmp(line + 3, "/") == 0 ? "" : line + 3; + if (snprintf(path, sizeof path, "/sys/fs/cgroup%s/%s", dir, name) < (int)sizeof path) + read_first_line(path, out, size); + break; + } + fclose(f); +} + +static int count_allowed_cpus(void) { +#ifdef __linux__ + cpu_set_t set; + + if (sched_getaffinity(0, sizeof set, &set) == 0) + return CPU_COUNT(&set); +#endif + return -1; +} + static void cpu_model(char *out, size_t size) { char line[512]; FILE *f = fopen("/proc/cpuinfo", "r"); @@ -395,6 +431,8 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure json_key_string(f, " ", "label", o->label, 1); json_key_string(f, " ", "command", ctx->command, 1); fprintf(f, " \"measured_runs\": %" PRIu64 ",\n \"worker_retries\": %u,\n", measured_runs, ctx->retries_used); + fprintf(f, " \"short_loops\": %" PRIu64 ",\n \"capped_loops\": %" PRIu64 ",\n", ctx->short_loops, + ctx->capped_loops); fputs(" \"clock\": \"CLOCK_MONOTONIC_RAW\",\n", f); fputs(" \"host\": {\n", f); @@ -406,6 +444,11 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure cpu_model(text, sizeof text); json_key_string(f, " ", "cpu_model", text, 1); fprintf(f, " \"online_cpus\": %ld,\n", sysconf(_SC_NPROCESSORS_ONLN)); + fprintf(f, " \"allowed_cpus\": %d,\n", ctx->allowed_cpus); + cgroup_value("cpu.max", text, sizeof text); + json_key_string(f, " ", "cgroup_cpu_max", text, 1); + cgroup_value("memory.max", text, sizeof text); + json_key_string(f, " ", "cgroup_memory_max", text, 1); char governor_path[128]; snprintf(governor_path, sizeof governor_path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", o->cpu_count > 0 ? o->cpus[0] : 0); @@ -792,6 +835,13 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { return error_message("cannot write %s: %s", ctx->csv_path, strerror(errno)), -1; } *measured = i + 1; + if (t->algorithm != ALGO_NONE) { + ctx->sorting_runs++; + if (m.result.loop_ns < o->min_loop_ns) { + ctx->short_loops++; + ctx->capped_loops += t->inner_reps >= o->max_reps; + } + } if (!o->quiet && progress_due(&progress, i + 1 == total)) { uint64_t elapsed = timing_now_ns() - started; @@ -995,6 +1045,7 @@ static int command_run(int argc, char **argv, const char *argv0) { else fprintf(stderr, "energy unavailable: %s\nMeasuring time and operation counts only.\n", ctx.energy_reason); } + ctx.allowed_cpus = count_allowed_cpus(); if (o->cpu_count > 0 && pin_cpus(o->cpus, o->cpu_count) != 0) { cleanup(&ctx); return EXIT_USAGE; @@ -1030,6 +1081,13 @@ static int command_run(int argc, char **argv, const char *argv0) { error_message("interrupted after %" PRIu64 " measured runs; they are in %s", measured, ctx.csv_path); if (write_meta(&ctx, status == 0 ? "complete" : g_interrupted ? "interrupted" : "failed", measured) != 0) status = 1; + if (ctx.short_loops > 0 && !o->quiet) + fprintf(stderr, + "%s: note: %" PRIu64 " of %" PRIu64 " measured sorting runs sorted for less than the %" PRIu64 + " ms target (%" PRIu64 " of them held back by --max-reps %" PRIu64 "). Calibration extrapolates " + "from the warm-up runs; more of them, or a quieter machine, bring the loops closer to the target\n", + g_program, ctx.short_loops, ctx.sorting_runs, o->min_loop_ns / 1000000u, ctx.capped_loops, + o->max_reps); if (status == 0 && !o->quiet) fprintf(stderr, "wrote %s and %s\n", ctx.csv_path, ctx.meta_path); cleanup(&ctx); diff --git a/src/options.h b/src/options.h index 0face33..232a62d 100644 --- a/src/options.h +++ b/src/options.h @@ -29,7 +29,7 @@ typedef struct s_options { unsigned warmup; /* unmeasured runs per trial; the first also calibrates */ uint32_t seed; /* input generator seed, the same for every language */ uint32_t order_seed; /* seed for the randomised run order */ - uint64_t min_loop_ns; /* each measured sort loop should last at least this long */ + uint64_t min_loop_ns; /* target length of each sort loop and startup batch */ uint64_t max_reps; /* upper bound on sorts per process */ int startup_trials; /* also measure processes that exit without sorting */ unsigned retries; /* times to rerun a worker process that fails, before giving up */ diff --git a/src/plan.h b/src/plan.h index 9b604ae..ed73354 100644 --- a/src/plan.h +++ b/src/plan.h @@ -42,8 +42,8 @@ void plan_runs(size_t trial_count, unsigned runs, t_run_item *items); so the same seed always gives the same order. Elements must be at most 64 bytes. */ void plan_shuffle(void *base, size_t count, size_t size, uint32_t seed); -/* Sorts per process so that one process's sort loop lasts at least min_loop_ns, given that - reps sorts took loop_ns. Always at least 1 and at most max_reps. */ +/* Sorts per process that make one sort loop last at least min_loop_ns if every sort takes as + long as the reps sorts that took loop_ns did. Always at least 1 and at most max_reps. */ uint64_t plan_calibrate(uint64_t loop_ns, uint64_t reps, uint64_t min_loop_ns, uint64_t max_reps); #endif diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index cb37451..0a85ced 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -71,6 +71,10 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert meta["clock"] == "CLOCK_MONOTONIC_RAW" assert meta["settings"]["runs"] == 3 assert meta["settings"]["cpus"] == [] + assert meta["host"]["allowed_cpus"] >= 1 + assert "cgroup_cpu_max" in meta["host"] + assert "cgroup_memory_max" in meta["host"] + assert 0 <= meta["capped_loops"] <= meta["short_loops"] assert meta["command"].startswith(f"{HARNESS} run --profile smoke") assert len(meta["calibration"]) == 48 assert set(meta["workers"]) == {"c", "python", "node"} From 4ca24d3a6e0930a8de08cbae4d75bac38ec96a82 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:14:17 +0400 Subject: [PATCH 25/51] fix: give the Node.js sorts a packed array, as C and Python get makeInput built the array with new Array(n) and filled it, which V8 stores as HOLEY_SMI_ELEMENTS, and data.slice() kept that kind, so every element access in the sorts paid a hole check: a small, systematic cost against Node.js in a comparison of languages. Array.from gives a packed array (checked with %HasHoleyElements on Node.js 24.21 for every order, before and after slice). Parity tests still pass. Co-Authored-By: Claude Opus 5.5 --- workers/node/lib/inputs.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workers/node/lib/inputs.js b/workers/node/lib/inputs.js index f8db4a0..7bc2540 100644 --- a/workers/node/lib/inputs.js +++ b/workers/node/lib/inputs.js @@ -53,10 +53,10 @@ export function isOrder(name) { * @returns {number[]} */ export function makeInput(n, order, seed) { - const arr = new Array(n); - for (let i = 0; i < n; i++) { - arr[i] = order === "reversed" ? n - 1 - i : i; - } + // Array.from gives V8 a packed array of small integers, the everyday kind that C and Python + // are compared with. new Array(n) filled afterwards is "holey" (as are its copies made with + // slice), and every element access in the sorts would then pay a check for holes. + const arr = Array.from({ length: n }, (_, i) => (order === "reversed" ? n - 1 - i : i)); let state = seed; if (order === "random") { for (let i = n; i > 1; i--) { From b151706b752fef2d477273af48fc88e2a74eea9d Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:33:02 +0400 Subject: [PATCH 26/51] feat: answer the proxy question with power and ratios, not pooled rho study.md decides the question by whether energy ratios match time ratios and whether any language or algorithm draws a different power, but the report computed neither. Its main test, a pooled Spearman rho between process time and process energy, was described as "dominated by input size" although calibration holds process time nearly constant. The analysis now computes, from one set of bootstrap draws per configuration (each run's time and energy kept together): - power while sorting (energy per sort / time per sort) with a 95% CI; - time, energy and power ratios with CIs between languages (Python/C, Node.js/C, Python/Node.js) and between each input order and random, with a plain reading: same power, or who draws how much more; - rank agreement across configurations (median time per sort against median energy per sort) at each size, per language and overall; - the within-configuration rho, kept and described as a noise check. The report also gains a power column, an energy-against-time chart with constant-power lines, net and including-idle carbon side by side, a warning for unfinished runs or fewer than 30 runs per configuration, the share of sort loops that fell short of the target, and the cgroup limits. It refuses a run whose meta.json status is not complete unless --allow-partial is given, and refuses energy that is all zeros. Tests use synthetic data in which one language draws 50% more power. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/charts.py | 85 +++-- analysis/greenbench_analysis/cli.py | 14 +- analysis/greenbench_analysis/report.py | 470 +++++++++++++++++++----- analysis/greenbench_analysis/stats.py | 81 ++-- analysis/greenbench_analysis/summary.py | 294 ++++++++++++--- analysis/tests/synthetic.py | 33 +- analysis/tests/test_charts.py | 14 +- analysis/tests/test_report.py | 64 +++- analysis/tests/test_stats.py | 38 +- analysis/tests/test_summary.py | 142 ++++++- 10 files changed, 959 insertions(+), 276 deletions(-) diff --git a/analysis/greenbench_analysis/charts.py b/analysis/greenbench_analysis/charts.py index 8d9226c..6d5e717 100644 --- a/analysis/greenbench_analysis/charts.py +++ b/analysis/greenbench_analysis/charts.py @@ -191,50 +191,69 @@ def energy_per_sort_chart(configurations: list[Configuration], n: int, order: st ) -def time_energy_chart( - points_by_language: dict[str, list[tuple[str, float, float]]], baseline_subtracted: bool, path: Path -) -> Path | None: - """Process time against process energy for every measured run, one panel per language. +def energy_time_chart(configurations: list[Configuration], baseline_subtracted: bool, path: Path) -> Path | None: + """Median energy per sort against median time per sort, one point per configuration. - points_by_language maps a language to (algorithm, wall_ns, energy_uj) for each run. + Points on one straight line through the origin would mean every configuration draws the + same power, so that time predicts energy exactly. The dashed line is the median power of + all the configurations drawn; the dotted lines are half and twice that power. """ - # Log axes cannot show zero or negative energy (a run that used no more than idle power). - positive = { - lang: [(a, t, e) for a, t, e in points if t > 0 and e > 0] for lang, points in points_by_language.items() - } - dropped = sum(len(points_by_language[lang]) - len(positive[lang]) for lang in positive) - languages = [lang for lang in LANGUAGE_ORDER if positive.get(lang)] - if not languages: + points = [ + (c, c.time_per_sort_ns.value, c.energy_per_sort_uj.value) + for c in configurations + if c.energy_per_sort_uj is not None and c.time_per_sort_ns.value > 0 and c.energy_per_sort_uj.value > 0 + ] + if len(points) < 2: return None - fig, axes = _figure(3.0 * len(languages) + 0.6, 3.2, len(languages)) - for ax, language in zip(axes, languages, strict=True): - points = positive[language] - for colour, algorithm in zip(SERIES, ALGORITHM_ORDER, strict=True): - kept = [(t / 1e6, e / 1e6) for a, t, e in points if a == algorithm] - if kept: + dropped = sum(1 for c in configurations if c.energy_per_sort_uj is not None) - len(points) + fig, (ax,) = _figure(6.4, 4.4) + xs = [t / 1e6 for _, t, _ in points] # ms + ys = [e / 1e3 for _, _, e in points] # mJ, so that mJ per ms is watts + watts = sorted(y / x for x, y in zip(xs, ys, strict=True)) + typical = watts[len(watts) // 2] + span = (min(xs) / 1.5, max(xs) * 1.5) + for factor, style in ((0.5, ":"), (1.0, "--"), (2.0, ":")): + ax.plot(span, [typical * factor * x for x in span], color=MUTED, linestyle=style, linewidth=1, zorder=1) + for colour, language in zip(SERIES, LANGUAGE_ORDER, strict=True): + for style, algorithm in zip(("o", "s", "^"), ALGORITHM_ORDER, strict=True): + chosen = [ + (x, y) + for (c, _, _), x, y in zip(points, xs, ys, strict=True) + if c.language == language and c.algorithm == algorithm + ] + if chosen: ax.scatter( - [k[0] for k in kept], - [k[1] for k in kept], - s=16, + [p[0] for p in chosen], + [p[1] for p in chosen], + s=36, + marker=style, color=colour, edgecolors=SURFACE, - linewidths=0.8, - label=algorithm, + linewidths=1, + zorder=3, + label=f"{LANGUAGE_NAMES[language]} {algorithm}", ) - ax.set_xscale("log") - ax.set_yscale("log") - ax.set_title(LANGUAGE_NAMES[language], fontsize=10, loc="left") - ax.set_xlabel("Process time (ms)", fontsize=8) - ylabel = "Process energy, net of idle (J)" if baseline_subtracted else "Process energy (J)" - axes[0].set_ylabel(ylabel, fontsize=8) - handles, labels = axes[0].get_legend_handles_labels() - if handles: - fig.legend(handles, labels, loc="upper center", ncol=3, frameon=False, fontsize=8, bbox_to_anchor=(0.5, 1.08)) + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlim(span) + ax.set_xlabel("Time per sort (ms, log scale)", fontsize=8) + energy = ( + "Energy per sort, net of idle (mJ, log scale)" if baseline_subtracted else "Energy per sort (mJ, log scale)" + ) + ax.set_ylabel(energy, fontsize=8) + ax.set_title( + f"One point per configuration; dashed line: constant power of {typical:.3g} W (the median)," + " dotted: half and double", + fontsize=8, + loc="left", + color=INK_SECONDARY, + ) + ax.legend(frameon=False, fontsize=7, loc="upper left", ncol=3) if dropped: fig.text( 0.01, -0.02, - f"Not shown: {dropped:,} runs whose net energy was zero or below.", + f"Not shown: {dropped} configurations whose median time or energy per sort was zero or below.", fontsize=7, color=INK_SECONDARY, va="top", diff --git a/analysis/greenbench_analysis/cli.py b/analysis/greenbench_analysis/cli.py index b38cb1c..e4c0e4f 100644 --- a/analysis/greenbench_analysis/cli.py +++ b/analysis/greenbench_analysis/cli.py @@ -56,6 +56,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="mark time and energy results as pending (for runs on a busy machine)", ) + report.add_argument( + "--allow-partial", + action="store_true", + help="report on a run that did not finish (interrupted or failed), with a warning at the top", + ) report.add_argument("--resamples", type=_positive, default=10_000, help="bootstrap resamples (default %(default)s)") report.add_argument("--seed", type=_positive, default=1, help="bootstrap seed (default %(default)s)") report.add_argument("--summary-csv", type=Path, help="also write every configuration's estimates here") @@ -64,7 +69,14 @@ def build_parser() -> argparse.ArgumentParser: def _report(args: argparse.Namespace, argv: list[str]) -> int: runs = load_runs(args.runs) - meta = load_meta(args.meta or args.runs.with_name("meta.json")) + meta_path = args.meta or args.runs.with_name("meta.json") + meta = load_meta(meta_path) + status = meta.get("status") + if status != "complete" and not args.allow_partial: + raise DataError( + f"{meta_path}: the run's status is {status!r}, not 'complete', so a report would describe only the " + "runs that finished. Pass --allow-partial to write it anyway, with a warning at the top" + ) grid = load_grid(args.grid) codes = [code.strip() for code in args.regions.split(",") if code.strip()] regions = select_regions(grid, codes) + [parse_custom(spec) for spec in args.intensity] diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 7a96024..340ed2b 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -9,7 +9,6 @@ import csv import os -from collections import defaultdict from collections.abc import Callable, Iterable from dataclasses import dataclass from pathlib import Path @@ -19,9 +18,10 @@ from .carbon import GridIntensity, grams_co2e from .load import ORDERS, Run from .stats import Estimate -from .summary import Configuration, Summary +from .summary import Comparison, Configuration, Summary LANGUAGE_NAMES = {"c": "C", "python": "Python", "node": "Node.js"} +STUDY_RUNS = 30 # measured runs per configuration in the study profile def _pending(options: ReportOptions) -> str: @@ -60,6 +60,7 @@ def _digits(value: float) -> str: TIME_UNITS = [(1.0, "ns"), (1e3, "µs"), (1e6, "ms"), (1e9, "s")] ENERGY_UNITS = [(1.0, "µJ"), (1e3, "mJ"), (1e6, "J"), (1e9, "kJ")] +POWER_UNITS = [(1e-3, "mW"), (1.0, "W")] def format_time(ns: float) -> str: @@ -92,6 +93,19 @@ def format_estimate(estimate: Estimate, units: list[tuple[float, str]]) -> str: ) +def format_ratio(estimate: Estimate | None) -> str: + """A ratio and its interval, to three significant figures: "10.2 (9.87 to 10.5)".""" + if estimate is None: + return "undefined" + return f"{estimate.value:.3g} ({estimate.low:.3g} to {estimate.high:.3g})" + + +def _names(languages: Iterable[str]) -> str: + """ "C", "C and Python", "C, Python and Node.js".""" + names = [LANGUAGE_NAMES[lang] for lang in languages] + return names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1] + + def _table(header: list[str], align: list[str], rows: Iterable[list[str]]) -> list[str]: lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(align) + " |"] lines.extend("| " + " | ".join(row) + " |" for row in rows) @@ -105,7 +119,42 @@ def _pinning(settings: dict[str, Any]) -> str: return ("to CPU " if len(cpus) == 1 else "to CPUs ") + ", ".join(str(c) for c in cpus) -def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: +def _limits(host: dict[str, Any]) -> str: + """The CPUs and memory the harness could actually use, when less than the machine has.""" + parts = [] + online, allowed = host.get("online_cpus"), host.get("allowed_cpus") + if isinstance(online, int) and isinstance(allowed, int) and 0 < allowed < online: + parts.append(f"{allowed} of them allowed to the harness") + quota = str(host.get("cgroup_cpu_max") or "").split() + if len(quota) == 2 and quota[0].isdigit() and quota[1].isdigit() and int(quota[1]) > 0: + parts.append(f"a cgroup CPU limit of {int(quota[0]) / int(quota[1]):g} CPUs") + memory = str(host.get("cgroup_memory_max") or "") + if memory.isdigit(): + parts.append(f"a cgroup memory limit of {int(memory) / 2**30:.1f} GiB") + return "".join(f", {part}" for part in parts) + + +def _loops_line(runs: list[Run], settings: dict[str, Any]) -> str | None: + target = settings.get("min_loop_ns") + if not isinstance(target, int) or target <= 0: + return None + sorting = [r for r in runs if not r.is_startup] + if not sorting: + return None + short = [r for r in sorting if r.loop_ns < target] + if not short: + return f"- Sort loops: every sorting run reached the {format_time(target)} target." + max_reps = settings.get("max_reps") + capped = sum(1 for r in short if isinstance(max_reps, int) and r.inner_reps >= max_reps) + return ( + f"- Sort loops: {len(short):,} of {len(sorting):,} sorting runs sorted for less than the " + f"{format_time(target)} target (shortest {format_time(min(r.loop_ns for r in short))}" + + (f"; {capped:,} of them held back by the limit of {max_reps:,} sorts per process" if capped else "") + + "). Calibration extrapolates from the warm-up runs." + ) + + +def _environment(meta: dict[str, Any], runs: list[Run], summary: Summary, options: ReportOptions) -> list[str]: host = meta.get("host", {}) settings = meta.get("settings", {}) energy = meta.get("energy", {}) @@ -118,18 +167,19 @@ def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: if version.startswith("Python "): version = version[len("Python ") :] versions.append(f"{LANGUAGE_NAMES.get(language, language)}: {version}") + retries = meta.get("worker_retries", 0) + label = f"; label: {meta['label']}" if meta.get("label") else "" lines = [ f"- Harness: greenbench {meta.get('version', '?')}, profile `{settings.get('profile', '?')}`, " f"{settings.get('runs', '?')} measured runs per configuration after {warmup} " f"warm-up run{'' if warmup == 1 else 's'}, in a random order (order seed {settings.get('order_seed', '?')}).", f"- Inputs: sizes {', '.join(f'{s:,}' for s in settings.get('sizes', []))}; orders " f"{', '.join(o.replace('_', ' ') for o in settings.get('orders', []))}; xorshift32 seed " - f"{settings.get('seed', '?')}. Each process sorts for at least " + f"{settings.get('seed', '?')}. Sort loops and startup batches are calibrated to about " f"{settings.get('min_loop_ns', 0) / 1e6:g} ms.", f"- Machine: {cpu}; {host.get('sysname', '?')} {host.get('release', '?')} {host.get('machine', '')}; " - f"{host.get('online_cpus', '?')} CPUs online; scaling governor " - f"{host.get('scaling_governor') or 'not reported'}; " - f"CPU pinning {_pinning(settings)}.", + f"{host.get('online_cpus', '?')} CPUs online{_limits(host)}; scaling governor " + f"{host.get('scaling_governor') or 'not reported'}; CPU pinning {_pinning(settings)}.", "- Workers: " + "; ".join(versions) + ".", ] if energy.get("status") == "ok": @@ -140,30 +190,63 @@ def _environment(meta: dict[str, Any], summary: Summary) -> list[str]: reason = energy.get("reason") lines.append(f"- Energy: {energy.get('status', 'unknown')}." + (f" Reason: {reason}." if reason else "")) lines.append( - f"- Run: {summary.run_count:,} measured worker processes, started {meta.get('started_utc', '?')}, " - f"finished {meta.get('finished_utc') or 'not recorded'}" - + (f"; label: {meta['label']}" if meta.get("label") else "") - + "." + f"- Run: {summary.run_count:,} measured runs, status {meta.get('status', 'not recorded')}, " + f"{retries} worker process{'' if retries == 1 else 'es'} retried; started {meta.get('started_utc', '?')}, " + f"finished {meta.get('finished_utc') or 'not recorded'}{label}." ) + if not options.withhold_timings: + loops = _loops_line(runs, settings) + if loops: + lines.append(loops) if meta.get("command"): lines.append(f"- Harness command: `{meta['command']}`") return lines +def _banners(meta: dict[str, Any], summary: Summary, options: ReportOptions, reason: str) -> list[str]: + lines = [] + status = meta.get("status") + if status != "complete": + lines += [ + f"> **Warning: this run did not finish (status: {status}).** It stopped after " + f"{summary.run_count:,} measured runs, so some configurations have fewer runs than planned or " + "none. The figures describe only the runs that finished.", + "", + ] + runs = meta.get("settings", {}).get("runs") + if isinstance(runs, int) and runs < STUDY_RUNS: + lines += [ + f"> **Note:** {runs} measured run{'' if runs == 1 else 's'} per configuration; the study uses " + f"{STUDY_RUNS}. Confidence intervals from so few runs are not reliable.", + "", + ] + if options.withhold_timings: + lines += [ + "> **Status: time and energy results are pending.** This file was generated from a run that " + "checked the pipeline end to end. Operation counts are exact and shown below; time, energy, " + "power, ratio, correlation and carbon figures are withheld until the study runs on a quiet " + f"bare-metal Linux machine ([how]({options.guide_link})). The run's raw data is not in the " + "repository, because it holds those timings; the harness command below reproduces the counts.", + "", + ] + elif not summary.energy_measured: + lines += [f"> **Energy was not measured in this run:** {reason}", ""] + return lines + + def _operations_section(summary: Summary, image: str | None) -> list[str]: lines = ["## Operation counts", ""] if not summary.operations: return [*lines, "No sorting runs in this data.", ""] languages = sorted({lang for op in summary.operations for lang in op.languages}, key=list(LANGUAGE_NAMES).index) - names = ", ".join(LANGUAGE_NAMES[lang] for lang in languages) lines += [ "Comparisons between elements and writes into the array, for one sort. These are exact counts, " "not measurements, so machine load does not affect them. " + ( - f"{names} reported identical counts and identical input hashes for every configuration; " + f"{_names(languages)} reported identical counts and identical input hashes for every configuration; " "the harness checks this during the run and the report checks it again." if len(languages) > 1 - else f"Only {names} was run." + else f"Only {_names(languages)} was run." ), "", ] @@ -183,19 +266,24 @@ def _operations_section(summary: Summary, image: str | None) -> list[str]: def _per_sort_rows(configurations: list[Configuration], energy: bool) -> list[list[str]]: rows = [] for c in configurations: - rows.append( - [ - LANGUAGE_NAMES[c.language], - c.algorithm, - c.order.replace("_", " "), - f"{c.n:,}", - f"{c.inner_reps:,}", - format_estimate(c.time_per_sort_ns, TIME_UNITS), - format_estimate(c.energy_per_sort_uj, ENERGY_UNITS) - if energy and c.energy_per_sort_uj is not None - else "energy unavailable", - ] - ) + row = [ + LANGUAGE_NAMES[c.language], + c.algorithm, + c.order.replace("_", " "), + f"{c.n:,}", + f"{c.inner_reps:,}", + format_estimate(c.time_per_sort_ns, TIME_UNITS), + ] + if energy: + row.append( + "energy unavailable" + if c.energy_per_sort_uj is None + else format_estimate(c.energy_per_sort_uj, ENERGY_UNITS) + ) + row.append("undefined" if c.power_w is None else format_estimate(c.power_w, POWER_UNITS)) + else: + row.append("energy unavailable") + rows.append(row) return rows @@ -205,15 +293,21 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str return [*lines, _pending(options), ""] largest = max((c.n for c in summary.configurations), default=0) subtract = ( - "minus the median startup trial for the same language, order and size, divided by the number of sorts " - "in the process" + "minus the median startup cost per process for the same language, order and size, divided by the " + "number of sorts in the process" if summary.startup_subtracted else "divided by the number of sorts in the process (no startup trials were run, so interpreter " "start-up is included)" ) + idle = ( + " Energy is net of the idle baseline: the extra energy the sort drew above an idle machine." + if summary.baseline_subtracted + else "" + ) + power = " Power while sorting is energy per sort divided by time per sort." if summary.energy_measured else "" lines += [ f"Median over the measured runs of each configuration, with a 95% bootstrap confidence interval in " - f"brackets. Per-sort figures are the whole-process measurement {subtract}.", + f"brackets. Per-sort figures are the whole-process measurement {subtract}.{idle}{power}", "", ] for key in ("time_per_sort", "energy_per_sort"): @@ -221,6 +315,9 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str lines += [f"![{key.replace('_', ' ').capitalize()}, largest input, random order]({images[key]})", ""] header = ["Language", "Algorithm", "Order", "Size", "Sorts per process", "Time per sort", "Energy per sort"] align = [":---", ":---", ":---", "---:", "---:", "---:", "---:"] + if summary.energy_measured: + header.append("Power while sorting") + align.append("---:") lines += [f"Inputs of {largest:,} elements:", ""] lines += _table( header, align, _per_sort_rows([c for c in summary.configurations if c.n == largest], summary.energy_measured) @@ -230,32 +327,186 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str return [*lines, "", "", ""] -def _correlation_section(summary: Summary, options: ReportOptions, image: str | None, reason: str) -> list[str]: +def _reading(comparison: Comparison, subject: str) -> str: + power = comparison.power_ratio + if power is None: + return "undefined (an energy or time per sort at or below zero)" + if power.low <= 1 <= power.high: + return "same power: the time ratio predicts the energy ratio" + share = abs(power.value - 1) * 100 + return f"{subject} draws {share:.0f}% {'more' if power.value > 1 else 'less'} power" + + +def _language_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: + rows = [] + for c in comparisons: + language, algorithm, order, n = c.subject + rows.append( + [ + algorithm, + order.replace("_", " "), + f"{n:,}", + f"{LANGUAGE_NAMES[language]} / {LANGUAGE_NAMES[c.reference[0]]}", + format_ratio(c.time_ratio), + format_ratio(c.energy_ratio), + format_ratio(c.power_ratio), + _reading(c, LANGUAGE_NAMES[language]), + ] + ) + return rows + + +def _order_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: + rows = [] + for c in comparisons: + language, algorithm, order, n = c.subject + name = order.replace("_", " ") + rows.append( + [ + LANGUAGE_NAMES[language], + algorithm, + f"{n:,}", + f"{name} / random", + format_ratio(c.time_ratio), + format_ratio(c.energy_ratio), + format_ratio(c.power_ratio), + _reading(c, f"{name.capitalize()} input"), + ] + ) + return rows + + +RATIO_HEADER = ["Time ratio", "Energy ratio", "Power ratio", "Reading"] +RATIO_ALIGN = ["---:", "---:", "---:", ":---"] + + +def _describe(c: Configuration) -> str: + return f"the {LANGUAGE_NAMES[c.language]} {c.algorithm} sort of a {c.order.replace('_', ' ')} input of {c.n:,}" + + +def _power_paragraph(summary: Summary) -> list[str]: + powered = [(c, c.power_w) for c in summary.configurations if c.power_w is not None] + if not powered: + return ["Power is undefined for every configuration: no time per sort was above zero.", ""] + if len(powered) == 1: + ((only, power),) = powered + return [ + f"Power while sorting is defined for one configuration: {format_estimate(power, POWER_UNITS)} for " + f"{_describe(only)}.", + "", + ] + low_c, low = min(powered, key=lambda item: item[1].value) + high_c, high = max(powered, key=lambda item: item[1].value) + values = sorted(p.value for _, p in powered) + half = len(values) // 2 + middle = values[half] if len(values) % 2 else (values[half - 1] + values[half]) / 2 + return [ + f"Across the {len(powered):,} configurations where it is defined, power while sorting ranged from " + f"{format_estimate(low, POWER_UNITS)} for {_describe(low_c)}, to {format_estimate(high, POWER_UNITS)} " + f"for {_describe(high_c)}. The median over configurations was {middle:.3g} W. The table in the previous " + "section lists every configuration, with its 95% interval in brackets.", + "", + ] + + +def _proxy_section(summary: Summary, options: ReportOptions, image: str | None, reason: str) -> list[str]: lines = ["## Is run time a good stand-in for energy?", ""] if options.withhold_timings: return [*lines, _pending(options), ""] if not summary.energy_measured: return [*lines, f"Energy was not measured in this run ({reason}), so this question stays open.", ""] + largest = max((c.n for c in summary.configurations), default=0) lines += [ - "Spearman's rank correlation between process time and process energy. *Pooled* uses every run of " - "every order and size, with a 95% bootstrap interval; it is dominated by input size. *Within* is the " - "median of the correlations computed inside each configuration, where only run-to-run variation is " - "left.", + "If every configuration drew the same average power while sorting, energy would be proportional to " + "time, and time would be a perfect stand-in for it. The sections below test that directly, through " + "power and through ratios between configurations, and then more weakly, through rank correlation.", "", ] if image: - lines += [f"![Process time against process energy for every run]({image})", ""] - rows = [] - for c in summary.correlations: - pooled = ( - "undefined" if c.pooled is None else f"{c.pooled.value:.3f} ({c.pooled.low:.3f} to {c.pooled.high:.3f})" - ) - within = "undefined" if c.within is None else f"{c.within:.3f} over {c.within_configurations} configurations" - rows.append([LANGUAGE_NAMES[c.language], c.algorithm, f"{c.pairs:,}", pooled, within]) + lines += [f"![Energy per sort against time per sort, one point per configuration]({image})", ""] + lines += ["### Power while sorting", "", *_power_paragraph(summary)] + + lines += [ + "### Between languages", + "", + "For the same algorithm and input, how many times the second language's time and energy per sort " + "the first language needs. The power ratio is the energy ratio divided by the time ratio: when its " + "95% interval includes 1, the two languages draw the same power and the time ratio predicts the " + "energy ratio. Random input, largest size:", + "", + ] + header = ["Algorithm", "Order", "Size", "Languages", *RATIO_HEADER] + align = [":---", ":---", "---:", ":---", *RATIO_ALIGN] + main = [c for c in summary.language_comparisons if c.subject[2] == "random" and c.subject[3] == largest] + lines += _table(header, align, _language_rows(main)) if main else ["No language pairs in this run."] + if summary.language_comparisons: + lines += ["", "
Every order and size", ""] + lines += _table(header, align, _language_rows(summary.language_comparisons)) + lines += ["", "
"] + lines += [""] + + lines += [ + "### Between input orders", + "", + "For the same language and algorithm, each input order against random input. Bubble sort compares " + "as often on sorted input as on random input but writes nothing; this quicksort does far more work " + "on sorted input. The question is whether energy follows time. Sorted against random, largest size:", + "", + ] + header = ["Language", "Algorithm", "Size", "Orders", *RATIO_HEADER] + align = [":---", ":---", "---:", ":---", *RATIO_ALIGN] + main = [c for c in summary.order_comparisons if c.subject[2] == "sorted" and c.subject[3] == largest] + lines += _table(header, align, _order_rows(main)) if main else ["No sorted and random pairs in this run."] + if summary.order_comparisons: + lines += ["", "
Every order and size", ""] + lines += _table(header, align, _order_rows(summary.order_comparisons)) + lines += ["", "
"] + lines += [""] + + scope_names = {"c:": "C:", "python:": "Python:", "node:": "Node.js:"} + lines += [ + "### Rank agreement across configurations", + "", + "Spearman's rank correlation between median time per sort and median energy per sort, over " + "configurations, with a 95% bootstrap interval: does energy put configurations in the same order as " + "time does? This is a weaker test than the ratios above. Configurations differ in cost by large " + "factors, so rho can stay close to 1 even when their power differs.", + "", + ] lines += _table( - ["Language", "Algorithm", "Runs", "Pooled rho", "Within-configuration rho (median)"], - [":---", ":---", "---:", "---:", "---:"], - rows, + ["Configurations compared", "Count", "Spearman's rho"], + [":---", "---:", "---:"], + ( + [ + next((a.scope.replace(k, v, 1) for k, v in scope_names.items() if a.scope.startswith(k)), a.scope), + f"{a.configurations:,}", + "undefined" if a.rho is None else f"{a.rho.value:.3f} ({a.rho.low:.3f} to {a.rho.high:.3f})", + ] + for a in summary.rank_agreement + ), + ) + lines += [ + "", + "### Run-to-run coupling (a noise check)", + "", + "Within each configuration, Spearman's rho between the process time and process energy of its runs; " + "the median over configurations is shown. With steady power a run that takes longer uses more " + "energy, so a positive value is expected: it checks measurement noise rather than answering the " + "question.", + "", + ] + lines += _table( + ["Language", "Algorithm", "Configurations", "Median rho"], + [":---", ":---", "---:", "---:"], + ( + [ + LANGUAGE_NAMES[n.language], + n.algorithm, + f"{n.configurations:,}", + "undefined" if n.rho is None else f"{n.rho:.3f}", + ] + for n in summary.noise_coupling + ), ) return [*lines, ""] @@ -276,6 +527,7 @@ def _startup_section(summary: Summary, options: ReportOptions) -> list[str]: [ LANGUAGE_NAMES[s.language], f"{s.n:,}", + f"{s.launches:,}", format_estimate(s.time_ns, TIME_UNITS), "energy unavailable" if s.energy_uj is None else format_estimate(s.energy_uj, ENERGY_UNITS), ] @@ -283,11 +535,37 @@ def _startup_section(summary: Summary, options: ReportOptions) -> list[str]: if s.order == "random" ] lines += _table( - ["Language", "Size", "Time per process", "Energy per process"], [":---", "---:", "---:", "---:"], rows + ["Language", "Size", "Processes per batch", "Time per process", "Energy per process"], + [":---", "---:", "---:", "---:", "---:"], + rows, ) return [*lines, ""] +def _carbon_rows(summary: Summary, options: ReportOptions, largest: int, order: str) -> list[list[str]]: + rows = [] + net = summary.baseline_subtracted + for c in summary.configurations: + if c.n != largest or c.order != order or c.energy_per_sort_uj is None: + continue + gross = c.gross_energy_per_sort_uj.value if c.gross_energy_per_sort_uj else c.energy_per_sort_uj.value + net_energy = c.energy_per_sort_uj.value * options.per_sorts + gross_energy = gross * options.per_sorts + row = [LANGUAGE_NAMES[c.language], c.algorithm] + if net: + row += [format_energy(net_energy), format_energy(gross_energy)] + row += [ + f"{format_grams(grams_co2e(net_energy, r.g_per_kwh))} to " + f"{format_grams(grams_co2e(gross_energy, r.g_per_kwh))}" + for r in options.regions + ] + else: + row += [format_energy(gross_energy)] + row += [format_grams(grams_co2e(gross_energy, r.g_per_kwh)) for r in options.regions] + rows.append(row) + return rows + + def _carbon_section(summary: Summary, options: ReportOptions, reason: str) -> list[str]: lines = ["## Carbon estimates", ""] if options.withhold_timings: @@ -297,29 +575,33 @@ def _carbon_section(summary: Summary, options: ReportOptions, reason: str) -> li regions = options.regions largest = max((c.n for c in summary.configurations), default=0) order = "random" if any(c.order == "random" for c in summary.configurations) else ORDERS[0] + net = summary.baseline_subtracted + basis = ( + "*Net* counts only the power the sort drew above idle, which is what the sort adds. *Including idle* " + "also counts the idle power the machine draws while it runs the sort, which is attributable to the " + "time the sort occupies it. " + if net + else "No idle baseline was measured, so the energy includes the machine's idle power. " + ) lines += [ - f"Energy converted with annual grid averages (see the table below). Per {options.per_sorts:,} sorts of " - f"a {order.replace('_', ' ')} input of {largest:,} elements:", + f"Energy converted with annual grid averages (see the table below), per {options.per_sorts:,} sorts of " + f"a {order.replace('_', ' ')} input of {largest:,} elements. {basis}RAPL covers the processor package " + "and DRAM only, not the rest of the machine or its power supply, so both figures understate the " + "electricity used.", "", ] - rows = [] - for c in summary.configurations: - if c.n != largest or c.order != order or c.energy_per_sort_uj is None: - continue - energy = c.energy_per_sort_uj.value * options.per_sorts - rows.append( - [LANGUAGE_NAMES[c.language], c.algorithm, format_energy(energy)] - + [format_grams(grams_co2e(energy, r.g_per_kwh)) for r in regions] - ) - lines += _table( - ["Language", "Algorithm", "Energy"] + [f"CO2e, {r.region}" for r in regions], - [":---", ":---", "---:"] + ["---:"] * len(regions), - rows, - ) + if net: + header = ["Language", "Algorithm", "Energy, net", "Energy, including idle"] + header += [f"CO2e, {r.region} (net to including idle)" for r in regions] + align = [":---", ":---", "---:", "---:"] + ["---:"] * len(regions) + else: + header = ["Language", "Algorithm", "Energy, including idle"] + [f"CO2e, {r.region}" for r in regions] + align = [":---", ":---", "---:"] + ["---:"] * len(regions) + lines += _table(header, align, _carbon_rows(summary, options, largest, order)) total = summary.total_energy_uj lines += [ "", - f"The measured runs themselves used {format_energy(total)} (package and DRAM, before subtracting idle), " + f"The measured runs themselves used {format_energy(total)} (package and DRAM, including idle), " + "; ".join(f"{format_grams(grams_co2e(total, r.g_per_kwh))} CO2e on the {r.region} grid" for r in regions) + ".", "", @@ -352,14 +634,6 @@ def _relative(path: Path, start: Path) -> str: return Path(os.path.relpath(path, start)).as_posix() -def _points_by_language(runs: list[Run]) -> dict[str, list[tuple[str, float, float]]]: - points: dict[str, list[tuple[str, float, float]]] = defaultdict(list) - for run in runs: - if not run.is_startup and run.energy_uj is not None: - points[run.language].append((run.algorithm, float(run.wall_ns), run.energy_uj)) - return dict(points) - - def write_report( runs: list[Run], meta: dict[str, Any], @@ -399,9 +673,9 @@ def draw(key: str, make: Callable[[Path], Path | None]) -> None: ), ) draw( - "time_vs_energy", + "energy_vs_time", lambda p: ( - charts.time_energy_chart(_points_by_language(runs), summary.baseline_subtracted, p) + charts.energy_time_chart(summary.configurations, summary.baseline_subtracted, p) if publish and summary.energy_measured else None ), @@ -414,23 +688,13 @@ def draw(key: str, make: Callable[[Path], Path | None]) -> None: "", "", "", + *_banners(meta, summary, options, reason), ] - if options.withhold_timings: - lines += [ - "> **Status: time and energy results are pending.** This file was generated from a run that " - "checked the pipeline end to end. Operation counts are exact and shown below; time, energy, " - "correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux " - f"machine ([how]({options.guide_link})). The run's raw data is not in the repository, because it " - "holds those timings; the harness command below reproduces the counts.", - "", - ] - elif not summary.energy_measured: - lines += [f"> **Energy was not measured in this run:** {reason}", ""] - lines += ["## How these results were produced", "", *_environment(meta, summary)] + lines += ["## How these results were produced", "", *_environment(meta, runs, summary, options)] lines += [f"- Report command: `{options.command}`", ""] lines += _operations_section(summary, images.get("operations")) lines += _per_sort_section(summary, options, images) - lines += _correlation_section(summary, options, images.get("time_vs_energy"), reason) + lines += _proxy_section(summary, options, images.get("energy_vs_time"), reason) lines += _startup_section(summary, options) lines += _carbon_section(summary, options, reason) lines += _grid_section(options) @@ -441,6 +705,12 @@ def draw(key: str, make: Callable[[Path], Path | None]) -> None: return written +def _cells(estimate: Estimate | None, digits: int) -> list[str]: + if estimate is None: + return ["", "", ""] + return [f"{estimate.value:.{digits}f}", f"{estimate.low:.{digits}f}", f"{estimate.high:.{digits}f}"] + + def write_summary_csv(summary: Summary, path: Path) -> Path: """Every configuration's estimates as a CSV, for further analysis.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -462,26 +732,22 @@ def write_summary_csv(summary: Summary, path: Path) -> Path: "energy_per_sort_uj", "energy_per_sort_uj_low", "energy_per_sort_uj_high", + "gross_energy_per_sort_uj", + "gross_energy_per_sort_uj_low", + "gross_energy_per_sort_uj_high", + "power_w", + "power_w_low", + "power_w_high", ] ) for c in summary.configurations: - e = c.energy_per_sort_uj writer.writerow( [ - c.language, - c.algorithm, - c.order, - c.n, - c.runs, - c.inner_reps, - c.comparisons, - c.writes, - f"{c.time_per_sort_ns.value:.1f}", - f"{c.time_per_sort_ns.low:.1f}", - f"{c.time_per_sort_ns.high:.1f}", - "" if e is None else f"{e.value:.3f}", - "" if e is None else f"{e.low:.3f}", - "" if e is None else f"{e.high:.3f}", + *(c.language, c.algorithm, c.order, c.n, c.runs, c.inner_reps, c.comparisons, c.writes), + *_cells(c.time_per_sort_ns, 1), + *_cells(c.energy_per_sort_uj, 3), + *_cells(c.gross_energy_per_sort_uj, 3), + *_cells(c.power_w, 4), ] ) return path diff --git a/analysis/greenbench_analysis/stats.py b/analysis/greenbench_analysis/stats.py index fc2cabd..3b9d9d6 100644 --- a/analysis/greenbench_analysis/stats.py +++ b/analysis/greenbench_analysis/stats.py @@ -72,6 +72,56 @@ def bootstrap_median( return Estimate(point, float(low), float(high)) +def resample_medians( + columns: Sequence[Sequence[float] | FloatArray], + *, + rng: np.random.Generator, + resamples: int, +) -> FloatArray: + """Bootstrap medians of paired columns: shape (number of columns, resamples). + + The columns hold one value per run, in the same order. Every resample draws runs with + replacement and takes the median of each column over the same runs, so quantities + computed from two columns (such as energy divided by time) keep each run's pairing. + """ + if resamples < 1: + raise ValueError("resamples must be at least 1") + samples = [_as_array(column) for column in columns] + size = samples[0].size + if any(sample.size != size for sample in samples): + raise ValueError("paired columns must have the same length") + medians = np.empty((len(samples), resamples), dtype=np.float64) + for start in range(0, resamples, _CHUNK): + stop = min(start + _CHUNK, resamples) + indices = rng.integers(0, size, size=(stop - start, size)) + for row, sample in enumerate(samples): + medians[row, start:stop] = np.median(sample[indices], axis=1) + return medians + + +def interval(point: float, draws: FloatArray, *, confidence: float = 0.95) -> Estimate: + """A percentile interval from bootstrap draws around a point estimate. + + Draws that are not finite (a ratio whose denominator fell to zero in that resample) are + left out; with none left, the interval collapses to the point. + """ + if not 0 < confidence < 1: + raise ValueError("confidence must be between 0 and 1") + finite = draws[np.isfinite(draws)] + if finite.size == 0: + return Estimate(point, point, point) + tail = (1 - confidence) / 2 + low, high = np.quantile(finite, [tail, 1 - tail]) + return Estimate(point, float(low), float(high)) + + +def ratio_draws(numerator: FloatArray, denominator: FloatArray) -> FloatArray: + """numerator / denominator element by element, NaN where the denominator is not positive.""" + with np.errstate(divide="ignore", invalid="ignore"): + result = np.where(denominator > 0, numerator / denominator, np.nan) + return np.asarray(result, dtype=np.float64) + + def rank_average(values: Sequence[float] | FloatArray) -> FloatArray: """Ranks starting at 1, with tied values sharing the average of their ranks.""" sample = _as_array(values) @@ -104,34 +154,3 @@ def spearman(x: Sequence[float] | FloatArray, y: Sequence[float] | FloatArray) - if denominator == 0.0: return None return float(np.sum(rx * ry) / denominator) - - -def bootstrap_spearman( - x: Sequence[float] | FloatArray, - y: Sequence[float] | FloatArray, - *, - rng: np.random.Generator, - confidence: float = 0.95, - resamples: int = 2_000, -) -> Estimate | None: - """Spearman's rho with a percentile-bootstrap interval from resampling pairs. - - Resamples in which rho is undefined (for example, every pair drawn the same) are - skipped. Returns None when rho itself is undefined. - """ - point = spearman(x, y) - if point is None: - return None - xs = _as_array(x) - ys = _as_array(y) - values = [] - for _ in range(resamples): - indices = rng.integers(0, xs.size, size=xs.size) - rho = spearman(xs[indices], ys[indices]) - if rho is not None: - values.append(rho) - if not values: - return Estimate(point, point, point) - tail = (1 - confidence) / 2 - low, high = np.quantile(np.asarray(values), [tail, 1 - tail]) - return Estimate(point, float(low), float(high)) diff --git a/analysis/greenbench_analysis/summary.py b/analysis/greenbench_analysis/summary.py index 4aa1394..412e1fc 100644 --- a/analysis/greenbench_analysis/summary.py +++ b/analysis/greenbench_analysis/summary.py @@ -1,13 +1,23 @@ -"""Per-configuration summaries, startup costs and time-energy correlations. +"""Per-configuration summaries, power, ratios between configurations, and rank agreement. A configuration is one language running one algorithm on one input (order and size). Every -measured run is one worker process that sorted the input inner_reps times, so: +measured sorting run is one worker process that sorted the input inner_reps times, so: - process time and energy cover the whole process, as the harness measured them; - per-sort time and energy subtract the median startup cost for the same language, order and size, then divide by inner_reps. A startup run starts a batch of processes that generate the input and exit, back to back, so its cost per process is the batch's time and energy divided by the number of launches. + +Energy is net of the idle baseline when one was measured. Power while sorting is energy per +sort divided by time per sort. If every configuration drew the same power, energy would be +proportional to time and time would be a perfect stand-in for it; the ratios between +configurations test exactly that. + +Every confidence interval is a percentile bootstrap. Each configuration's runs are resampled +once, with each run's time and energy kept together, and every estimate for that +configuration (medians, power, and its side of any ratio) is computed from the same draws. +The startup medians that are subtracted are treated as fixed. """ from __future__ import annotations @@ -19,10 +29,17 @@ import numpy as np from .load import ALGORITHMS, LANGUAGES, ORDERS, DataError, Run -from .stats import Estimate, bootstrap_median, bootstrap_spearman, median, spearman +from .stats import Estimate, FloatArray, bootstrap_median, interval, median, ratio_draws, resample_medians, spearman Key = tuple[str, str, str, int] # language, algorithm, order, n +# Energy in microjoules over time in nanoseconds is in kilowatts; this gives watts. +WATTS_PER_UJ_PER_NS = 1e3 + +# Language pairs compared, as (subject, reference): how many times the reference's cost the +# subject's is. C is the reference for both interpreters; Python is compared with Node.js. +LANGUAGE_PAIRS = (("python", "c"), ("node", "c"), ("python", "node")) + @dataclass(frozen=True) class OperationCounts: @@ -47,7 +64,13 @@ class Configuration: process_time_ns: Estimate process_energy_uj: Estimate | None time_per_sort_ns: Estimate - energy_per_sort_uj: Estimate | None + energy_per_sort_uj: Estimate | None # net of idle when an idle baseline was measured + gross_energy_per_sort_uj: Estimate | None # including the idle power drawn while sorting + power_w: Estimate | None # energy per sort / time per sort: average power while sorting + + @property + def key(self) -> Key: + return (self.language, self.algorithm, self.order, self.n) @dataclass(frozen=True) @@ -56,25 +79,48 @@ class Startup: order: str n: int runs: int - time_ns: Estimate - energy_uj: Estimate | None + launches: int # processes per measured run (the median, if calibration differed) + time_ns: Estimate # per process + energy_uj: Estimate | None # per process @dataclass(frozen=True) -class Correlation: - """Spearman's rho between process time and process energy, for one language and algorithm. +class Comparison: + """How a subject configuration compares with a reference one: subject over reference. - ``pooled`` uses every run of every order and size; it is dominated by input size, since - bigger inputs take longer and use more energy. ``within`` is the median of rho computed - separately inside each configuration, where only run-to-run variation is left. + power_ratio is energy_ratio / time_ratio. It is 1 when both configurations draw the same + power while sorting, which is when their time ratio predicts their energy ratio; an + interval that excludes 1 means time alone misjudges the energy difference. """ + subject: Key + reference: Key + time_ratio: Estimate | None + energy_ratio: Estimate | None + power_ratio: Estimate | None + + +@dataclass(frozen=True) +class RankAgreement: + """Spearman's rho between median time per sort and median energy per sort, across + configurations: does energy put them in the same order as time does?""" + + scope: str + configurations: int + rho: Estimate | None + + +@dataclass(frozen=True) +class NoiseCoupling: + """Median, over the configurations of one language and algorithm, of Spearman's rho between + process time and process energy across the runs of a configuration. A diagnostic of + run-to-run noise: with steady power, a run that takes longer uses more energy, so a + positive value is expected and says little about whether time is a good proxy.""" + language: str algorithm: str - pairs: int - pooled: Estimate | None - within: float | None - within_configurations: int + rho: float | None + configurations: int @dataclass(frozen=True) @@ -82,14 +128,25 @@ class Summary: configurations: list[Configuration] startups: list[Startup] operations: list[OperationCounts] - correlations: list[Correlation] + language_comparisons: list[Comparison] + order_comparisons: list[Comparison] + rank_agreement: list[RankAgreement] + noise_coupling: list[NoiseCoupling] run_count: int languages: list[str] energy_measured: bool baseline_subtracted: bool startup_subtracted: bool total_wall_ns: int - total_energy_uj: float | None + total_energy_uj: float | None # gross, package and DRAM, over every measured run + + +@dataclass(frozen=True) +class _Draws: + """Bootstrap medians of one configuration's per-sort time and energy, from the same draws.""" + + time: FloatArray + energy: FloatArray | None def _sort_key(language: str, algorithm: str, order: str, n: int) -> tuple[int, int, int, int]: @@ -122,6 +179,15 @@ def _check_inputs_agree(runs: list[Run]) -> None: ) +def _check_counters_moved(runs: list[Run]) -> None: + measured = [run for run in runs if run.pkg_uj is not None] + if measured and all(run.pkg_uj == 0 for run in measured): + raise DataError( + "every pkg_uj is 0: the RAPL counters did not move during this run, so its energy figures would " + "be fake. greenbench 2.1 and later refuse frozen counters; run the harness again" + ) + + def _operation_counts(groups: dict[Key, list[Run]]) -> list[OperationCounts]: by_input: dict[tuple[str, str, int], dict[str, tuple[int, int]]] = defaultdict(dict) for key, runs in groups.items(): @@ -151,10 +217,131 @@ def _energies(runs: list[Run]) -> list[float] | None: return [float(v) for v in values if v is not None] +def _gross(runs: list[Run]) -> list[float] | None: + if any(run.pkg_uj is None for run in runs): + return None + return [(run.pkg_uj or 0.0) + (run.dram_uj or 0.0) for run in runs] + + +def _ratio(subject: float, reference: float, subject_draws: FloatArray, reference_draws: FloatArray) -> Estimate | None: + if subject <= 0 or reference <= 0: + return None + return interval(subject / reference, ratio_draws(subject_draws, reference_draws)) + + +def _compare(a: Configuration, b: Configuration, draws: dict[Key, _Draws]) -> Comparison: + da, db = draws[a.key], draws[b.key] + time_ratio = _ratio(a.time_per_sort_ns.value, b.time_per_sort_ns.value, da.time, db.time) + energy_ratio = power_ratio = None + if ( + a.energy_per_sort_uj is not None + and b.energy_per_sort_uj is not None + and da.energy is not None + and db.energy is not None + ): + energy_ratio = _ratio(a.energy_per_sort_uj.value, b.energy_per_sort_uj.value, da.energy, db.energy) + if time_ratio is not None and energy_ratio is not None: + # (energy ratio) / (time ratio), resample by resample. + power_draws = ratio_draws(ratio_draws(da.energy, db.energy), ratio_draws(da.time, db.time)) + power_ratio = interval(energy_ratio.value / time_ratio.value, power_draws) + return Comparison(a.key, b.key, time_ratio, energy_ratio, power_ratio) + + +def _language_comparisons(configurations: list[Configuration], draws: dict[Key, _Draws]) -> list[Comparison]: + by_key = {c.key: c for c in configurations} + result = [] + for c in configurations: + for subject, reference in LANGUAGE_PAIRS: + if c.language != subject: + continue + other = by_key.get((reference, c.algorithm, c.order, c.n)) + if other is not None: + result.append(_compare(c, other, draws)) + result.sort( + key=lambda r: ( + ALGORITHMS.index(r.subject[1]), + ORDERS.index(r.subject[2]), + r.subject[3], + LANGUAGE_PAIRS.index((r.subject[0], r.reference[0])), + ) + ) + return result + + +def _order_comparisons(configurations: list[Configuration], draws: dict[Key, _Draws]) -> list[Comparison]: + by_key = {c.key: c for c in configurations} + result = [] + for c in configurations: + if c.order == "random": + continue + random = by_key.get((c.language, c.algorithm, "random", c.n)) + if random is not None: + result.append(_compare(c, random, draws)) + return result + + +def _rank_agreement( + configurations: list[Configuration], draws: dict[Key, _Draws], resamples: int +) -> list[RankAgreement]: + measured = [c for c in configurations if c.energy_per_sort_uj is not None] + scopes: list[tuple[str, list[Configuration]]] = [] + for n in sorted({c.n for c in measured}): + scopes.append((f"every language, algorithm and order at {n:,} elements", [c for c in measured if c.n == n])) + for language in LANGUAGES: + chosen = [c for c in measured if c.language == language] + if chosen: + scopes.append((f"{language}: every algorithm, order and size", chosen)) + if measured: + scopes.append(("every configuration", measured)) + + result = [] + for scope, chosen in scopes: + times = np.array([c.time_per_sort_ns.value for c in chosen]) + energies = np.array([c.energy_per_sort_uj.value if c.energy_per_sort_uj else 0.0 for c in chosen]) + point = spearman(times, energies) + estimate = None + if point is not None: + time_draws = np.stack([draws[c.key].time[:resamples] for c in chosen]) + energy_draws = np.stack([_energy_draws(draws[c.key])[:resamples] for c in chosen]) + rhos = [spearman(time_draws[:, r], energy_draws[:, r]) for r in range(time_draws.shape[1])] + estimate = interval(point, np.array([np.nan if rho is None else rho for rho in rhos])) + result.append(RankAgreement(scope, len(chosen), estimate)) + return result + + +def _energy_draws(draws: _Draws) -> FloatArray: + if draws.energy is None: + raise ValueError("energy draws are missing for a configuration with energy") + return draws.energy + + +def _noise_coupling(groups: dict[Key, list[Run]]) -> list[NoiseCoupling]: + within: dict[tuple[str, str], list[float]] = defaultdict(list) + for (language, algorithm, _order, _n), group in groups.items(): + energies = _energies(group) + if algorithm == "none" or energies is None: + continue + rhos = within[(language, algorithm)] + rho = spearman([float(r.wall_ns) for r in group], energies) if len(group) >= 3 else None + if rho is not None: + rhos.append(rho) + keys = sorted(within, key=lambda k: (LANGUAGES.index(k[0]), ALGORITHMS.index(k[1]))) + return [ + NoiseCoupling( + language, + algorithm, + median(within[(language, algorithm)]) if within[(language, algorithm)] else None, + len(within[(language, algorithm)]), + ) + for language, algorithm in keys + ] + + def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = 2_000, seed: int = 1) -> Summary: """Summarise measured runs. The bootstrap is seeded, so the same data gives the same report.""" rng = np.random.default_rng(seed) _check_inputs_agree(runs) + _check_counters_moved(runs) groups = _group(runs) operations = _operation_counts(groups) energy_measured = all(run.energy_uj is not None for run in runs) @@ -162,27 +349,35 @@ def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = startups: list[Startup] = [] startup_time: dict[tuple[str, str, int], float] = {} startup_energy: dict[tuple[str, str, int], float] = {} + startup_gross: dict[tuple[str, str, int], float] = {} for (language, algorithm, order, n), group in groups.items(): if algorithm != "none": continue times = [r.wall_ns / r.launches for r in group] batches = _energies(group) energies = None if batches is None else [e / r.launches for e, r in zip(batches, group, strict=True)] + gross_batches = _gross(group) startup_time[(language, order, n)] = median(times) if energies is not None: startup_energy[(language, order, n)] = median(energies) + if gross_batches is not None: + startup_gross[(language, order, n)] = median( + [e / r.launches for e, r in zip(gross_batches, group, strict=True)] + ) startups.append( Startup( language=language, order=order, n=n, runs=len(group), + launches=round(median([float(r.launches) for r in group])), time_ns=bootstrap_median(times, rng=rng, resamples=resamples), energy_uj=None if energies is None else bootstrap_median(energies, rng=rng, resamples=resamples), ) ) configurations: list[Configuration] = [] + draws: dict[Key, _Draws] = {} startup_subtracted = True for key, group in groups.items(): language, algorithm, order, n = key @@ -191,13 +386,26 @@ def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = reps = _constant(group, lambda r: r.inner_reps, "inner_reps", key) times = [float(r.wall_ns) for r in group] energies = _energies(group) + gross = _gross(group) base_key = (language, order, n) if base_key not in startup_time: startup_subtracted = False - base_time = startup_time.get(base_key, 0.0) - base_energy = startup_energy.get(base_key, 0.0) - per_sort_time = [(t - base_time) / reps for t in times] - per_sort_energy = None if energies is None else [(e - base_energy) / reps for e in energies] + per_sort_time = [(t - startup_time.get(base_key, 0.0)) / reps for t in times] + per_sort_energy = ( + None if energies is None else [(e - startup_energy.get(base_key, 0.0)) / reps for e in energies] + ) + per_sort_gross = None if gross is None else [(e - startup_gross.get(base_key, 0.0)) / reps for e in gross] + + columns = [per_sort_time] if per_sort_energy is None else [per_sort_time, per_sort_energy] + medians = resample_medians(columns, rng=rng, resamples=resamples) + draws[key] = _Draws(medians[0], None if per_sort_energy is None else medians[1]) + time_per_sort = interval(median(per_sort_time), medians[0]) + energy_per_sort = power = None + if per_sort_energy is not None: + energy_per_sort = interval(median(per_sort_energy), medians[1]) + if time_per_sort.value > 0: + power_draws = ratio_draws(medians[1], medians[0]) * WATTS_PER_UJ_PER_NS + power = interval(energy_per_sort.value / time_per_sort.value * WATTS_PER_UJ_PER_NS, power_draws) configurations.append( Configuration( language=language, @@ -212,42 +420,15 @@ def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = process_energy_uj=None if energies is None else bootstrap_median(energies, rng=rng, resamples=resamples), - time_per_sort_ns=bootstrap_median(per_sort_time, rng=rng, resamples=resamples), - energy_per_sort_uj=None - if per_sort_energy is None - else bootstrap_median(per_sort_energy, rng=rng, resamples=resamples), + time_per_sort_ns=time_per_sort, + energy_per_sort_uj=energy_per_sort, + gross_energy_per_sort_uj=None + if per_sort_gross is None + else bootstrap_median(per_sort_gross, rng=rng, resamples=resamples), + power_w=power, ) ) - correlations: list[Correlation] = [] - if energy_measured: - pairs: dict[tuple[str, str], list[tuple[float, float]]] = defaultdict(list) - within: dict[tuple[str, str], list[float]] = defaultdict(list) - for (language, algorithm, _order, _n), group in groups.items(): - if algorithm == "none": - continue - xs = [float(r.wall_ns) for r in group] - ys = [float(r.energy_uj or 0.0) for r in group] - pairs[(language, algorithm)].extend(zip(xs, ys, strict=True)) - rho = spearman(xs, ys) if len(group) >= 3 else None - if rho is not None: - within[(language, algorithm)].append(rho) - for language, algorithm in sorted(pairs, key=lambda k: (LANGUAGES.index(k[0]), ALGORITHMS.index(k[1]))): - points = pairs[(language, algorithm)] - rhos = within.get((language, algorithm), []) - correlations.append( - Correlation( - language=language, - algorithm=algorithm, - pairs=len(points), - pooled=bootstrap_spearman( - [p[0] for p in points], [p[1] for p in points], rng=rng, resamples=rho_resamples - ), - within=median(rhos) if rhos else None, - within_configurations=len(rhos), - ) - ) - total_energy = None if energy_measured: total_energy = sum((r.pkg_uj or 0.0) + (r.dram_uj or 0.0) for r in runs) @@ -255,7 +436,10 @@ def summarise(runs: list[Run], *, resamples: int = 10_000, rho_resamples: int = configurations=configurations, startups=startups, operations=operations, - correlations=correlations, + language_comparisons=_language_comparisons(configurations, draws), + order_comparisons=_order_comparisons(configurations, draws), + rank_agreement=_rank_agreement(configurations, draws, rho_resamples) if energy_measured else [], + noise_coupling=_noise_coupling(groups) if energy_measured else [], run_count=len(runs), languages=sorted({r.language for r in runs}, key=LANGUAGES.index), energy_measured=energy_measured, diff --git a/analysis/tests/synthetic.py b/analysis/tests/synthetic.py index 4c84ee0..4bc0cdc 100644 --- a/analysis/tests/synthetic.py +++ b/analysis/tests/synthetic.py @@ -63,6 +63,10 @@ def make_run( ) +IDLE_WATTS = 1.0 # idle package power in the synthetic data; 1 W is 1e-3 µJ per ns +TIME_SCALE = {"c": 1, "python": 10, "node": 5} + + def study( *, energy: bool, @@ -70,13 +74,25 @@ def study( languages: Iterable[str] = ("c", "python"), startup: bool = True, sizes: Iterable[int] = (100,), + power: dict[str, float] | None = None, + startup_launches: int = 1, ) -> list[Run]: - """A small, fully known data set: per-sort cost 1 ms and 2 mJ for C, ten times that for Python, - plus a startup cost of 5 ms and 10 mJ per process for C and 50 ms and 100 mJ for Python.""" + """A small, fully known data set. + + Per sort, C takes 1 ms and 2 mJ above idle, so it draws 2 W while sorting; Python takes ten + times as long and Node.js five times, at the same power unless ``power`` gives a language a + factor on its energy. Starting a process costs 5 ms and 10 mJ for C, scaled the same way; + each startup run is a batch of ``startup_launches`` processes. The idle package draws 1 W + throughout, so gross energy is the net energy plus 1 µJ per µs, and every idle window has + the length of its run. The runs of a configuration differ by a small jitter that raises + time and energy together. + """ + factors = power or {} result: list[Run] = [] sequence = 0 for language in languages: - scale = 1 if language == "c" else 10 + scale = TIME_SCALE[language] + factor = factors.get(language, 1.0) for order, n in ((o, n) for o in ("sorted", "random") for n in sizes): algorithms = ["bubble", "quick", *(["none"] if startup else [])] for algorithm in algorithms: @@ -84,8 +100,10 @@ def study( sequence += 1 jitter = repetition - (runs + 1) // 2 # symmetric around the middle run reps = 0 if algorithm == "none" else 10 - wall = scale * (5_000_000 + reps * 1_000_000) + jitter * 1000 - gross = scale * (10_000.0 + reps * 2_000.0) + jitter * 10.0 + launches = startup_launches if algorithm == "none" else 1 + wall = scale * (launches * 5_000_000 + reps * 1_000_000) + jitter * 1000 + net = factor * scale * (launches * 10_000.0 + reps * 2_000.0) + jitter * 10.0 + idle = IDLE_WATTS * 1e-3 * wall result.append( make_run( language=language, @@ -94,9 +112,10 @@ def study( n=n, repetition=repetition, inner_reps=reps, + launches=launches, wall_ns=wall, - energy_uj=gross + 500.0 if energy else None, - idle_uj=500.0 if energy else None, + energy_uj=net + idle if energy else None, + idle_uj=idle if energy else None, sequence=sequence, ) ) diff --git a/analysis/tests/test_charts.py b/analysis/tests/test_charts.py index 607fe9b..4409a16 100644 --- a/analysis/tests/test_charts.py +++ b/analysis/tests/test_charts.py @@ -18,11 +18,15 @@ def test_operations_chart_needs_two_sizes(tmp_path: Path) -> None: assert drawn.read_bytes().startswith(PNG_SIGNATURE) -def test_time_energy_chart_skips_runs_it_cannot_draw(tmp_path: Path) -> None: - nothing_positive = {"c": [("quick", 1e6, 0.0), ("quick", 2e6, -5.0)]} - assert charts.time_energy_chart(nothing_positive, True, tmp_path / "a.png") is None - some_positive = {"c": [("quick", 1e6, 10.0), ("quick", 2e6, -5.0)], "python": [("bubble", 5e6, 40.0)]} - assert charts.time_energy_chart(some_positive, True, tmp_path / "b.png") is not None +def test_energy_time_chart_needs_two_drawable_configurations(tmp_path: Path) -> None: + summary = summarise(synthetic.study(energy=True, power={"python": 1.5}), resamples=100) + drawn = charts.energy_time_chart(summary.configurations, True, tmp_path / "a.png") + assert drawn is not None + assert drawn.read_bytes().startswith(PNG_SIGNATURE) + one = [c for c in summary.configurations if c.language == "c"][:1] + assert charts.energy_time_chart(one, True, tmp_path / "b.png") is None + without_energy = summarise(synthetic.study(energy=False), resamples=100) + assert charts.energy_time_chart(without_energy.configurations, False, tmp_path / "c.png") is None def test_dot_charts_from_a_summary(tmp_path: Path) -> None: diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index a6f4b16..4eace98 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -43,13 +43,14 @@ def test_unit_formatting() -> None: assert format_estimate(Estimate(2_000, 1_900, 2_100), ENERGY_UNITS) == "2.00 mJ (1.90 to 2.10)" -def render(tmp_path: Path, *, energy: bool, withhold: bool) -> str: - runs = synthetic.study(energy=energy, sizes=(100, 200)) +def render( + tmp_path: Path, *, energy: bool, withhold: bool, power: dict[str, float] | None = None, status: str = "complete" +) -> str: + runs = synthetic.study(energy=energy, sizes=(100, 200), power=power) summary = summarise(runs, resamples=200) out = tmp_path / "docs" / "results.md" - write_report( - runs, synthetic.meta(energy=energy), summary, out, tmp_path / "docs" / "img", options(withhold=withhold) - ) + meta = {**synthetic.meta(energy=energy), "status": status} + write_report(runs, meta, summary, out, tmp_path / "docs" / "img", options(withhold=withhold)) return out.read_text(encoding="utf-8") @@ -57,16 +58,36 @@ def test_report_with_energy(tmp_path: Path) -> None: text = render(tmp_path, energy=True, withhold=False) assert "Pending" not in text assert "1.00 ms (" in text # C quick sort: 1 ms per sort - assert "2.00 mJ (" in text # and 2 mJ per sort + assert "2.00 mJ (" in text # and 2 mJ per sort above idle + assert "2.00 W (" in text # so 2 W while sorting assert "## Is run time a good stand-in for energy?" in text - assert "1.000 over 4 configurations" in text - # 2 mJ per sort x 1e6 sorts = 2 kJ = 0.000556 kWh; at 467.51 g/kWh that is 0.260 g. - assert "| C | quick | 2.00 kJ | 260 mg |" in text - for chart in ("operations", "time_per_sort", "energy_per_sort", "time_vs_energy"): + assert "| quick | random | 200 | Python / C | 10 (" in text + assert "same power: the time ratio predicts the energy ratio" in text + assert "### Between input orders" in text + assert "| every configuration | 16 |" in text + assert "| C | quick | 4 | 1.000 |" in text # run-to-run coupling over four configurations + # Per 1e6 sorts: 2 kJ net and 3 kJ including idle; at 467.51 g/kWh, 260 mg and 390 mg. + assert "| C | quick | 2.00 kJ | 3.00 kJ | 260 mg to 390 mg |" in text + assert "Sort loops: every sorting run reached the 2.00 ms target." in text + assert "C and Python reported identical counts" in text + for chart in ("operations", "time_per_sort", "energy_per_sort", "energy_vs_time"): assert (tmp_path / "docs" / "img" / f"{chart}.png").stat().st_size > 1000 assert f"](img/{chart}.png)" in text +def test_report_names_a_language_that_draws_more_power(tmp_path: Path) -> None: + text = render(tmp_path, energy=True, withhold=False, power={"python": 1.5}) + assert "| quick | random | 200 | Python / C | 10 (" in text + assert "Python draws 50% more power" in text + assert "3.00 W (" in text + + +def test_report_warns_about_unfinished_and_short_runs(tmp_path: Path) -> None: + text = render(tmp_path, energy=True, withhold=False, status="interrupted") + assert "Warning: this run did not finish (status: interrupted)" in text + assert "5 measured runs per configuration; the study uses 30" in text + + def test_report_without_energy_says_so(tmp_path: Path) -> None: text = render(tmp_path, energy=False, withhold=False) assert "Energy was not measured in this run" in text @@ -82,15 +103,17 @@ def test_withheld_report_shows_counts_but_no_timings(tmp_path: Path) -> None: assert "| quick | random | 100 | 700 | 800 |" in text assert " ms (" not in text assert " mJ (" not in text + assert " W (" not in text + assert "- Sort loops:" not in text # loop lengths are timings too assert (tmp_path / "docs" / "img" / "operations.png").exists() # exact counts are always drawn assert not (tmp_path / "docs" / "img" / "time_per_sort.png").exists() def test_stale_charts_are_removed(tmp_path: Path) -> None: render(tmp_path, energy=True, withhold=False) - assert (tmp_path / "docs" / "img" / "time_vs_energy.png").exists() + assert (tmp_path / "docs" / "img" / "energy_vs_time.png").exists() render(tmp_path, energy=True, withhold=True) - assert not (tmp_path / "docs" / "img" / "time_vs_energy.png").exists() + assert not (tmp_path / "docs" / "img" / "energy_vs_time.png").exists() def test_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -121,10 +144,25 @@ def test_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> Non assert "CO2e, United Kingdom" in text assert "CO2e, Solar farm" in text assert "PYTHONPATH=analysis python3 -m greenbench_analysis report --runs" in text - assert (tmp_path / "summary.csv").read_text(encoding="utf-8").startswith("language,algorithm,order,n,") + summary_csv = (tmp_path / "summary.csv").read_text(encoding="utf-8") + assert summary_csv.startswith("language,algorithm,order,n,") + assert summary_csv.splitlines()[0].endswith(",power_w,power_w_low,power_w_high") assert "wrote" in capsys.readouterr().out +def test_command_line_refuses_an_unfinished_run(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + runs_path = synthetic.write_csv(synthetic.study(energy=True), tmp_path / "runs.csv") + synthetic.write_meta({**synthetic.meta(energy=True), "status": "interrupted"}, tmp_path / "meta.json") + out = tmp_path / "results.md" + common = ["report", "--runs", str(runs_path), "--out", str(out), "--img-dir", str(tmp_path / "img")] + common += ["--grid", str(GRID), "--resamples", "100"] + assert main(common) == 1 + assert "status is 'interrupted'" in capsys.readouterr().err + assert not out.exists() + assert main([*common, "--allow-partial"]) == 0 + assert "this run did not finish" in out.read_text(encoding="utf-8") + + def test_command_line_errors(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: assert main(["report", "--runs", str(tmp_path / "missing.csv")]) == 1 assert "file not found" in capsys.readouterr().err diff --git a/analysis/tests/test_stats.py b/analysis/tests/test_stats.py index 370e992..78b1095 100644 --- a/analysis/tests/test_stats.py +++ b/analysis/tests/test_stats.py @@ -6,9 +6,11 @@ from greenbench_analysis.stats import ( Estimate, bootstrap_median, - bootstrap_spearman, + interval, median, rank_average, + ratio_draws, + resample_medians, spearman, ) @@ -95,12 +97,28 @@ def test_spearman_undefined_cases() -> None: spearman([1, 2, 3], [1, 2]) -def test_bootstrap_spearman() -> None: - source = np.random.default_rng(5) - x = source.uniform(0, 1, 200) - y = x + source.normal(0, 0.1, 200) - estimate = bootstrap_spearman(x, y, rng=rng(), resamples=500) - assert estimate is not None - assert 0.8 < estimate.value < 1.0 - assert estimate.low <= estimate.value <= estimate.high - assert bootstrap_spearman([1, 1, 1], [1, 2, 3], rng=rng()) is None +def test_resample_medians_keeps_pairs_together() -> None: + times = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + medians = resample_medians([times, 2 * times], rng=rng(), resamples=500) + assert medians.shape == (2, 500) + # Energy is exactly twice time in every run, so it is twice time in every resample too. + assert np.allclose(medians[1], 2 * medians[0]) + assert medians[0].min() >= 1.0 + assert medians[0].max() <= 5.0 + with pytest.raises(ValueError, match="same length"): + resample_medians([times, times[:3]], rng=rng(), resamples=10) + + +def test_interval_ignores_undefined_draws() -> None: + draws = np.array([np.nan, 1.0, 2.0, 3.0, np.inf]) + estimate = interval(2.0, draws, confidence=0.5) + assert estimate.value == 2.0 + assert 1.0 <= estimate.low <= estimate.high <= 3.0 + assert interval(4.0, np.array([np.nan])) == Estimate(4.0, 4.0, 4.0) + + +def test_ratio_draws_are_undefined_below_zero() -> None: + ratios = ratio_draws(np.array([2.0, 3.0, 4.0]), np.array([1.0, 0.0, -2.0])) + assert ratios[0] == 2.0 + assert np.isnan(ratios[1]) + assert np.isnan(ratios[2]) diff --git a/analysis/tests/test_summary.py b/analysis/tests/test_summary.py index 01d00b7..7a816d4 100644 --- a/analysis/tests/test_summary.py +++ b/analysis/tests/test_summary.py @@ -6,15 +6,21 @@ import synthetic from greenbench_analysis.load import DataError -from greenbench_analysis.summary import summarise +from greenbench_analysis.summary import Comparison, Configuration, Summary, summarise + + +def configuration(summary: Summary, language: str, algorithm: str, order: str = "random") -> Configuration: + return next(c for c in summary.configurations if (c.language, c.algorithm, c.order) == (language, algorithm, order)) + + +def comparison(comparisons: list[Comparison], subject: str, reference: str, algorithm: str = "quick") -> Comparison: + return next(c for c in comparisons if c.subject[:2] == (subject, algorithm) and c.reference[0] == reference) def test_per_sort_values_subtract_startup_and_divide_by_reps() -> None: summary = summarise(synthetic.study(energy=True), resamples=500) - c_quick = next(c for c in summary.configurations if (c.language, c.algorithm, c.order) == ("c", "quick", "random")) - py_quick = next( - c for c in summary.configurations if (c.language, c.algorithm, c.order) == ("python", "quick", "random") - ) + c_quick = configuration(summary, "c", "quick") + py_quick = configuration(summary, "python", "quick") assert c_quick.inner_reps == 10 assert c_quick.time_per_sort_ns.value == pytest.approx(1_000_000) assert c_quick.energy_per_sort_uj is not None @@ -27,43 +33,135 @@ def test_per_sort_values_subtract_startup_and_divide_by_reps() -> None: assert c.time_per_sort_ns.low <= c.time_per_sort_ns.value <= c.time_per_sort_ns.high -def test_startups_are_reported_separately() -> None: - summary = summarise(synthetic.study(energy=True), resamples=200) +def test_gross_energy_adds_the_idle_power_drawn_while_sorting() -> None: + summary = summarise(synthetic.study(energy=True), resamples=300) + c_quick = configuration(summary, "c", "quick") + # 2 mJ above idle, plus 1 W of idle power for the 1 ms the sort takes. + assert c_quick.gross_energy_per_sort_uj is not None + assert c_quick.gross_energy_per_sort_uj.value == pytest.approx(3_000) + + +def test_startup_batches_are_divided_by_the_number_of_launches() -> None: + summary = summarise(synthetic.study(energy=True, startup_launches=4), resamples=200) startup = next(s for s in summary.startups if (s.language, s.order) == ("python", "random")) - assert startup.time_ns.value == pytest.approx(50_000_000) + assert startup.launches == 4 + assert startup.time_ns.value == pytest.approx(50_000_000) # per process, not per batch of four assert startup.energy_uj is not None assert startup.energy_uj.value == pytest.approx(100_000) + assert configuration(summary, "c", "quick").time_per_sort_ns.value == pytest.approx(1_000_000) assert all(c.algorithm != "none" for c in summary.configurations) def test_without_startup_trials_per_sort_includes_startup() -> None: summary = summarise(synthetic.study(energy=False, startup=False), resamples=200) - c_quick = next(c for c in summary.configurations if (c.language, c.algorithm, c.order) == ("c", "quick", "random")) assert not summary.startup_subtracted - assert c_quick.time_per_sort_ns.value == pytest.approx(1_500_000) + assert configuration(summary, "c", "quick").time_per_sort_ns.value == pytest.approx(1_500_000) def test_without_energy_there_are_no_energy_figures() -> None: summary = summarise(synthetic.study(energy=False), resamples=200) assert not summary.energy_measured assert summary.total_energy_uj is None - assert summary.correlations == [] - assert all(c.energy_per_sort_uj is None and c.process_energy_uj is None for c in summary.configurations) + assert summary.rank_agreement == [] + assert summary.noise_coupling == [] + for c in summary.configurations: + assert c.energy_per_sort_uj is None + assert c.process_energy_uj is None + assert c.power_w is None + python_c = comparison(summary.language_comparisons, "python", "c") + assert python_c.time_ratio is not None + assert python_c.time_ratio.value == pytest.approx(10.0) + assert python_c.energy_ratio is None + assert python_c.power_ratio is None + + +def test_power_while_sorting() -> None: + summary = summarise(synthetic.study(energy=True, power={"python": 1.5}), resamples=500) + c_power = configuration(summary, "c", "quick").power_w + python_power = configuration(summary, "python", "quick").power_w + assert c_power is not None + assert python_power is not None + assert c_power.value == pytest.approx(2.0) # 2 mJ in 1 ms + assert python_power.value == pytest.approx(3.0) # 30 mJ in 10 ms + assert c_power.low <= 2.0 <= c_power.high + + +def test_equal_power_means_the_time_ratio_predicts_the_energy_ratio() -> None: + summary = summarise(synthetic.study(energy=True), resamples=1000) + python_c = comparison(summary.language_comparisons, "python", "c") + assert python_c.time_ratio is not None + assert python_c.energy_ratio is not None + assert python_c.power_ratio is not None + assert python_c.time_ratio.value == pytest.approx(10.0) + assert python_c.energy_ratio.value == pytest.approx(10.0) + assert python_c.power_ratio.value == pytest.approx(1.0) + assert python_c.power_ratio.low <= 1.0 <= python_c.power_ratio.high + + +def test_a_language_that_draws_more_power_is_caught() -> None: + summary = summarise(synthetic.study(energy=True, power={"python": 1.5}), resamples=1000) + python_c = comparison(summary.language_comparisons, "python", "c") + assert python_c.time_ratio is not None + assert python_c.energy_ratio is not None + assert python_c.power_ratio is not None + assert python_c.time_ratio.value == pytest.approx(10.0) # time alone says ten times the energy + assert python_c.energy_ratio.value == pytest.approx(15.0) # but it is fifteen + assert python_c.power_ratio.value == pytest.approx(1.5) + assert python_c.power_ratio.low > 1.0 # the interval excludes equal power + + +def test_language_pairs_and_their_order() -> None: + summary = summarise(synthetic.study(energy=True, languages=("c", "python", "node")), resamples=200) + pairs = [ + (c.subject[0], c.reference[0]) + for c in summary.language_comparisons + if c.subject[1:] == ("quick", "random", 100) + ] + assert pairs == [("python", "c"), ("node", "c"), ("python", "node")] + python_node = comparison(summary.language_comparisons, "python", "node") + assert python_node.time_ratio is not None + assert python_node.time_ratio.value == pytest.approx(2.0) + + +def test_input_orders_are_compared_with_random_input() -> None: + summary = summarise(synthetic.study(energy=True), resamples=200) + keys = {(c.subject, c.reference) for c in summary.order_comparisons} + assert (("c", "bubble", "sorted", 100), ("c", "bubble", "random", 100)) in keys + assert all(c.reference[2] == "random" and c.subject[2] != "random" for c in summary.order_comparisons) + sorted_random = next(c for c in summary.order_comparisons if c.subject == ("c", "quick", "sorted", 100)) + assert sorted_random.time_ratio is not None + assert sorted_random.time_ratio.value == pytest.approx(1.0) # the synthetic costs ignore the order + + +def test_rank_agreement_across_configurations() -> None: + summary = summarise(synthetic.study(energy=True, sizes=(100, 200)), resamples=300, rho_resamples=200) + scopes = [a.scope for a in summary.rank_agreement] + assert scopes == [ + "every language, algorithm and order at 100 elements", + "every language, algorithm and order at 200 elements", + "c: every algorithm, order and size", + "python: every algorithm, order and size", + "every configuration", + ] + everything = summary.rank_agreement[-1] + assert everything.configurations == 16 + assert everything.rho is not None + assert everything.rho.value > 0.8 # C is ten times cheaper than Python in time and energy + # Within one language every configuration costs the same, so the ranks are ties: undefined. + assert summary.rank_agreement[2].rho is None -def test_correlations_within_and_pooled() -> None: +def test_noise_coupling_within_configurations() -> None: summary = summarise(synthetic.study(energy=True, runs=7), resamples=200) - assert [(c.language, c.algorithm) for c in summary.correlations] == [ + assert [(n.language, n.algorithm) for n in summary.noise_coupling] == [ ("c", "bubble"), ("c", "quick"), ("python", "bubble"), ("python", "quick"), ] - for correlation in summary.correlations: - assert correlation.pairs == 14 # 7 runs x 2 orders - assert correlation.within == pytest.approx(1.0) # time and energy rise together in the synthetic data - assert correlation.within_configurations == 2 - assert correlation.pooled is not None + for coupling in summary.noise_coupling: + assert coupling.rho == pytest.approx(1.0) # time and energy rise together in the synthetic data + assert coupling.configurations == 2 def test_operation_counts_are_checked_across_languages() -> None: @@ -94,6 +192,12 @@ def test_inputs_must_agree_across_languages() -> None: summarise(runs, resamples=100) +def test_frozen_counters_are_refused() -> None: + runs = [dataclasses.replace(r, pkg_uj=0.0, idle_pkg_uj=0.0, net_pkg_uj=0.0) for r in synthetic.study(energy=True)] + with pytest.raises(DataError, match="the RAPL counters did not move"): + summarise(runs, resamples=100) + + def test_results_are_reproducible_with_the_same_seed() -> None: first = summarise(synthetic.study(energy=True), resamples=300, seed=3) second = summarise(synthetic.study(energy=True), resamples=300, seed=3) From c8e8793ef108ccaa8951291256f5bed766513980 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:33:02 +0400 Subject: [PATCH 27/51] test: run moving, wrapping counters through the harness into the report Every harness-level energy test used frozen counters, so non-zero energy had never gone from the harness through the net-of-idle columns into the report. The new pipeline test moves a fake package counter faster while a worker runs than during the idle windows, wraps it several times, and checks the CSV (net formula, no mishandled wrap) and every energy section of the report: power, ratios, rank agreement, carbon and both charts. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_pipeline.py | 118 ++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 6759bae..2dd2c45 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -2,13 +2,29 @@ from __future__ import annotations +import csv import os import subprocess import sys from pathlib import Path +import pytest + +from powercap import CounterWriter +from powercap import build as build_powercap from workers import REPOSITORY, interpreter_options, require_build, run_harness +# A small package counter (100 J) that wraps every few seconds at these powers, so that some +# measured intervals span a wrap, and a DRAM zone beside it. One run uses a few joules. +WRAPPING_TREE = """ +intel-rapl:0/name package-0 +intel-rapl:0/energy_uj 95000000 +intel-rapl:0/max_energy_range_uj 99999999 +intel-rapl:0:0/name dram +intel-rapl:0:0/energy_uj 500000 +intel-rapl:0:0/max_energy_range_uj 99999999 +""" + def report(*args: str) -> subprocess.CompletedProcess[str]: env = dict(os.environ, PYTHONPATH=str(REPOSITORY / "analysis")) @@ -57,7 +73,7 @@ def test_harness_then_report(tmp_path: Path) -> None: assert reported.returncode == 0, reported.stderr text = results.read_text(encoding="utf-8") assert "Energy was not measured in this run" in text - assert "C, Python, Node.js reported identical counts and identical input hashes" in text + assert "C, Python and Node.js reported identical counts and identical input hashes" in text assert "| quick | sorted | 90 | 4,005 | 0 |" in text # n(n-1)/2 comparisons: sorted input is the worst case assert "energy unavailable" in text assert (tmp_path / "img" / "operations.png").exists() @@ -78,3 +94,103 @@ def test_harness_then_report(tmp_path: Path) -> None: text = results.read_text(encoding="utf-8") assert "time and energy results are pending" in text assert not (tmp_path / "img" / "time_per_sort.png").exists() + + +def test_moving_counters_through_the_harness_into_the_report(tmp_path: Path) -> None: + """Non-zero energy from end to end: counters that move (faster while a worker runs, and + wrapping), through the harness's net-of-idle columns, into every energy section.""" + require_build() + manifest = tmp_path / "tree.txt" + manifest.write_text(WRAPPING_TREE, encoding="utf-8") + root = build_powercap(manifest, tmp_path / "powercap") + + # A Python worker that marks itself busy while it runs, so the fake package draws more + # power during runs than during the idle windows that follow them. + busy = tmp_path / "busy" + real = REPOSITORY / "workers" / "python" / "greenbench_worker.py" + wrapper = tmp_path / "busy_worker.py" + wrapper.write_text( + "\n".join( + [ + "import pathlib, runpy, sys", + f"busy = pathlib.Path({str(busy)!r})", + "busy.touch()", + "try:", + f" sys.argv[0] = {str(real)!r}", + f" runpy.run_path({str(real)!r}, run_name='__main__')", + "finally:", + " busy.unlink(missing_ok=True)", + ] + ) + + "\n", + encoding="utf-8", + ) + out = tmp_path / "run" + writer = CounterWriter(root, idle_watts=10.0, busy_watts=30.0, busy_marker=busy, dram_share=0.1) + with writer: + completed = run_harness( + "run", + "--profile", + "smoke", + "--languages", + "python", + "--algorithms", + "bubble,quick", + "--orders", + "random,sorted", + "--sizes", + "30,90", + "--runs", + "3", + "--min-loop-ms", + "20", + "--energy", + "require", + "--powercap-root", + str(root), + "--python-worker", + str(wrapper), + "--quiet", + "--out", + str(out), + *interpreter_options(), + ) + wraps = writer.package_wraps + assert completed.returncode == 0, completed.stderr + assert wraps >= 1, "the package counter should have wrapped during the run" + + with (out / "runs.csv").open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + for row in rows: + pkg, idle = int(row["pkg_uj"]), int(row["idle_pkg_uj"]) + wall, idle_ns = int(row["wall_ns"]), int(row["idle_ns"]) + assert 0 < pkg < 50_000_000, "a wrap handled wrongly would give a value near 100 J or above" + assert int(row["dram_uj"]) > 0 + assert float(row["net_pkg_uj"]) == pytest.approx(pkg - idle * wall / idle_ns, abs=0.051) + # The worker draws 20 W more than idle while it runs, so most runs have positive net energy. + assert sum(float(r["net_pkg_uj"]) > 0 for r in rows) > len(rows) * 0.8 + + results = tmp_path / "results.md" + reported = report( + "--runs", + str(out / "runs.csv"), + "--out", + str(results), + "--img-dir", + str(tmp_path / "img"), + "--resamples", + "200", + ) + assert reported.returncode == 0, reported.stderr + text = results.read_text(encoding="utf-8") + assert "energy unavailable" not in text + assert "Energy was not measured" not in text + assert "- Energy: RAPL zones intel-rapl:0 (package-0), intel-rapl:0:0 (dram)" in text + assert " W (" in text # power while sorting + assert "### Between input orders" in text + assert "| Python | quick | 90 | sorted / random |" in text + assert "Spearman's rho" in text + assert "| every configuration | 8 |" in text + assert "CO2e, United Arab Emirates (net to including idle)" in text + for chart in ("energy_per_sort", "energy_vs_time"): + assert (tmp_path / "img" / f"{chart}.png").exists() From 4f31e7c8ca8e59ba4f84afd61fafdb3ce0cfcd3c Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:40:20 +0400 Subject: [PATCH 28/51] fix: make the study script's checks catch what they claim to Checked on stock Ubuntu 24.04 packages (Node.js 18.19.1, Python 3.12.3): - "python3 -m venv --help" exits 0 without python3-venv, so the check passed and the script failed later. It now creates a throwaway venv. - A fake --powercap-root in study mode was refused only after make clean, the build, the tests, the probe and the pip install. It is now refused right after the arguments are parsed. - The chmod o+r prompt now says why it matters: until reboot any local program can read the counters, reopening the PLATYPUS side channel. - A probe that finds frozen counters now stops the script with a reason. - Node.js older than the 24 LTS the guide installs gets a warning, since the results describe whichever version ran. - The closing message tells Farah to copy the results to a USB stick and commit from her usual computer, not from the live session. Co-Authored-By: Claude Opus 5.5 --- scripts/run-study.sh | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/scripts/run-study.sh b/scripts/run-study.sh index 779706a..750deab 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -69,6 +69,10 @@ while (($# > 0)); do shift done +if [[ "$MODE" == study ]] && ((FAKE_POWERCAP)); then + fail "a fake powercap tree can only be used with --check or --rehearse" +fi + check_platform() { [[ "$(uname -s)" == Linux ]] || fail "RAPL counters need Linux on real hardware; see docs/bare-metal.md" if [[ -f /.dockerenv ]] || grep -qE 'docker|containerd|kubepods' /proc/1/cgroup 2>/dev/null; then @@ -86,16 +90,28 @@ check_platform() { check_tools() { local tool - for tool in cc make python3 node; do + for tool in cc make python3; do command -v "$tool" >/dev/null || - fail "$tool not found; install the tools with: sudo apt-get install -y git build-essential python3-venv nodejs" + fail "$tool not found; install the tools with: sudo apt-get install -y git build-essential python3-venv" done + command -v node >/dev/null || fail "node not found; install Node.js 24 as docs/bare-metal.md shows" python3 -c 'import sys; sys.exit(sys.version_info < (3, 12))' || fail "Python 3.12 or later is needed for the analysis (found $(python3 --version))" - python3 -m venv --help >/dev/null 2>&1 || fail "python3-venv is missing; sudo apt-get install -y python3-venv" + # "python3 -m venv --help" succeeds even without python3-venv; only making one shows it works. + local scratch + scratch="$(mktemp -d)" + if ! python3 -m venv "$scratch/venv" >/dev/null 2>&1; then + rm -rf "$scratch" + fail "python3 cannot create a virtual environment; sudo apt-get install -y python3-venv" + fi + rm -rf "$scratch" local node_major node_major="$(node -p 'process.versions.node.split(".")[0]')" ((node_major >= 18)) || fail "Node.js 18 or later is needed (found $(node --version))" + if ((node_major < 24)); then + warn "Node.js $(node --version) is older than the Node.js 24 LTS that docs/bare-metal.md installs;" \ + "the results will describe Node.js $(node --version)" + fi } check_rapl() { @@ -112,7 +128,10 @@ check_rapl() { say "Since the fix for CVE-2020-8694 (late 2020), only root can read the RAPL energy counters." say "The next step runs: sudo chmod o+r on ${#unreadable[@]} energy_uj file(s)." - say "The change lasts until the next reboot. Do this only on a machine you control." + say "Until the next reboot, every local user and program can then read them, which reopens the" + say "PLATYPUS power side channel (CVE-2020-8694): fine-grained energy readings can leak secrets," + say "such as cryptographic keys, from other programs. Do this only on a machine you control, such" + say "as a live USB session, and reboot when the study has finished." if ((ASSUME_YES == 0)); then local answer read -r -p "Make the counters readable? [y/N] " answer @@ -201,7 +220,8 @@ make -s clean make -s all test say "== RAPL counters" -build/greenbench probe --powercap-root "$POWERCAP" +build/greenbench probe --powercap-root "$POWERCAP" || + fail "the RAPL counters cannot be used for the study (see the reason above and docs/bare-metal.md)" if [[ "$MODE" == check ]]; then say "This machine can run the study: ./scripts/run-study.sh" exit 0 @@ -212,9 +232,6 @@ python3 -m venv .venv .venv/bin/python -m pip install --quiet --require-hashes -r requirements.txt stamp="$(date -u +%Y%m%dT%H%M%SZ)" -if [[ "$MODE" == study ]] && ((FAKE_POWERCAP)); then - fail "a fake powercap tree can only be used with --check or --rehearse" -fi if [[ "$MODE" == study ]]; then out="data/runs/$stamp" profile=study @@ -242,6 +259,8 @@ PYTHONPATH=analysis .venv/bin/python -m greenbench_analysis report \ say say "Done. Raw data: $out/ Report: $report" if [[ "$MODE" == study ]]; then - say "Next: read $report, then commit it with the data:" + say "Next: read $report. A live session forgets everything when it shuts down, so copy the" + say "whole repository folder to a second USB stick, then commit from your usual computer:" say " git add $report $images $out && git commit -m 'docs: add measured study results'" + say "Reboot afterwards to make the RAPL counters root-only again." fi From 59a6e7c45a7cac7ebdd782719862662494f9adc1 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:46:52 +0400 Subject: [PATCH 29/51] fix: pin the Docker base images by digest so Dependabot can update them Every FROM line used ${REGISTRY}/..., which Dependabot cannot resolve, so its docker entry did nothing, and the images tracked floating tags. The FROM lines are now literal and pinned by digest (the current Docker Hub digests, which Amazon's mirror of the official images serves too). The comment gives a command that builds from that mirror when Docker Hub limits pulls, deriving each --build-context name from the FROM lines so that it stays right when Dependabot changes a digest. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2eb4f90..456b220 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,15 @@ # Node.js 24 and ShellCheck. RAPL counters are not visible inside a container, so this image # runs the harness with energy off. It is for trying the project on Windows or macOS and for # running the same checks as CI; the energy study itself needs bare-metal Linux. -# REGISTRY can point at a mirror of the Docker official images, for example -# --build-arg REGISTRY=public.ecr.aws/docker/library when Docker Hub rate-limits pulls. -ARG REGISTRY=docker.io/library -FROM ${REGISTRY}/node:24-trixie-slim AS node +# +# The base images are pinned by digest, and Dependabot proposes updates. If Docker Hub limits +# pulls, build from Amazon's mirror of the Docker official images, which serves the same +# digests. Each --build-context name must match a FROM line exactly, digest included, so this +# command derives them from the FROM lines (in bash, from the repository root): +# docker buildx build $(sed -n 's|^FROM \([^ ]*\).*|--build-context \1=docker-image://public.ecr.aws/docker/library/\1|p' Dockerfile) -t greenbench . +FROM node:24-trixie-slim@sha256:8ec5d7557396cfe32d21c3f9c13072355ceab22b584578ca4bb28af31120cffe AS node -FROM ${REGISTRY}/python:3.12-slim-trixie +FROM python:3.12-slim-trixie@sha256:f77ac9e44ae96ef2c90b8053ea08c31f8be030f824196b0ae4db6d462c84e51f RUN apt-get update \ && apt-get install -y --no-install-recommends clang gcc libc6-dev make shellcheck \ && rm -rf /var/lib/apt/lists/* From 0ca001b8dc2bcd3faed2d773092995e751dfb13b Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:48:05 +0400 Subject: [PATCH 30/51] docs: cite the grid intensity data the way Our World in Data asks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The source column now reads "Ember (2026) – with major processing by Our World in Data", OWID's short citation for indicator 1295504 (checked in the chart's metadata on 2026-09-26), and data/README.md quotes the full citation. The values and the CC BY 4.0 links are unchanged. Co-Authored-By: Claude Opus 5.5 --- data/README.md | 5 +++++ data/grid_intensity.csv | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/data/README.md b/data/README.md index 15f7e8a..5d334da 100644 --- a/data/README.md +++ b/data/README.md @@ -17,6 +17,11 @@ grams of CO2-equivalent. Add a row to use another region, or pass The two rows come from Ember's yearly electricity data, as published by Our World in Data in the chart "Lifecycle carbon intensity of electricity" (indicator 1295504, last updated 2026-06-30), read on 2026-09-26. 2024 is the latest year with a value for both countries. +The `source` column uses the short citation Our World in Data asks for; the full one is: + +> Ember (2026) – with major processing by Our World in Data. "Lifecycle carbon intensity of +> electricity generation – Ember" [dataset]. Ember, "Yearly Electricity Data Europe"; Ember, +> "Yearly Electricity Data" [original data]. - Ember, *Yearly Electricity Data* and *Yearly Electricity Data Europe* (2026 release, published 2026-04-21), . diff --git a/data/grid_intensity.csv b/data/grid_intensity.csv index b05c6d7..5ede787 100644 --- a/data/grid_intensity.csv +++ b/data/grid_intensity.csv @@ -1,3 +1,3 @@ region_code,region,year,g_co2e_per_kwh,basis,source,source_url,licence,retrieved -ARE,United Arab Emirates,2024,467.51,"lifecycle, whole grid, annual average","Ember (2026), yearly electricity data, via Our World in Data indicator 1295504",https://ourworldindata.org/grapher/carbon-intensity-electricity,CC BY 4.0,2026-09-26 -GBR,United Kingdom,2024,216.50,"lifecycle, whole grid, annual average","Ember (2026), yearly electricity data, via Our World in Data indicator 1295504",https://ourworldindata.org/grapher/carbon-intensity-electricity,CC BY 4.0,2026-09-26 +ARE,United Arab Emirates,2024,467.51,"lifecycle, whole grid, annual average",Ember (2026) – with major processing by Our World in Data,https://ourworldindata.org/grapher/carbon-intensity-electricity,CC BY 4.0,2026-09-26 +GBR,United Kingdom,2024,216.50,"lifecycle, whole grid, annual average",Ember (2026) – with major processing by Our World in Data,https://ourworldindata.org/grapher/carbon-intensity-electricity,CC BY 4.0,2026-09-26 From ca63170f27a2fe703b8ba9b3d1cf1ffe7020bdc7 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:57:03 +0400 Subject: [PATCH 31/51] docs: describe the study as it now measures and decides - study.md: "What would count as an answer" now points at power ratios, power while sorting and rank agreement, and the analysis section describes them; the pooled rho and its wrong "dominated by input size" description are gone. The method says loops are calibrated to about 200 ms (a target, not a minimum), which clocks the workers use and why, and how startup batches work. New threats: helper threads and pinning, warm starts in startup batches, calibration shortfalls, startup medians left out of the intervals, and net against including-idle carbon. - decisions.md: ADR 5 reworded; ADR 16 (Node.js 24 LTS for the study) and ADR 17 (power ratios rather than correlation decide the question). - bare-metal.md: install Node.js 24 from nodejs.org, checked against the published SHA-256 sums; copy the results to a USB stick and commit from the usual computer (or set a git identity first); the new duration estimate; the side-channel risk of chmod; rehearsing without RAPL. Followed step by step on Ubuntu 24.04 in a container. - README: operations chart first, then the probe output; plain feature names; the new analyses, options and limitations. Co-Authored-By: Claude Opus 5.5 --- README.md | 80 ++++++++++++++++++++++-------------- docs/bare-metal.md | 70 +++++++++++++++++++++++-------- docs/decisions.md | 66 +++++++++++++++++++++++++----- docs/study.md | 100 +++++++++++++++++++++++++++++++++------------ 4 files changed, 234 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index b4bb55b..7d51d0b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,14 @@ written method. ## Sample output -In Docker Desktop, `greenbench probe` reports honestly that there is nothing to measure: +Operation counts are exact, so they are already in [docs/results.md](docs/results.md), +generated from a full run of the study profile. They are identical in all three languages, +which the harness checks on every run: + +![Comparisons per sort against input size for bubble, insertion and quick sort, one panel per input order](docs/img/operations.png) + +Energy is another matter. In Docker Desktop, where this version was developed, +`greenbench probe` reports that there is nothing to measure: ```text $ build/greenbench probe @@ -30,11 +37,7 @@ reason: no RAPL package zone (intel-rapl:N named package-N) under /sys/class/pow ``` The harness then measures time and operation counts only, and leaves every energy cell -empty rather than writing zeros. Operation counts are exact, so they are already in -[docs/results.md](docs/results.md), generated from a full run of the study profile. They are -identical in all three languages, which the harness checks on every run: - -![Comparisons per sort against input size for bubble, insertion and quick sort, one panel per input order](docs/img/operations.png) +empty rather than writing zeros. ## The problem @@ -50,21 +53,25 @@ compare?** The research question, method and threats to validity are in - **Energy from RAPL in C.** Reads package and DRAM zones from `/sys/class/powercap`, handles counter wraparound with `max_energy_range_uj`, and reports "energy unavailable" with a - reason when the counters are missing or unreadable, never a fake value. + reason when the counters are missing, unreadable or frozen, never a fake value. - **Identical inputs everywhere.** xorshift32 and the four input orders are implemented in C, Python and JavaScript; tests check that all three produce identical sequences, inputs and operation counts, and the harness stops if a worker ever disagrees. -- **A careful harness.** One process per measured run for every language, time from - `CLOCK_MONOTONIC_RAW`, repetitions calibrated so each process sorts for at least 200 ms - (well above the ~1 ms RAPL update interval), an idle baseline of the same length after every - run, 30 measured runs per configuration after warm-up, randomised run order, optional CPU - pinning, startup trials that measure each interpreter's fixed cost, and tidy CSV output. -- **Statistics.** Medians with 95% bootstrap confidence intervals, and Spearman's rank - correlation between time and energy per language and algorithm, in a Python package with - its own tests. +- **Measurement harness.** One process per measured run for every language, time from + `CLOCK_MONOTONIC_RAW`, sorts per process calibrated so each loop lasts about 200 ms (far + above the ~1 ms RAPL update interval), an idle baseline of the same length after every run, + 30 measured runs per configuration after warm-up, randomised run order, optional pinning to + a set of CPUs, startup trials measured over batches of launches, and tidy CSV output. +- **Statistics that answer the question.** Power while sorting, and time, energy and power + ratios between languages and between input orders, each with a 95% bootstrap interval: a + power ratio of 1 means time predicts energy. Also medians with intervals, rank agreement + across configurations, and a run-to-run noise check, in a Python package with its own tests. - **Carbon estimates.** Energy converted to grams of CO2-equivalent for the UAE and UK grids, - from a cited table (Ember data, CC BY 4.0), with any other region or value on request. + net of idle and including idle, from a cited table (Ember data, CC BY 4.0), with any other + region or value on request. - **A generated report.** `docs/results.md` and its charts come from a script, never by hand. + It refuses a run that did not finish unless told otherwise, and says when a run has fewer + than 30 runs per configuration or loops shorter than their target. - **Runs anywhere, measures where it can.** The same harness runs on WSL2, in Docker and in CI with energy off; `scripts/run-study.sh` runs the full study on bare-metal Linux. @@ -84,15 +91,16 @@ flowchart LR node["Node.js"] end workers -- "result line: counts, loop time, input hash" --> run - files --> analysis["greenbench_analysis (Python):
medians, bootstrap CIs, Spearman, carbon"] + files --> analysis["greenbench_analysis (Python):
medians, power, ratios, bootstrap CIs, carbon"] grid[("data/grid_intensity.csv")] --> analysis analysis --> report["docs/results.md
docs/img/*.png"] ``` -For each measured run the harness reads the energy counters and the clock, starts a worker, -waits for it to exit, and reads them again. It then sleeps for the same duration and reads -the counters once more to measure idle energy. The worker prints one line; the harness checks -that its input hash and counts match the other languages, and writes a CSV row. +For each measured run the harness reads the energy counters and the clock, starts a worker +(for a startup trial, a batch of them back to back), waits for it to exit, and reads them +again. It then sleeps for the same duration and reads the counters once more to measure idle +energy. Each worker prints one line; the harness checks that its input hash and counts match +the other languages, and writes a CSV row. ## Tech stack and why @@ -101,8 +109,8 @@ that its input hash and counts match the other languages, and writes a CSV row. | Harness and C worker | C11, POSIX (`posix_spawnp`, `pread`, `clock_gettime`) | Small, predictable overhead around each measured process, and direct access to sysfs. | | Energy | Linux powercap sysfs | No kernel modules, no per-model unit decoding; the kernel reports microjoules ([ADR 1](docs/decisions.md#1-read-energy-from-the-powercap-sysfs-files)). | | Python worker | Python standard library | Runs on the Python a Linux distribution ships, with nothing to install. | -| Node.js worker | JavaScript with JSDoc types, checked by `tsc --strict` | Type-checked like TypeScript, but no build step between the source and what is measured ([ADR 9](docs/decisions.md#9-no-runtime-dependencies-in-the-workers)). | -| Statistics and report | Python 3.12+, numpy, matplotlib | The bootstrap and Spearman's rho are short enough to write out and test; no SciPy needed. | +| Node.js worker | JavaScript with JSDoc types, checked by `tsc --strict`; Node.js 24 LTS for the study | Type-checked like TypeScript, but no build step between the source and what is measured ([ADR 9](docs/decisions.md#9-no-runtime-dependencies-in-the-workers), [ADR 16](docs/decisions.md#16-measure-nodejs-24-lts-not-the-nodejs-that-ubuntu-ships)). | +| Statistics and report | Python 3.12+, numpy, matplotlib | The bootstrap, the ratios and Spearman's rho are short enough to write out and test; no SciPy needed. | | Quality | GCC and Clang with `-Werror`, ASan and UBSan, pytest, mypy `--strict`, ruff, ESLint, ShellCheck, actionlint | Every language in the repository is linted, type-checked where possible, and tested in CI. | | Portability | Dockerfile, GitHub Actions | Try it on Windows or macOS; the energy study itself needs bare-metal Linux. | @@ -128,6 +136,9 @@ docker build -t greenbench . docker run --rm -v "$PWD/results:/work/results" greenbench ``` +The base images are pinned by digest. If Docker Hub limits your pulls, the +[Dockerfile](Dockerfile) gives a build command that takes the same images from Amazon's mirror. + ## Configuration `build/greenbench run` takes these options; the defaults are the study profile. @@ -135,10 +146,10 @@ docker run --rm -v "$PWD/results:/work/results" greenbench | Option | Default | Meaning | | :--- | :--- | :--- | -| `--profile study\|smoke` | `study` | Study: sizes 250, 500, 1,000, 2,000; 30 runs; 3 warm-up runs; 200 ms minimum sort loop. Smoke: sizes 64, 256; 3 runs; 1 warm-up; 2 ms. | +| `--profile study\|smoke` | `study` | Study: sizes 250, 500, 1,000, 2,000; 30 runs; 3 warm-up runs; 200 ms loops. Smoke: sizes 64, 256; 3 runs; 1 warm-up; 2 ms. | | `--languages`, `--algorithms`, `--orders`, `--sizes` | all | Comma-separated subsets, for example `--languages c,node --sizes 100,1000`. | | `--runs N`, `--warmup N` | 30, 3 | Measured and warm-up runs per configuration. | -| `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Minimum sort loop per process, and the cap on sorts per process. | +| `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Target length of each sort loop and startup batch, set during warm-up; and the cap on sorts per process. | | `--energy auto\|require\|off` | `auto` | `require` fails with exit status 3 if RAPL cannot be read. | | `--idle-baseline auto\|on\|off` | `auto` | On whenever energy is measured. | | `--cpus LIST` | off | Pin the harness and every worker to these CPUs, such as `2,3` or `2-3`. | @@ -150,12 +161,14 @@ docker run --rm -v "$PWD/results:/work/results" greenbench | `--powercap-root DIR` | `/sys/class/powercap` | Read RAPL zones from elsewhere, for tests. | | `--dry-run` | | Print the plan and exit. | -`build/greenbench probe` checks the counters and prints idle power over one second. +`build/greenbench probe` checks that the counters exist, are readable and move, and prints idle +power over one second; it exits with status 3 when energy cannot be measured. The report is `python -m greenbench_analysis report --runs DIR/runs.csv` with `PYTHONPATH=analysis`. It takes `--out`, `--img-dir`, `--regions ARE,GBR` (codes from [data/grid_intensity.csv](data/grid_intensity.csv)), `--intensity NAME=G_PER_KWH` for any -other grid, `--withhold-timings` for runs on a busy machine, and `--summary-csv`. +other grid, `--withhold-timings` for runs on a busy machine, `--allow-partial` to report on a +run that did not finish, and `--summary-csv`. ## Running the tests @@ -172,7 +185,10 @@ make lint-shell # ShellCheck on the study script The RAPL tests build fake powercap trees from the manifests in [tests/fixtures/powercap](tests/fixtures/powercap), including a two-socket server, an AMD processor without DRAM, a missing tree, corrupt values and an unreadable counter, and rewrite -counters between readings to test wraparound. CI runs all of the above and lints the workflow +counters between readings to test wraparound. The integration tests also move fake counters +while the harness runs, faster during runs than during idle windows and wrapping several +times, so that non-zero energy goes from the harness through to every section of the report; +counters that never move must be refused. CI runs all of the above and lints the workflow with actionlint; the C build and tests run with both GCC and Clang, and the statistics tests on Python 3.12 and 3.14. @@ -193,8 +209,10 @@ docs/ study.md, bare-metal.md, decisions.md, results.md (generat ## Design decisions Recorded in [docs/decisions.md](docs/decisions.md): why powercap rather than MSRs, why every -language is measured as a whole process, how repetitions are calibrated, why the idle -baseline follows every run, why missing energy is never written as zero, and more. +language is measured as a whole process, how repetitions and startup batches are calibrated, +why the idle baseline follows every run, why missing or frozen energy is never written as +zero, why the study pins to two cores and measures Node.js 24, and why power ratios rather +than correlation decide whether time is a good stand-in for energy. ## Limitations and roadmap @@ -205,6 +223,8 @@ baseline follows every run, why missing energy is never written as zero, and mor [threats to validity](docs/study.md#threats-to-validity). - **A small workload.** Three single-threaded sorts on integer arrays of up to 2,000 elements in the study profile (100,000 at most). +- **One setup per run.** Results will describe the recorded compiler, Python and Node.js + versions and the two pinned CPUs. Pinned and unpinned runs have not been compared. Next: diff --git a/docs/bare-metal.md b/docs/bare-metal.md index 67823c3..cf5d722 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -16,19 +16,34 @@ disk, and the study leaves the machine as it found it after a reboot. - A USB stick of 8 GB or more with Ubuntu 24.04 LTS (desktop image), written with [balenaEtcher](https://etcher.balena.io/) or [Rufus](https://rufus.ie/). - Mains power and a network connection. -- A second USB stick, or a way to push to GitHub, to take the results away: a live session - forgets everything when it shuts down. +- A second USB stick to take the results away: a live session forgets everything when it + shuts down. ## Steps 1. Boot from the USB stick (usually F12, F10 or Esc at power-on) and choose **Try Ubuntu**. 2. Connect to the network and open a terminal (Ctrl+Alt+T). -3. Install the tools. `nodejs` is in the universe repository, which the first line enables: +3. Install the tools. The C compiler, make, git and Python come from Ubuntu: ```bash sudo add-apt-repository -y universe sudo apt-get update - sudo apt-get install -y git build-essential python3-venv nodejs + sudo apt-get install -y git build-essential python3-venv curl + ``` + + Node.js comes from nodejs.org rather than Ubuntu, whose Node.js 18 is past its end of life + ([ADR 16](decisions.md#16-measure-nodejs-24-lts-not-the-nodejs-that-ubuntu-ships)). These + lines download the latest Node.js 24 LTS release, check it against the release's published + SHA-256 sums, and install it under `/usr/local`: + + ```bash + cd /tmp + curl -fsSLO https://nodejs.org/dist/latest-v24.x/SHASUMS256.txt + tarball="$(grep -o 'node-v24[.0-9]*-linux-x64[.]tar[.]xz' SHASUMS256.txt | head -n 1)" + curl -fsSLO "https://nodejs.org/dist/latest-v24.x/$tarball" + grep " $tarball$" SHASUMS256.txt | sha256sum -c - + sudo tar -xJf "$tarball" -C /usr/local --strip-components=1 + cd - && node --version ``` 4. Get the code and check the machine: @@ -41,9 +56,11 @@ disk, and the study leaves the machine as it found it after a reboot. `--check` confirms that this is Linux on real hardware, that the tools are present and that the RAPL counters can be read. Since the fix for CVE-2020-8694 in late 2020, only root can - read them, so it asks before running `sudo chmod o+r` on the counter files. That change - lasts until the next reboot. It then builds the project, runs the unit tests and prints the RAPL zones with - the idle power over one second. + read them, so it asks before running `sudo chmod o+r` on the counter files. Until the next + reboot, any program on the machine can then read them, which is the power side channel the + fix closed; that is why this belongs on a live session you control. The script then builds + the project, runs the unit tests, and prints the RAPL zones and the idle power over one + second, stopping if the counters do not move. 5. Run the study. This is the one command: @@ -53,24 +70,33 @@ disk, and the study leaves the machine as it found it after a reboot. Leave the machine alone while it runs: no browser, no typing, lid open, on mains power. After the warm-up the harness prints an estimate of the time left and updates it as it goes. - The study profile makes 5,760 measured runs. The 4,320 that sort each sort for at least - 200 ms and are followed by an idle window of the same length, so those alone take at least - 29 minutes (4,320 x 2 x 200 ms); warm-up, process start-up and the slow configurations add - to that. + The study profile makes 5,760 measured runs: 4,320 that sort and 1,440 startup batches. + Each is calibrated to about 200 ms and followed by an idle window of the same length, so + the runs alone take about 38 minutes (5,760 x 2 x 200 ms); warm-up, process start-up and + the slow configurations add to that. 6. Take the results away. The script prints where they are: - `data/runs//`: `runs.csv` (every run), `meta.json` (settings, RAPL zones, - versions), `environment.txt` (kernel, `lscpu`, governor, power source) and `summary.csv`. + versions, limits), `environment.txt` (kernel, `lscpu`, governor, power source) and + `summary.csv`. - `docs/results.md` and `docs/img/`: the report, now with measured time and energy. - Copy the repository folder to the second USB stick, or commit and push: + Copy the whole repository folder to the second USB stick, then shut down; the reboot also + makes the counters root-only again. On your usual computer, copy `data/runs//`, + `docs/results.md` and `docs/img/` into your clone and commit them there: ```bash git add docs/results.md docs/img data/runs git commit -m "docs: add measured study results" ``` + A live session has no git identity and no GitHub credentials, which is why the commit + happens elsewhere. To commit from the live session instead, first run + `git config user.name "Farah Sharif"` and + `git config user.email 129452512+fasharif@users.noreply.github.com`, and push with a + fine-grained personal access token limited to this repository, revoked afterwards. + ## Before you start: reducing noise - Close every other application. A live session may start update checks in the background; @@ -88,18 +114,30 @@ disk, and the study leaves the machine as it found it after a reboot. ## A shorter rehearsal `./scripts/run-study.sh --rehearse` runs the smoke profile (sizes 64 and 256, three runs per -configuration) with energy, and writes everything under `results/` instead of `docs/`. Use it -to see that the machine works before committing to the full run. +configuration, 2 ms loops) with energy, and writes everything under `results/` instead of +`docs/`. It takes a few minutes. Use it to see that the machine works before committing to +the full run; its numbers are too short and too few to mean anything. + +The whole script can also be rehearsed without RAPL, for example in a container, against a +fake powercap tree whose counters a helper keeps moving: + +```bash +python3 tests/integration/powercap.py build tests/fixtures/powercap/laptop-intel.txt /tmp/pc +python3 tests/integration/powercap.py advance /tmp/pc -- ./scripts/run-study.sh --rehearse --yes --powercap-root /tmp/pc +``` ## If something goes wrong | Message | Meaning | | :--- | :--- | | `no RAPL zones under /sys/class/powercap` | The processor, firmware or kernel does not expose RAPL. Check that you booted the USB stick and not a virtual machine. | +| `python3 cannot create a virtual environment` | Install `python3-venv` (step 3). | +| `Node.js v18... is older than the Node.js 24 LTS` | The `nodejs` package was installed instead of Node.js 24 from step 3. The study still runs, but describes that version. | | `this is WSL` or `this is a container` | The script refuses environments without real counters. | | `permission denied reading .../energy_uj` from the harness | Run `--check` again and accept the `chmod`, or reboot and start again. | | `the package energy counter did not change` | The counter files exist but their values never change, as some virtual machines and firmware leave them. The probe exits with status 3, the script stops, and this machine cannot measure energy. | | `the implementations disagree` | A worker produced different counts from the others. This is a bug; please open an issue with the message. | The harness stops cleanly on Ctrl+C: the runs measured so far stay in `runs.csv`, and -`meta.json` records that the run was interrupted. +`meta.json` records that the run was interrupted. The report refuses such a run unless you +pass `--allow-partial`, and then says at the top that the run did not finish. diff --git a/docs/decisions.md b/docs/decisions.md index 5b22a44..5f2298c 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -75,16 +75,17 @@ dividing by the number of sorts. needs. Interpreter start-up is measured explicitly instead of being hidden. The cost is some extra overhead per run, and per-sort figures depend on subtracting a median startup cost. -## 5. Calibrate repetitions so each process sorts for at least 200 ms +## 5. Calibrate repetitions so each process sorts for about 200 ms **Context.** RAPL counters update about once a millisecond. A sort that takes 50 microseconds cannot be measured on its own. **Decision.** During warm-up the harness asks each worker for one sort, reads the worker's -own loop time, and sets the number of sorts per process so that the loop lasts at least -`--min-loop-ms` (200 ms in the study profile, about 200 counter updates). It refines the -number on each further warm-up run, then keeps it fixed for every measured run of that -configuration. +own loop time, and sets the number of sorts per process so that the loop would last +`--min-loop-ms` (200 ms in the study profile, about 200 counter updates) if every sort took as +long as those just measured. It refines the number on each further warm-up run, then keeps it +fixed for every measured run of that configuration. The target is not a guarantee: a measured +loop can run faster or slower than the warm-up that set it. Startup trials are calibrated the same way, by the number of processes launched back to back in one measured interval (ADR 4). @@ -92,7 +93,9 @@ in one measured interval (ADR 4). **Consequences.** Every measured interval, sorting or startup, spans about 200 counter updates, so the update granularity is well under 1% of the measurement. Fast configurations sort thousands of times per process, slow ones once. The numbers of sorts and launches are -recorded in the CSV and in `meta.json`. +recorded in the CSV and in `meta.json`. Loops that fall short of the target are counted in +`meta.json` (`short_loops`, and `capped_loops` for those held back by `--max-reps`), noted by +the harness, and reported with the shortest loop. ## 6. Subtract an idle baseline of the same length after every run @@ -137,8 +140,8 @@ C sorts still produce the exact counts Algorithm-Energy-C published, which the t ## 9. No runtime dependencies in the workers -**Context.** The study has to run on a freshly booted Linux, where Python and Node.js come -from the distribution (Ubuntu 24.04 ships Python 3.12 and Node.js 18). +**Context.** The study has to run on a freshly booted Linux, where Python comes from the +distribution (Python 3.12 on Ubuntu 24.04) and Node.js is one download away (ADR 16). **Decision.** The Python worker uses only the standard library and runs on Python 3.10 or later. The Node.js worker is plain JavaScript with JSDoc types, checked by the TypeScript @@ -150,8 +153,9 @@ numpy and matplotlib, installed from hashed, pinned requirements into a virtual ## 10. Statistics written out rather than imported -**Context.** The report needs medians, bootstrap confidence intervals and Spearman's rank -correlation. SciPy provides them but is a large dependency for three functions. +**Context.** The report needs medians, bootstrap confidence intervals for medians and for +ratios of medians, and Spearman's rank correlation. SciPy provides most of them but is a +large dependency for a handful of functions. **Decision.** Implement the percentile bootstrap and Spearman's rho (Pearson's correlation of average ranks) with numpy, seeded so the same data gives the same report, and test them @@ -172,7 +176,9 @@ a custom value. **Consequences.** The estimates are transparent and easy to change, but they are annual averages: they do not describe the marginal emissions of running a benchmark at a given hour. -The report says so. +The report says so. Per-sort carbon is shown twice, from energy net of idle (what the sort +adds) and including idle (the idle power drawn while it runs), because the choice between +them is a matter of accounting, not of measurement. ## 12. Results stay pending until a quiet bare-metal run @@ -236,3 +242,41 @@ desktop. C and Python can move between the two CPUs, which are the same kind of changes little. RAPL counts the whole package, so the helper threads' energy is included either way. No pinned-against-unpinned comparison has been measured; the threats to validity say so. + +## 16. Measure Node.js 24 LTS, not the Node.js that Ubuntu ships + +**Context.** Ubuntu 24.04, the suggested live system, ships Node.js 18, whose support ended in +April 2025. A 2026 comparison of languages measured on it would describe an old V8 that CI +does not test. CI and the Docker image use Node.js 24, the current long-term support line. + +**Decision.** [bare-metal.md](bare-metal.md) installs the latest Node.js 24 release from +nodejs.org, checked against the release's published SHA-256 sums, instead of the `nodejs` +package. The study script warns when the Node.js on the path is older than 24, and the harness +records the version it ran in `meta.json`. The workers still run on Node.js 18 or later, so +the rest of the project keeps working with a distribution's Node.js. + +**Consequences.** The Node.js results describe the same major version that CI tests. The +guide has one more step, with a download that is verified before use. Python stays the +distribution's 3.12, which is supported until 2028. + +## 17. Decide the proxy question with power ratios, not with correlation + +**Context.** The first report answered "is time a good stand-in for energy?" with Spearman's +rho between process time and process energy, pooled over sizes and orders. That test is weak. +Calibration holds every process near 200 ms whatever the input, so the pooled figure mostly +reflects calibration granularity and noise; and within a configuration, energy is power +integrated over time, so a longer run uses more energy and rho is positive almost by +construction. + +**Decision.** The report decides the question with power: energy per sort divided by time per +sort. For each pair of languages on the same input, and for each input order against random +input, it gives the time ratio, the energy ratio and the power ratio, with bootstrap intervals +computed from the same resamples of each configuration's runs. Rank agreement between median +time and median energy across configurations is kept as a weaker test, and the +within-configuration rho as a check on noise. Synthetic tests include a language that draws +50% more power, which the power ratio must detect. + +**Consequences.** A reader can see not only whether time misjudges energy but by how much, for +which languages and inputs. The report is longer, with tables of ratios that are collapsed +under the main ones. Treating the subtracted startup medians as fixed makes the intervals +slightly too narrow; the threats to validity say so. diff --git a/docs/study.md b/docs/study.md index bcc9b62..94c5e18 100644 --- a/docs/study.md +++ b/docs/study.md @@ -15,10 +15,9 @@ identically in three languages, sorting identical inputs. ## Questions -1. **Time as a proxy.** For each language and algorithm, does energy rank runs in the same - order as time does? Measured by Spearman's rank correlation between process time and - process energy, both over all input sizes and orders together and within each - configuration. +1. **Time as a proxy.** Does energy follow time? If every configuration drew the same + average power while sorting, energy per sort would be proportional to time per sort, and + time would be a perfect stand-in for energy. 2. **Languages.** How much energy does one sort take in C, Python and Node.js, for the same algorithm and input? Is the energy ratio between two languages the same as their time ratio? @@ -28,10 +27,26 @@ identically in three languages, sorting identical inputs. ### What would count as an answer -If time is a good proxy, the within-configuration correlation should be clearly positive in -every language, and energy ratios between languages should match time ratios within their -confidence intervals. A language or algorithm whose energy ratio differs from its time ratio -would mean it draws a different average power, and that time alone would mislead. +Time is a good stand-in for energy to the extent that the power drawn while sorting (energy +per sort divided by time per sort) is the same in every configuration. The report tests this +in three ways, from the most direct to the weakest: + +- **Power ratios.** For each pair of languages on the same algorithm and input, and for each + input order against random input in the same language, the report gives the time ratio, the + energy ratio and their quotient, the power ratio, each with a 95% confidence interval. When + the power ratio's interval includes 1, the time ratio predicts the energy ratio. When it + excludes 1, one side draws more power, and time alone misjudges the energy difference by + that factor. The report prints the size of every power ratio, because a difference of a few + per cent can be real without mattering. +- **Power while sorting** for every configuration, and a chart of energy per sort against time + per sort, in which constant power is a straight line. +- **Rank agreement.** Spearman's rank correlation between median time per sort and median + energy per sort across configurations. This is weaker: configurations differ in cost by + large factors, so rho stays near 1 unless power differs enough to reorder them. + +Power ratios close to 1 for every pair of languages and every input order would answer the +first question "yes". Power ratios clearly away from 1 would answer the second: for those +languages, comparing time is not comparing energy. ## Method @@ -70,10 +85,17 @@ lists, JavaScript arrays). None uses a built-in sort. after it exits. C is measured the same way as Python and Node.js. - **Energy.** Package and DRAM zones from `/sys/class/powercap/intel-rapl:*`, summed over sockets, with one counter wrap handled using `max_energy_range_uj`. If the counters are - missing or unreadable the harness records "energy unavailable", never a zero. + missing, unreadable or frozen (present but never changing), the harness records "energy + unavailable", never a zero. - **Enough work per process.** RAPL counters update about once a millisecond (Hähnel et al., 2012). During warm-up the harness sets the number of sorts per process so that each worker's - sort loop lasts at least 200 ms. + sort loop lasts about 200 ms: it extrapolates from the last warm-up run, so a measured loop + can fall somewhat short on a busy machine. The report counts the loops that did. +- **Clocks.** Every reported time comes from the harness's `CLOCK_MONOTONIC_RAW`, read around + the whole process. Each worker also times its own sort loop (C with `CLOCK_MONOTONIC_RAW`, + Python with `time.perf_counter_ns` and Node.js with `process.hrtime`, both `CLOCK_MONOTONIC` + on Linux); the harness uses these loop times only to calibrate the number of sorts, and the + report only to count loops that fell short of the target. - **Idle baseline.** After every run the harness sleeps for the same duration and measures the idle energy, which is subtracted, scaled to the run's exact length. - **Startup cost.** For each language, order and size, a startup trial runs a worker that @@ -95,18 +117,34 @@ lists, JavaScript arrays). None uses a built-in sort. ### Analysis -- **Central value.** The median of the 30 runs of each configuration. -- **Uncertainty.** A 95% percentile-bootstrap confidence interval for each median (10,000 - resamples, seeded, so the same data always gives the same report). -- **Time against energy.** Spearman's rho between process time and process energy for each - language and algorithm, pooled over all sizes and orders (with a 95% bootstrap interval from - 2,000 resamples of the pairs), and the median of the rho values computed within each - configuration. The pooled figure is dominated by input size; the within-configuration figure - shows whether run-to-run variation in time tracks variation in energy. +- **Central value.** The median of the 30 runs of each configuration. Per-sort time and energy + are the whole-process values minus the median startup cost per process, divided by the + number of sorts. Energy is net of the idle baseline. +- **Uncertainty.** 95% percentile-bootstrap confidence intervals from 10,000 resamples, seeded + so that the same data always gives the same report. Each configuration's runs are resampled + once, keeping each run's time and energy together, and every estimate for that + configuration comes from the same resamples. The startup medians that are subtracted are + treated as fixed. +- **Power while sorting.** Median energy per sort divided by median time per sort, in watts. +- **Ratios.** Between languages (Python against C, Node.js against C, Python against Node.js) + for the same algorithm and input, and between each input order and random input for the + same language and algorithm: the time ratio, the energy ratio and the power ratio (energy + ratio divided by time ratio). The two configurations are resampled independently. +- **Rank agreement.** Spearman's rho between median time per sort and median energy per sort + across configurations: at each input size, for each language, and over all of them, with an + interval from 2,000 of the resamples. +- **Run-to-run coupling.** Within each configuration, Spearman's rho between the process time + and process energy of its 30 runs; the report gives the median per language and algorithm. + With steady power, a run that takes longer uses more energy, so a positive value is expected + almost by construction. It is a check on measurement noise, not an answer to the question. - **Carbon.** Energy converted to grams of CO2-equivalent with annual grid intensities for - the UAE and the UK (Ember, via Our World in Data, CC BY 4.0; see - [data/README.md](../data/README.md)). Other regions or a custom intensity can be passed to - the report. + the UAE and the UK (Ember, with major processing by Our World in Data, CC BY 4.0; see + [data/README.md](../data/README.md)). Per-sort figures are given twice: net of idle, which is + what the sort adds, and including the idle power drawn while it runs. Other regions or a + custom intensity can be passed to the report. +- **Checks on the data.** The report refuses a run that did not finish unless told otherwise + (`--allow-partial`), refuses energy readings that are all zero, and warns when a + configuration has fewer than 30 runs. ## Threats to validity @@ -126,7 +164,7 @@ lists, JavaScript arrays). None uses a built-in sort. processor is still warm and may be leaving a turbo state, so idle power can be slightly higher than true idle, and net energy slightly low. - **Counter security filtering.** After the PLATYPUS power side-channel attack (Lipp et al., - 2021), some Intel systems add noise to RAPL readings. The 200 ms minimum per process and the + 2021), some Intel systems add noise to RAPL readings. Measured intervals of about 200 ms and 30 repetitions reduce, but do not remove, its effect. - **Frequency scaling and heat.** Turbo and the scaling governor are left as the system sets them and recorded. Long runs heat the processor, which raises its power at the same work; @@ -147,9 +185,18 @@ lists, JavaScript arrays). None uses a built-in sort. different choice of CPUs, Node.js could be somewhat faster. Pinned and unpinned runs have not been compared. - **Startup subtraction.** Per-sort figures subtract a median startup cost measured in - separate processes. Noise in that median shifts every per-sort value of the configuration. -- **Language versions.** Results describe the recorded compiler and interpreter versions - (Ubuntu 24.04 ships Python 3.12 and Node.js 18), not the languages in general. + separate processes. Noise in that median shifts every per-sort value of the configuration, + and the confidence intervals do not include it. Startup is measured over batches of + processes started back to back; after the first, the files each process reads are in the + page cache, so the cost per process describes a warm start. The sorting runs start warm too, + because warm-up runs come first. +- **Calibration.** Sorts per process are set from the warm-up runs and can leave a measured + loop shorter than the 200 ms target on a busy machine. Shorter loops are still far longer + than the counter's update interval; the report says how many there were. +- **Language versions.** Results describe the recorded compiler and interpreter versions, not + the languages in general. The study script uses the system Python (3.12 on Ubuntu 24.04) and + the Node.js 24 LTS that [bare-metal.md](bare-metal.md) installs from nodejs.org, and warns + if an older Node.js is on the path. ### Scope @@ -158,6 +205,9 @@ lists, JavaScript arrays). None uses a built-in sort. memory footprints or I/O-bound programs. - Carbon estimates use annual, location-based averages that include lifecycle emissions. They do not describe the marginal emissions of running the benchmark at a particular hour. + Per-sort carbon net of idle counts only the extra power the sort draws; including idle, it + also counts the idle power drawn while the sort runs. Neither includes the rest of the + machine, so both understate the electricity used. ## Reproducing From b8b4407d3496b3c022adde6d982d21d383be9d4c Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 05:20:55 +0400 Subject: [PATCH 32/51] docs: give the statistics feature a plain name Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d51d0b..012eb10 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ compare?** The research question, method and threats to validity are in above the ~1 ms RAPL update interval), an idle baseline of the same length after every run, 30 measured runs per configuration after warm-up, randomised run order, optional pinning to a set of CPUs, startup trials measured over batches of launches, and tidy CSV output. -- **Statistics that answer the question.** Power while sorting, and time, energy and power +- **Statistics.** Power while sorting, and time, energy and power ratios between languages and between input orders, each with a 95% bootstrap interval: a power ratio of 1 means time predicts energy. Also medians with intervals, rank agreement across configurations, and a run-to-run noise check, in a Python package with its own tests. From ae844dc7e210ab98e10159b36040502a88934519 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 05:20:56 +0400 Subject: [PATCH 33/51] docs: regenerate results.md with harness 2.1, timings still pending Generated with --withhold-timings from a full study-profile run of commit ca63170 in Docker Desktop (container limited to 2 CPUs and 2 GiB, energy unavailable): 5,760 measured runs, status complete, no retries. The operation counts are unchanged. The provenance now records the cgroup limits, the run status and a harness command that pastes back from the repository root; the grid table uses OWID's citation. The raw CSV is not committed, because it holds the withheld timings. Co-Authored-By: Claude Opus 5.5 --- docs/results.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/results.md b/docs/results.md index ebb3718..4e5ed9d 100644 --- a/docs/results.md +++ b/docs/results.md @@ -2,22 +2,22 @@ -> **Status: time and energy results are pending.** This file was generated from a run that checked the pipeline end to end. Operation counts are exact and shown below; time, energy, correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux machine ([how](bare-metal.md)). The run's raw data is not in the repository, because it holds those timings; the harness command below reproduces the counts. +> **Status: time and energy results are pending.** This file was generated from a run that checked the pipeline end to end. Operation counts are exact and shown below; time, energy, power, ratio, correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux machine ([how](bare-metal.md)). The run's raw data is not in the repository, because it holds those timings; the harness command below reproduces the counts. ## How these results were produced -- Harness: greenbench 2.0.0, profile `study`, 30 measured runs per configuration after 3 warm-up runs, in a random order (order seed 1234567). -- Inputs: sizes 250, 500, 1,000, 2,000; orders random, sorted, reversed, nearly sorted; xorshift32 seed 2463534242. Each process sorts for at least 200 ms. -- Machine: AMD Ryzen 7 6800H with Radeon Graphics; Linux 6.18.33.2-microsoft-standard-WSL2 x86_64; 16 CPUs online; scaling governor not reported; CPU pinning off. +- Harness: greenbench 2.1.0, profile `study`, 30 measured runs per configuration after 3 warm-up runs, in a random order (order seed 1234567). +- Inputs: sizes 250, 500, 1,000, 2,000; orders random, sorted, reversed, nearly sorted; xorshift32 seed 2463534242. Sort loops and startup batches are calibrated to about 200 ms. +- Machine: AMD Ryzen 7 6800H with Radeon Graphics; Linux 6.18.33.2-microsoft-standard-WSL2 x86_64; 16 CPUs online, a cgroup CPU limit of 2 CPUs, a cgroup memory limit of 2.0 GiB; scaling governor not reported; CPU pinning off. - Workers: C: gcc 14.2.0; Python: 3.12.14; Node.js: v24.21.0. - Energy: unavailable. Reason: no RAPL package zone (intel-rapl:N named package-N) under /sys/class/powercap; this CPU, kernel or virtual machine does not expose RAPL. -- Run: 5,760 measured worker processes, started 2026-09-25T22:37:21Z, finished 2026-09-25T23:00:54Z; label: pipeline check in Docker Desktop; timings withheld. -- Harness command: `greenbench run --profile study --energy auto --label 'pipeline check in Docker Desktop; timings withheld' --out results/study-docker` +- Run: 5,760 measured runs, status complete, 0 worker processes retried; started 2026-09-26T00:57:33Z, finished 2026-09-26T01:18:53Z; label: pipeline check in Docker Desktop; timings withheld. +- Harness command: `build/greenbench run --profile study --energy auto --label 'pipeline check in Docker Desktop; timings withheld' --out results/study-docker` - Report command: `PYTHONPATH=analysis python3 -m greenbench_analysis report --runs results/study-docker/runs.csv --out docs/results.md --img-dir docs/img --withhold-timings` ## Operation counts -Comparisons between elements and writes into the array, for one sort. These are exact counts, not measurements, so machine load does not affect them. C, Python, Node.js reported identical counts and identical input hashes for every configuration; the harness checks this during the run and the report checks it again. +Comparisons between elements and writes into the array, for one sort. These are exact counts, not measurements, so machine load does not affect them. C, Python and Node.js reported identical counts and identical input hashes for every configuration; the harness checks this during the run and the report checks it again. ![Comparisons per sort against input size, by input order](img/operations.png) @@ -92,5 +92,5 @@ Comparisons between elements and writes into the array, for one sort. These are | Region | Year | g CO2e per kWh | Basis | Source | Licence | | :--- | ---: | ---: | :--- | :--- | :--- | -| United Arab Emirates | 2024 | 467.51 | lifecycle, whole grid, annual average | [Ember (2026), yearly electricity data, via Our World in Data indicator 1295504](https://ourworldindata.org/grapher/carbon-intensity-electricity) | CC BY 4.0 | -| United Kingdom | 2024 | 216.5 | lifecycle, whole grid, annual average | [Ember (2026), yearly electricity data, via Our World in Data indicator 1295504](https://ourworldindata.org/grapher/carbon-intensity-electricity) | CC BY 4.0 | +| United Arab Emirates | 2024 | 467.51 | lifecycle, whole grid, annual average | [Ember (2026) – with major processing by Our World in Data](https://ourworldindata.org/grapher/carbon-intensity-electricity) | CC BY 4.0 | +| United Kingdom | 2024 | 216.5 | lifecycle, whole grid, annual average | [Ember (2026) – with major processing by Our World in Data](https://ourworldindata.org/grapher/carbon-intensity-electricity) | CC BY 4.0 | From 9b2b83fa191011f690f1183ab3743bcc24348101 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:49:14 +0400 Subject: [PATCH 34/51] fix: judge each socket's counter on its own, and refuse implausible jumps A frozen package counter on one socket of a two-socket machine passed the frozen check, because the check looked at the sum over sockets. Each package domain is now judged on its own, and the error names the zone. A reading that rises by more than 1 kW per package over its interval now stops the run: a counter that reset after a suspend would otherwise be read as a wrap worth hundreds of kilojoules. meta.json also records how many counter wraps fell inside measured runs and idle windows. Co-Authored-By: Claude Opus 5.5 --- docs/bare-metal.md | 3 +- docs/decisions.md | 13 ++++-- src/greenbench.c | 68 ++++++++++++++++++++----------- src/rapl.c | 28 ++++++++++++- src/rapl.h | 28 ++++++++++--- tests/integration/test_harness.py | 3 +- tests/test_rapl.c | 63 ++++++++++++++++++++++++---- 7 files changed, 162 insertions(+), 44 deletions(-) diff --git a/docs/bare-metal.md b/docs/bare-metal.md index cf5d722..f5a4685 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -135,7 +135,8 @@ python3 tests/integration/powercap.py advance /tmp/pc -- ./scripts/run-study.sh | `Node.js v18... is older than the Node.js 24 LTS` | The `nodejs` package was installed instead of Node.js 24 from step 3. The study still runs, but describes that version. | | `this is WSL` or `this is a container` | The script refuses environments without real counters. | | `permission denied reading .../energy_uj` from the harness | Run `--check` again and accept the `chmod`, or reboot and start again. | -| `the package energy counter did not change` | The counter files exist but their values never change, as some virtual machines and firmware leave them. The probe exits with status 3, the script stops, and this machine cannot measure energy. | +| `the package energy counter intel-rapl:N (package-N) did not change` | The counter files exist but their values do not change, as some virtual machines and firmware leave them. The probe exits with status 3, the script stops, and this machine cannot measure energy. | +| `the RAPL counters rose by ... more than ... package(s) can draw` | The counters jumped, usually because the machine was suspended during the run and the counters reset. Turn off automatic suspend and start again. | | `the implementations disagree` | A worker produced different counts from the others. This is a bug; please open an issue with the message. | The harness stops cleanly on Ctrl+C: the runs measured so far stay in `runs.csv`, and diff --git a/docs/decisions.md b/docs/decisions.md index 5f2298c..d4913cd 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -31,8 +31,9 @@ harness records `energy_status=unavailable` and a reason, and leaves every energ Counters that exist but never move, as some virtual machines and firmware leave them, count as missing: before measuring, the harness reads the counters 200 ms apart, and a package counter that has not changed makes energy unavailable. During the run, a measured interval or -idle window of 100 ms or more with no package change stops the run with an error; a live -counter moves about once a millisecond. `--energy require` turns unavailable energy into exit +idle window of 100 ms or more in which any package counter did not change stops the run with +an error naming the zone; a live counter moves about once a millisecond. Each socket is judged +on its own, so one frozen counter on a two-socket machine cannot hide behind the other. `--energy require` turns unavailable energy into exit status 3, and so does `greenbench probe` when the counters do not move. The report prints "energy unavailable" and keeps the energy sections empty. @@ -51,9 +52,15 @@ counter can wrap in well under an hour. end`. Readings above the maximum are treated as corrupt. Every measured interval is one worker process, seconds long at most, so at most one wrap can occur. +A reading that rose by more than 1 kW per package over its interval (plus 10 ms for the +counter's coarse updates) stops the run. No processor draws that much, but a counter that +reset, as counters can after a suspend, would otherwise be read as a wrap worth hundreds of +kilojoules. + **Consequences.** Correct deltas across a wrap, tested with fixture trees whose counters are rewritten between readings. The formula loses the single step from the maximum back to zero, -one energy unit (about 15 to 61 microjoules), which is far below the noise. +one energy unit (about 15 to 61 microjoules), which is far below the noise. `meta.json` +records how many wraps fell inside measured runs and idle windows (`counter_wraps`). ## 4. Measure whole worker processes, for every language diff --git a/src/greenbench.c b/src/greenbench.c index 0749278..b75a8a3 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -82,6 +82,7 @@ typedef struct s_context { uint64_t sorting_runs; /* measured runs that sorted */ uint64_t short_loops; /* of those, runs whose sort loop fell short of --min-loop-ms */ uint64_t capped_loops; /* of those, runs held back by --max-reps */ + uint64_t counter_wraps; /* RAPL counter wraps inside measured runs and idle windows */ int allowed_cpus; /* CPUs this process could use before any pinning, or -1 */ char started_utc[32]; char meta_path[PATH_MAX]; @@ -465,6 +466,7 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure json_key_string(f, " ", "reason", ctx->energy_reason, 1); json_key_string(f, " ", "powercap_root", o->powercap_root, 1); fprintf(f, " \"idle_baseline\": %s,\n", ctx->baseline ? "true" : "false"); + fprintf(f, " \"counter_wraps\": %" PRIu64 ",\n", ctx->counter_wraps); fputs(" \"domains\": [", f); for (size_t i = 0; ctx->energy_ok && i < ctx->rapl.count; i++) { const t_rapl_domain *d = &ctx->rapl.domains[i]; @@ -585,6 +587,28 @@ static int check_consistency(t_context *ctx, const t_trial *t, const t_worker_re return 0; } +/* Energy between two readings taken interval_ns apart, while `what` happened. Stops the run + (returns -1) when a reading is corrupt, when a package counter stood still, or when the + counters jumped by more than a processor can draw, as they can after a suspend. */ +static int check_energy(const t_context *ctx, const t_rapl_sample *before, const t_rapl_sample *after, + uint64_t interval_ns, const char *what, t_rapl_energy *energy) { + if (rapl_energy_between(&ctx->rapl, before, after, energy) != 0) + return error_message("a RAPL counter reading was above its max_energy_range_uj"), -1; + const t_rapl_domain *frozen = rapl_frozen_package(&ctx->rapl, before, after, interval_ns); + if (frozen != NULL) + return error_message("the package energy counter %s (%s) did not change %s (%.0f ms); the RAPL readings " + "have frozen, so energy cannot be measured", + frozen->zone, frozen->name, what, (double)interval_ns / 1e6), + -1; + if (!rapl_is_plausible(&ctx->rapl, energy, interval_ns)) + return error_message("the RAPL counters rose by %.1f J %s (%.0f ms), more than %zu package(s) can draw; " + "the machine may have been suspended or the counters reset, so the run stops", + (double)(energy->package_uj + energy->dram_uj) / 1e6, what, + (double)interval_ns / 1e6, rapl_package_count(&ctx->rapl)), + -1; + return 0; +} + /* Runs the worker `launches` times back to back and measures the batch as one interval. Sorting runs launch one process; startup trials launch enough processes to span about --min-loop-ms, because a single start can be shorter than the RAPL update interval. @@ -665,15 +689,13 @@ static int run_worker(t_context *ctx, const t_trial *t, uint64_t reps, uint64_t return -1; m->wall_ns = end - start; m->launches = launches; - m->energy.package_uj = 0; - m->energy.dram_uj = 0; - if (ctx->energy_ok && rapl_energy_between(&ctx->rapl, &before, &after, &m->energy) != 0) - return error_message("a RAPL counter reading was above its max_energy_range_uj"), -1; - if (ctx->energy_ok && rapl_is_frozen(&m->energy, m->wall_ns)) - return error_message("the package energy counter did not change while %s ran for %.0f ms; the RAPL " - "readings have frozen, so energy cannot be measured", - what, (double)m->wall_ns / 1e6), - -1; + memset(&m->energy, 0, sizeof m->energy); + if (ctx->energy_ok) { + char during[TEXT_LEN + 16]; + snprintf(during, sizeof during, "while %s ran", what); + if (check_energy(ctx, &before, &after, m->wall_ns, during, &m->energy) != 0) + return -1; + } return 0; } @@ -807,23 +829,22 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { record.package_uj = m.energy.package_uj; record.dram_uj = m.energy.dram_uj; + ctx->counter_wraps += m.energy.wraps; if (ctx->baseline) { t_rapl_sample before, after; - t_rapl_energy idle = {0, 0}; + t_rapl_energy idle; + memset(&idle, 0, sizeof idle); if (ctx->energy_ok && rapl_read(&ctx->rapl, &before) != 0) return free(items), error_message("reading the RAPL counters failed: %s", strerror(errno)), -1; uint64_t idle_start = timing_now_ns(); timing_sleep_ns(m.wall_ns); uint64_t idle_end = timing_now_ns(); - if (ctx->energy_ok && (rapl_read(&ctx->rapl, &after) != 0 || - rapl_energy_between(&ctx->rapl, &before, &after, &idle) != 0)) + if (ctx->energy_ok && rapl_read(&ctx->rapl, &after) != 0) return free(items), error_message("reading the RAPL counters failed during the idle window"), -1; - if (ctx->energy_ok && rapl_is_frozen(&idle, idle_end - idle_start)) - return free(items), - error_message("the package energy counter did not change during a %.0f ms idle window; the " - "RAPL readings have frozen, so energy cannot be measured", - (double)(idle_end - idle_start) / 1e6), - -1; + if (ctx->energy_ok && + check_energy(ctx, &before, &after, idle_end - idle_start, "during an idle window", &idle) != 0) + return free(items), -1; + ctx->counter_wraps += idle.wraps; record.has_idle = 1; record.idle_ns = idle_end - idle_start; record.idle_package_uj = idle.package_uj; @@ -901,11 +922,12 @@ static int counters_move(const t_rapl *rapl, uint64_t sample_ns, t_rapl_energy * return -1; } *elapsed = took; - if (rapl_is_frozen(energy, *elapsed)) { + const t_rapl_domain *frozen = rapl_frozen_package(rapl, &before, &after, took); + if (frozen != NULL) { snprintf(reason, size, - "the package energy counter did not change over %.0f ms: the RAPL files exist but their readings " - "are frozen, as some virtual machines and firmware leave them", - (double)*elapsed / 1e6); + "the package energy counter %s (%s) did not change over %.0f ms: the RAPL files exist but their " + "readings are frozen, as some virtual machines and firmware leave them", + frozen->zone, frozen->name, (double)took / 1e6); return -1; } return 0; @@ -1129,7 +1151,7 @@ static int command_probe(int argc, char **argv) { rapl_close(&rapl); return 0; } - t_rapl_energy energy = {0, 0}; + t_rapl_energy energy = {0, 0, 0}; uint64_t elapsed = 0; char reason[RAPL_REASON_LEN] = ""; int moved = counters_move(&rapl, sample_ms * 1000000u, &energy, &elapsed, reason, sizeof reason); diff --git a/src/rapl.c b/src/rapl.c index ee597ed..d72f51b 100644 --- a/src/rapl.c +++ b/src/rapl.c @@ -271,10 +271,13 @@ int rapl_energy_between(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl t_rapl_energy *energy) { energy->package_uj = 0; energy->dram_uj = 0; + energy->wraps = 0; for (size_t i = 0; i < rapl->count; i++) { uint64_t delta; if (rapl_delta(a->uj[i], b->uj[i], rapl->domains[i].max_range_uj, &delta) != 0) return -1; + if (b->uj[i] < a->uj[i]) + energy->wraps++; if (rapl->domains[i].kind == RAPL_PACKAGE) energy->package_uj += delta; else @@ -283,14 +286,35 @@ int rapl_energy_between(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl return 0; } +size_t rapl_package_count(const t_rapl *rapl) { + size_t packages = 0; + + for (size_t i = 0; i < rapl->count; i++) + packages += rapl->domains[i].kind == RAPL_PACKAGE; + return packages; +} + double rapl_net_uj(uint64_t gross_uj, uint64_t idle_uj, uint64_t run_ns, uint64_t idle_ns) { if (idle_ns == 0) return (double)gross_uj; return (double)gross_uj - (double)idle_uj * ((double)run_ns / (double)idle_ns); } -int rapl_is_frozen(const t_rapl_energy *energy, uint64_t interval_ns) { - return energy->package_uj == 0 && interval_ns >= RAPL_FROZEN_NS; +const t_rapl_domain *rapl_frozen_package(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl_sample *b, + uint64_t interval_ns) { + if (interval_ns < RAPL_FROZEN_NS) + return NULL; + for (size_t i = 0; i < rapl->count; i++) + if (rapl->domains[i].kind == RAPL_PACKAGE && a->uj[i] == b->uj[i]) + return &rapl->domains[i]; + return NULL; +} + +int rapl_is_plausible(const t_rapl *rapl, const t_rapl_energy *energy, uint64_t interval_ns) { + /* W x ns / 1000 is microjoules. */ + double limit_uj = (double)rapl_package_count(rapl) * RAPL_PLAUSIBLE_WATTS * + (double)(interval_ns + RAPL_PLAUSIBLE_SLACK_NS) / 1000.0; + return (double)energy->package_uj + (double)energy->dram_uj <= limit_uj; } void rapl_close(t_rapl *rapl) { diff --git a/src/rapl.h b/src/rapl.h index f616156..b34763f 100644 --- a/src/rapl.h +++ b/src/rapl.h @@ -26,6 +26,13 @@ the readings are not real. */ #define RAPL_FROZEN_NS 100000000u /* 100 ms */ +/* No processor package draws 1 kW. More than that per package over an interval, allowing + RAPL_PLAUSIBLE_SLACK_NS for the counter's coarse updates, means the readings jumped: after + a suspend, for example, the counters can reset, and the reset would otherwise be read as a + wrap worth hundreds of kilojoules. */ +#define RAPL_PLAUSIBLE_WATTS 1000u +#define RAPL_PLAUSIBLE_SLACK_NS 10000000u /* 10 ms */ + typedef enum e_rapl_kind { RAPL_PACKAGE, RAPL_DRAM } t_rapl_kind; typedef struct s_rapl_domain { @@ -52,6 +59,7 @@ typedef struct s_rapl_sample { typedef struct s_rapl_energy { uint64_t package_uj; uint64_t dram_uj; /* 0 when there is no DRAM zone; check has_dram */ + unsigned wraps; /* counters that wrapped between the two samples */ } t_rapl_energy; /* Parses a sysfs counter: decimal digits, then optional whitespace (sysfs ends with "\n"). @@ -75,19 +83,29 @@ int rapl_open(t_rapl *rapl, const char *root); /* Reads every open counter. Returns 0, or -1 with errno set. */ int rapl_read(const t_rapl *rapl, t_rapl_sample *sample); -/* Energy from a to b. Returns 0, or -1 if a counter reading was corrupt. */ +/* Energy from a to b, and how many counters wrapped. Returns 0, or -1 if a counter reading + was corrupt. */ int rapl_energy_between(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl_sample *b, t_rapl_energy *energy); +/* Number of package domains (sockets). */ +size_t rapl_package_count(const t_rapl *rapl); + /* Energy left after removing the idle baseline: gross minus idle energy scaled from the idle window to the run's duration. Can be negative when the run used no more power than idle; the value is kept, not clamped, so the noise stays visible. */ double rapl_net_uj(uint64_t gross_uj, uint64_t idle_uj, uint64_t run_ns, uint64_t idle_ns); -/* 1 when the package energy over an interval of interval_ns shows frozen counters: no - change at all over at least RAPL_FROZEN_NS. Shorter intervals are never judged, because a - live counter can legitimately stand still between two updates. */ -int rapl_is_frozen(const t_rapl_energy *energy, uint64_t interval_ns); +/* The first package domain whose counter did not change at all between a and b, when the + interval of interval_ns is at least RAPL_FROZEN_NS; NULL otherwise. Each socket is judged + on its own, so one frozen counter is caught while another moves. Shorter intervals are + never judged, because a live counter can legitimately stand still between two updates. */ +const t_rapl_domain *rapl_frozen_package(const t_rapl *rapl, const t_rapl_sample *a, const t_rapl_sample *b, + uint64_t interval_ns); + +/* 1 when energy over an interval of interval_ns is physically possible: at most + RAPL_PLAUSIBLE_WATTS per package, over the interval plus RAPL_PLAUSIBLE_SLACK_NS. */ +int rapl_is_plausible(const t_rapl *rapl, const t_rapl_energy *energy, uint64_t interval_ns); void rapl_close(t_rapl *rapl); diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 0a85ced..86fc5de 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -68,6 +68,7 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert meta["measured_runs"] == len(rows) assert meta["energy"]["status"] == "off" assert meta["energy"]["idle_baseline"] is False + assert meta["energy"]["counter_wraps"] == 0 assert meta["clock"] == "CLOCK_MONOTONIC_RAW" assert meta["settings"]["runs"] == 3 assert meta["settings"]["cpus"] == [] @@ -207,7 +208,7 @@ def test_frozen_counters_are_not_energy(tmp_path: Path) -> None: "run", *SMALL, "--languages", "c", "--energy", "require", "--powercap-root", str(root), "--out", str(tmp_path) ) assert required.returncode == 3 - assert "energy unavailable: the package energy counter did not change" in required.stderr + assert "energy unavailable: the package energy counter intel-rapl:0 (package-0) did not change" in required.stderr assert "frozen" in required.stderr out = tmp_path / "auto" diff --git a/tests/test_rapl.c b/tests/test_rapl.c index dc78e43..53c6239 100644 --- a/tests/test_rapl.c +++ b/tests/test_rapl.c @@ -91,6 +91,7 @@ static void test_laptop(void) { CHECK(rapl_energy_between(&rapl, &a, &b, &e) == 0, "energy between readings"); CHECK(e.package_uj == 5000000u, "package: 5 J"); CHECK(e.dram_uj == (65712999613u - 9876543210u) + 1000u, "DRAM: wrapped once"); + CHECK(e.wraps == 1, "the DRAM wrap is counted"); fixture_write(root, "intel-rapl:0/energy_uj", "garbage"); CHECK(rapl_read(&rapl, &b) == -1, "an unreadable counter is an error, not zero"); @@ -115,6 +116,8 @@ static void test_two_sockets(void) { CHECK(rapl_read(&rapl, &b) == 0 && rapl_energy_between(&rapl, &a, &b, &e) == 0, "second reading"); CHECK(e.package_uj == 3000000u, "packages summed over sockets"); CHECK(e.dram_uj == 700000u, "DRAM summed over sockets"); + CHECK(e.wraps == 0, "no wraps"); + CHECK(rapl_package_count(&rapl) == 2, "two packages"); rapl_close(&rapl); fixture_remove(root); } @@ -182,15 +185,56 @@ static void test_net(void) { } static void test_frozen(void) { - t_rapl_energy still = {0, 0}; - t_rapl_energy moved = {1, 0}; - t_rapl_energy dram_only = {0, 500}; - - CHECK(rapl_is_frozen(&still, RAPL_FROZEN_NS), "no package change over 100 ms is frozen"); - CHECK(rapl_is_frozen(&still, 250000000u), "no package change over 250 ms is frozen"); - CHECK(!rapl_is_frozen(&still, RAPL_FROZEN_NS - 1), "a shorter interval can fall between two updates"); - CHECK(!rapl_is_frozen(&moved, 250000000u), "one microjoule is movement"); - CHECK(rapl_is_frozen(&dram_only, 250000000u), "only the package counter decides"); + t_rapl rapl; + char root[512]; + t_rapl_sample a, b; + const t_rapl_domain *frozen; + + CHECK(open_fixture(FIXTURES "server-two-socket.txt", &rapl, root, sizeof root) == 0, "server opens"); + CHECK(rapl_read(&rapl, &a) == 0, "first reading"); + frozen = rapl_frozen_package(&rapl, &a, &a, RAPL_FROZEN_NS); + CHECK(frozen != NULL && strcmp(frozen->zone, "intel-rapl:0") == 0, "nothing moved: the first package is named"); + CHECK(rapl_frozen_package(&rapl, &a, &a, RAPL_FROZEN_NS - 1) == NULL, + "a shorter interval can fall between two updates"); + + /* Socket 0 moves and socket 1 stands still: their sum moves, but socket 1 is frozen. */ + fixture_write(root, "intel-rapl:0/energy_uj", "150001000000"); + CHECK(rapl_read(&rapl, &b) == 0, "second reading"); + frozen = rapl_frozen_package(&rapl, &a, &b, 250000000u); + CHECK(frozen != NULL && strcmp(frozen->zone, "intel-rapl:1") == 0 && strcmp(frozen->name, "package-1") == 0, + "a frozen second socket is caught while the first moves"); + + /* Both packages move; the DRAM counters stand still, which is not judged. */ + fixture_write(root, "intel-rapl:1/energy_uj", "160000000001"); + CHECK(rapl_read(&rapl, &b) == 0, "third reading"); + CHECK(rapl_frozen_package(&rapl, &a, &b, 250000000u) == NULL, "one microjoule on every package is movement"); + rapl_close(&rapl); + fixture_remove(root); +} + +static void test_plausible(void) { + t_rapl rapl; + char root[512]; + t_rapl_energy e = {0, 0, 0}; + + CHECK(open_fixture(FIXTURES "server-two-socket.txt", &rapl, root, sizeof root) == 0, "server opens"); + /* Two packages at 1 kW each, over 100 ms plus 10 ms of slack: 220 J. */ + e.package_uj = 220000000u; + CHECK(rapl_is_plausible(&rapl, &e, 100000000u), "at the limit"); + e.package_uj = 220000001u; + CHECK(!rapl_is_plausible(&rapl, &e, 100000000u), "just above the limit"); + e.package_uj = 120000000u; + e.dram_uj = 100000001u; + CHECK(!rapl_is_plausible(&rapl, &e, 100000000u), "DRAM counts towards the limit"); + /* A counter that reset after a suspend, read as a wrap: nearly the whole range at once. */ + e.package_uj = 262143328850u - 150000000000u; + e.dram_uj = 0; + CHECK(!rapl_is_plausible(&rapl, &e, 200000000u), "a reset counter is implausible"); + /* 2 mJ in a 50 us interval is 40 W, but one coarse counter update can bring that much. */ + e.package_uj = 2000u; + CHECK(rapl_is_plausible(&rapl, &e, 50000u), "short intervals get slack for coarse updates"); + rapl_close(&rapl); + fixture_remove(root); } int main(void) { @@ -204,5 +248,6 @@ int main(void) { test_permission_denied(); test_net(); test_frozen(); + test_plausible(); return check_report("rapl"); } From 037dbf56db98dc10e8381bf9d9af6dfc42edfa47 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:18:58 +0400 Subject: [PATCH 35/51] test: simulate RAPL readings from the clock so energy tests cannot flake The energy tests moved fake counters from a Python thread. On a busy machine the thread was not always scheduled in time, so a short interval saw no energy and a long stall tripped the frozen-counter check; the pipeline test failed in 4 of 11 runs on a loaded machine. make simulated now builds build/greenbench-simulated with -DGREENBENCH_SIMULATED_RAPL. It reads a fake tree as usual and adds energy computed at each reading from CLOCK_MONOTONIC_RAW and from the processor time of its finished workers, wrapping at max_energy_range_uj. The integration tests, the sanitizer run and the study script's fake-tree rehearsal use it; the frozen-counter tests keep using build/greenbench. The simulated build says so on standard error, meta.json records "simulated": true, and the report opens with a warning that such energy is not a measurement. The report also shows how many counter wraps were handled. ADR 18 records the decision. Co-Authored-By: Claude Opus 5.5 --- Makefile | 24 ++-- README.md | 14 ++- analysis/greenbench_analysis/report.py | 12 +- analysis/tests/test_report.py | 12 ++ docs/bare-metal.md | 5 +- docs/decisions.md | 28 ++++- scripts/run-study.sh | 15 ++- src/greenbench.c | 7 ++ src/rapl.c | 69 ++++++++++++ src/rapl.h | 12 +- tests/fixtures/powercap/README.md | 11 +- tests/integration/powercap.py | 150 ++----------------------- tests/integration/test_harness.py | 77 +++++++++---- tests/integration/test_pipeline.py | 122 +++++++++----------- tests/integration/workers.py | 23 +++- 15 files changed, 319 insertions(+), 262 deletions(-) diff --git a/Makefile b/Makefile index d463b22..1edfa7b 100644 --- a/Makefile +++ b/Makefile @@ -17,13 +17,18 @@ LIB_OBJ := $(LIB:%=$(BUILD)/obj/%.o) HEADERS := $(wildcard src/*.h) HARNESS := $(BUILD)/greenbench WORKER := $(BUILD)/greenbench-worker +# A test build of the harness whose RAPL readings are computed from the clock (see src/rapl.c), +# so that energy tests do not depend on another program being scheduled to move fake counters. +SIMULATED := $(BUILD)/greenbench-simulated TESTS := test_sorts test_inputs test_rapl test_options test_plan test_protocol test_output test_spawn TEST_BIN := $(TESTS:%=$(BUILD)/tests/%) -.PHONY: all test sanitize smoke parity integration check-python check-node lint-shell check clean study +.PHONY: all simulated test sanitize smoke parity integration check-python check-node lint-shell check clean study all: $(HARNESS) $(WORKER) +simulated: $(SIMULATED) + $(BUILD)/obj $(BUILD)/tests: mkdir -p $@ @@ -39,6 +44,12 @@ $(HARNESS): $(BUILD)/obj/greenbench.o $(LIB_OBJ) $(WORKER): $(BUILD)/obj/worker.o $(BUILD)/obj/sorts.o $(BUILD)/obj/inputs.o $(BUILD)/obj/protocol.o $(CC) $(ALL_CFLAGS) $(LDFLAGS) -o $@ $^ +$(BUILD)/obj/%-simulated.o: src/%.c $(HEADERS) | $(BUILD)/obj + $(CC) $(ALL_CFLAGS) -DGREENBENCH_SIMULATED_RAPL -c -o $@ $< + +$(SIMULATED): $(BUILD)/obj/greenbench-simulated.o $(BUILD)/obj/rapl-simulated.o $(filter-out %/rapl.o,$(LIB_OBJ)) + $(CC) $(ALL_CFLAGS) $(LDFLAGS) -o $@ $^ + $(BUILD)/tests/%: tests/%.c tests/check.h $(LIB_OBJ) $(BUILD)/obj/fixture.o $(HEADERS) | $(BUILD)/tests $(CC) $(ALL_CFLAGS) -Isrc -Itests $(LDFLAGS) -o $@ $< $(LIB_OBJ) $(BUILD)/obj/fixture.o @@ -46,13 +57,12 @@ $(BUILD)/tests/%: tests/%.c tests/check.h $(LIB_OBJ) $(BUILD)/obj/fixture.o $(HE test: $(TEST_BIN) @for t in $(TEST_BIN); do ./$$t || exit 1; done -# Unit tests and a short C-only harness run under AddressSanitizer and UBSan. The harness -# reads a fake powercap tree whose counters move, so the energy and idle-baseline code runs too. +# Unit tests and a short C-only harness run under AddressSanitizer and UBSan. The run uses the +# simulated-RAPL build on a fake two-socket tree, so the energy and idle-baseline code runs too. sanitize: - $(MAKE) BUILD=build/sanitize CFLAGS="$(SANITIZE)" LDFLAGS="$(SANITIZE)" all test + $(MAKE) BUILD=build/sanitize CFLAGS="$(SANITIZE)" LDFLAGS="$(SANITIZE)" all simulated test $(PYTHON) tests/integration/powercap.py build tests/fixtures/powercap/server-two-socket.txt build/sanitize/powercap - $(PYTHON) tests/integration/powercap.py advance build/sanitize/powercap -- \ - build/sanitize/greenbench run --profile smoke --languages c --quiet --energy require \ + build/sanitize/greenbench-simulated run --profile smoke --languages c --quiet --energy require \ --powercap-root build/sanitize/powercap --out build/sanitize/smoke # A short run of every language with energy off, then the report. Works anywhere. @@ -66,7 +76,7 @@ parity: all $(PYRUN) -m pytest -q tests/integration/test_parity.py # End-to-end tests of the harness binary, the workers and the report. -integration: all +integration: all simulated $(PYRUN) -m pytest -q tests/integration check-python: diff --git a/README.md b/README.md index 012eb10..bce19bc 100644 --- a/README.md +++ b/README.md @@ -185,12 +185,14 @@ make lint-shell # ShellCheck on the study script The RAPL tests build fake powercap trees from the manifests in [tests/fixtures/powercap](tests/fixtures/powercap), including a two-socket server, an AMD processor without DRAM, a missing tree, corrupt values and an unreadable counter, and rewrite -counters between readings to test wraparound. The integration tests also move fake counters -while the harness runs, faster during runs than during idle windows and wrapping several -times, so that non-zero energy goes from the harness through to every section of the report; -counters that never move must be refused. CI runs all of the above and lints the workflow -with actionlint; the C build and tests run with both GCC and Clang, and the statistics tests on -Python 3.12 and 3.14. +counters between readings to test wraparound. The integration tests also run +`build/greenbench-simulated`, a test build whose readings are computed from the clock +([ADR 18](docs/decisions.md#18-test-energy-with-a-simulated-rapl-build-not-with-moving-files)). +With it, energy that is higher during runs than during idle windows goes from the harness +through to every section of the report, and a counter wraps inside a measured interval, +however busy the machine is. The real build must refuse counters that never move. CI runs all +of the above and lints the workflow with actionlint; the C build and tests run with both GCC +and Clang, and the statistics tests on Python 3.12 and 3.14. ## Folder structure diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 340ed2b..3ff715c 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -185,7 +185,10 @@ def _environment(meta: dict[str, Any], runs: list[Run], summary: Summary, option if energy.get("status") == "ok": domains = ", ".join(f"{d['zone']} ({d['name']})" for d in energy.get("domains", [])) baseline = "an idle window of the same length after every run" if energy.get("idle_baseline") else "none" - lines.append(f"- Energy: RAPL zones {domains}; idle baseline: {baseline}.") + simulated = "; readings simulated by a test build, not measured" if energy.get("simulated") else "" + wraps = energy.get("counter_wraps") + wrapped = f"; {wraps:,} counter wrap{'' if wraps == 1 else 's'} handled" if isinstance(wraps, int) else "" + lines.append(f"- Energy: RAPL zones {domains}; idle baseline: {baseline}{wrapped}{simulated}.") else: reason = energy.get("reason") lines.append(f"- Energy: {energy.get('status', 'unknown')}." + (f" Reason: {reason}." if reason else "")) @@ -205,6 +208,13 @@ def _environment(meta: dict[str, Any], runs: list[Run], summary: Summary, option def _banners(meta: dict[str, Any], summary: Summary, options: ReportOptions, reason: str) -> list[str]: lines = [] + if meta.get("energy", {}).get("simulated"): + lines += [ + "> **Simulated energy.** This run used build/greenbench-simulated, a test build of the harness " + "that computes its RAPL readings from the clock. Its energy figures test the pipeline; they are not " + "measurements and must not be published as results.", + "", + ] status = meta.get("status") if status != "complete": lines += [ diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index 4eace98..1be5a75 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -75,6 +75,18 @@ def test_report_with_energy(tmp_path: Path) -> None: assert f"](img/{chart}.png)" in text +def test_simulated_energy_is_labelled(tmp_path: Path) -> None: + runs = synthetic.study(energy=True, sizes=(100, 200)) + summary = summarise(runs, resamples=200) + meta = synthetic.meta(energy=True) + meta["energy"] = {**meta["energy"], "simulated": True, "counter_wraps": 2} + out = tmp_path / "results.md" + write_report(runs, meta, summary, out, tmp_path / "img", options()) + text = out.read_text(encoding="utf-8") + assert "> **Simulated energy.**" in text + assert "2 counter wraps handled; readings simulated by a test build, not measured." in text + + def test_report_names_a_language_that_draws_more_power(tmp_path: Path) -> None: text = render(tmp_path, energy=True, withhold=False, power={"python": 1.5}) assert "| quick | random | 200 | Python / C | 10 (" in text diff --git a/docs/bare-metal.md b/docs/bare-metal.md index f5a4685..4beccef 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -119,11 +119,12 @@ configuration, 2 ms loops) with energy, and writes everything under `results/` i the full run; its numbers are too short and too few to mean anything. The whole script can also be rehearsed without RAPL, for example in a container, against a -fake powercap tree whose counters a helper keeps moving: +fake powercap tree. The script then uses `build/greenbench-simulated`, a test build whose +energy readings are computed from the clock, and the report says the energy is simulated: ```bash python3 tests/integration/powercap.py build tests/fixtures/powercap/laptop-intel.txt /tmp/pc -python3 tests/integration/powercap.py advance /tmp/pc -- ./scripts/run-study.sh --rehearse --yes --powercap-root /tmp/pc +./scripts/run-study.sh --rehearse --yes --powercap-root /tmp/pc ``` ## If something goes wrong diff --git a/docs/decisions.md b/docs/decisions.md index d4913cd..efab3b6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -40,8 +40,7 @@ status 3, and so does `greenbench probe` when the counters do not move. The repo **Consequences.** The same harness runs everywhere with time and operation counts, and no file can be mistaken for an energy measurement. The unit tests check that a corrupt counter is an error rather than a zero, and the integration tests check that frozen counters are -refused. Tests that need energy move the fake counters with a writer thread, as the kernel -would. +refused. Tests that need moving counters use a simulated build of the harness (ADR 18). ## 3. Handle one counter wrap; keep intervals short @@ -287,3 +286,28 @@ within-configuration rho as a check on noise. Synthetic tests include a language which languages and inputs. The report is longer, with tables of ratios that are collapsed under the main ones. Treating the subtracted startup medians as fixed makes the intervals slightly too narrow; the threats to validity say so. + +## 18. Test energy with a simulated-RAPL build, not with moving files + +**Context.** The energy code needs counters that move to be tested end to end, and RAPL is +missing wherever the tests run. The first tests moved the counters of a fake powercap tree +from a Python thread that rewrote the files every half millisecond. On a busy machine the +thread was sometimes not scheduled for tens of milliseconds, so a 20 ms measured interval saw +no energy at all, the next one a burst, and a pause of 100 ms tripped the frozen-counter check. +A review saw the pipeline test fail in 4 of 11 runs on a loaded machine. + +**Decision.** `make simulated` builds `build/greenbench-simulated` from the same sources with +`-DGREENBENCH_SIMULATED_RAPL`. It reads a fake tree's files as usual and adds, at the moment +of each reading, energy computed from `CLOCK_MONOTONIC_RAW` at a set idle power plus extra +power for the processor time its finished worker processes used, wrapping the sum at +`max_energy_range_uj`. The integration tests, the sanitizer run and the study script's +rehearsal on a fake tree use it; the frozen-counter tests keep using `build/greenbench` on +files that stand still. The simulated build says so on standard error, records +`"simulated": true` in `meta.json`, and the report opens with a warning that its energy is +not a measurement. + +**Consequences.** Every measured interval has energy of at least the idle power times its +length, so the tests can check exact lower bounds on every row and do not depend on +scheduling. The simulation code sits behind a compile-time flag and cannot be switched on +in `build/greenbench`. The file-reading code still runs in the simulated build, and the unit +tests rewrite counter files between readings to test parsing and wraparound. diff --git a/scripts/run-study.sh b/scripts/run-study.sh index 750deab..0bdcf18 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -15,6 +15,7 @@ CPUS=auto ASSUME_YES=0 POWERCAP=/sys/class/powercap FAKE_POWERCAP=0 +HARNESS=build/greenbench usage() { cat <<'EOF' @@ -27,9 +28,10 @@ Usage: scripts/run-study.sh [--check | --rehearse] [--cpus LIST | --no-pin] [--y --no-pin do not pin to any CPUs --yes do not ask before making the RAPL counters readable --powercap-root DIR - read RAPL zones from DIR instead of /sys/class/powercap. Only for testing this - script with a fake tree whose counters move (tests/integration/powercap.py - advance); it skips the bare-metal checks, and the energy values are not real + read RAPL zones from DIR instead of /sys/class/powercap. Only for rehearsing this + script without RAPL, on a fake tree (tests/integration/powercap.py build): it + skips the bare-metal checks and uses build/greenbench-simulated, a test build + whose energy readings are computed from the clock, so they are not real EOF } @@ -72,6 +74,8 @@ done if [[ "$MODE" == study ]] && ((FAKE_POWERCAP)); then fail "a fake powercap tree can only be used with --check or --rehearse" fi +# A fake tree's counters stand still; the simulated test build moves them from the clock. +if ((FAKE_POWERCAP)); then HARNESS=build/greenbench-simulated; fi check_platform() { [[ "$(uname -s)" == Linux ]] || fail "RAPL counters need Linux on real hardware; see docs/bare-metal.md" @@ -218,9 +222,10 @@ check_rapl say "== Building and running the unit tests" make -s clean make -s all test +if ((FAKE_POWERCAP)); then make -s simulated; fi say "== RAPL counters" -build/greenbench probe --powercap-root "$POWERCAP" || +"$HARNESS" probe --powercap-root "$POWERCAP" || fail "the RAPL counters cannot be used for the study (see the reason above and docs/bare-metal.md)" if [[ "$MODE" == check ]]; then say "This machine can run the study: ./scripts/run-study.sh" @@ -249,7 +254,7 @@ mapfile -t pin < <(pin_options) if ((${#pin[@]} > 0)); then say "== Pinning the harness and workers to CPUs ${pin[1]}"; else say "== Not pinning to CPUs"; fi say "== Measuring (profile $profile); close other programs and keep the machine on mains power" -build/greenbench run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ +"$HARNESS" run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ --label "run-study.sh $MODE $stamp" "${pin[@]}" say "== Report" diff --git a/src/greenbench.c b/src/greenbench.c index b75a8a3..f119f6e 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -467,6 +467,7 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure json_key_string(f, " ", "powercap_root", o->powercap_root, 1); fprintf(f, " \"idle_baseline\": %s,\n", ctx->baseline ? "true" : "false"); fprintf(f, " \"counter_wraps\": %" PRIu64 ",\n", ctx->counter_wraps); + fprintf(f, " \"simulated\": %s,\n", RAPL_SIMULATED && ctx->energy_ok ? "true" : "false"); fputs(" \"domains\": [", f); for (size_t i = 0; ctx->energy_ok && i < ctx->rapl.count; i++) { const t_rapl_domain *d = &ctx->rapl.domains[i]; @@ -1058,6 +1059,10 @@ static int command_run(int argc, char **argv, const char *argv0) { cleanup(&ctx); return EXIT_ENERGY; } + if (ctx.energy_ok && RAPL_SIMULATED) + fprintf(stderr, "%s: this test build SIMULATES the RAPL readings from the clock; its energy values are not " + "measurements (build/greenbench makes real ones)\n", + g_program); if (!o->quiet) { if (ctx.energy_ok) fprintf(stderr, "energy: reading %zu RAPL zone(s) under %s%s\n", ctx.rapl.count, o->powercap_root, @@ -1142,6 +1147,8 @@ static int command_probe(int argc, char **argv) { printf("energy: unavailable\nreason: %s\n", rapl.reason); return EXIT_ENERGY; } + if (RAPL_SIMULATED) + printf("note: this test build simulates the readings from the clock; they are not measurements\n"); printf("zones:\n"); for (size_t i = 0; i < rapl.count; i++) printf(" %-16s %-12s %-8s wraps at %" PRIu64 " uJ\n", rapl.domains[i].zone, rapl.domains[i].name, diff --git a/src/rapl.c b/src/rapl.c index d72f51b..fadc366 100644 --- a/src/rapl.c +++ b/src/rapl.c @@ -14,6 +14,68 @@ #define MAX_ZONES 64 #define ZONE_PREFIX "intel-rapl:" +#ifdef GREENBENCH_SIMULATED_RAPL +/* Test build only (build/greenbench-simulated, never build/greenbench). Tests need counters + that move, and a program rewriting fake counter files cannot keep up on a busy machine: when + it is not scheduled for a while, a measured interval sees no energy at all. So this build + reads the fake tree's files as usual and adds energy computed at the moment of reading: + package: GREENBENCH_SIMULATED_IDLE_W watts (10 by default) for the time since rapl_open, + plus GREENBENCH_SIMULATED_BUSY_W minus that for the processor time used by the + harness's finished child processes, the workers (busy = idle by default); + DRAM: GREENBENCH_SIMULATED_DRAM_SHARE (0.1 by default) of its package's energy. + The sum wraps at max_energy_range_uj like a hardware counter, so the wrap code runs too. */ +#include + +#include "timing.h" + +static struct { + uint64_t start_ns; + double start_busy_s; + double idle_w; + double busy_w; + double dram_share; +} g_simulation; + +static double children_seconds(void) { + struct rusage usage; + + if (getrusage(RUSAGE_CHILDREN, &usage) != 0) + return 0.0; + return (double)usage.ru_utime.tv_sec + (double)usage.ru_utime.tv_usec / 1e6 + (double)usage.ru_stime.tv_sec + + (double)usage.ru_stime.tv_usec / 1e6; +} + +static double setting(const char *name, double fallback) { + const char *text = getenv(name); + char *end; + + if (text == NULL || *text == '\0') + return fallback; + double value = strtod(text, &end); + return *end == '\0' && value >= 0 ? value : fallback; +} + +static void simulation_start(void) { + g_simulation.idle_w = setting("GREENBENCH_SIMULATED_IDLE_W", 10.0); + g_simulation.busy_w = setting("GREENBENCH_SIMULATED_BUSY_W", g_simulation.idle_w); + if (g_simulation.busy_w < g_simulation.idle_w) + g_simulation.busy_w = g_simulation.idle_w; /* energy must never run backwards */ + g_simulation.dram_share = setting("GREENBENCH_SIMULATED_DRAM_SHARE", 0.1); + g_simulation.start_ns = timing_now_ns(); + g_simulation.start_busy_s = children_seconds(); +} + +static uint64_t simulated_uj(t_rapl_kind kind) { + double seconds = (double)(timing_now_ns() - g_simulation.start_ns) / 1e9; + double busy = children_seconds() - g_simulation.start_busy_s; + double joules = g_simulation.idle_w * seconds + (g_simulation.busy_w - g_simulation.idle_w) * busy; + + if (kind == RAPL_DRAM) + joules *= g_simulation.dram_share; + return joules > 0 ? (uint64_t)(joules * 1e6) : 0; +} +#endif + int rapl_parse_u64(const char *text, uint64_t *out) { const char *p = text; uint64_t value = 0; @@ -209,6 +271,9 @@ int rapl_open(t_rapl *rapl, const char *root) { memset(rapl, 0, sizeof *rapl); for (size_t i = 0; i < RAPL_MAX_DOMAINS; i++) rapl->domains[i].fd = -1; +#ifdef GREENBENCH_SIMULATED_RAPL + simulation_start(); +#endif if (root == NULL) root = RAPL_DEFAULT_ROOT; if (strlen(root) >= sizeof rapl->root - RAPL_ZONE_LEN - 32) { @@ -263,6 +328,10 @@ int rapl_read(const t_rapl *rapl, t_rapl_sample *sample) { errno = EINVAL; return -1; } +#ifdef GREENBENCH_SIMULATED_RAPL + if (sample->uj[i] <= rapl->domains[i].max_range_uj) + sample->uj[i] = (sample->uj[i] + simulated_uj(rapl->domains[i].kind)) % (rapl->domains[i].max_range_uj + 1); +#endif } return 0; } diff --git a/src/rapl.h b/src/rapl.h index b34763f..0b838cc 100644 --- a/src/rapl.h +++ b/src/rapl.h @@ -33,6 +33,15 @@ #define RAPL_PLAUSIBLE_WATTS 1000u #define RAPL_PLAUSIBLE_SLACK_NS 10000000u /* 10 ms */ +/* 1 only in the test build made with -DGREENBENCH_SIMULATED_RAPL (build/greenbench-simulated), + whose readings are computed from the clock (see rapl_read in rapl.c); 0 in every real build. + The harness records it in meta.json, and the report says that such energy is simulated. */ +#ifdef GREENBENCH_SIMULATED_RAPL +#define RAPL_SIMULATED 1 +#else +#define RAPL_SIMULATED 0 +#endif + typedef enum e_rapl_kind { RAPL_PACKAGE, RAPL_DRAM } t_rapl_kind; typedef struct s_rapl_domain { @@ -80,7 +89,8 @@ int rapl_classify(const char *zone, const char *name, t_rapl_kind *kind); rapl->reason; nothing is left open in that case. */ int rapl_open(t_rapl *rapl, const char *root); -/* Reads every open counter. Returns 0, or -1 with errno set. */ +/* Reads every open counter. Returns 0, or -1 with errno set. In the simulated test build, + each reading is the file's value plus energy computed from the clock. */ int rapl_read(const t_rapl *rapl, t_rapl_sample *sample); /* Energy from a to b, and how many counters wrapped. Returns 0, or -1 if a counter reading diff --git a/tests/fixtures/powercap/README.md b/tests/fixtures/powercap/README.md index 80028d4..4bf30a0 100644 --- a/tests/fixtures/powercap/README.md +++ b/tests/fixtures/powercap/README.md @@ -19,8 +19,9 @@ The values are invented but shaped like real sysfs output: counters in microjoul | `counter-above-max.txt` | A package whose counter is above its wrap value | | `dram-only.txt` | A DRAM zone without a package zone | -The counters in a built tree stand still. The harness treats a package counter that does not -move as frozen and refuses it, so tests that need energy run with `CounterWriter` from -`tests/integration/powercap.py`, or with `powercap.py advance ROOT -- COMMAND`. It advances -the package and DRAM counters in place, as a live system does, and wraps them at -`max_energy_range_uj`. +The counters in a built tree stand still. `build/greenbench` treats a package counter that +does not move as frozen and refuses it, which the frozen-counter tests rely on. Tests that +need moving counters run `build/greenbench-simulated` (`make simulated`) on the tree instead: +a test build that adds energy computed from the clock, and from the processor time its +workers use, to every reading, and wraps the sum at `max_energy_range_uj` +([ADR 18](../../../docs/decisions.md#18-test-energy-with-a-simulated-rapl-build-not-with-moving-files)). diff --git a/tests/integration/powercap.py b/tests/integration/powercap.py index 775d63f..80ab3db 100644 --- a/tests/integration/powercap.py +++ b/tests/integration/powercap.py @@ -1,28 +1,23 @@ -"""Fake powercap trees for testing: build one from a manifest, and make its counters move. +"""Fake powercap trees for testing, built from a manifest. Each manifest line in tests/fixtures/powercap is a path relative to the powercap root, whitespace, then the file's contents. Zone directories are named like ``intel-rapl:0``, which -Windows cannot store, so the trees are kept as manifests and built on demand. - -A built tree's counters stand still, which the harness rejects as frozen. CounterWriter moves -them the way live RAPL counters move, so the energy code runs end to end: +Windows cannot store, so the trees are kept as manifests and built on demand: python3 tests/integration/powercap.py build tests/fixtures/powercap/laptop-intel.txt /tmp/pc - python3 tests/integration/powercap.py advance /tmp/pc -- build/greenbench run --powercap-root /tmp/pc + +A built tree's counters stand still, which build/greenbench rejects as frozen. For moving +counters, point build/greenbench-simulated (``make simulated``) at the tree: it adds energy +computed from the clock to every reading, so tests do not depend on another program being +scheduled in time to rewrite the files. """ from __future__ import annotations import argparse -import os import shutil -import subprocess import sys -import threading -import time -from dataclasses import dataclass from pathlib import Path -from types import TracebackType def build(manifest: Path, root: Path) -> Path: @@ -44,142 +39,15 @@ def build(manifest: Path, root: Path) -> Path: return root -@dataclass -class _Counter: - zone: str - kind: str # "package" or "dram" - fd: int - maximum: int - width: int - value: float - wraps: int = 0 - - -class CounterWriter: - """Moves the package and DRAM counters of a fake powercap tree, like live RAPL counters. - - Package counters gain ``idle_watts`` (or ``busy_watts`` while ``busy_marker`` exists, which - a fake worker can create while it runs), DRAM counters gain ``dram_share`` of that, and - every counter wraps to zero after its ``max_energy_range_uj``. - - The harness keeps the counter files open and re-reads them with pread, so values are - written in place, never by replacing the file. Each is one fixed-width, zero-padded write - of as many digits as the maximum has; a read that overlaps a write therefore still parses - and stays within range. - """ - - def __init__( - self, - root: Path, - *, - idle_watts: float = 10.0, - busy_watts: float | None = None, - busy_marker: Path | None = None, - dram_share: float = 0.1, - interval_s: float = 0.0005, - ) -> None: - self.root = root - self.idle_watts = idle_watts - self.busy_watts = idle_watts if busy_watts is None else busy_watts - self.busy_marker = busy_marker - self.dram_share = dram_share - self.interval_s = interval_s - self._counters: list[_Counter] = [] - self._stop = threading.Event() - self._thread: threading.Thread | None = None - self._last_ns = 0 - - @property - def package_wraps(self) -> int: - """How many times a package counter has wrapped since start().""" - return sum(c.wraps for c in self._counters if c.kind == "package") - - def _open(self) -> None: - for zone in sorted(self.root.glob("intel-rapl:*")): - name_file = zone / "name" - if not name_file.is_file(): - continue - name = name_file.read_text(encoding="utf-8").strip() - kind = "package" if name.startswith("package") else "dram" if name == "dram" else None - if kind is None: - continue - maximum = int((zone / "max_energy_range_uj").read_text(encoding="utf-8").strip()) - value = int((zone / "energy_uj").read_text(encoding="utf-8").strip()) - fd = os.open(zone / "energy_uj", os.O_WRONLY) - self._counters.append(_Counter(zone.name, kind, fd, maximum, len(str(maximum)), float(value))) - if not any(c.kind == "package" for c in self._counters): - raise ValueError(f"{self.root}: no package zone to move") - - def _write(self, counter: _Counter) -> None: - text = f"{int(counter.value):0{counter.width}d}\n".encode("ascii") - os.pwrite(counter.fd, text, 0) - - def tick(self) -> None: - """Advance every counter by the energy used since the previous tick.""" - now = time.monotonic_ns() - elapsed_s = (now - self._last_ns) / 1e9 - self._last_ns = now - busy = self.busy_marker is not None and self.busy_marker.exists() - watts = self.busy_watts if busy else self.idle_watts - for counter in self._counters: - share = 1.0 if counter.kind == "package" else self.dram_share - counter.value += watts * share * elapsed_s * 1e6 - while counter.value > counter.maximum: - counter.value -= counter.maximum + 1 - counter.wraps += 1 - self._write(counter) - - def _run(self) -> None: - while not self._stop.wait(self.interval_s): - self.tick() - - def start(self) -> CounterWriter: - """Rewrite every counter at its fixed width, then keep them moving in a thread.""" - self._open() - for counter in self._counters: - self._write(counter) - self._last_ns = time.monotonic_ns() - self._thread = threading.Thread(target=self._run, name="powercap-writer", daemon=True) - self._thread.start() - return self - - def stop(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join() - for counter in self._counters: - os.close(counter.fd) - self._counters.clear() - - def __enter__(self) -> CounterWriter: - return self.start() - - def __exit__( - self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None - ) -> None: - self.stop() - - def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(prog="powercap.py", description="Build fake powercap trees for testing.") commands = parser.add_subparsers(dest="command", required=True) make = commands.add_parser("build", help="build a tree from a manifest, replacing OUTPUT_DIR") make.add_argument("manifest", type=Path) make.add_argument("output_dir", type=Path) - advance = commands.add_parser("advance", help="run COMMAND while the tree's counters move") - advance.add_argument("root", type=Path) - advance.add_argument("--watts", type=float, default=10.0, help="package power (default %(default)s)") - advance.add_argument("run", nargs=argparse.REMAINDER, metavar="-- COMMAND", help="the command to run") args = parser.parse_args(argv[1:]) - - if args.command == "build": - build(args.manifest, args.output_dir) - return 0 - command = args.run[1:] if args.run[:1] == ["--"] else args.run - if not command: - parser.error("advance needs a command after --") - with CounterWriter(args.root, idle_watts=args.watts): - return subprocess.run(command, check=False).returncode + build(args.manifest, args.output_dir) + return 0 if __name__ == "__main__": diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 86fc5de..e5e3f13 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -5,15 +5,15 @@ import csv import json import os +import re import shlex from pathlib import Path from typing import Any import pytest -from powercap import CounterWriter from powercap import build as build_powercap -from workers import HARNESS, REPOSITORY, interpreter_options, require_build, run_harness +from workers import HARNESS, REPOSITORY, interpreter_options, require_build, run_harness, run_simulated_harness FIXTURES = REPOSITORY / "tests" / "fixtures" / "powercap" SMALL = ["--profile", "smoke", "--sizes", "20,50", "--orders", "random,sorted", "--runs", "3", "--quiet"] @@ -160,38 +160,43 @@ def test_counts_match_across_languages_in_the_csv(tmp_path: Path) -> None: def test_moving_counters_fill_the_energy_columns(tmp_path: Path) -> None: + # The simulated build: each package draws exactly 10 W, however busy this machine is. root = build_powercap(FIXTURES / "server-two-socket.txt", tmp_path / "powercap") out = tmp_path / "run" - with CounterWriter(root, idle_watts=10.0, dram_share=0.2): - completed = run_harness( - "run", - *SMALL, - "--languages", - "c", - "--energy", - "require", - "--powercap-root", - str(root), - "--out", - str(out), - *interpreter_options(), - ) + completed = run_simulated_harness( + "run", + *SMALL, + "--languages", + "c", + "--energy", + "require", + "--powercap-root", + str(root), + "--out", + str(out), + *interpreter_options(), + idle_watts=10.0, + dram_share=0.2, + ) assert completed.returncode == 0, completed.stderr + assert "SIMULATES the RAPL readings" in completed.stderr rows = read_runs(out) for row in rows: assert row["energy_status"] == "ok" - for column in ("pkg_uj", "dram_uj", "idle_pkg_uj", "idle_dram_uj"): - assert int(row[column]) >= 0, f"{column} is a whole number of microjoules" - assert int(row["idle_ns"]) >= int(row["wall_ns"]) - expected = int(row["pkg_uj"]) - int(row["idle_pkg_uj"]) * int(row["wall_ns"]) / int(row["idle_ns"]) + wall, idle_ns = int(row["wall_ns"]), int(row["idle_ns"]) + assert idle_ns >= wall + # Two packages at 10 W: at least 20 uJ per microsecond, less at most one unit per counter. + assert int(row["pkg_uj"]) >= 20 * wall // 1000 - 2 + assert int(row["idle_pkg_uj"]) >= 20 * idle_ns // 1000 - 2 + assert int(row["dram_uj"]) >= 4 * wall // 1000 - 2 # two DRAM zones at 20% of 10 W + expected = int(row["pkg_uj"]) - int(row["idle_pkg_uj"]) * wall / idle_ns assert float(row["net_pkg_uj"]) == pytest.approx(expected, abs=0.051) - # Both sockets' packages move at 10 W, so the package total is about 20 W times the time measured. seconds = sum(int(r["wall_ns"]) + int(r["idle_ns"]) for r in rows) / 1e9 joules = sum(int(r["pkg_uj"]) + int(r["idle_pkg_uj"]) for r in rows) / 1e6 assert joules == pytest.approx(20.0 * seconds, rel=0.5) - assert sum(int(r["dram_uj"]) for r in rows) > 0 meta = read_meta(out) assert meta["energy"]["status"] == "ok" + assert meta["energy"]["simulated"] is True assert meta["energy"]["idle_baseline"] is True assert [d["zone"] for d in meta["energy"]["domains"]] == [ "intel-rapl:0", @@ -257,16 +262,19 @@ def test_energy_auto_falls_back_to_time_only(tmp_path: Path) -> None: def test_probe(tmp_path: Path) -> None: root = build_powercap(FIXTURES / "laptop-intel.txt", tmp_path / "powercap") - with CounterWriter(root, idle_watts=5.0): - live = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "200") + live = run_simulated_harness("probe", "--powercap-root", str(root), "--sample-ms", "200", idle_watts=5.0) assert live.returncode == 0, live.stdout + live.stderr + assert "note: this test build simulates the readings" in live.stdout assert "intel-rapl:0:2" in live.stdout assert "psys" not in live.stdout assert "idle power over" in live.stdout assert live.stdout.rstrip().endswith("energy: available") + # The real build reads the same files, which stand still: frozen. + frozen = run_harness("probe", "--powercap-root", str(root), "--sample-ms", "200") assert frozen.returncode == 3 + assert "simulates" not in frozen.stdout assert "energy: unavailable" in frozen.stdout assert "did not change over" in frozen.stdout @@ -279,6 +287,27 @@ def test_probe(tmp_path: Path) -> None: assert "energy: unavailable" in missing.stdout +def test_a_counter_wrap_is_handled(tmp_path: Path) -> None: + # 1 J below the wrap of a 1 kJ counter. The probe reads the counters just after opening + # them, and again 200 ms later at 10 W, so the wrap falls inside that interval every time, + # and no interval on this machine can last the 100 s that a second wrap would need. + manifest = tmp_path / "tree.txt" + zone = "intel-rapl:0" + manifest.write_text( + f"{zone}/name package-0\n{zone}/energy_uj 999000000\n{zone}/max_energy_range_uj 999999999\n", + encoding="utf-8", + ) + root = build_powercap(manifest, tmp_path / "powercap") + probe = run_simulated_harness("probe", "--powercap-root", str(root), "--sample-ms", "200", idle_watts=10.0) + assert probe.returncode == 0, probe.stdout + probe.stderr + match = re.search(r"idle power over [0-9.]+ s: package ([0-9.]+) W", probe.stdout) + assert match is not None, probe.stdout + # About 10 W: the probe divides by its own clock readings, which can differ a little from + # the moments the counters were read if this process is paused in between. A wrap read + # wrongly would give about 5 kW, or fail as implausible. + assert 8 <= float(match.group(1)) < 100 + + def test_dry_run_prints_the_plan() -> None: completed = run_harness("run", "--dry-run") assert completed.returncode == 0 diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 2dd2c45..f4e92f3 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv +import json import os import subprocess import sys @@ -10,19 +11,19 @@ import pytest -from powercap import CounterWriter from powercap import build as build_powercap -from workers import REPOSITORY, interpreter_options, require_build, run_harness +from workers import REPOSITORY, interpreter_options, require_build, run_harness, run_simulated_harness -# A small package counter (100 J) that wraps every few seconds at these powers, so that some -# measured intervals span a wrap, and a DRAM zone beside it. One run uses a few joules. -WRAPPING_TREE = """ +# One package and a DRAM zone beside it. The range (1 kJ) is far more than any one interval +# can use, however slow the machine, so no interval spans two wraps; test_harness.py checks a +# wrap on its own, where it can be placed exactly. +TREE = """ intel-rapl:0/name package-0 -intel-rapl:0/energy_uj 95000000 -intel-rapl:0/max_energy_range_uj 99999999 +intel-rapl:0/energy_uj 500000000 +intel-rapl:0/max_energy_range_uj 999999999 intel-rapl:0:0/name dram intel-rapl:0:0/energy_uj 500000 -intel-rapl:0:0/max_energy_range_uj 99999999 +intel-rapl:0:0/max_energy_range_uj 999999999 """ @@ -97,77 +98,63 @@ def test_harness_then_report(tmp_path: Path) -> None: def test_moving_counters_through_the_harness_into_the_report(tmp_path: Path) -> None: - """Non-zero energy from end to end: counters that move (faster while a worker runs, and - wrapping), through the harness's net-of-idle columns, into every energy section.""" + """Non-zero energy from end to end: counters that move faster while a worker runs, through + the harness's net-of-idle columns, into every energy section. + + The simulated build computes each reading from the clock and from the processor time the + workers used, so the checks below hold however busy the machine is. The C worker starts in + about a millisecond of processor time, so what a sort adds stays clear of the noise in + subtracting startup, even when the machine is busy.""" require_build() manifest = tmp_path / "tree.txt" - manifest.write_text(WRAPPING_TREE, encoding="utf-8") + manifest.write_text(TREE, encoding="utf-8") root = build_powercap(manifest, tmp_path / "powercap") - - # A Python worker that marks itself busy while it runs, so the fake package draws more - # power during runs than during the idle windows that follow them. - busy = tmp_path / "busy" - real = REPOSITORY / "workers" / "python" / "greenbench_worker.py" - wrapper = tmp_path / "busy_worker.py" - wrapper.write_text( - "\n".join( - [ - "import pathlib, runpy, sys", - f"busy = pathlib.Path({str(busy)!r})", - "busy.touch()", - "try:", - f" sys.argv[0] = {str(real)!r}", - f" runpy.run_path({str(real)!r}, run_name='__main__')", - "finally:", - " busy.unlink(missing_ok=True)", - ] - ) - + "\n", - encoding="utf-8", - ) out = tmp_path / "run" - writer = CounterWriter(root, idle_watts=10.0, busy_watts=30.0, busy_marker=busy, dram_share=0.1) - with writer: - completed = run_harness( - "run", - "--profile", - "smoke", - "--languages", - "python", - "--algorithms", - "bubble,quick", - "--orders", - "random,sorted", - "--sizes", - "30,90", - "--runs", - "3", - "--min-loop-ms", - "20", - "--energy", - "require", - "--powercap-root", - str(root), - "--python-worker", - str(wrapper), - "--quiet", - "--out", - str(out), - *interpreter_options(), - ) - wraps = writer.package_wraps + completed = run_simulated_harness( + "run", + "--profile", + "smoke", + "--languages", + "c", + "--algorithms", + "bubble,quick", + "--orders", + "random,sorted", + "--sizes", + "30,90", + "--runs", + "3", + "--min-loop-ms", + "20", + "--energy", + "require", + "--powercap-root", + str(root), + "--quiet", + "--out", + str(out), + *interpreter_options(), + idle_watts=10.0, + busy_watts=30.0, + dram_share=0.1, + ) assert completed.returncode == 0, completed.stderr - assert wraps >= 1, "the package counter should have wrapped during the run" + energy = json.loads((out / "meta.json").read_text(encoding="utf-8"))["energy"] + assert energy["simulated"] is True with (out / "runs.csv").open(newline="", encoding="utf-8") as handle: rows = list(csv.DictReader(handle)) for row in rows: pkg, idle = int(row["pkg_uj"]), int(row["idle_pkg_uj"]) wall, idle_ns = int(row["wall_ns"]), int(row["idle_ns"]) - assert 0 < pkg < 50_000_000, "a wrap handled wrongly would give a value near 100 J or above" + # 10 W is 10 uJ per microsecond, less one unit of rounding and, if the counter wrapped + # during the interval, the one unit that the wrap formula drops. + assert 10 * wall // 1000 - 2 <= pkg < 500_000_000 + assert 10 * idle_ns // 1000 - 2 <= idle < 500_000_000 assert int(row["dram_uj"]) > 0 assert float(row["net_pkg_uj"]) == pytest.approx(pkg - idle * wall / idle_ns, abs=0.051) - # The worker draws 20 W more than idle while it runs, so most runs have positive net energy. + # The workers draw 20 W more than idle for the processor time they use, so most runs have + # positive net energy. assert sum(float(r["net_pkg_uj"]) > 0 for r in rows) > len(rows) * 0.8 results = tmp_path / "results.md" @@ -185,10 +172,11 @@ def test_moving_counters_through_the_harness_into_the_report(tmp_path: Path) -> text = results.read_text(encoding="utf-8") assert "energy unavailable" not in text assert "Energy was not measured" not in text + assert "Simulated energy" in text assert "- Energy: RAPL zones intel-rapl:0 (package-0), intel-rapl:0:0 (dram)" in text assert " W (" in text # power while sorting assert "### Between input orders" in text - assert "| Python | quick | 90 | sorted / random |" in text + assert "| C | quick | 90 | sorted / random |" in text assert "Spearman's rho" in text assert "| every configuration | 8 |" in text assert "CO2e, United Arab Emirates (net to including idle)" in text diff --git a/tests/integration/workers.py b/tests/integration/workers.py index 69bdb0e..3c81454 100644 --- a/tests/integration/workers.py +++ b/tests/integration/workers.py @@ -1,7 +1,9 @@ """How the integration tests find and start the three workers and the harness. The binaries come from `make` (build/ by default; set GREENBENCH_BUILD to test another -build, such as the sanitizer one). Node.js and Python are found on PATH. +build, such as the sanitizer one). Node.js and Python are found on PATH. Tests that need +moving energy counters use build/greenbench-simulated (`make simulated`), whose RAPL +readings are computed from the clock, so they do not depend on how busy the machine is. """ from __future__ import annotations @@ -15,6 +17,7 @@ REPOSITORY = Path(__file__).resolve().parents[2] BUILD = Path(os.environ.get("GREENBENCH_BUILD", REPOSITORY / "build")) HARNESS = BUILD / "greenbench" +SIMULATED_HARNESS = BUILD / "greenbench-simulated" C_WORKER = BUILD / "greenbench-worker" PYTHON_WORKER = REPOSITORY / "workers" / "python" / "greenbench_worker.py" NODE_WORKER = REPOSITORY / "workers" / "node" / "worker.js" @@ -61,5 +64,23 @@ def run_harness(*args: str, cwd: Path | None = None) -> subprocess.CompletedProc ) +def run_simulated_harness( + *args: str, idle_watts: float = 10.0, busy_watts: float | None = None, dram_share: float = 0.1 +) -> subprocess.CompletedProcess[str]: + """Run build/greenbench-simulated: each package draws idle_watts, plus busy_watts minus that + for the processor time its worker processes use; each DRAM zone draws dram_share of that.""" + if not SIMULATED_HARNESS.exists(): + raise RuntimeError(f"missing {SIMULATED_HARNESS}; run 'make simulated' before the integration tests") + env = dict( + os.environ, + GREENBENCH_SIMULATED_IDLE_W=str(idle_watts), + GREENBENCH_SIMULATED_BUSY_W=str(idle_watts if busy_watts is None else busy_watts), + GREENBENCH_SIMULATED_DRAM_SHARE=str(dram_share), + ) + return subprocess.run( + [str(SIMULATED_HARNESS), *args], capture_output=True, text=True, check=False, timeout=600, env=env + ) + + def interpreter_options() -> list[str]: return ["--python", sys.executable, "--node", node()] From fde62e79933dcbce3d7d94d586562f5f7312788a Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:31:27 +0400 Subject: [PATCH 36/51] feat: give rank agreement for each language and algorithm The study asks for Spearman's correlation between time and energy per language and algorithm. The only figure at that level was the median of within-configuration rho values, which is a noise check. Rank agreement across configurations now also covers each language and algorithm, across input orders and sizes, with the same bootstrap interval. RankAgreement now holds its scope as language, algorithm and size, and the report puts it into words. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/report.py | 22 ++++++++++--- analysis/greenbench_analysis/summary.py | 25 ++++++++++---- analysis/tests/test_report.py | 2 ++ analysis/tests/test_summary.py | 44 ++++++++++++++++++++----- docs/study.md | 5 +-- 5 files changed, 76 insertions(+), 22 deletions(-) diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 3ff715c..2c5abf1 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -18,7 +18,7 @@ from .carbon import GridIntensity, grams_co2e from .load import ORDERS, Run from .stats import Estimate -from .summary import Comparison, Configuration, Summary +from .summary import Comparison, Configuration, RankAgreement, Summary LANGUAGE_NAMES = {"c": "C", "python": "Python", "node": "Node.js"} STUDY_RUNS = 30 # measured runs per configuration in the study profile @@ -390,6 +390,17 @@ def _order_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: RATIO_ALIGN = ["---:", "---:", "---:", ":---"] +def _scope(agreement: RankAgreement) -> str: + """Which configurations a rank agreement covers, in words.""" + if agreement.language is not None and agreement.algorithm is not None: + return f"{LANGUAGE_NAMES[agreement.language]} {agreement.algorithm}: every order and size" + if agreement.language is not None: + return f"{LANGUAGE_NAMES[agreement.language]}: every algorithm, order and size" + if agreement.n is not None: + return f"every language, algorithm and order at {agreement.n:,} elements" + return "every configuration" + + def _describe(c: Configuration) -> str: return f"the {LANGUAGE_NAMES[c.language]} {c.algorithm} sort of a {c.order.replace('_', ' ')} input of {c.n:,}" @@ -473,14 +484,15 @@ def _proxy_section(summary: Summary, options: ReportOptions, image: str | None, lines += ["", ""] lines += [""] - scope_names = {"c:": "C:", "python:": "Python:", "node:": "Node.js:"} lines += [ "### Rank agreement across configurations", "", "Spearman's rank correlation between median time per sort and median energy per sort, over " "configurations, with a 95% bootstrap interval: does energy put configurations in the same order as " - "time does? This is a weaker test than the ratios above. Configurations differ in cost by large " - "factors, so rho can stay close to 1 even when their power differs.", + "time does? It is given at each input size, for each language, for each language and algorithm " + "across input orders and sizes, and over everything. This is a weaker test than the ratios above. " + "Configurations differ in cost by large factors, so rho can stay close to 1 even when their power " + "differs.", "", ] lines += _table( @@ -488,7 +500,7 @@ def _proxy_section(summary: Summary, options: ReportOptions, image: str | None, [":---", "---:", "---:"], ( [ - next((a.scope.replace(k, v, 1) for k, v in scope_names.items() if a.scope.startswith(k)), a.scope), + _scope(a), f"{a.configurations:,}", "undefined" if a.rho is None else f"{a.rho.value:.3f} ({a.rho.low:.3f} to {a.rho.high:.3f})", ] diff --git a/analysis/greenbench_analysis/summary.py b/analysis/greenbench_analysis/summary.py index 412e1fc..ad3026a 100644 --- a/analysis/greenbench_analysis/summary.py +++ b/analysis/greenbench_analysis/summary.py @@ -32,6 +32,7 @@ from .stats import Estimate, FloatArray, bootstrap_median, interval, median, ratio_draws, resample_medians, spearman Key = tuple[str, str, str, int] # language, algorithm, order, n +_Scope = tuple[str | None, str | None, int | None] # language, algorithm, n; None for all # Energy in microjoules over time in nanoseconds is in kilowatts; this gives watts. WATTS_PER_UJ_PER_NS = 1e3 @@ -103,9 +104,14 @@ class Comparison: @dataclass(frozen=True) class RankAgreement: """Spearman's rho between median time per sort and median energy per sort, across - configurations: does energy put them in the same order as time does?""" + configurations: does energy put them in the same order as time does? - scope: str + The scope is every configuration with the given language, algorithm and size; None + stands for all of them.""" + + language: str | None + algorithm: str | None + n: int | None configurations: int rho: Estimate | None @@ -284,15 +290,20 @@ def _rank_agreement( configurations: list[Configuration], draws: dict[Key, _Draws], resamples: int ) -> list[RankAgreement]: measured = [c for c in configurations if c.energy_per_sort_uj is not None] - scopes: list[tuple[str, list[Configuration]]] = [] + scopes: list[tuple[_Scope, list[Configuration]]] = [] for n in sorted({c.n for c in measured}): - scopes.append((f"every language, algorithm and order at {n:,} elements", [c for c in measured if c.n == n])) + scopes.append(((None, None, n), [c for c in measured if c.n == n])) for language in LANGUAGES: chosen = [c for c in measured if c.language == language] if chosen: - scopes.append((f"{language}: every algorithm, order and size", chosen)) + scopes.append(((language, None, None), chosen)) + for language in LANGUAGES: + for algorithm in ALGORITHMS: + chosen = [c for c in measured if c.language == language and c.algorithm == algorithm] + if chosen: + scopes.append(((language, algorithm, None), chosen)) if measured: - scopes.append(("every configuration", measured)) + scopes.append(((None, None, None), measured)) result = [] for scope, chosen in scopes: @@ -305,7 +316,7 @@ def _rank_agreement( energy_draws = np.stack([_energy_draws(draws[c.key])[:resamples] for c in chosen]) rhos = [spearman(time_draws[:, r], energy_draws[:, r]) for r in range(time_draws.shape[1])] estimate = interval(point, np.array([np.nan if rho is None else rho for rho in rhos])) - result.append(RankAgreement(scope, len(chosen), estimate)) + result.append(RankAgreement(*scope, len(chosen), estimate)) return result diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index 1be5a75..7145ab9 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -65,6 +65,8 @@ def test_report_with_energy(tmp_path: Path) -> None: assert "same power: the time ratio predicts the energy ratio" in text assert "### Between input orders" in text assert "| every configuration | 16 |" in text + assert "| C quick: every order and size | 4 |" in text + assert "| Python: every algorithm, order and size | 8 |" in text assert "| C | quick | 4 | 1.000 |" in text # run-to-run coupling over four configurations # Per 1e6 sorts: 2 kJ net and 3 kJ including idle; at 467.51 g/kWh, 260 mg and 390 mg. assert "| C | quick | 2.00 kJ | 3.00 kJ | 260 mg to 390 mg |" in text diff --git a/analysis/tests/test_summary.py b/analysis/tests/test_summary.py index 7a816d4..20325b0 100644 --- a/analysis/tests/test_summary.py +++ b/analysis/tests/test_summary.py @@ -5,7 +5,7 @@ import pytest import synthetic -from greenbench_analysis.load import DataError +from greenbench_analysis.load import DataError, Run from greenbench_analysis.summary import Comparison, Configuration, Summary, summarise @@ -135,22 +135,50 @@ def test_input_orders_are_compared_with_random_input() -> None: def test_rank_agreement_across_configurations() -> None: summary = summarise(synthetic.study(energy=True, sizes=(100, 200)), resamples=300, rho_resamples=200) - scopes = [a.scope for a in summary.rank_agreement] + scopes = [(a.language, a.algorithm, a.n, a.configurations) for a in summary.rank_agreement] assert scopes == [ - "every language, algorithm and order at 100 elements", - "every language, algorithm and order at 200 elements", - "c: every algorithm, order and size", - "python: every algorithm, order and size", - "every configuration", + (None, None, 100, 8), + (None, None, 200, 8), + ("c", None, None, 8), + ("python", None, None, 8), + ("c", "bubble", None, 4), + ("c", "quick", None, 4), + ("python", "bubble", None, 4), + ("python", "quick", None, 4), + (None, None, None, 16), ] everything = summary.rank_agreement[-1] - assert everything.configurations == 16 assert everything.rho is not None assert everything.rho.value > 0.8 # C is ten times cheaper than Python in time and energy # Within one language every configuration costs the same, so the ranks are ties: undefined. assert summary.rank_agreement[2].rho is None +def test_rank_agreement_for_each_language_and_algorithm() -> None: + # Larger inputs cost more, in time and energy alike (startup included), so that the ranks + # within one language and algorithm are defined, and agree exactly. + def scaled(run: Run) -> Run: + k = run.n // 100 + assert run.pkg_uj is not None + assert run.idle_pkg_uj is not None + assert run.net_pkg_uj is not None + return dataclasses.replace( + run, + wall_ns=run.wall_ns * k, + idle_ns=run.wall_ns * k, + pkg_uj=run.pkg_uj * k, + idle_pkg_uj=run.idle_pkg_uj * k, + net_pkg_uj=run.net_pkg_uj * k, + ) + + runs = [scaled(r) for r in synthetic.study(energy=True, sizes=(100, 200, 300))] + summary = summarise(runs, resamples=300, rho_resamples=200) + c_quick = next(a for a in summary.rank_agreement if (a.language, a.algorithm) == ("c", "quick")) + assert c_quick.configurations == 6 # two orders at three sizes + assert c_quick.rho is not None + assert c_quick.rho.value == pytest.approx(1.0) + + def test_noise_coupling_within_configurations() -> None: summary = summarise(synthetic.study(energy=True, runs=7), resamples=200) assert [(n.language, n.algorithm) for n in summary.noise_coupling] == [ diff --git a/docs/study.md b/docs/study.md index 94c5e18..107ae22 100644 --- a/docs/study.md +++ b/docs/study.md @@ -131,8 +131,9 @@ lists, JavaScript arrays). None uses a built-in sort. same language and algorithm: the time ratio, the energy ratio and the power ratio (energy ratio divided by time ratio). The two configurations are resampled independently. - **Rank agreement.** Spearman's rho between median time per sort and median energy per sort - across configurations: at each input size, for each language, and over all of them, with an - interval from 2,000 of the resamples. + across configurations: at each input size, for each language, for each language and + algorithm across input orders and sizes, and over all of them, with an interval from 2,000 + of the resamples. - **Run-to-run coupling.** Within each configuration, Spearman's rho between the process time and process energy of its 30 runs; the report gives the median per language and algorithm. With steady power, a run that takes longer uses more energy, so a positive value is expected From 944b88a915060fc38635d0f7211b7ec43dbd6ccc Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:31:37 +0400 Subject: [PATCH 37/51] feat: read each power ratio against a margin set before measuring The report called two configurations "same power" whenever the 95% interval of their power ratio included 1, which reads a result that is not significant as proof of equality: a rehearsal printed "same power" for an interval from 0.0006 to 15.7. It also printed about 250 such intervals with no word on how many would exclude 1 by chance. Each power ratio now gets one of four readings against a margin of 10% either way (1/1.1 to 1.1), fixed in docs/study.md: same power within the margin (the whole interval inside it), differs (the interval excludes 1), inconclusive (includes 1 but reaches beyond the margin), or undefined. A ratio is also undefined when a time or energy per sort fell to zero or below in more than 2.5% of the resamples, because an interval from the rest would look too narrow. The report tabulates the readings and states how many intervals exclude 1 against the 5% that chance would give. --equivalence-margin sets another margin. Power is labelled "net power above idle" wherever it is net of the idle baseline, in the tables, the text and the chart. study.md explains the margin, why the question is decided on net power, and the many comparisons, and ADR 19 records the decision. Co-Authored-By: Claude Opus 5.5 --- README.md | 19 +-- analysis/greenbench_analysis/charts.py | 5 +- analysis/greenbench_analysis/cli.py | 21 ++- analysis/greenbench_analysis/report.py | 182 ++++++++++++++++++++---- analysis/greenbench_analysis/summary.py | 14 +- analysis/tests/synthetic.py | 5 +- analysis/tests/test_report.py | 66 ++++++++- analysis/tests/test_summary.py | 15 ++ docs/decisions.md | 25 ++++ docs/study.md | 63 ++++++-- 10 files changed, 358 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index bce19bc..3c7ef42 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,11 @@ compare?** The research question, method and threats to validity are in above the ~1 ms RAPL update interval), an idle baseline of the same length after every run, 30 measured runs per configuration after warm-up, randomised run order, optional pinning to a set of CPUs, startup trials measured over batches of launches, and tidy CSV output. -- **Statistics.** Power while sorting, and time, energy and power - ratios between languages and between input orders, each with a 95% bootstrap interval: a - power ratio of 1 means time predicts energy. Also medians with intervals, rank agreement - across configurations, and a run-to-run noise check, in a Python package with its own tests. +- **Statistics.** Net power above idle, and time, energy and power ratios between languages + and between input orders, each with a 95% bootstrap interval. Each power ratio is read + against a 10% margin set before measuring: the same power (so time predicts energy), + different, or inconclusive. Also medians with intervals, rank agreement across + configurations and a run-to-run noise check, in a Python package with its own tests. - **Carbon estimates.** Energy converted to grams of CO2-equivalent for the UAE and UK grids, net of idle and including idle, from a cited table (Ember data, CC BY 4.0), with any other region or value on request. @@ -167,8 +168,9 @@ power over one second; it exits with status 3 when energy cannot be measured. The report is `python -m greenbench_analysis report --runs DIR/runs.csv` with `PYTHONPATH=analysis`. It takes `--out`, `--img-dir`, `--regions ARE,GBR` (codes from [data/grid_intensity.csv](data/grid_intensity.csv)), `--intensity NAME=G_PER_KWH` for any -other grid, `--withhold-timings` for runs on a busy machine, `--allow-partial` to report on a -run that did not finish, and `--summary-csv`. +other grid, `--equivalence-margin PERCENT` (10 by default; see +[study.md](docs/study.md#what-would-count-as-an-answer)), `--withhold-timings` for runs on a +busy machine, `--allow-partial` to report on a run that did not finish, and `--summary-csv`. ## Running the tests @@ -213,8 +215,9 @@ docs/ study.md, bare-metal.md, decisions.md, results.md (generat Recorded in [docs/decisions.md](docs/decisions.md): why powercap rather than MSRs, why every language is measured as a whole process, how repetitions and startup batches are calibrated, why the idle baseline follows every run, why missing or frozen energy is never written as -zero, why the study pins to two cores and measures Node.js 24, and why power ratios rather -than correlation decide whether time is a good stand-in for energy. +zero, why the study pins to two cores and measures Node.js 24, why power ratios rather +than correlation decide whether time is a good stand-in for energy, and why an interval that +includes 1 is not enough to call two powers the same. ## Limitations and roadmap diff --git a/analysis/greenbench_analysis/charts.py b/analysis/greenbench_analysis/charts.py index 6d5e717..f01891d 100644 --- a/analysis/greenbench_analysis/charts.py +++ b/analysis/greenbench_analysis/charts.py @@ -241,9 +241,10 @@ def energy_time_chart(configurations: list[Configuration], baseline_subtracted: "Energy per sort, net of idle (mJ, log scale)" if baseline_subtracted else "Energy per sort (mJ, log scale)" ) ax.set_ylabel(energy, fontsize=8) + power = f"constant {'net ' if baseline_subtracted else ''}power of {typical:.3g} W" + power += " above idle" if baseline_subtracted else "" ax.set_title( - f"One point per configuration; dashed line: constant power of {typical:.3g} W (the median)," - " dotted: half and double", + f"One point per configuration; dashed line: {power} (the median), dotted: half and double", fontsize=8, loc="left", color=INK_SECONDARY, diff --git a/analysis/greenbench_analysis/cli.py b/analysis/greenbench_analysis/cli.py index e4c0e4f..70d004b 100644 --- a/analysis/greenbench_analysis/cli.py +++ b/analysis/greenbench_analysis/cli.py @@ -10,7 +10,7 @@ from .carbon import load_grid, parse_custom, select_regions from .load import DataError, load_meta, load_runs -from .report import ReportOptions, write_report, write_summary_csv +from .report import EQUIVALENCE_MARGIN, ReportOptions, write_report, write_summary_csv from .summary import summarise @@ -24,6 +24,16 @@ def _positive(text: str) -> int: return value +def _margin(text: str) -> float: + try: + value = float(text) + except ValueError: + raise argparse.ArgumentTypeError(f"expected a percentage, got {text!r}") from None + if not 0 < value < 100: + raise argparse.ArgumentTypeError("must be above 0 and below 100") + return value / 100 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m greenbench_analysis", @@ -61,6 +71,14 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="report on a run that did not finish (interrupted or failed), with a warning at the top", ) + report.add_argument( + "--equivalence-margin", + type=_margin, + default=EQUIVALENCE_MARGIN, + metavar="PERCENT", + help="call two powers the same when their ratio's interval lies within PERCENT either way " + f"(default {EQUIVALENCE_MARGIN * 100:g}); set it before measuring", + ) report.add_argument("--resamples", type=_positive, default=10_000, help="bootstrap resamples (default %(default)s)") report.add_argument("--seed", type=_positive, default=1, help="bootstrap seed (default %(default)s)") report.add_argument("--summary-csv", type=Path, help="also write every configuration's estimates here") @@ -91,6 +109,7 @@ def _report(args: argparse.Namespace, argv: list[str]) -> int: per_sorts=args.per_sorts, command="PYTHONPATH=analysis python3 -m greenbench_analysis " + shlex.join(argv), guide_link=guide, + margin=args.equivalence_margin, ) written = write_report(runs, meta, summary, args.out, args.img_dir, options) if args.summary_csv: diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 2c5abf1..3b07cfa 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -18,11 +18,17 @@ from .carbon import GridIntensity, grams_co2e from .load import ORDERS, Run from .stats import Estimate -from .summary import Comparison, Configuration, RankAgreement, Summary +from .summary import CONFIDENCE, Comparison, Configuration, RankAgreement, Summary LANGUAGE_NAMES = {"c": "C", "python": "Python", "node": "Node.js"} STUDY_RUNS = 30 # measured runs per configuration in the study profile +# Two configurations count as drawing the same power when the whole 95% interval of their +# power ratio lies within this margin either way: from 1 / 1.1 to 1.1 by default. It is set +# before measuring (docs/study.md), and the report can be given another with +# --equivalence-margin. +EQUIVALENCE_MARGIN = 0.10 + def _pending(options: ReportOptions) -> str: return ( @@ -39,6 +45,7 @@ class ReportOptions: per_sorts: int command: str guide_link: str = "bare-metal.md" # docs/bare-metal.md, relative to the report + margin: float = EQUIVALENCE_MARGIN def _scaled(value: float, units: list[tuple[float, str]]) -> tuple[float, str]: @@ -273,6 +280,11 @@ def _operations_section(summary: Summary, image: str | None) -> list[str]: return [*lines, ""] +def _power_label(summary: Summary) -> str: + """What the power column holds: net of idle when an idle baseline was subtracted.""" + return "Net power above idle" if summary.baseline_subtracted else "Power while sorting" + + def _per_sort_rows(configurations: list[Configuration], energy: bool) -> list[list[str]]: rows = [] for c in configurations: @@ -314,7 +326,14 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str if summary.baseline_subtracted else "" ) - power = " Power while sorting is energy per sort divided by time per sort." if summary.energy_measured else "" + power = "" + if summary.energy_measured and summary.baseline_subtracted: + power = ( + " Net power above idle is the net energy per sort divided by the time per sort: the power the sort " + "drew on top of an idle machine, not the processor's total power." + ) + elif summary.energy_measured: + power = " Power while sorting is energy per sort divided by time per sort." lines += [ f"Median over the measured runs of each configuration, with a 95% bootstrap confidence interval in " f"brackets. Per-sort figures are the whole-process measurement {subtract}.{idle}{power}", @@ -326,7 +345,7 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str header = ["Language", "Algorithm", "Order", "Size", "Sorts per process", "Time per sort", "Energy per sort"] align = [":---", ":---", ":---", "---:", "---:", "---:", "---:"] if summary.energy_measured: - header.append("Power while sorting") + header.append(_power_label(summary)) align.append("---:") lines += [f"Inputs of {largest:,} elements:", ""] lines += _table( @@ -337,17 +356,66 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str return [*lines, "", "", ""] -def _reading(comparison: Comparison, subject: str) -> str: - power = comparison.power_ratio - if power is None: - return "undefined (an energy or time per sort at or below zero)" - if power.low <= 1 <= power.high: - return "same power: the time ratio predicts the energy ratio" - share = abs(power.value - 1) * 100 - return f"{subject} draws {share:.0f}% {'more' if power.value > 1 else 'less'} power" +SAME, DIFFERS, INCONCLUSIVE, UNDEFINED = "same", "differs", "inconclusive", "undefined" +READINGS = (SAME, DIFFERS, INCONCLUSIVE, UNDEFINED) + + +def _percent(margin: float) -> str: + return f"{margin * 100:g}%" + + +def classify_power_ratio(power: Estimate | None, margin: float, *, reliable: bool = True) -> str: + """One of four readings of a power ratio and its interval, checked in this order: + - UNDEFINED: no ratio (an energy or time per sort at or below zero), an interval that + reaches zero or below, or one that is not reliable (Comparison.power_reliable); all of + these happen when energy or time per sort is lost in the noise; + - SAME: the whole interval lies within the margin either way, from 1 / (1 + margin) to + 1 + margin, so the time ratio predicts the energy ratio to within the margin; + - DIFFERS: the interval excludes 1, so one side draws more power; + - INCONCLUSIVE: the interval includes 1 but reaches beyond the margin. Not finding a + difference is not evidence that there is none. + """ + if power is None or power.low <= 0 or not reliable: + return UNDEFINED + if 1 / (1 + margin) <= power.low and power.high <= 1 + margin: + return SAME + if power.high < 1 or power.low > 1: + return DIFFERS + return INCONCLUSIVE -def _language_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: + +def excludes_one(power: Estimate | None) -> bool: + return power is not None and (power.high < 1 or power.low > 1) + + +def _difference(power: Estimate, subject: str) -> str: + """ "Python draws 50% more power", from the ratio's point estimate.""" + if power.value >= 1: + return f"{subject} draws {(power.value - 1) * 100:.0f}% more power" + share = (1 - power.value) * 100 + return f"{subject} draws {share:.0f}% less power" if share < 99.5 else f"{subject} draws under 1% of the power" + + +def power_reading(power: Estimate | None, subject: str, margin: float, *, reliable: bool = True) -> str: + """The plain-words reading of a power ratio, subject over reference (classify_power_ratio).""" + kind = classify_power_ratio(power, margin, reliable=reliable) + if power is None: + return "undefined: an energy or time per sort at or below zero" + if power.low <= 0: + return "undefined: the interval reaches zero, so energy per sort is within the noise" + if kind == UNDEFINED: + share = (1 - CONFIDENCE) / 2 + return f"undefined: energy or time per sort fell to zero or below in more than {share:.1%} of the resamples" + if kind == SAME: + detail = f" ({_difference(power, subject)})" if excludes_one(power) else "" + return f"same power within {_percent(margin)}{detail}: time predicts energy" + if kind == DIFFERS: + return f"{_difference(power, subject)}: time misjudges energy" + return f"inconclusive: the interval is wider than the {_percent(margin)} margin and includes 1" + + +def _language_rows(comparisons: Iterable[Comparison], margin: float) -> list[list[str]]: rows = [] for c in comparisons: language, algorithm, order, n = c.subject @@ -360,13 +428,13 @@ def _language_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: format_ratio(c.time_ratio), format_ratio(c.energy_ratio), format_ratio(c.power_ratio), - _reading(c, LANGUAGE_NAMES[language]), + power_reading(c.power_ratio, LANGUAGE_NAMES[language], margin, reliable=c.power_reliable), ] ) return rows -def _order_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: +def _order_rows(comparisons: Iterable[Comparison], margin: float) -> list[list[str]]: rows = [] for c in comparisons: language, algorithm, order, n = c.subject @@ -380,12 +448,57 @@ def _order_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: format_ratio(c.time_ratio), format_ratio(c.energy_ratio), format_ratio(c.power_ratio), - _reading(c, f"{name.capitalize()} input"), + power_reading(c.power_ratio, f"{name.capitalize()} input", margin, reliable=c.power_reliable), ] ) return rows +def _reading_counts(comparisons: list[Comparison], margin: float) -> dict[str, int]: + counts = dict.fromkeys(READINGS, 0) + for c in comparisons: + counts[classify_power_ratio(c.power_ratio, margin, reliable=c.power_reliable)] += 1 + return counts + + +def _readings_table(summary: Summary, margin: float) -> list[str]: + """How many power ratios got each reading, between languages and between input orders.""" + groups = [ + ("Between languages", summary.language_comparisons), + ("Between input orders", summary.order_comparisons), + ] + rows = [] + for name, comparisons in groups: + if comparisons: + counts = _reading_counts(comparisons, margin) + rows.append([name, f"{len(comparisons):,}", *(f"{counts[k]:,}" for k in READINGS)]) + return _table( + ["Power ratios", "Count", f"Same power within {_percent(margin)}", "Differs", "Inconclusive", "Undefined"], + [":---", "---:", "---:", "---:", "---:", "---:"], + rows, + ) + + +def _chance_paragraph(summary: Summary) -> list[str]: + """How many intervals exclude 1, against how many would by chance alone.""" + comparisons = summary.language_comparisons + summary.order_comparisons + defined = [ + c for c in comparisons if classify_power_ratio(c.power_ratio, 1.0, reliable=c.power_reliable) != UNDEFINED + ] + if not defined: + return [] + excluding = sum(1 for c in defined if excludes_one(c.power_ratio)) + expected = len(defined) * (1 - CONFIDENCE) + return [ + f"{excluding:,} of the {len(defined):,} defined power-ratio intervals exclude 1. These are separate " + f"{CONFIDENCE:.0%} intervals, not adjusted for making many comparisons at once: even if every " + f"configuration drew exactly the same power, about {1 - CONFIDENCE:.0%} of them, around {expected:.1f} here, " + "would exclude 1 by chance. A difference that is larger than the margin and recurs across sizes and " + "orders is more credible than one that appears once.", + "", + ] + + RATIO_HEADER = ["Time ratio", "Energy ratio", "Power ratio", "Reading"] RATIO_ALIGN = ["---:", "---:", "---:", ":---"] @@ -406,14 +519,14 @@ def _describe(c: Configuration) -> str: def _power_paragraph(summary: Summary) -> list[str]: + label = _power_label(summary) powered = [(c, c.power_w) for c in summary.configurations if c.power_w is not None] if not powered: - return ["Power is undefined for every configuration: no time per sort was above zero.", ""] + return [f"{label} is undefined for every configuration: no time per sort was above zero.", ""] if len(powered) == 1: ((only, power),) = powered return [ - f"Power while sorting is defined for one configuration: {format_estimate(power, POWER_UNITS)} for " - f"{_describe(only)}.", + f"{label} is defined for one configuration: {format_estimate(power, POWER_UNITS)} for {_describe(only)}.", "", ] low_c, low = min(powered, key=lambda item: item[1].value) @@ -422,8 +535,8 @@ def _power_paragraph(summary: Summary) -> list[str]: half = len(values) // 2 middle = values[half] if len(values) % 2 else (values[half - 1] + values[half]) / 2 return [ - f"Across the {len(powered):,} configurations where it is defined, power while sorting ranged from " - f"{format_estimate(low, POWER_UNITS)} for {_describe(low_c)}, to {format_estimate(high, POWER_UNITS)} " + f"Across the {len(powered):,} configurations where it is defined, {label[0].lower() + label[1:]} ranged " + f"from {format_estimate(low, POWER_UNITS)} for {_describe(low_c)}, to {format_estimate(high, POWER_UNITS)} " f"for {_describe(high_c)}. The median over configurations was {middle:.3g} W. The table in the previous " "section lists every configuration, with its 95% interval in brackets.", "", @@ -437,32 +550,45 @@ def _proxy_section(summary: Summary, options: ReportOptions, image: str | None, if not summary.energy_measured: return [*lines, f"Energy was not measured in this run ({reason}), so this question stays open.", ""] largest = max((c.n for c in summary.configurations), default=0) + margin = options.margin + lower, upper = 1 / (1 + margin), 1 + margin lines += [ "If every configuration drew the same average power while sorting, energy would be proportional to " "time, and time would be a perfect stand-in for it. The sections below test that directly, through " "power and through ratios between configurations, and then more weakly, through rank correlation.", "", + "### Readings", + "", + "The power ratio of two configurations is their energy ratio divided by their time ratio. Each one " + f"gets one of four readings. *Same power within {_percent(margin)}*: the whole {CONFIDENCE:.0%} interval " + f"lies between {lower:.3g} and {upper:.3g}, a margin set before measuring, so the time ratio predicts " + "the energy ratio to within that margin. *Differs*: the interval excludes 1, so one side draws more " + "power and time alone misjudges the energy difference. *Inconclusive*: the interval includes 1 but " + "reaches beyond the margin, so this data cannot tell. *Undefined*: an energy or time per sort, or the " + "lower end of the interval, is at or below zero.", + "", + *_readings_table(summary, margin), + "", + *_chance_paragraph(summary), ] if image: lines += [f"![Energy per sort against time per sort, one point per configuration]({image})", ""] - lines += ["### Power while sorting", "", *_power_paragraph(summary)] + lines += [f"### {_power_label(summary)}", "", *_power_paragraph(summary)] lines += [ "### Between languages", "", "For the same algorithm and input, how many times the second language's time and energy per sort " - "the first language needs. The power ratio is the energy ratio divided by the time ratio: when its " - "95% interval includes 1, the two languages draw the same power and the time ratio predicts the " - "energy ratio. Random input, largest size:", + "the first language needs, and their power ratio. Random input, largest size:", "", ] header = ["Algorithm", "Order", "Size", "Languages", *RATIO_HEADER] align = [":---", ":---", "---:", ":---", *RATIO_ALIGN] main = [c for c in summary.language_comparisons if c.subject[2] == "random" and c.subject[3] == largest] - lines += _table(header, align, _language_rows(main)) if main else ["No language pairs in this run."] + lines += _table(header, align, _language_rows(main, margin)) if main else ["No language pairs in this run."] if summary.language_comparisons: lines += ["", "
Every order and size", ""] - lines += _table(header, align, _language_rows(summary.language_comparisons)) + lines += _table(header, align, _language_rows(summary.language_comparisons, margin)) lines += ["", "
"] lines += [""] @@ -477,10 +603,10 @@ def _proxy_section(summary: Summary, options: ReportOptions, image: str | None, header = ["Language", "Algorithm", "Size", "Orders", *RATIO_HEADER] align = [":---", ":---", "---:", ":---", *RATIO_ALIGN] main = [c for c in summary.order_comparisons if c.subject[2] == "sorted" and c.subject[3] == largest] - lines += _table(header, align, _order_rows(main)) if main else ["No sorted and random pairs in this run."] + lines += _table(header, align, _order_rows(main, margin)) if main else ["No sorted and random pairs in this run."] if summary.order_comparisons: lines += ["", "
Every order and size", ""] - lines += _table(header, align, _order_rows(summary.order_comparisons)) + lines += _table(header, align, _order_rows(summary.order_comparisons, margin)) lines += ["", "
"] lines += [""] diff --git a/analysis/greenbench_analysis/summary.py b/analysis/greenbench_analysis/summary.py index ad3026a..f7ad724 100644 --- a/analysis/greenbench_analysis/summary.py +++ b/analysis/greenbench_analysis/summary.py @@ -37,6 +37,8 @@ # Energy in microjoules over time in nanoseconds is in kilowatts; this gives watts. WATTS_PER_UJ_PER_NS = 1e3 +CONFIDENCE = 0.95 # of every bootstrap interval + # Language pairs compared, as (subject, reference): how many times the reference's cost the # subject's is. C is the reference for both interpreters; Python is compared with Node.js. LANGUAGE_PAIRS = (("python", "c"), ("node", "c"), ("python", "node")) @@ -92,6 +94,10 @@ class Comparison: power_ratio is energy_ratio / time_ratio. It is 1 when both configurations draw the same power while sorting, which is when their time ratio predicts their energy ratio; an interval that excludes 1 means time alone misjudges the energy difference. + + power_reliable is False when a time or energy per sort of either configuration fell to zero + or below in more than (1 - CONFIDENCE) / 2 of the resamples. The power ratio is undefined + in those resamples, and an interval from the rest would look narrower than it is. """ subject: Key @@ -99,6 +105,7 @@ class Comparison: time_ratio: Estimate | None energy_ratio: Estimate | None power_ratio: Estimate | None + power_reliable: bool = True @dataclass(frozen=True) @@ -239,6 +246,7 @@ def _compare(a: Configuration, b: Configuration, draws: dict[Key, _Draws]) -> Co da, db = draws[a.key], draws[b.key] time_ratio = _ratio(a.time_per_sort_ns.value, b.time_per_sort_ns.value, da.time, db.time) energy_ratio = power_ratio = None + reliable = True if ( a.energy_per_sort_uj is not None and b.energy_per_sort_uj is not None @@ -249,8 +257,10 @@ def _compare(a: Configuration, b: Configuration, draws: dict[Key, _Draws]) -> Co if time_ratio is not None and energy_ratio is not None: # (energy ratio) / (time ratio), resample by resample. power_draws = ratio_draws(ratio_draws(da.energy, db.energy), ratio_draws(da.time, db.time)) - power_ratio = interval(energy_ratio.value / time_ratio.value, power_draws) - return Comparison(a.key, b.key, time_ratio, energy_ratio, power_ratio) + power_ratio = interval(energy_ratio.value / time_ratio.value, power_draws, confidence=CONFIDENCE) + defined = (da.time > 0) & (db.time > 0) & (da.energy > 0) & (db.energy > 0) + reliable = bool(np.mean(defined) >= 1 - (1 - CONFIDENCE) / 2) + return Comparison(a.key, b.key, time_ratio, energy_ratio, power_ratio, reliable) def _language_comparisons(configurations: list[Configuration], draws: dict[Key, _Draws]) -> list[Comparison]: diff --git a/analysis/tests/synthetic.py b/analysis/tests/synthetic.py index 4bc0cdc..fa6fac9 100644 --- a/analysis/tests/synthetic.py +++ b/analysis/tests/synthetic.py @@ -76,6 +76,7 @@ def study( sizes: Iterable[int] = (100,), power: dict[str, float] | None = None, startup_launches: int = 1, + energy_jitter: float = 10.0, ) -> list[Run]: """A small, fully known data set. @@ -85,7 +86,7 @@ def study( each startup run is a batch of ``startup_launches`` processes. The idle package draws 1 W throughout, so gross energy is the net energy plus 1 µJ per µs, and every idle window has the length of its run. The runs of a configuration differ by a small jitter that raises - time and energy together. + time and energy together; ``energy_jitter`` (µJ per step of jitter) widens the energy's. """ factors = power or {} result: list[Run] = [] @@ -102,7 +103,7 @@ def study( reps = 0 if algorithm == "none" else 10 launches = startup_launches if algorithm == "none" else 1 wall = scale * (launches * 5_000_000 + reps * 1_000_000) + jitter * 1000 - net = factor * scale * (launches * 10_000.0 + reps * 2_000.0) + jitter * 10.0 + net = factor * scale * (launches * 10_000.0 + reps * 2_000.0) + jitter * energy_jitter idle = IDLE_WATTS * 1e-3 * wall result.append( make_run( diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index 7145ab9..6b9ad28 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -8,13 +8,19 @@ from greenbench_analysis.carbon import GridIntensity from greenbench_analysis.cli import main from greenbench_analysis.report import ( + DIFFERS, ENERGY_UNITS, + INCONCLUSIVE, + SAME, TIME_UNITS, + UNDEFINED, ReportOptions, + classify_power_ratio, format_energy, format_estimate, format_grams, format_time, + power_reading, write_report, ) from greenbench_analysis.stats import Estimate @@ -62,11 +68,18 @@ def test_report_with_energy(tmp_path: Path) -> None: assert "2.00 W (" in text # so 2 W while sorting assert "## Is run time a good stand-in for energy?" in text assert "| quick | random | 200 | Python / C | 10 (" in text - assert "same power: the time ratio predicts the energy ratio" in text + assert "same power within 10%: time predicts energy" in text + # Every language pair and every order pair draws the same power in the synthetic data. + assert "| Between languages | 8 | 8 | 0 | 0 | 0 |" in text + assert "| Between input orders | 8 | 8 | 0 | 0 | 0 |" in text + assert "of the 16 defined power-ratio intervals exclude 1" in text + assert "about 5% of them, around 0.8 here, would exclude 1 by chance" in text assert "### Between input orders" in text assert "| every configuration | 16 |" in text assert "| C quick: every order and size | 4 |" in text assert "| Python: every algorithm, order and size | 8 |" in text + assert "| Net power above idle |" in text # the per-sort table's power column + assert "### Net power above idle" in text assert "| C | quick | 4 | 1.000 |" in text # run-to-run coupling over four configurations # Per 1e6 sorts: 2 kJ net and 3 kJ including idle; at 467.51 g/kWh, 260 mg and 390 mg. assert "| C | quick | 2.00 kJ | 3.00 kJ | 260 mg to 390 mg |" in text @@ -92,10 +105,53 @@ def test_simulated_energy_is_labelled(tmp_path: Path) -> None: def test_report_names_a_language_that_draws_more_power(tmp_path: Path) -> None: text = render(tmp_path, energy=True, withhold=False, power={"python": 1.5}) assert "| quick | random | 200 | Python / C | 10 (" in text - assert "Python draws 50% more power" in text + assert "Python draws 50% more power: time misjudges energy" in text + assert "| Between languages | 8 | 0 | 8 | 0 | 0 |" in text assert "3.00 W (" in text +def test_power_ratio_readings() -> None: + margin = 0.10 + cases = [ + (Estimate(1.02, 0.95, 1.08), SAME, "same power within 10%: time predicts energy"), + # Excludes 1 but stays inside the margin: a real difference, too small to matter. + ( + Estimate(1.04, 1.01, 1.07), + SAME, + "same power within 10% (Python draws 4% more power): time predicts energy", + ), + (Estimate(1.0, 1 / 1.1, 1.1), SAME, "same power within 10%: time predicts energy"), + (Estimate(1.3, 1.2, 1.4), DIFFERS, "Python draws 30% more power: time misjudges energy"), + (Estimate(0.8, 0.7, 0.95), DIFFERS, "Python draws 20% less power: time misjudges energy"), + (Estimate(0.001, 0.0005, 0.002), DIFFERS, "Python draws under 1% of the power: time misjudges energy"), + # Includes 1 but reaches beyond the margin: not evidence of the same power. + (Estimate(1.0, 0.9, 1.1), INCONCLUSIVE, "inconclusive: the interval is wider than the 10% margin"), + (Estimate(0.0252, 0.000626, 15.7), INCONCLUSIVE, "inconclusive"), + (Estimate(0.000415, -0.351, 0.22), UNDEFINED, "undefined: the interval reaches zero"), + (None, UNDEFINED, "undefined: an energy or time per sort at or below zero"), + ] + for estimate, kind, text in cases: + assert classify_power_ratio(estimate, margin) == kind, estimate + assert power_reading(estimate, "Python", margin).startswith(text), estimate + # An interval built from too few defined resamples is not trusted, whatever it says. + assert classify_power_ratio(Estimate(1.0, 0.99, 1.01), margin, reliable=False) == UNDEFINED + assert power_reading(Estimate(1.0, 0.99, 1.01), "Python", margin, reliable=False).startswith( + "undefined: energy or time per sort fell to zero or below in more than 2.5% of the resamples" + ) + # A narrower margin turns the first case inconclusive. + assert classify_power_ratio(Estimate(1.02, 0.95, 1.08), 0.05) == INCONCLUSIVE + assert classify_power_ratio(Estimate(1.0, 0.96, 1.04), 0.05) == SAME + + +def test_wide_intervals_are_inconclusive_not_the_same(tmp_path: Path) -> None: + runs = synthetic.study(energy=True, sizes=(100, 200), energy_jitter=4_000.0) + summary = summarise(runs, resamples=500) + out = tmp_path / "results.md" + write_report(runs, synthetic.meta(energy=True), summary, out, tmp_path / "img", options()) + text = out.read_text(encoding="utf-8") + assert "inconclusive: the interval is wider than the 10% margin and includes 1" in text + + def test_report_warns_about_unfinished_and_short_runs(tmp_path: Path) -> None: text = render(tmp_path, energy=True, withhold=False, status="interrupted") assert "Warning: this run did not finish (status: interrupted)" in text @@ -151,10 +207,13 @@ def test_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> Non "200", "--summary-csv", str(tmp_path / "summary.csv"), + "--equivalence-margin", + "5", ] ) assert status == 0 text = out.read_text(encoding="utf-8") + assert "Same power within 5%" in text assert "CO2e, United Kingdom" in text assert "CO2e, Solar farm" in text assert "PYTHONPATH=analysis python3 -m greenbench_analysis report --runs" in text @@ -187,3 +246,6 @@ def test_command_line_errors(tmp_path: Path, capsys: pytest.CaptureFixture[str]) ) assert status == 1 assert "would publish the withheld timings" in capsys.readouterr().err + with pytest.raises(SystemExit): + main(["report", "--runs", str(runs_path), "--equivalence-margin", "0"]) + assert "must be above 0 and below 100" in capsys.readouterr().err diff --git a/analysis/tests/test_summary.py b/analysis/tests/test_summary.py index 20325b0..0949b22 100644 --- a/analysis/tests/test_summary.py +++ b/analysis/tests/test_summary.py @@ -123,6 +123,21 @@ def test_language_pairs_and_their_order() -> None: assert python_node.time_ratio.value == pytest.approx(2.0) +def test_a_power_ratio_is_unreliable_when_energy_per_sort_reaches_zero_in_resamples() -> None: + # C's energy per sort is 2 mJ at the middle run but below zero at the lowest two, which a + # resample's median lands on often enough. Python's stays well above zero. + summary = summarise(synthetic.study(energy=True, energy_jitter=15_000.0), resamples=2_000) + c_energy = configuration(summary, "c", "quick").energy_per_sort_uj + assert c_energy is not None + assert c_energy.value == pytest.approx(2_000) + python_c = comparison(summary.language_comparisons, "python", "c") + assert python_c.power_ratio is not None + assert not python_c.power_reliable + python_orders = [c for c in summary.order_comparisons if c.subject[0] == "python"] + assert python_orders + assert all(c.power_reliable for c in python_orders) + + def test_input_orders_are_compared_with_random_input() -> None: summary = summarise(synthetic.study(energy=True), resamples=200) keys = {(c.subject, c.reference) for c in summary.order_comparisons} diff --git a/docs/decisions.md b/docs/decisions.md index efab3b6..ecfb5d2 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -311,3 +311,28 @@ length, so the tests can check exact lower bounds on every row and do not depend scheduling. The simulation code sits behind a compile-time flag and cannot be switched on in `build/greenbench`. The file-reading code still runs in the simulated build, and the unit tests rewrite counter files between readings to test parsing and wraparound. + +## 19. Call two powers the same only within a margin set in advance + +**Context.** The first version of the report read every power-ratio interval that included 1 +as "same power: the time ratio predicts the energy ratio". That treats a result that is not +significant as proof of equality: in a rehearsal on fake counters, an interval from 0.0006 to +15.7 was read as the same power. A review also +pointed out that the study profile prints 252 such intervals, of which about 13 would exclude +1 by chance even if power were the same everywhere. + +**Decision.** Each power ratio gets one of four readings against a margin of 10% either way, +from 1/1.1 to 1.1, fixed in [study.md](study.md#what-would-count-as-an-answer) before any +energy is measured. *Same power within 10%* needs the whole 95% interval inside the margin; +*differs* needs the interval to exclude 1; an interval that includes 1 but reaches beyond the +margin is *inconclusive*. A ratio is *undefined* when a time or energy per sort is at or below +zero, when the interval reaches zero, or when either configuration's time or energy per sort +fell to zero or below in more than 2.5% of the resamples. The report tabulates the readings, +and states how many intervals exclude 1 against the 5% expected by chance, instead of +adjusting each interval for multiple comparisons. `--equivalence-margin` changes the margin. + +**Consequences.** Wide intervals now say that the data cannot tell, rather than answering the +question "yes". A study run on a noisy machine may end with many inconclusive readings, which +is the honest outcome. The margin is a judgement, stated with its reason, and a reader can +apply another one without measuring again. Tests cover every reading, the boundaries of the +margin and the resampling rule. diff --git a/docs/study.md b/docs/study.md index 107ae22..c59c048 100644 --- a/docs/study.md +++ b/docs/study.md @@ -33,20 +33,30 @@ in three ways, from the most direct to the weakest: - **Power ratios.** For each pair of languages on the same algorithm and input, and for each input order against random input in the same language, the report gives the time ratio, the - energy ratio and their quotient, the power ratio, each with a 95% confidence interval. When - the power ratio's interval includes 1, the time ratio predicts the energy ratio. When it - excludes 1, one side draws more power, and time alone misjudges the energy difference by - that factor. The report prints the size of every power ratio, because a difference of a few - per cent can be real without mattering. -- **Power while sorting** for every configuration, and a chart of energy per sort against time - per sort, in which constant power is a straight line. + energy ratio and their quotient, the power ratio, each with a 95% confidence interval. Each + power ratio gets one of four readings against a margin of 10% either way (from 1/1.1, about + 0.909, to 1.1), which is set here, before any energy has been measured: + - *same power within 10%*: the whole interval lies inside the margin, so the time ratio + predicts the energy ratio to within 10%; + - *differs*: the interval excludes 1, so one side draws more power, and time alone + misjudges the energy difference by that factor; + - *inconclusive*: the interval includes 1 but reaches beyond the margin. Failing to find a + difference is not evidence that there is none, so this is never read as the same power; + - *undefined*: energy or time per sort is lost in the noise. +- **Net power above idle** for every configuration, and a chart of energy per sort against + time per sort, in which constant power is a straight line. - **Rank agreement.** Spearman's rank correlation between median time per sort and median energy per sort across configurations. This is weaker: configurations differ in cost by large factors, so rho stays near 1 unless power differs enough to reorder them. -Power ratios close to 1 for every pair of languages and every input order would answer the -first question "yes". Power ratios clearly away from 1 would answer the second: for those -languages, comparing time is not comparing energy. +The margin is a judgement: a power difference of less than 10% would rarely change a decision +made from time alone, when the languages differ in time by factors of ten or more. The study +profile gives 144 power ratios between languages and 108 between input orders. If most of them +read "same power within 10%", and the ones that differ are no more than chance would produce +(see [Analysis](#analysis)), the answer to the first question is "yes". Power ratios that +differ by more than the margin, consistently across sizes and orders, answer the second: for +those languages or inputs, comparing time is not comparing energy. Many inconclusive readings +would mean that the measurements were too noisy to decide, and the report would say so. ## Method @@ -125,11 +135,24 @@ lists, JavaScript arrays). None uses a built-in sort. once, keeping each run's time and energy together, and every estimate for that configuration comes from the same resamples. The startup medians that are subtracted are treated as fixed. -- **Power while sorting.** Median energy per sort divided by median time per sort, in watts. +- **Net power above idle.** Median net energy per sort divided by median time per sort, in + watts: the power the sort draws on top of an idle machine, not the processor's total power. + The proxy question is decided on net power, because the idle power is drawn whether the sort + runs or not (see [threats to validity](#threats-to-validity)). - **Ratios.** Between languages (Python against C, Node.js against C, Python against Node.js) for the same algorithm and input, and between each input order and random input for the same language and algorithm: the time ratio, the energy ratio and the power ratio (energy - ratio divided by time ratio). The two configurations are resampled independently. + ratio divided by time ratio). The two configurations are resampled independently. Each + power ratio is read against the 10% margin as described above; `--equivalence-margin` + changes it, which a published result should state and justify. A power ratio also counts as + undefined when a time or energy per sort of either configuration falls to zero or below in + more than 2.5% of the resamples, because the interval from the remaining resamples would + look narrower than it is. +- **Many comparisons.** The intervals are separate 95% intervals, not adjusted for being many. + The report counts how many exclude 1 and sets that against the number that chance alone + would give if every configuration drew the same power: 5% of them. It does not apply a + Bonferroni or Holm correction, which would widen every interval and push more readings to + "inconclusive"; the question is about the overall pattern, not about any single pair. - **Rank agreement.** Spearman's rho between median time per sort and median energy per sort across configurations: at each input size, for each language, for each language and algorithm across input orders and sizes, and over all of them, with an interval from 2,000 @@ -171,6 +194,22 @@ lists, JavaScript arrays). None uses a built-in sort. them and recorded. Long runs heat the processor, which raises its power at the same work; the random order keeps this from favouring one configuration. +### Statistics + +- **Net rather than total power.** The power ratios use energy net of idle: what the sort + adds. Idle power is drawn whether or not the sort runs, so adding it to both sides of a ratio + pulls every ratio towards 1. With a package that idles at 5 W, sorts that draw 10 W and 12 W + above idle have a power ratio of 1.2, but the package's total power (15 W against 17 W) gives + 1.13. Judged on total power, time would look like a better stand-in for energy than it is + for the work itself. The carbon section gives energy both ways. +- **Many comparisons.** The report prints 252 intervals for the study profile. About 13 would + exclude 1 by chance even if every configuration drew the same power, so a single "differs" + reading proves little on its own. The report states how many exclude 1 against that + expectation, and does not adjust the intervals. +- **The margin.** Whether a 10% difference in power matters is a judgement, fixed before + measuring. A reader who prefers another margin can rerun the report with + `--equivalence-margin`; the intervals do not change, only their readings. + ### Workload and implementations - **Instrumentation.** Counting comparisons and writes adds work in every language, and From c8a89001c1f6ea0efe99252839409b850cfcfe2b Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:35:37 +0400 Subject: [PATCH 38/51] fix: aim calibration 20% above the minimum loop length Calibration set the sorts per process so that a loop would last exactly --min-loop-ms. A measured run is as likely to be a little faster than the warm-up that set it as a little slower, so about half of all loops fell just short of the target on any machine, and the count of short loops meant nothing. Reviews found 128 to 217 short loops per 216 to 360 in busy containers. Warm-up now aims 20% higher (240 ms for the study profile's 200 ms), for sort loops and startup batches alike, so ordinary variation stays above the target and a short loop points to a busy or throttled machine. meta.json records the aim as settings.aim_loop_ns and the report describes it; reports of earlier runs read as before. The harness is now version 2.2.0. ADR 5, study.md and the bare-metal time estimate (about 46 minutes of runs) are updated. Co-Authored-By: Claude Opus 5.5 --- README.md | 13 ++++++------ analysis/greenbench_analysis/report.py | 16 ++++++++++++--- analysis/tests/test_report.py | 11 ++++++++++ docs/bare-metal.md | 6 +++--- docs/decisions.md | 28 ++++++++++++++++---------- docs/study.md | 12 ++++++----- src/greenbench.c | 22 +++++++++++--------- src/plan.c | 6 ++++++ src/plan.h | 8 ++++++++ tests/integration/test_harness.py | 2 ++ tests/test_plan.c | 12 +++++++++++ 11 files changed, 99 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 3c7ef42..767256b 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,11 @@ compare?** The research question, method and threats to validity are in Python and JavaScript; tests check that all three produce identical sequences, inputs and operation counts, and the harness stops if a worker ever disagrees. - **Measurement harness.** One process per measured run for every language, time from - `CLOCK_MONOTONIC_RAW`, sorts per process calibrated so each loop lasts about 200 ms (far - above the ~1 ms RAPL update interval), an idle baseline of the same length after every run, - 30 measured runs per configuration after warm-up, randomised run order, optional pinning to - a set of CPUs, startup trials measured over batches of launches, and tidy CSV output. + `CLOCK_MONOTONIC_RAW`, sorts per process calibrated to about 240 ms so that each loop lasts + at least 200 ms (far above the ~1 ms RAPL update interval), an idle baseline of the same + length after every run, 30 measured runs per configuration after warm-up, randomised run + order, optional pinning to a set of CPUs, startup trials measured over batches of launches, + and tidy CSV output. - **Statistics.** Net power above idle, and time, energy and power ratios between languages and between input orders, each with a 95% bootstrap interval. Each power ratio is read against a 10% margin set before measuring: the same power (so time predicts energy), @@ -147,10 +148,10 @@ The base images are pinned by digest. If Docker Hub limits your pulls, the | Option | Default | Meaning | | :--- | :--- | :--- | -| `--profile study\|smoke` | `study` | Study: sizes 250, 500, 1,000, 2,000; 30 runs; 3 warm-up runs; 200 ms loops. Smoke: sizes 64, 256; 3 runs; 1 warm-up; 2 ms. | +| `--profile study\|smoke` | `study` | Study: sizes 250, 500, 1,000, 2,000; 30 runs; 3 warm-up runs; loops of at least 200 ms. Smoke: sizes 64, 256; 3 runs; 1 warm-up; 2 ms. | | `--languages`, `--algorithms`, `--orders`, `--sizes` | all | Comma-separated subsets, for example `--languages c,node --sizes 100,1000`. | | `--runs N`, `--warmup N` | 30, 3 | Measured and warm-up runs per configuration. | -| `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Target length of each sort loop and startup batch, set during warm-up; and the cap on sorts per process. | +| `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Minimum length of each sort loop and startup batch (calibration during warm-up aims 20% higher); and the cap on sorts per process. | | `--energy auto\|require\|off` | `auto` | `require` fails with exit status 3 if RAPL cannot be read. | | `--idle-baseline auto\|on\|off` | `auto` | On whenever energy is measured. | | `--cpus LIST` | off | Pin the harness and every worker to these CPUs, such as `2,3` or `2-3`. | diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 3b07cfa..41f9961 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -141,6 +141,17 @@ def _limits(host: dict[str, Any]) -> str: return "".join(f", {part}" for part in parts) +def _calibration(settings: dict[str, Any]) -> str: + target = settings.get("min_loop_ns", 0) / 1e6 + aim = settings.get("aim_loop_ns") + if isinstance(aim, int) and aim > settings.get("min_loop_ns", 0): + return ( + f"Sort loops and startup batches are calibrated to about {aim / 1e6:g} ms, so that they last at least " + f"{target:g} ms." + ) + return f"Sort loops and startup batches are calibrated to about {target:g} ms." + + def _loops_line(runs: list[Run], settings: dict[str, Any]) -> str | None: target = settings.get("min_loop_ns") if not isinstance(target, int) or target <= 0: @@ -157,7 +168,7 @@ def _loops_line(runs: list[Run], settings: dict[str, Any]) -> str | None: f"- Sort loops: {len(short):,} of {len(sorting):,} sorting runs sorted for less than the " f"{format_time(target)} target (shortest {format_time(min(r.loop_ns for r in short))}" + (f"; {capped:,} of them held back by the limit of {max_reps:,} sorts per process" if capped else "") - + "). Calibration extrapolates from the warm-up runs." + + "). Calibration extrapolates from the warm-up runs, so a busy machine leaves more loops short." ) @@ -182,8 +193,7 @@ def _environment(meta: dict[str, Any], runs: list[Run], summary: Summary, option f"warm-up run{'' if warmup == 1 else 's'}, in a random order (order seed {settings.get('order_seed', '?')}).", f"- Inputs: sizes {', '.join(f'{s:,}' for s in settings.get('sizes', []))}; orders " f"{', '.join(o.replace('_', ' ') for o in settings.get('orders', []))}; xorshift32 seed " - f"{settings.get('seed', '?')}. Sort loops and startup batches are calibrated to about " - f"{settings.get('min_loop_ns', 0) / 1e6:g} ms.", + f"{settings.get('seed', '?')}. {_calibration(settings)}", f"- Machine: {cpu}; {host.get('sysname', '?')} {host.get('release', '?')} {host.get('machine', '')}; " f"{host.get('online_cpus', '?')} CPUs online{_limits(host)}; scaling governor " f"{host.get('scaling_governor') or 'not reported'}; CPU pinning {_pinning(settings)}.", diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index 6b9ad28..dafc9ec 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -84,12 +84,23 @@ def test_report_with_energy(tmp_path: Path) -> None: # Per 1e6 sorts: 2 kJ net and 3 kJ including idle; at 467.51 g/kWh, 260 mg and 390 mg. assert "| C | quick | 2.00 kJ | 3.00 kJ | 260 mg to 390 mg |" in text assert "Sort loops: every sorting run reached the 2.00 ms target." in text + assert "Sort loops and startup batches are calibrated to about 2 ms." in text # meta without an aim assert "C and Python reported identical counts" in text for chart in ("operations", "time_per_sort", "energy_per_sort", "energy_vs_time"): assert (tmp_path / "docs" / "img" / f"{chart}.png").stat().st_size > 1000 assert f"](img/{chart}.png)" in text +def test_calibration_aim_is_described(tmp_path: Path) -> None: + runs = synthetic.study(energy=False) + meta = synthetic.meta(energy=False) + meta["settings"] = {**meta["settings"], "aim_loop_ns": 2_400_000} + out = tmp_path / "results.md" + write_report(runs, meta, summarise(runs, resamples=100), out, tmp_path / "img", options(withhold=True)) + text = out.read_text(encoding="utf-8") + assert "calibrated to about 2.4 ms, so that they last at least 2 ms." in text + + def test_simulated_energy_is_labelled(tmp_path: Path) -> None: runs = synthetic.study(energy=True, sizes=(100, 200)) summary = summarise(runs, resamples=200) diff --git a/docs/bare-metal.md b/docs/bare-metal.md index 4beccef..d66b838 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -71,9 +71,9 @@ disk, and the study leaves the machine as it found it after a reboot. Leave the machine alone while it runs: no browser, no typing, lid open, on mains power. After the warm-up the harness prints an estimate of the time left and updates it as it goes. The study profile makes 5,760 measured runs: 4,320 that sort and 1,440 startup batches. - Each is calibrated to about 200 ms and followed by an idle window of the same length, so - the runs alone take about 38 minutes (5,760 x 2 x 200 ms); warm-up, process start-up and - the slow configurations add to that. + Each is calibrated to about 240 ms, so that it lasts at least 200 ms, and is followed by an + idle window of the same length, so the runs alone take about 46 minutes (5,760 x 2 x + 240 ms); warm-up, process start-up and the slow configurations add to that. 6. Take the results away. The script prints where they are: diff --git a/docs/decisions.md b/docs/decisions.md index ecfb5d2..65b1919 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -87,21 +87,27 @@ extra overhead per run, and per-sort figures depend on subtracting a median star microseconds cannot be measured on its own. **Decision.** During warm-up the harness asks each worker for one sort, reads the worker's -own loop time, and sets the number of sorts per process so that the loop would last -`--min-loop-ms` (200 ms in the study profile, about 200 counter updates) if every sort took as -long as those just measured. It refines the number on each further warm-up run, then keeps it -fixed for every measured run of that configuration. The target is not a guarantee: a measured -loop can run faster or slower than the warm-up that set it. +own loop time, and sets the number of sorts per process so that the loop would last 20% longer +than `--min-loop-ms` (so 240 ms for the study profile's 200 ms, which is about 200 counter +updates) if every sort took as long as those just measured. It refines the number on each +further warm-up run, then keeps it fixed for every measured run of that configuration. + +The first version aimed at `--min-loop-ms` itself. A measured loop is as likely to run a +little faster than the warm-up that set it as a little slower, so about half of the loops fell +just short of the target even when nothing was wrong, and the count of short loops said +nothing. With 20% of headroom, ordinary variation leaves loops above the target, and a loop +that falls short points to a busy or throttled machine. The target is still not a guarantee. Startup trials are calibrated the same way, by the number of processes launched back to back in one measured interval (ADR 4). -**Consequences.** Every measured interval, sorting or startup, spans about 200 counter -updates, so the update granularity is well under 1% of the measurement. Fast configurations -sort thousands of times per process, slow ones once. The numbers of sorts and launches are -recorded in the CSV and in `meta.json`. Loops that fall short of the target are counted in -`meta.json` (`short_loops`, and `capped_loops` for those held back by `--max-reps`), noted by -the harness, and reported with the shortest loop. +**Consequences.** A measured interval, sorting or startup, typically spans 200 to 240 counter +updates, so the update granularity is about 0.5% of the measurement. The study takes about 20% +longer than it would at the target itself. Fast configurations sort thousands of times per +process, slow ones once. The numbers of sorts and launches are recorded in the CSV, and the +target and the aim in `meta.json` (`min_loop_ns`, `aim_loop_ns`). Loops that fall short of the +target are counted in `meta.json` (`short_loops`, and `capped_loops` for those held back by +`--max-reps`), noted by the harness, and reported with the shortest loop. ## 6. Subtract an idle baseline of the same length after every run diff --git a/docs/study.md b/docs/study.md index c59c048..792574f 100644 --- a/docs/study.md +++ b/docs/study.md @@ -99,8 +99,9 @@ lists, JavaScript arrays). None uses a built-in sort. unavailable", never a zero. - **Enough work per process.** RAPL counters update about once a millisecond (Hähnel et al., 2012). During warm-up the harness sets the number of sorts per process so that each worker's - sort loop lasts about 200 ms: it extrapolates from the last warm-up run, so a measured loop - can fall somewhat short on a busy machine. The report counts the loops that did. + sort loop lasts at least 200 ms: it extrapolates from the last warm-up run and aims 20% + higher, at 240 ms, so that ordinary run-to-run variation stays above 200 ms. A measured loop + can still fall short on a busy machine; the report counts the loops that did. - **Clocks.** Every reported time comes from the harness's `CLOCK_MONOTONIC_RAW`, read around the whole process. Each worker also times its own sort loop (C with `CLOCK_MONOTONIC_RAW`, Python with `time.perf_counter_ns` and Node.js with `process.hrtime`, both `CLOCK_MONOTONIC` @@ -230,9 +231,10 @@ lists, JavaScript arrays). None uses a built-in sort. processes started back to back; after the first, the files each process reads are in the page cache, so the cost per process describes a warm start. The sorting runs start warm too, because warm-up runs come first. -- **Calibration.** Sorts per process are set from the warm-up runs and can leave a measured - loop shorter than the 200 ms target on a busy machine. Shorter loops are still far longer - than the counter's update interval; the report says how many there were. +- **Calibration.** Sorts per process are set from the warm-up runs, aiming 20% above the + 200 ms target, and can still leave a measured loop shorter than the target on a busy + machine. Shorter loops are still far longer than the counter's update interval; the report + says how many there were and gives the shortest. - **Language versions.** Results describe the recorded compiler and interpreter versions, not the languages in general. The study script uses the system Python (3.12 on Ubuntu 24.04) and the Node.js 24 LTS that [bare-metal.md](bare-metal.md) installs from nodejs.org, and warns diff --git a/src/greenbench.c b/src/greenbench.c index f119f6e..4bbc53f 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -33,7 +33,7 @@ #include "spawn.h" #include "timing.h" -#define GREENBENCH_VERSION "2.1.0" +#define GREENBENCH_VERSION "2.2.0" #define EXIT_USAGE 2 #define EXIT_ENERGY 3 #define TEXT_LEN 256 @@ -138,7 +138,8 @@ static void print_usage(FILE *out) { " --sizes LIST input sizes, each 1 to %d\n" " --runs N measured runs per trial (the study uses at least %d)\n" " --warmup N unmeasured runs per trial before measuring; the first calibrates\n" - " --min-loop-ms N calibrate each sort loop and startup batch to about N ms (RAPL: ~1 ms updates)\n" + " --min-loop-ms N make each sort loop and startup batch last at least about N ms; calibration\n" + " aims 20%% higher (RAPL counters update about once a millisecond)\n" " --max-reps N never sort more than N times in one process\n" " --no-startup skip the startup trials (processes that exit without sorting)\n" " --retries N rerun a worker process that fails up to N times (default 1)\n" @@ -498,9 +499,10 @@ static int write_meta(const t_context *ctx, const char *status, uint64_t measure fputs("],\n", f); fprintf(f, " \"runs\": %u,\n \"warmup\": %u,\n \"seed\": %" PRIu32 ",\n \"order_seed\": %" PRIu32 - ",\n \"min_loop_ns\": %" PRIu64 ",\n \"max_reps\": %" PRIu64 ",\n \"startup_trials\": %s,\n" + ",\n \"min_loop_ns\": %" PRIu64 ",\n \"aim_loop_ns\": %" PRIu64 ",\n \"max_reps\": %" PRIu64 + ",\n \"startup_trials\": %s,\n" " \"retries\": %u,\n \"cpus\": [", - o->runs, o->warmup, o->seed, o->order_seed, o->min_loop_ns, o->max_reps, + o->runs, o->warmup, o->seed, o->order_seed, o->min_loop_ns, plan_aim(o->min_loop_ns), o->max_reps, o->startup_trials ? "true" : "false", o->retries); for (size_t i = 0; i < o->cpu_count; i++) fprintf(f, "%s%d", i ? ", " : "", o->cpus[i]); @@ -754,6 +756,7 @@ static int warm_up(t_context *ctx) { t_trial *t = &ctx->trials[order[i]]; uint64_t reps = t->algorithm == ALGO_NONE ? 0 : 1; uint64_t launches = 1; + uint64_t aim = plan_aim(o->min_loop_ns); t_measurement m; for (unsigned w = 0; w < o->warmup; w++) { @@ -763,11 +766,11 @@ static int warm_up(t_context *ctx) { } /* Calibrate, and estimate the next run's length for the progress report. */ if (t->algorithm == ALGO_NONE) { - uint64_t next = plan_calibrate(m.wall_ns, launches, o->min_loop_ns, PLAN_MAX_LAUNCHES); + uint64_t next = plan_calibrate(m.wall_ns, launches, aim, PLAN_MAX_LAUNCHES); t->warm_wall_ns = m.wall_ns / launches * next; launches = next; } else { - uint64_t next = plan_calibrate(m.result.loop_ns, reps, o->min_loop_ns, o->max_reps); + uint64_t next = plan_calibrate(m.result.loop_ns, reps, aim, o->max_reps); uint64_t overhead = m.wall_ns > m.result.loop_ns ? m.wall_ns - m.result.loop_ns : 0; t->warm_wall_ns = overhead + m.result.loop_ns / reps * next; reps = next; @@ -1041,7 +1044,7 @@ static int command_run(int argc, char **argv, const char *argv0) { o->warmup, o->runs, intervals, ctx.trial_count * o->runs); if (o->startup_trials) printf("each startup run launches a batch of processes lasting about %" PRIu64 " ms in total\n", - o->min_loop_ns / 1000000u); + plan_aim(o->min_loop_ns) / 1000000u); cleanup(&ctx); return 0; } @@ -1111,8 +1114,9 @@ static int command_run(int argc, char **argv, const char *argv0) { if (ctx.short_loops > 0 && !o->quiet) fprintf(stderr, "%s: note: %" PRIu64 " of %" PRIu64 " measured sorting runs sorted for less than the %" PRIu64 - " ms target (%" PRIu64 " of them held back by --max-reps %" PRIu64 "). Calibration extrapolates " - "from the warm-up runs; more of them, or a quieter machine, bring the loops closer to the target\n", + " ms target (%" PRIu64 " of them held back by --max-reps %" PRIu64 "). Calibration aims 20%% " + "above the target from the warm-up runs; more of them, or a quieter machine, make fewer loops fall " + "short\n", g_program, ctx.short_loops, ctx.sorting_runs, o->min_loop_ns / 1000000u, ctx.capped_loops, o->max_reps); if (status == 0 && !o->quiet) diff --git a/src/plan.c b/src/plan.c index c437a25..79bb0c3 100644 --- a/src/plan.c +++ b/src/plan.c @@ -57,6 +57,12 @@ void plan_shuffle(void *base, size_t count, size_t size, uint32_t seed) { } } +uint64_t plan_aim(uint64_t min_loop_ns) { + uint64_t extra = min_loop_ns / 100u * PLAN_HEADROOM_PERCENT + min_loop_ns % 100u * PLAN_HEADROOM_PERCENT / 100u; + + return extra > UINT64_MAX - min_loop_ns ? UINT64_MAX : min_loop_ns + extra; +} + uint64_t plan_calibrate(uint64_t loop_ns, uint64_t reps, uint64_t min_loop_ns, uint64_t max_reps) { if (reps == 0) reps = 1; diff --git a/src/plan.h b/src/plan.h index ed73354..158516a 100644 --- a/src/plan.h +++ b/src/plan.h @@ -46,4 +46,12 @@ void plan_shuffle(void *base, size_t count, size_t size, uint32_t seed); long as the reps sorts that took loop_ns did. Always at least 1 and at most max_reps. */ uint64_t plan_calibrate(uint64_t loop_ns, uint64_t reps, uint64_t min_loop_ns, uint64_t max_reps); +/* Calibration aims this many per cent above --min-loop-ms. Aiming at the target itself would + leave about half of the measured loops just short of it, since a measured run is as likely to + be a little faster than the warm-up that set it as a little slower. */ +#define PLAN_HEADROOM_PERCENT 20u + +/* The loop length that calibration aims for: min_loop_ns plus PLAN_HEADROOM_PERCENT. */ +uint64_t plan_aim(uint64_t min_loop_ns); + #endif diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index e5e3f13..2f10563 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -72,6 +72,8 @@ def test_run_without_energy_writes_tidy_csv(tmp_path: Path) -> None: assert meta["clock"] == "CLOCK_MONOTONIC_RAW" assert meta["settings"]["runs"] == 3 assert meta["settings"]["cpus"] == [] + # Calibration aims 20% above the target, so that loops rarely fall short of it. + assert meta["settings"]["aim_loop_ns"] == meta["settings"]["min_loop_ns"] * 6 // 5 assert meta["host"]["allowed_cpus"] >= 1 assert "cgroup_cpu_max" in meta["host"] assert "cgroup_memory_max" in meta["host"] diff --git a/tests/test_plan.c b/tests/test_plan.c index 311edc7..491ef4b 100644 --- a/tests/test_plan.c +++ b/tests/test_plan.c @@ -108,9 +108,21 @@ static void test_calibrate(void) { CHECK(plan_calibrate(5 * ms, 1, 0, 1000000) == 1, "no minimum means one sort"); } +static void test_aim(void) { + const uint64_t ms = 1000000u; + + CHECK(plan_aim(200 * ms) == 240 * ms, "a 200 ms target is aimed at 240 ms"); + CHECK(plan_aim(2 * ms) == 2400000u, "a 2 ms target is aimed at 2.4 ms"); + CHECK(plan_aim(0) == 0, "no minimum, no headroom"); + CHECK(plan_aim(99) == 118, "small values round down"); + CHECK(plan_aim(UINT64_MAX) == UINT64_MAX, "the aim saturates instead of wrapping"); + CHECK(plan_calibrate(1 * ms, 1, plan_aim(200 * ms), 1000000) == 240, "1 ms per sort: 240 sorts"); +} + int main(void) { test_trials(); test_runs_and_shuffle(); test_calibrate(); + test_aim(); return check_report("plan"); } From 4721180110bfbe57e343ac79be788a2f5a0dc794 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:37:35 +0400 Subject: [PATCH 39/51] fix: refuse --cpus when the kernel leaves out a listed CPU sched_setaffinity quietly drops CPUs that are offline, do not exist or lie outside the process's cpuset, as long as one listed CPU remains. The harness then pinned to fewer CPUs than meta.json recorded. It now reads the set back with sched_getaffinity and stops with a usage error naming the CPU, and says plainly when none of the listed CPUs can be used. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/greenbench.c | 21 ++++++++++++++++++--- tests/integration/test_harness.py | 12 ++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 767256b..c8dffe2 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ The base images are pinned by digest. If Docker Hub limits your pulls, the | `--min-loop-ms N`, `--max-reps N` | 200, 1,000,000 | Minimum length of each sort loop and startup batch (calibration during warm-up aims 20% higher); and the cap on sorts per process. | | `--energy auto\|require\|off` | `auto` | `require` fails with exit status 3 if RAPL cannot be read. | | `--idle-baseline auto\|on\|off` | `auto` | On whenever energy is measured. | -| `--cpus LIST` | off | Pin the harness and every worker to these CPUs, such as `2,3` or `2-3`. | +| `--cpus LIST` | off | Pin the harness and every worker to these CPUs, such as `2,3` or `2-3`; each must be online and allowed to the harness. | | `--seed N`, `--order-seed N` | 2463534242, 1234567 | Input generator seed, and run-order seed. | | `--no-startup` | | Skip the startup trials. | | `--retries N` | 1 | Rerun a worker process that fails (cannot start, crashes) up to N times; disagreeing results are never retried. | diff --git a/src/greenbench.c b/src/greenbench.c index 4bbc53f..f6c1fe7 100644 --- a/src/greenbench.c +++ b/src/greenbench.c @@ -882,16 +882,31 @@ static int measure(t_context *ctx, FILE *csv, uint64_t *measured) { return 0; } -/* Pins the harness to the CPUs; the workers it starts inherit the set. */ +/* Pins the harness to the CPUs; the workers it starts inherit the set. The kernel quietly + leaves out CPUs that are offline, do not exist or lie outside this process's cpuset, as + long as one listed CPU remains, so the set in effect is read back and must hold them all: + meta.json records the CPUs asked for, which must be the CPUs used. */ static int pin_cpus(const int *cpus, size_t count) { #ifdef __linux__ - cpu_set_t set; + cpu_set_t set, granted; CPU_ZERO(&set); for (size_t i = 0; i < count; i++) CPU_SET((size_t)cpus[i], &set); if (sched_setaffinity(0, sizeof set, &set) != 0) - return error_message("cannot pin to the CPUs given with --cpus: %s", strerror(errno)), -1; + return error_message("cannot pin to the CPUs given with --cpus (%s): none of them is online and allowed " + "to this process", + strerror(errno)), + -1; + CPU_ZERO(&granted); + if (sched_getaffinity(0, sizeof granted, &granted) != 0) + return error_message("cannot read back the CPUs set with --cpus: %s", strerror(errno)), -1; + for (size_t i = 0; i < count; i++) + if (!CPU_ISSET((size_t)cpus[i], &granted)) + return error_message("cannot pin to CPU %d given with --cpus: it is offline, does not exist or is not " + "allowed to this process", + cpus[i]), + -1; return 0; #else (void)cpus; diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 2f10563..fb9b328 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -138,6 +138,18 @@ def test_pinning_to_a_cpu_set(tmp_path: Path) -> None: assert read_meta(tmp_path / "run")["status"] == "complete" +def test_pinning_refuses_cpus_that_cannot_be_used(tmp_path: Path) -> None: + # CPU 1000 does not exist here. Pinning to 0 and 1000 would quietly give CPU 0 alone, + # while meta.json recorded both, so the harness refuses. + common = [*SMALL, "--languages", "c", "--energy", "off", *interpreter_options()] + mixed = run_harness("run", *common, "--cpus", "0,1000", "--out", str(tmp_path / "mixed")) + assert mixed.returncode == 2 # a usage error + assert "cannot pin to CPU 1000 given with --cpus: it is offline, does not exist" in mixed.stderr + missing = run_harness("run", *common, "--cpus", "1000", "--out", str(tmp_path / "missing")) + assert missing.returncode == 2 + assert "none of them is online and allowed to this process" in missing.stderr + + def test_meta_json_stays_valid_and_the_command_can_be_pasted(tmp_path: Path) -> None: out = tmp_path / "run" label = os.fsdecode(b"it's caf\xe9") # a quote, and a byte that is not UTF-8 From cf4aa73304c74d84f2454d142ee251baec22a356 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:42:30 +0400 Subject: [PATCH 40/51] feat: generate the README's results block with the report After the documented bare-metal run, docs/results.md would show measured energy while the README's status banner, its limitations and study.md still said "pending". The report now takes --readme README.md and rewrites the block between and from the same run: a pending notice while timings are withheld, and otherwise the readings of the power ratios and the main table of ratios between languages. It refuses simulated energy and a README without exactly one pair of markers, before writing anything. run-study.sh passes --readme in study mode and adds README.md to the commit line; bare-metal.md lists it with the files to take away. The block in this README was generated by the report from a withheld run. The README's limitations and study.md's status box now read correctly before and after the study. Co-Authored-By: Claude Opus 5.5 --- README.md | 19 +++--- analysis/greenbench_analysis/cli.py | 29 ++++++++- analysis/greenbench_analysis/report.py | 87 +++++++++++++++++++++++++- analysis/tests/test_report.py | 50 +++++++++++++++ docs/bare-metal.md | 9 ++- docs/decisions.md | 3 +- docs/study.md | 9 +-- scripts/run-study.sh | 9 ++- 8 files changed, 195 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c8dffe2..98898b0 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ counters as the next step. This is that step, done as a small study: an energy-m harness in C, the same algorithms in three languages, statistics, carbon estimates and a written method. -> **Status: energy results are pending.** Everything is built and tested, but the energy -> study needs Linux running directly on hardware, and this version was developed in Docker -> Desktop, where the energy counters do not exist. [docs/results.md](docs/results.md) shows -> the exact operation counts and marks every time, energy and carbon figure as pending until -> the study runs on bare-metal Linux with one command ([how](docs/bare-metal.md)). + + + +> **Status: time and energy results are pending.** Everything is built and tested, but the energy study needs Linux running directly on hardware. [docs/results.md](docs/results.md) comes from a run that checked the pipeline, so it shows the exact operation counts and marks every time, energy and carbon figure as pending until the study runs on bare-metal Linux with one command ([how](docs/bare-metal.md)). + ## Sample output @@ -171,7 +171,8 @@ The report is `python -m greenbench_analysis report --runs DIR/runs.csv` with [data/grid_intensity.csv](data/grid_intensity.csv)), `--intensity NAME=G_PER_KWH` for any other grid, `--equivalence-margin PERCENT` (10 by default; see [study.md](docs/study.md#what-would-count-as-an-answer)), `--withhold-timings` for runs on a -busy machine, `--allow-partial` to report on a run that did not finish, and `--summary-csv`. +busy machine, `--allow-partial` to report on a run that did not finish, `--summary-csv`, and +`--readme README.md`, which rewrites the results block at the top of this file. ## Running the tests @@ -222,8 +223,10 @@ includes 1 is not enough to call two powers the same. ## Limitations and roadmap -- **No energy data yet.** The study has to run on bare-metal Linux; until then the results - are pending, and the README makes no claims about energy. +- **Energy needs bare-metal Linux.** RAPL counters are hidden in containers, WSL2 and most + virtual machines, so the energy study runs only on Linux on real hardware. The block at the + top of this README, generated with [docs/results.md](docs/results.md), says whether the + published results include energy. - **RAPL is not a power meter.** It covers the processor package and DRAM, not the whole machine, and its accuracy varies between processors. See [threats to validity](docs/study.md#threats-to-validity). diff --git a/analysis/greenbench_analysis/cli.py b/analysis/greenbench_analysis/cli.py index 70d004b..a3e0140 100644 --- a/analysis/greenbench_analysis/cli.py +++ b/analysis/greenbench_analysis/cli.py @@ -10,7 +10,15 @@ from .carbon import load_grid, parse_custom, select_regions from .load import DataError, load_meta, load_runs -from .report import EQUIVALENCE_MARGIN, ReportOptions, write_report, write_summary_csv +from .report import ( + EQUIVALENCE_MARGIN, + ReportOptions, + check_readme, + readme_block, + write_readme_block, + write_report, + write_summary_csv, +) from .summary import summarise @@ -82,6 +90,12 @@ def build_parser() -> argparse.ArgumentParser: report.add_argument("--resamples", type=_positive, default=10_000, help="bootstrap resamples (default %(default)s)") report.add_argument("--seed", type=_positive, default=1, help="bootstrap seed (default %(default)s)") report.add_argument("--summary-csv", type=Path, help="also write every configuration's estimates here") + report.add_argument( + "--readme", + type=Path, + help="also rewrite the results block between the and lines " + "of this file (README.md)", + ) return parser @@ -100,6 +114,13 @@ def _report(args: argparse.Namespace, argv: list[str]) -> int: regions = select_regions(grid, codes) + [parse_custom(spec) for spec in args.intensity] if args.withhold_timings and args.summary_csv: raise DataError("--summary-csv would publish the withheld timings; drop one of the two options") + if args.readme and meta.get("energy", {}).get("simulated"): + raise DataError( + "--readme: this run's energy was simulated by a test build (meta.json energy.simulated), so it must " + "not be published" + ) + if args.readme: + check_readme(args.readme) summary = summarise(runs, resamples=args.resamples, seed=args.seed) guide = Path(os.path.relpath(Path("docs/bare-metal.md"), args.out.parent)).as_posix() @@ -114,6 +135,12 @@ def _report(args: argparse.Namespace, argv: list[str]) -> int: written = write_report(runs, meta, summary, args.out, args.img_dir, options) if args.summary_csv: written.append(write_summary_csv(summary, args.summary_csv)) + if args.readme: + start = args.readme.parent + report_link = Path(os.path.relpath(args.out, start)).as_posix() + guide_link = Path(os.path.relpath(Path("docs/bare-metal.md"), start)).as_posix() + block = readme_block(meta, summary, options, report_link, guide_link) + written.append(write_readme_block(args.readme, block)) for path in written: print(f"wrote {path.as_posix()}") return 0 diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 41f9961..12fd6a3 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -16,7 +16,7 @@ from . import charts from .carbon import GridIntensity, grams_co2e -from .load import ORDERS, Run +from .load import ORDERS, DataError, Run from .stats import Estimate from .summary import CONFIDENCE, Comparison, Configuration, RankAgreement, Summary @@ -909,3 +909,88 @@ def write_summary_csv(summary: Summary, path: Path) -> Path: ] ) return path + + +README_START = "" +README_END = "" + + +def readme_block( + meta: dict[str, Any], summary: Summary, options: ReportOptions, report_link: str, guide_link: str +) -> list[str]: + """The short results block for the top of the README, linking to the full report. + + It says that results are pending when timings are withheld, and otherwise gives the + readings of the power ratios and the main table of ratios between languages, so that the + README never says "pending" beside a finished study, or the reverse. + """ + lines = [ + f"", + "", + ] + if options.withhold_timings: + return [ + *lines, + "> **Status: time and energy results are pending.** Everything is built and tested, but the energy " + f"study needs Linux running directly on hardware. [{report_link}]({report_link}) comes from a run " + "that checked the pipeline, so it shows the exact operation counts and marks every time, energy and " + f"carbon figure as pending until the study runs on bare-metal Linux with one command " + f"([how]({guide_link})).", + ] + host = meta.get("host", {}) + started = str(meta.get("started_utc") or "")[:10] or "an unrecorded date" + lines.append( + f"**Results** from [{report_link}]({report_link}): {host.get('cpu_model') or 'CPU not recorded'}, " + f"{host.get('sysname', '?')} {host.get('release', '?')}, {summary.run_count:,} measured runs on {started}." + ) + lines.append("") + status = meta.get("status") + if status != "complete": + lines += [f"> **Warning: this run did not finish (status: {status}).** See the full report.", ""] + runs = meta.get("settings", {}).get("runs") + if isinstance(runs, int) and runs < STUDY_RUNS: + lines += [f"> **Note:** {runs} measured runs per configuration; the study uses {STUDY_RUNS}.", ""] + if not summary.energy_measured: + reason = meta.get("energy", {}).get("reason") or "energy unavailable" + return [*lines, f"Energy was not measured in this run ({reason}); the report gives time and operation counts."] + margin = options.margin + lines += [ + f"Is run time a good stand-in for energy? Each power ratio (energy ratio over time ratio) read against " + f"a {_percent(margin)} margin set before measuring:", + "", + *_readings_table(summary, margin), + "", + *_chance_paragraph(summary), + ] + largest = max((c.n for c in summary.configurations), default=0) + main = [c for c in summary.language_comparisons if c.subject[2] == "random" and c.subject[3] == largest] + if main: + lines += [f"Between languages, random input of {largest:,} elements:", ""] + lines += _table( + ["Algorithm", "Order", "Size", "Languages", *RATIO_HEADER], + [":---", ":---", "---:", ":---", *RATIO_ALIGN], + _language_rows(main, margin), + ) + return lines + + +def check_readme(path: Path) -> str: + """The README's text, if it has exactly one pair of results markers in the right order.""" + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + raise DataError(f"{path}: {error.strerror or error}") from None + start, end = text.find(README_START), text.find(README_END) + if start < 0 or end < start or text.count(README_START) != 1 or text.count(README_END) != 1: + raise DataError(f"{path}: expected one {README_START} line and one {README_END} line after it") + return text + + +def write_readme_block(path: Path, block: list[str]) -> Path: + """Replace what lies between the README's results markers with the block.""" + text = check_readme(path) + start, end = text.find(README_START), text.find(README_END) + head = text[: start + len(README_START)] + path.write_text(head + "\n" + "\n".join(block).rstrip() + "\n" + text[end:], encoding="utf-8") + return path diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index dafc9ec..184af63 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -260,3 +260,53 @@ def test_command_line_errors(tmp_path: Path, capsys: pytest.CaptureFixture[str]) with pytest.raises(SystemExit): main(["report", "--runs", str(runs_path), "--equivalence-margin", "0"]) assert "must be above 0 and below 100" in capsys.readouterr().err + + +README_TEXT = "# Project\n\n\nold block\n\n\n## Rest\n" + + +def _readme_run(tmp_path: Path, *, energy: bool, meta_changes: dict[str, object] | None = None) -> list[str]: + runs_path = synthetic.write_csv(synthetic.study(energy=energy, sizes=(100, 200)), tmp_path / "runs.csv") + synthetic.write_meta({**synthetic.meta(energy=energy), **(meta_changes or {})}, tmp_path / "meta.json") + (tmp_path / "README.md").write_text(README_TEXT, encoding="utf-8") + args = ["report", "--runs", str(runs_path), "--out", str(tmp_path / "docs" / "results.md")] + return [*args, "--img-dir", str(tmp_path / "docs" / "img"), "--grid", str(GRID), "--resamples", "200"] + + +def test_readme_block_says_pending_while_timings_are_withheld(tmp_path: Path) -> None: + args = _readme_run(tmp_path, energy=False) + assert main([*args, "--withhold-timings", "--readme", str(tmp_path / "README.md")]) == 0 + text = (tmp_path / "README.md").read_text(encoding="utf-8") + assert text.startswith("# Project\n\n\n\n\n## Rest\n") + + +def test_readme_block_gives_the_answer_after_a_measured_run(tmp_path: Path) -> None: + args = _readme_run(tmp_path, energy=True) + assert main([*args, "--readme", str(tmp_path / "README.md")]) == 0 + text = (tmp_path / "README.md").read_text(encoding="utf-8") + assert "pending" not in text + assert "**Results** from [docs/results.md](docs/results.md): Test CPU, Linux 6.8.0" in text + assert "| Between languages | 8 | 8 | 0 | 0 | 0 |" in text + assert "Between languages, random input of 200 elements:" in text + assert "| quick | random | 200 | Python / C | 10 (" in text + assert "same power within 10%: time predicts energy" in text + + +def test_readme_block_refuses_simulated_energy_and_missing_markers( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + energy = {**synthetic.meta(energy=True)["energy"], "simulated": True} + args = _readme_run(tmp_path, energy=True, meta_changes={"energy": energy}) + assert main([*args, "--readme", str(tmp_path / "README.md")]) == 1 + assert "must not be published" in capsys.readouterr().err + assert (tmp_path / "README.md").read_text(encoding="utf-8") == README_TEXT + + args = _readme_run(tmp_path, energy=True) + (tmp_path / "README.md").write_text("# Project without markers\n", encoding="utf-8") + assert main([*args, "--readme", str(tmp_path / "README.md")]) == 1 + assert "expected one line" in capsys.readouterr().err + assert not (tmp_path / "docs" / "results.md").exists() # nothing is written when the README is wrong diff --git a/docs/bare-metal.md b/docs/bare-metal.md index d66b838..628ded1 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -81,16 +81,21 @@ disk, and the study leaves the machine as it found it after a reboot. versions, limits), `environment.txt` (kernel, `lscpu`, governor, power source) and `summary.csv`. - `docs/results.md` and `docs/img/`: the report, now with measured time and energy. + - `README.md`: the results block at the top, rewritten from the same run. Copy the whole repository folder to the second USB stick, then shut down; the reboot also makes the counters root-only again. On your usual computer, copy `data/runs//`, - `docs/results.md` and `docs/img/` into your clone and commit them there: + `docs/results.md`, `docs/img/` and `README.md` into your clone and commit them there: ```bash - git add docs/results.md docs/img data/runs + git add README.md docs/results.md docs/img data/runs git commit -m "docs: add measured study results" ``` + The other documents describe the method rather than the results, so they need no change. + Before publishing, replace the README's sample `greenbench probe` output with the one this + machine printed. + A live session has no git identity and no GitHub credentials, which is why the commit happens elsewhere. To commit from the live session instead, first run `git config user.name "Farah Sharif"` and diff --git a/docs/decisions.md b/docs/decisions.md index 65b1919..66174cb 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -203,7 +203,8 @@ counts and marks every time, energy and carbon figure as pending. **Consequences.** No number in the repository comes from a machine that could not measure it properly. One command (`./scripts/run-study.sh`) replaces the pending sections with -measured ones. +measured ones, and rewrites the results block at the top of the README from the same run, so +the front page cannot say "pending" beside measured results, or the reverse. ## 13. Keep the last-element pivot diff --git a/docs/study.md b/docs/study.md index 792574f..245e661 100644 --- a/docs/study.md +++ b/docs/study.md @@ -8,10 +8,11 @@ study tests it on a small, fully controlled workload: three sorting algorithms, identically in three languages, sorting identical inputs. > **Status.** The harness, the workers, the statistics and the report are built and tested. -> The energy measurements have not been made yet: they need Linux on real hardware, and this -> version was developed in Docker Desktop, where RAPL is unavailable. Until the study runs on -> bare-metal Linux ([how](bare-metal.md)), [results.md](results.md) shows exact operation -> counts and marks every time, energy and carbon figure as pending. +> The results, and whether they include energy yet, are in [results.md](results.md), which +> the report script generates from a recorded run. Energy can only be measured on Linux +> running directly on hardware ([how](bare-metal.md)); until the study has run there, +> results.md shows exact operation counts and marks every time, energy and carbon figure as +> pending. ## Questions diff --git a/scripts/run-study.sh b/scripts/run-study.sh index 0bdcf18..5c178db 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Runs the greenbench energy study on bare-metal Linux, from checks to report: # -# ./scripts/run-study.sh the full study; writes docs/results.md and docs/img/ +# ./scripts/run-study.sh the full study; writes docs/results.md, docs/img/ and the +# results block at the top of README.md # ./scripts/run-study.sh --check only check that this machine can run it # ./scripts/run-study.sh --rehearse a short run with energy, written under results/ # @@ -258,14 +259,16 @@ say "== Measuring (profile $profile); close other programs and keep the machine --label "run-study.sh $MODE $stamp" "${pin[@]}" say "== Report" +readme=() +if [[ "$MODE" == study ]]; then readme=(--readme README.md); fi PYTHONPATH=analysis .venv/bin/python -m greenbench_analysis report \ - --runs "$out/runs.csv" --out "$report" --img-dir "$images" --summary-csv "$out/summary.csv" + --runs "$out/runs.csv" --out "$report" --img-dir "$images" --summary-csv "$out/summary.csv" "${readme[@]}" say say "Done. Raw data: $out/ Report: $report" if [[ "$MODE" == study ]]; then say "Next: read $report. A live session forgets everything when it shuts down, so copy the" say "whole repository folder to a second USB stick, then commit from your usual computer:" - say " git add $report $images $out && git commit -m 'docs: add measured study results'" + say " git add README.md $report $images $out && git commit -m 'docs: add measured study results'" say "Reboot afterwards to make the RAPL counters root-only again." fi From 27629e6fa49ef80697c28ce360989ff2f1ba5d3c Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:48:41 +0400 Subject: [PATCH 41/51] fix: keep the study off CPU 0's sibling thread and efficiency cores The study script took the first online CPU after 0 and the next CPU on another core. On processors that number a core's two threads 0 and 1, as this project's own Ryzen 7 6800H does, that pinned the study to CPU 0's other thread, which shares CPU 0's interrupt load. The choice also ignored core types on hybrid processors. It now skips every CPU in CPU 0's thread_siblings_list, and uses only the performance cores in /sys/devices/cpu_core/cpus when Linux lists them. It prints the choice with the reason, and --show-cpus prints them without running anything. New tests check the choice on fake topologies through GREENBENCH_SYSFS: adjacent and split thread numbering, a hybrid processor, offline CPUs, a second package, no suitable pair, and the overrides. ADR 15, study.md and bare-metal.md describe the rule. Co-Authored-By: Claude Opus 5.5 --- docs/bare-metal.md | 8 +- docs/decisions.md | 14 +++- docs/study.md | 5 +- scripts/run-study.sh | 95 ++++++++++++++++----- tests/integration/test_study_script.py | 112 +++++++++++++++++++++++++ 5 files changed, 202 insertions(+), 32 deletions(-) create mode 100644 tests/integration/test_study_script.py diff --git a/docs/bare-metal.md b/docs/bare-metal.md index 628ded1..927c7eb 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -108,9 +108,11 @@ disk, and the study leaves the machine as it found it after a reboot. wait a few minutes after booting before running the study. - Keep the machine on mains power; laptops on battery often cap the processor. - The script pins the harness and every worker to two CPUs on different physical cores of - one processor, never CPU 0, and prints which. Choose others with `--cpus 4,6`, or turn - pinning off with `--no-pin`. On a processor with performance and efficiency cores, check - with `lscpu --extended` that both CPUs are the same kind. + one processor, never CPU 0 or a CPU that shares its core, and on a processor with + performance and efficiency cores only performance cores. It prints which, and why; + `./scripts/run-study.sh --show-cpus` shows the choice without running anything. Choose + others with `--cpus 4,6` (check with `lscpu --extended` that both are the same kind of core), + or turn pinning off with `--no-pin`. - Frequency scaling and turbo stay as they are, and are recorded. For steadier numbers you can fix them for the session, for example on Intel with `echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo`. Mention it when you diff --git a/docs/decisions.md b/docs/decisions.md index 66174cb..62a66fc 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -243,16 +243,22 @@ worker 1 (counted in `/proc/PID/task` in the Docker image). On one CPU those hel take turns with the sorting thread, which slows Node.js down in a way an ordinary machine with idle cores would not, and biases the comparison between languages against it. -**Decision.** The harness takes a CPU list, `--cpus 2,3`. The study script chooses two CPUs -on different physical cores of the same processor from `/sys/devices/system/cpu/*/topology`, -leaving out CPU 0, which handles most interrupts; `--cpus` overrides the choice and +**Decision.** The harness takes a CPU list, `--cpus 2,3`, and refuses one that the kernel +would quietly shorten. The study script chooses two CPUs on different physical cores of the +same processor from `/sys/devices/system/cpu/*/topology`. It leaves out CPU 0, which handles +most interrupts, and any CPU in CPU 0's `thread_siblings_list`: many processors number a +core's two threads 0 and 1, so "the first CPU after 0" can be CPU 0's other half. On a +processor with performance and efficiency cores it chooses only from the performance cores +that Linux lists in `/sys/devices/cpu_core/cpus`. It prints the choice and the reason; +`--show-cpus` prints them without running anything, `--cpus` overrides the choice and `--no-pin` turns pinning off. Different physical cores rather than the two hardware threads of one core, so that a helper thread does not compete with the sorting thread for the same execution units. **Consequences.** Node.js's helper threads run beside the sorting thread, as on an idle desktop. C and Python can move between the two CPUs, which are the same kind of core, so this -changes little. RAPL counts the whole package, so the helper threads' energy is included +changes little. The choice is tested on fake CPU topologies (`GREENBENCH_SYSFS`), including +adjacent sibling numbering and a hybrid processor. RAPL counts the whole package, so the helper threads' energy is included either way. No pinned-against-unpinned comparison has been measured; the threats to validity say so. diff --git a/docs/study.md b/docs/study.md index 245e661..e8e3bee 100644 --- a/docs/study.md +++ b/docs/study.md @@ -119,8 +119,9 @@ lists, JavaScript arrays). None uses a built-in sort. - **Repetitions and order.** Three warm-up runs per configuration, then 30 measured runs. All measured runs of all configurations are shuffled into one random order from a recorded seed. - **Pinning.** The study script pins the harness and all workers to two CPUs on different - physical cores of one processor (never CPU 0), so the measured processes stay on known - cores. Two rather than one, because Node.js runs its compiler and garbage collector on + physical cores of one processor (never CPU 0 or a CPU that shares its core, and only + performance cores on a processor that also has efficiency cores), so the measured processes + stay on known cores of one kind. Two rather than one, because Node.js runs its compiler and garbage collector on helper threads ([ADR 15](decisions.md#15-pin-to-two-cpus-on-different-physical-cores-not-one)). - **Records.** `runs.csv` holds one row per measured run (tidy data: one variable per column). `meta.json` holds the settings, the RAPL zones and their wrap values, the CPU model, the diff --git a/scripts/run-study.sh b/scripts/run-study.sh index 5c178db..6251b50 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -5,6 +5,7 @@ # results block at the top of README.md # ./scripts/run-study.sh --check only check that this machine can run it # ./scripts/run-study.sh --rehearse a short run with energy, written under results/ +# ./scripts/run-study.sh --show-cpus print the CPUs the study would pin to, and why # # See docs/bare-metal.md for preparing a machine (for example Ubuntu booted from a USB stick). set -euo pipefail @@ -17,6 +18,8 @@ ASSUME_YES=0 POWERCAP=/sys/class/powercap FAKE_POWERCAP=0 HARNESS=build/greenbench +# Where the CPU topology is read from. Only the tests point it at a fake tree. +SYSFS="${GREENBENCH_SYSFS:-/sys}" usage() { cat <<'EOF' @@ -25,8 +28,10 @@ Usage: scripts/run-study.sh [--check | --rehearse] [--cpus LIST | --no-pin] [--y --check check the machine (Linux on real hardware, tools, readable RAPL counters) and stop --rehearse run the short smoke profile with energy, to test the machine; results go to results/ --cpus LIST pin the harness and workers to these CPUs, such as 2,3 (default: two CPUs on - different physical cores of one processor, never CPU 0) + different physical cores of one processor, never CPU 0 or its sibling thread, + and only performance cores on a processor that also has efficiency cores) --no-pin do not pin to any CPUs + --show-cpus print the CPUs the study would pin to, and why, then stop --yes do not ask before making the RAPL counters readable --powercap-root DIR read RAPL zones from DIR instead of /sys/class/powercap. Only for rehearsing this @@ -53,6 +58,7 @@ while (($# > 0)); do shift ;; --no-pin) CPUS=none ;; + --show-cpus) MODE=show-cpus ;; --yes) ASSUME_YES=1 ;; --powercap-root) [[ $# -ge 2 && -d "$2" ]] || fail "--powercap-root needs an existing directory" @@ -145,33 +151,64 @@ check_rapl() { sudo chmod o+r "${unreadable[@]}" } -# Two CPUs on different physical cores of the same processor, leaving out CPU 0, which -# handles most interrupts. Two cores, not one: Node.js runs its JIT compiler and garbage -# collector on helper threads, which must not queue behind the sorting thread. Prints nothing -# when there are no two such CPUs. -default_cpus() { - local total cpu dir package core first="" first_package="" first_core="" - total="$(nproc --all)" - for ((cpu = 1; cpu < total; cpu++)); do - dir="/sys/devices/system/cpu/cpu$cpu" +# Whether CPU $1 is in a kernel CPU list such as "0-3,8,10-11". +cpu_in_list() { + local cpu="$1" part + local -a parts + IFS=, read -r -a parts <<<"$2" + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9]+(-[0-9]+)?$ ]] || continue + if ((cpu >= ${part%-*} && cpu <= ${part#*-})); then return 0; fi + done + return 1 +} + +# Chooses the CPUs to pin to and sets PIN_CPUS (empty for no pinning) and PIN_REASON. +# By default: two CPUs on different physical cores of the same processor. Two cores, not one: +# Node.js runs its JIT compiler and garbage collector on helper threads, which must not queue +# behind the sorting thread. CPU 0 handles most interrupts, so it is left out, and so is any +# CPU that shares a core with it: many processors number a core's two threads 0 and 1. On a +# processor with performance and efficiency cores, Linux lists the performance cores in +# devices/cpu_core/cpus, and only those are used, so that both CPUs are the same kind. +choose_cpus() { + PIN_CPUS="" PIN_REASON="" + if [[ "$CPUS" == none ]]; then + PIN_REASON="pinning turned off with --no-pin" + return 0 + fi + if [[ "$CPUS" != auto ]]; then + PIN_CPUS="$CPUS" PIN_REASON="given with --cpus" + return 0 + fi + local base="$SYSFS/devices/system/cpu" performance="" cpu dir siblings package core + local first="" first_package="" first_core="" + if [[ -r "$SYSFS/devices/cpu_core/cpus" ]]; then performance="$(<"$SYSFS/devices/cpu_core/cpus")"; fi + local -a cpus=() + for dir in "$base"/cpu[0-9]*; do + [[ -d "$dir" ]] && cpus+=("${dir##*/cpu}") + done + mapfile -t cpus < <(printf '%s\n' "${cpus[@]}" | sort -n) + for cpu in "${cpus[@]}"; do + ((cpu == 0)) && continue + dir="$base/cpu$cpu" [[ -r "$dir/topology/core_id" && -r "$dir/topology/physical_package_id" ]] || continue - if [[ -r "$dir/online" && "$(cat "$dir/online")" != 1 ]]; then continue; fi - package="$(cat "$dir/topology/physical_package_id")" - core="$(cat "$dir/topology/core_id")" + if [[ -r "$dir/online" && "$(<"$dir/online")" != 1 ]]; then continue; fi + siblings="$cpu" + if [[ -r "$dir/topology/thread_siblings_list" ]]; then siblings="$(<"$dir/topology/thread_siblings_list")"; fi + if cpu_in_list 0 "$siblings"; then continue; fi + if [[ -n "$performance" ]] && ! cpu_in_list "$cpu" "$performance"; then continue; fi + package="$(<"$dir/topology/physical_package_id")" + core="$(<"$dir/topology/core_id")" if [[ -z "$first" ]]; then first="$cpu" first_package="$package" first_core="$core" elif [[ "$package" == "$first_package" && "$core" != "$first_core" ]]; then - printf '%s,%s\n' "$first" "$cpu" + PIN_CPUS="$first,$cpu" + PIN_REASON="two physical cores of one processor, leaving out CPU 0 and any CPU that shares its core" + if [[ -n "$performance" ]]; then PIN_REASON+="; performance cores only ($performance)"; fi return 0 fi done -} - -pin_options() { - local cpus="$CPUS" - if [[ "$cpus" == none ]]; then return 0; fi - if [[ "$cpus" == auto ]]; then cpus="$(default_cpus)"; fi - if [[ -n "$cpus" ]]; then printf '%s\n' --cpus "$cpus"; fi + PIN_REASON="no two CPUs on different cores of one processor apart from CPU 0 and its core" } mains_power() { @@ -212,6 +249,12 @@ record_environment() { } >"$file" } +if [[ "$MODE" == show-cpus ]]; then + choose_cpus + say "CPUs for the study: ${PIN_CPUS:-none} ($PIN_REASON)" + exit 0 +fi + if ((FAKE_POWERCAP)); then warn "reading a fake powercap tree at $POWERCAP; the energy values are not real" else @@ -251,8 +294,14 @@ else fi mkdir -p "$out" record_environment "$out/environment.txt" -mapfile -t pin < <(pin_options) -if ((${#pin[@]} > 0)); then say "== Pinning the harness and workers to CPUs ${pin[1]}"; else say "== Not pinning to CPUs"; fi +choose_cpus +pin=() +if [[ -n "$PIN_CPUS" ]]; then + pin=(--cpus "$PIN_CPUS") + say "== Pinning the harness and workers to CPUs $PIN_CPUS ($PIN_REASON)" +else + say "== Not pinning to CPUs ($PIN_REASON)" +fi say "== Measuring (profile $profile); close other programs and keep the machine on mains power" "$HARNESS" run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ diff --git a/tests/integration/test_study_script.py b/tests/integration/test_study_script.py new file mode 100644 index 0000000..ace5c94 --- /dev/null +++ b/tests/integration/test_study_script.py @@ -0,0 +1,112 @@ +"""The study script's choice of CPUs, on fake CPU topologies (GREENBENCH_SYSFS).""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from workers import REPOSITORY + +SCRIPT = REPOSITORY / "scripts" / "run-study.sh" + + +@dataclass(frozen=True) +class Cpu: + package: int + core: int + siblings: str + online: bool = True + + +def fake_sysfs(root: Path, cpus: dict[int, Cpu], performance: str | None = None) -> Path: + for number, cpu in cpus.items(): + directory = root / "devices" / "system" / "cpu" / f"cpu{number}" + (directory / "topology").mkdir(parents=True) + (directory / "topology" / "physical_package_id").write_text(f"{cpu.package}\n", encoding="utf-8") + (directory / "topology" / "core_id").write_text(f"{cpu.core}\n", encoding="utf-8") + (directory / "topology" / "thread_siblings_list").write_text(f"{cpu.siblings}\n", encoding="utf-8") + if number != 0: # like Linux: cpu0 usually has no online file + (directory / "online").write_text("1\n" if cpu.online else "0\n", encoding="utf-8") + if performance is not None: + (root / "devices" / "cpu_core").mkdir(parents=True) + (root / "devices" / "cpu_core" / "cpus").write_text(f"{performance}\n", encoding="utf-8") + return root + + +def show_cpus(root: Path, *options: str) -> str: + completed = subprocess.run( + ["bash", str(SCRIPT), "--show-cpus", *options], + env={**os.environ, "GREENBENCH_SYSFS": str(root)}, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + assert completed.returncode == 0, completed.stderr + prefix = "CPUs for the study: " + assert completed.stdout.startswith(prefix), completed.stdout + return completed.stdout[len(prefix) :].strip() + + +def two_threads_per_core(cores: int, *, adjacent: bool) -> dict[int, Cpu]: + """Numbering 0,1 on core 0 (adjacent), or 0 and cores on core 0 (split).""" + cpus = {} + for core in range(cores): + pair = (2 * core, 2 * core + 1) if adjacent else (core, core + cores) + for number in pair: + cpus[number] = Cpu(0, core, f"{pair[0]}-{pair[1]}" if adjacent else f"{pair[0]},{pair[1]}") + return cpus + + +def test_adjacent_threads_leave_out_cpu_0s_sibling(tmp_path: Path) -> None: + # As on many AMD and Intel laptops: CPU 1 is CPU 0's second thread, so the first choice is 2. + root = fake_sysfs(tmp_path, two_threads_per_core(4, adjacent=True)) + assert show_cpus(root).startswith("2,4 (two physical cores of one processor, leaving out CPU 0") + + +def test_split_thread_numbering(tmp_path: Path) -> None: + root = fake_sysfs(tmp_path, two_threads_per_core(4, adjacent=False)) + assert show_cpus(root).startswith("1,2 (") + + +def test_only_performance_cores_on_a_hybrid_processor(tmp_path: Path) -> None: + cpus = {n: Cpu(0, n // 2, f"{n - n % 2}-{n - n % 2 + 1}") for n in range(4)} # P-cores 0-1 and 2-3 + cpus |= {n: Cpu(0, 10 + n, str(n)) for n in range(4, 8)} # four efficiency cores + cpus |= {8: Cpu(0, 4, "8-9"), 9: Cpu(0, 4, "8-9")} # one more P-core, numbered last + root = fake_sysfs(tmp_path, cpus, performance="0-3,8-9") + chosen = show_cpus(root) + assert chosen.startswith("2,8 (two physical cores of one processor") + assert chosen.endswith("; performance cores only (0-3,8-9))") + + +def test_offline_cpus_and_other_packages_are_skipped(tmp_path: Path) -> None: + cpus = { + 0: Cpu(0, 0, "0"), + 1: Cpu(0, 1, "1", online=False), + 2: Cpu(0, 2, "2"), + 3: Cpu(1, 0, "3"), + 4: Cpu(0, 3, "4"), + } + root = fake_sysfs(tmp_path, cpus) + assert show_cpus(root).startswith("2,4 (") + + +def test_no_pinning_without_two_suitable_cores(tmp_path: Path) -> None: + root = fake_sysfs(tmp_path, two_threads_per_core(1, adjacent=True)) + assert show_cpus(root) == "none (no two CPUs on different cores of one processor apart from CPU 0 and its core)" + + +@pytest.mark.parametrize( + ("options", "expected"), + [ + (["--cpus", "4,6"], "4,6 (given with --cpus)"), + (["--no-pin"], "none (pinning turned off with --no-pin)"), + ], +) +def test_the_choice_can_be_overridden(tmp_path: Path, options: list[str], expected: str) -> None: + root = fake_sysfs(tmp_path, two_threads_per_core(4, adjacent=True)) + assert show_cpus(root, *options) == expected From 45aeec150539ffb8e92140b7ff55a68de11fbc93 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:48:57 +0400 Subject: [PATCH 42/51] fix: block suspend while the study measures The full study runs for about an hour unattended. A suspend would stop CLOCK_MONOTONIC_RAW and can reset the RAPL counters; the harness already stops on the implausible jump that a reset causes, but the hour is lost. run-study.sh now runs the harness under systemd-inhibit, blocking suspend, idle sleep and the lid switch, when logind is reachable, and otherwise warns that nothing stops the machine from suspending. bare-metal.md asks for automatic suspend to be turned off as well. Checked in a container with a stub systemd-inhibit on the PATH: the rehearsal ran the harness through it with the expected options. Co-Authored-By: Claude Opus 5.5 --- docs/bare-metal.md | 4 ++++ scripts/run-study.sh | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/bare-metal.md b/docs/bare-metal.md index 927c7eb..a08bd13 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -113,6 +113,9 @@ disk, and the study leaves the machine as it found it after a reboot. `./scripts/run-study.sh --show-cpus` shows the choice without running anything. Choose others with `--cpus 4,6` (check with `lscpu --extended` that both are the same kind of core), or turn pinning off with `--no-pin`. +- A suspend in the middle of the study stops the clock and can reset the counters. The script + runs the measurements under `systemd-inhibit`, which blocks suspend, idle sleep and the lid + switch; turn off automatic suspend in Settings > Power as well, and keep the lid open. - Frequency scaling and turbo stay as they are, and are recorded. For steadier numbers you can fix them for the session, for example on Intel with `echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo`. Mention it when you @@ -145,6 +148,7 @@ python3 tests/integration/powercap.py build tests/fixtures/powercap/laptop-intel | `permission denied reading .../energy_uj` from the harness | Run `--check` again and accept the `chmod`, or reboot and start again. | | `the package energy counter intel-rapl:N (package-N) did not change` | The counter files exist but their values do not change, as some virtual machines and firmware leave them. The probe exits with status 3, the script stops, and this machine cannot measure energy. | | `the RAPL counters rose by ... more than ... package(s) can draw` | The counters jumped, usually because the machine was suspended during the run and the counters reset. Turn off automatic suspend and start again. | +| `systemd-inhibit is not available` | Nothing stops the machine from suspending during the study. Turn off automatic suspend in Settings > Power before you start. | | `the implementations disagree` | A worker produced different counts from the others. This is a bug; please open an issue with the message. | The harness stops cleanly on Ctrl+C: the runs measured so far stay in `runs.csv`, and diff --git a/scripts/run-study.sh b/scripts/run-study.sh index 6251b50..fd0157a 100755 --- a/scripts/run-study.sh +++ b/scripts/run-study.sh @@ -303,8 +303,20 @@ else say "== Not pinning to CPUs ($PIN_REASON)" fi +# A suspend would stop the clock and can reset the RAPL counters, so the harness runs under a +# systemd-inhibit lock that blocks suspend, idle sleep and the lid switch, where logind allows it. +inhibit=() +if command -v systemd-inhibit >/dev/null && systemd-inhibit --list >/dev/null 2>&1; then + inhibit=(systemd-inhibit --what=sleep:idle:handle-lid-switch --who=greenbench + --why="measuring energy; a suspend would spoil the run" --mode=block) + say "== Blocking suspend, idle sleep and the lid switch while measuring (systemd-inhibit)" +else + warn "systemd-inhibit is not available, so nothing stops the machine from suspending: turn off automatic" \ + "suspend (Settings > Power) and keep the lid open" +fi + say "== Measuring (profile $profile); close other programs and keep the machine on mains power" -"$HARNESS" run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ +"${inhibit[@]}" "$HARNESS" run --profile "$profile" --energy require --powercap-root "$POWERCAP" --out "$out" \ --label "run-study.sh $MODE $stamp" "${pin[@]}" say "== Report" From a940a04912a310cc2e5753edbdddebdcc74e62ad Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:50:38 +0400 Subject: [PATCH 43/51] build: add make setup, so the quick start is five commands The quick start chained eight commands on five lines. make setup creates .venv and installs the hashed analysis requirements, and make smoke already builds what it needs, so the block now reads: git clone, cd, make setup, make smoke, ./scripts/run-study.sh. Checked in the Docker image as a normal user: make setup and make smoke write results/smoke/results.md. Co-Authored-By: Claude Opus 5.5 --- Makefile | 8 +++++++- README.md | 13 +++++++------ docs/study.md | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 1edfa7b..31ec782 100644 --- a/Makefile +++ b/Makefile @@ -23,12 +23,18 @@ SIMULATED := $(BUILD)/greenbench-simulated TESTS := test_sorts test_inputs test_rapl test_options test_plan test_protocol test_output test_spawn TEST_BIN := $(TESTS:%=$(BUILD)/tests/%) -.PHONY: all simulated test sanitize smoke parity integration check-python check-node lint-shell check clean study +.PHONY: all setup simulated test sanitize smoke parity integration check-python check-node lint-shell check clean study all: $(HARNESS) $(WORKER) simulated: $(SIMULATED) +# The project's virtual environment with the analysis packages, installed from hashed pins. +# Afterwards the Makefile runs Python from .venv. +setup: + python3 -m venv .venv + .venv/bin/python -m pip install --require-hashes -r requirements.txt + $(BUILD)/obj $(BUILD)/tests: mkdir -p $@ diff --git a/README.md b/README.md index 98898b0..a9c1ee7 100644 --- a/README.md +++ b/README.md @@ -121,14 +121,15 @@ the other languages, and writes a CSV row. On Linux (including WSL2) with a C compiler, make, Python 3.12+ (with `venv`) and Node.js 18+: ```bash -git clone https://github.com/fasharif/Algorithm-Energy-C.git greenbench && cd greenbench -make && make test # build the harness and workers, run the C unit tests -python3 -m venv .venv && .venv/bin/pip install --require-hashes -r requirements.txt -make smoke # all three languages, energy off; report in results/smoke/results.md -./scripts/run-study.sh # the energy study, on bare-metal Linux only +git clone https://github.com/fasharif/Algorithm-Energy-C.git greenbench +cd greenbench +make setup # a virtual environment (.venv) with the analysis packages +make smoke # build, run all three languages with energy off, report in results/smoke/ +./scripts/run-study.sh # the energy study, on bare-metal Linux only ``` -The Makefile uses `.venv/bin/python` when it exists, and `python3` otherwise. +The Makefile uses `.venv/bin/python` when it exists, and `python3` otherwise. `make test` +runs the C unit tests; [Running the tests](#running-the-tests) lists the rest. On Windows or macOS, the Docker image runs the same smoke test and writes the report to `results/smoke/` on your machine (in PowerShell, write `${PWD}` instead of `$PWD`): diff --git a/docs/study.md b/docs/study.md index e8e3bee..f34887f 100644 --- a/docs/study.md +++ b/docs/study.md @@ -257,6 +257,7 @@ lists, JavaScript arrays). None uses a built-in sort. ```bash make && make test # build and unit tests, on any Linux +make setup # a virtual environment with the analysis packages make smoke # a short run of all three languages with energy off ./scripts/run-study.sh --check # on bare-metal Linux: can this machine measure energy? ./scripts/run-study.sh # the study itself; writes docs/results.md From f66878cde60ca3e8b55771fca71cfa3c176a17cf Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:53:13 +0400 Subject: [PATCH 44/51] ci: add job timeouts, test the Node.js worker on 18 and 20, drop a repeat - Every job has timeout-minutes, so a hung run cannot hold a runner for six hours. - The parity tests ran twice in the workers job: make parity, then make integration, which includes them. The job now runs make integration once. - package.json and the README say the workers run on Node.js 18 or later, which CI did not test. A new job runs the worker tests on Node.js 18 and 20 (they pass on 18.20.8 and 20.20.2 in containers). - The Python worker claimed Python 3.10 or later, which nothing tested; ADR 9 and its docstring now say 3.12 or later, as CI tests with 3.12 and 3.14. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++++---- README.md | 10 +++++---- docs/decisions.md | 12 +++++----- workers/python/greenbench_worker.py | 2 +- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c6039a..3cb9d4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: c: name: C build and unit tests (${{ matrix.compiler }}) runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -28,6 +29,7 @@ jobs: sanitizers: name: AddressSanitizer and UBSan runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v7 @@ -41,6 +43,7 @@ jobs: workers: name: Cross-language parity, harness run and pipeline runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v7 @@ -60,10 +63,9 @@ jobs: - name: Build run: make - - name: xorshift32, inputs and operation counts identical in C, Python and Node.js - run: make parity PYTHON=python - - - name: Harness and pipeline integration tests + # Includes the parity tests: xorshift32, inputs and operation counts identical in C, + # Python and Node.js. make parity runs them on their own. + - name: Parity, harness, study script and pipeline integration tests run: make integration PYTHON=python - name: Short harness run with energy off, then the report @@ -77,6 +79,7 @@ jobs: statistics: name: Statistics and Python worker (Python ${{ matrix.python }}) runs-on: ubuntu-latest + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -99,6 +102,7 @@ jobs: node: name: Node.js worker (lint, type-check, tests) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -111,9 +115,31 @@ jobs: - name: Lint, type-check and test run: make check-node + node-older: + name: Node.js worker tests on Node.js ${{ matrix.node }} + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # The workers support Node.js 18 and later (package.json engines), which is what + # Ubuntu 24.04 ships; the lint and type-check tools need a newer Node.js. + node: ["18", "20"] + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + + - name: Worker tests (no packages needed) + working-directory: workers/node + run: node --test + shell: name: ShellCheck runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v7 @@ -123,6 +149,7 @@ jobs: actionlint: name: Workflow lint runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v7 diff --git a/README.md b/README.md index a9c1ee7..ec171f6 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ make sanitize # the C tests and a harness run under AddressSanitiz .venv/bin/pip install --require-hashes -r requirements-dev.txt make check-python # ruff, mypy --strict, and the statistics and Python worker tests make check-node # ESLint, tsc --strict and the Node.js worker tests -make integration # cross-language parity, the harness end to end, and the report pipeline +make integration # cross-language parity, the harness, the study script and the report pipeline make lint-shell # ShellCheck on the study script ``` @@ -195,9 +195,11 @@ counters between readings to test wraparound. The integration tests also run ([ADR 18](docs/decisions.md#18-test-energy-with-a-simulated-rapl-build-not-with-moving-files)). With it, energy that is higher during runs than during idle windows goes from the harness through to every section of the report, and a counter wraps inside a measured interval, -however busy the machine is. The real build must refuse counters that never move. CI runs all -of the above and lints the workflow with actionlint; the C build and tests run with both GCC -and Clang, and the statistics tests on Python 3.12 and 3.14. +however busy the machine is. The real build must refuse counters that never move. Tests of +the study script check its choice of CPUs on fake CPU topologies. CI runs all of the above +and lints the workflow with actionlint; the C build and tests run with both GCC and Clang, the +statistics tests on Python 3.12 and 3.14, and the Node.js worker tests also on Node.js 18 +and 20. ## Folder structure diff --git a/docs/decisions.md b/docs/decisions.md index 62a66fc..63fec3e 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -155,9 +155,10 @@ C sorts still produce the exact counts Algorithm-Energy-C published, which the t **Context.** The study has to run on a freshly booted Linux, where Python comes from the distribution (Python 3.12 on Ubuntu 24.04) and Node.js is one download away (ADR 16). -**Decision.** The Python worker uses only the standard library and runs on Python 3.10 or -later. The Node.js worker is plain JavaScript with JSDoc types, checked by the TypeScript -compiler in strict mode (`checkJs`), and needs no build step or packages at run time. +**Decision.** The Python worker uses only the standard library and runs on Python 3.12 or +later, which CI tests with 3.12 and 3.14. The Node.js worker is plain JavaScript with JSDoc +types, checked by the TypeScript compiler in strict mode (`checkJs`), and needs no build step +or packages at run time. **Consequences.** Nothing to install for the workers, and no build step between source and what is measured. Type checking still covers the JavaScript. The analysis package needs @@ -271,8 +272,9 @@ does not test. CI and the Docker image use Node.js 24, the current long-term sup **Decision.** [bare-metal.md](bare-metal.md) installs the latest Node.js 24 release from nodejs.org, checked against the release's published SHA-256 sums, instead of the `nodejs` package. The study script warns when the Node.js on the path is older than 24, and the harness -records the version it ran in `meta.json`. The workers still run on Node.js 18 or later, so -the rest of the project keeps working with a distribution's Node.js. +records the version it ran in `meta.json`. The workers still run on Node.js 18 or later, and +CI runs their tests on Node.js 18 and 20, so the rest of the project keeps working with a +distribution's Node.js. **Consequences.** The Node.js results describe the same major version that CI tests. The guide has one more step, with a download that is verified before use. Python stays the diff --git a/workers/python/greenbench_worker.py b/workers/python/greenbench_worker.py index ff415fc..1eb1149 100755 --- a/workers/python/greenbench_worker.py +++ b/workers/python/greenbench_worker.py @@ -8,7 +8,7 @@ The sorts are line-for-line ports of src/sorts.c, including where comparisons and writes are counted, so all three languages report identical counts for identical inputs. They use plain lists and loops, never the built-in sort. Only the standard library is used, so the worker -runs on any Python 3.10 or later without installing anything. +runs on Python 3.12 or later (tested with 3.12 and 3.14) without installing anything. """ from __future__ import annotations From 60ddc84147aca471959593245b814ce6d08b7cdc Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:54:06 +0400 Subject: [PATCH 45/51] docs: keep the README and study notes true before and after the study - ADR 2's decision paragraph is wrapped like the rest again. - study.md's measurement section says that a run stops on a counter that stands still for 100 ms or jumps by more than 1 kW per package. - bare-metal.md says what the SHA-256 check of the Node.js download does and does not cover. - The README's features, limitations, roadmap and folder notes no longer assume that the study has not run yet; the design decisions list mentions the simulated test build. Co-Authored-By: Claude Opus 5.5 --- README.md | 21 ++++++++++++--------- docs/bare-metal.md | 4 +++- docs/decisions.md | 7 ++++--- docs/study.md | 4 +++- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ec171f6..276b75c 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,10 @@ compare?** The research question, method and threats to validity are in - **Carbon estimates.** Energy converted to grams of CO2-equivalent for the UAE and UK grids, net of idle and including idle, from a cited table (Ember data, CC BY 4.0), with any other region or value on request. -- **A generated report.** `docs/results.md` and its charts come from a script, never by hand. - It refuses a run that did not finish unless told otherwise, and says when a run has fewer - than 30 runs per configuration or loops shorter than their target. +- **A generated report.** `docs/results.md`, its charts and the results block at the top of + this README come from a script, never by hand. It refuses a run that did not finish unless + told otherwise, and says when a run has fewer than 30 runs per configuration or loops + shorter than their target. - **Runs anywhere, measures where it can.** The same harness runs on WSL2, in Docker and in CI with energy off; `scripts/run-study.sh` runs the full study on bare-metal Linux. @@ -211,7 +212,7 @@ workers/node/ Node.js worker (no runtime dependencies), tests, ESLint an analysis/ greenbench_analysis: loading, statistics, carbon, charts, report; tests tests/ C unit tests, fake powercap manifests, integration tests (pytest) scripts/run-study.sh the one-command study for bare-metal Linux -data/ grid carbon intensity with sources; data/runs/ will hold study runs +data/ grid carbon intensity with sources; the study script writes runs to data/runs/ docs/ study.md, bare-metal.md, decisions.md, results.md (generated), img/ ``` @@ -221,8 +222,9 @@ Recorded in [docs/decisions.md](docs/decisions.md): why powercap rather than MSR language is measured as a whole process, how repetitions and startup batches are calibrated, why the idle baseline follows every run, why missing or frozen energy is never written as zero, why the study pins to two cores and measures Node.js 24, why power ratios rather -than correlation decide whether time is a good stand-in for energy, and why an interval that -includes 1 is not enough to call two powers the same. +than correlation decide whether time is a good stand-in for energy, why an interval that +includes 1 is not enough to call two powers the same, and why the energy tests use a +simulated build of the harness. ## Limitations and roadmap @@ -235,12 +237,13 @@ includes 1 is not enough to call two powers the same. [threats to validity](docs/study.md#threats-to-validity). - **A small workload.** Three single-threaded sorts on integer arrays of up to 2,000 elements in the study profile (100,000 at most). -- **One setup per run.** Results will describe the recorded compiler, Python and Node.js - versions and the two pinned CPUs. Pinned and unpinned runs have not been compared. +- **One setup per run.** Results describe the recorded compiler, Python and Node.js versions + and the two pinned CPUs of one machine. Pinned and unpinned runs have not been compared. Next: -1. Run the study on bare-metal Linux and publish `docs/results.md` with the raw data. +1. Repeat the study on a second machine with a processor from the other vendor (Intel or + AMD), to see whether the answer holds across processors. 2. Check RAPL against a plug-in power meter on the same machine. 3. Add merge sort and a median-of-three quicksort, to show a worst case being avoided. 4. Add hardware counters (instructions, cache misses) as explanations for energy differences. diff --git a/docs/bare-metal.md b/docs/bare-metal.md index a08bd13..0331c04 100644 --- a/docs/bare-metal.md +++ b/docs/bare-metal.md @@ -34,7 +34,9 @@ disk, and the study leaves the machine as it found it after a reboot. Node.js comes from nodejs.org rather than Ubuntu, whose Node.js 18 is past its end of life ([ADR 16](decisions.md#16-measure-nodejs-24-lts-not-the-nodejs-that-ubuntu-ships)). These lines download the latest Node.js 24 LTS release, check it against the release's published - SHA-256 sums, and install it under `/usr/local`: + SHA-256 sums, and install it under `/usr/local`. The sums come over HTTPS from the same + server, so the check catches a damaged download; it does not verify the release's GPG + signature: ```bash cd /tmp diff --git a/docs/decisions.md b/docs/decisions.md index 63fec3e..9f580be 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -33,9 +33,10 @@ as missing: before measuring, the harness reads the counters 200 ms apart, and a counter that has not changed makes energy unavailable. During the run, a measured interval or idle window of 100 ms or more in which any package counter did not change stops the run with an error naming the zone; a live counter moves about once a millisecond. Each socket is judged -on its own, so one frozen counter on a two-socket machine cannot hide behind the other. `--energy require` turns unavailable energy into exit -status 3, and so does `greenbench probe` when the counters do not move. The report prints -"energy unavailable" and keeps the energy sections empty. +on its own, so one frozen counter on a two-socket machine cannot hide behind the other. +`--energy require` turns unavailable energy into exit status 3, and so does `greenbench probe` +when the counters do not move. The report prints "energy unavailable" and keeps the energy +sections empty. **Consequences.** The same harness runs everywhere with time and operation counts, and no file can be mistaken for an energy measurement. The unit tests check that a corrupt counter diff --git a/docs/study.md b/docs/study.md index f34887f..72a3e5f 100644 --- a/docs/study.md +++ b/docs/study.md @@ -97,7 +97,9 @@ lists, JavaScript arrays). None uses a built-in sort. - **Energy.** Package and DRAM zones from `/sys/class/powercap/intel-rapl:*`, summed over sockets, with one counter wrap handled using `max_energy_range_uj`. If the counters are missing, unreadable or frozen (present but never changing), the harness records "energy - unavailable", never a zero. + unavailable", never a zero. A run stops if a package counter stands still for 100 ms or + more, or if the counters rise by more than 1 kW per package over an interval, as they can + when a suspend resets them. - **Enough work per process.** RAPL counters update about once a millisecond (Hähnel et al., 2012). During warm-up the harness sets the number of sorts per process so that each worker's sort loop lasts at least 200 ms: it extrapolates from the last warm-up run and aims 20% From a48caef4e86d93d92bc5dc51422f0ee1ea540200 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:56:27 +0400 Subject: [PATCH 46/51] docs: drop an unmeasured claim about language speed, and rewrap study.md justified the 10% margin by saying that the languages differ in time by factors of ten or more, which this repository has not measured yet. The margin now rests on its own reasoning. Two paragraphs are rewrapped. Co-Authored-By: Claude Opus 5.5 --- docs/decisions.md | 6 +++--- docs/study.md | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index 9f580be..d86c854 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -260,9 +260,9 @@ execution units. **Consequences.** Node.js's helper threads run beside the sorting thread, as on an idle desktop. C and Python can move between the two CPUs, which are the same kind of core, so this changes little. The choice is tested on fake CPU topologies (`GREENBENCH_SYSFS`), including -adjacent sibling numbering and a hybrid processor. RAPL counts the whole package, so the helper threads' energy is included -either way. No pinned-against-unpinned comparison has been measured; the threats to validity -say so. +adjacent sibling numbering and a hybrid processor. RAPL counts the whole package, so the +helper threads' energy is included either way. No pinned-against-unpinned comparison has been +measured; the threats to validity say so. ## 16. Measure Node.js 24 LTS, not the Node.js that Ubuntu ships diff --git a/docs/study.md b/docs/study.md index 72a3e5f..39f8546 100644 --- a/docs/study.md +++ b/docs/study.md @@ -50,9 +50,9 @@ in three ways, from the most direct to the weakest: energy per sort across configurations. This is weaker: configurations differ in cost by large factors, so rho stays near 1 unless power differs enough to reorder them. -The margin is a judgement: a power difference of less than 10% would rarely change a decision -made from time alone, when the languages differ in time by factors of ten or more. The study -profile gives 144 power ratios between languages and 108 between input orders. If most of them +The margin is a judgement: a difference in power of less than 10% would rarely change a +decision made from time alone. The study profile gives 144 power ratios between languages and +108 between input orders. If most of them read "same power within 10%", and the ones that differ are no more than chance would produce (see [Analysis](#analysis)), the answer to the first question is "yes". Power ratios that differ by more than the margin, consistently across sizes and orders, answer the second: for @@ -123,8 +123,9 @@ lists, JavaScript arrays). None uses a built-in sort. - **Pinning.** The study script pins the harness and all workers to two CPUs on different physical cores of one processor (never CPU 0 or a CPU that shares its core, and only performance cores on a processor that also has efficiency cores), so the measured processes - stay on known cores of one kind. Two rather than one, because Node.js runs its compiler and garbage collector on - helper threads ([ADR 15](decisions.md#15-pin-to-two-cpus-on-different-physical-cores-not-one)). + stay on known cores of one kind. Two rather than one, because Node.js runs its compiler and + garbage collector on helper threads + ([ADR 15](decisions.md#15-pin-to-two-cpus-on-different-physical-cores-not-one)). - **Records.** `runs.csv` holds one row per measured run (tidy data: one variable per column). `meta.json` holds the settings, the RAPL zones and their wrap values, the CPU model, the kernel, the scaling governor, the turbo setting, the language versions and the exact From 1295633f867c3c39f7eb85ade6a67aa396a6e73a Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Mon, 28 Sep 2026 00:31:16 +0400 Subject: [PATCH 47/51] docs: regenerate results.md with harness 2.2.0, timings still pending A full study-profile run of commit a48caef in Docker Desktop (2 CPUs, 2 GiB, 5,760 measured runs, status complete, no retries), reported with --withhold-timings --readme README.md. The operation counts, the chart and the README's results block are unchanged; the provenance lines now name harness 2.2.0 and its calibration aim. The raw data is not committed. Co-Authored-By: Claude Opus 5.5 --- docs/results.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/results.md b/docs/results.md index 4e5ed9d..55ddb6a 100644 --- a/docs/results.md +++ b/docs/results.md @@ -6,14 +6,14 @@ ## How these results were produced -- Harness: greenbench 2.1.0, profile `study`, 30 measured runs per configuration after 3 warm-up runs, in a random order (order seed 1234567). -- Inputs: sizes 250, 500, 1,000, 2,000; orders random, sorted, reversed, nearly sorted; xorshift32 seed 2463534242. Sort loops and startup batches are calibrated to about 200 ms. +- Harness: greenbench 2.2.0, profile `study`, 30 measured runs per configuration after 3 warm-up runs, in a random order (order seed 1234567). +- Inputs: sizes 250, 500, 1,000, 2,000; orders random, sorted, reversed, nearly sorted; xorshift32 seed 2463534242. Sort loops and startup batches are calibrated to about 240 ms, so that they last at least 200 ms. - Machine: AMD Ryzen 7 6800H with Radeon Graphics; Linux 6.18.33.2-microsoft-standard-WSL2 x86_64; 16 CPUs online, a cgroup CPU limit of 2 CPUs, a cgroup memory limit of 2.0 GiB; scaling governor not reported; CPU pinning off. - Workers: C: gcc 14.2.0; Python: 3.12.14; Node.js: v24.21.0. - Energy: unavailable. Reason: no RAPL package zone (intel-rapl:N named package-N) under /sys/class/powercap; this CPU, kernel or virtual machine does not expose RAPL. -- Run: 5,760 measured runs, status complete, 0 worker processes retried; started 2026-09-26T00:57:33Z, finished 2026-09-26T01:18:53Z; label: pipeline check in Docker Desktop; timings withheld. +- Run: 5,760 measured runs, status complete, 0 worker processes retried; started 2026-09-27T19:54:53Z, finished 2026-09-27T20:30:34Z; label: pipeline check in Docker Desktop; timings withheld. - Harness command: `build/greenbench run --profile study --energy auto --label 'pipeline check in Docker Desktop; timings withheld' --out results/study-docker` -- Report command: `PYTHONPATH=analysis python3 -m greenbench_analysis report --runs results/study-docker/runs.csv --out docs/results.md --img-dir docs/img --withhold-timings` +- Report command: `PYTHONPATH=analysis python3 -m greenbench_analysis report --runs results/study-docker/runs.csv --out docs/results.md --img-dir docs/img --withhold-timings --readme README.md` ## Operation counts From 177f578e56aa831f5a2ce067ba4b0fadbc98e6dd Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:11:00 +0400 Subject: [PATCH 48/51] feat: report time and time ratios from a run without energy A run without energy, such as one in a container, now publishes what it measured and marks the rest as pending a run on bare-metal Linux: - time per sort and startup time per process, with their intervals, and no energy columns; - a "Time ratios" section: between languages and between input orders, each with a 95% bootstrap interval. A time ratio is undefined when a time per sort is at or below zero, or is in more than 2.5% of the resamples (Comparison.time_reliable); - a note at the top, and pending sections for the proxy question and carbon, saying why energy is missing and where it will come from; - in the README block, a table of time per sort by algorithm and language and a table of time ratios between languages. The report and the README block also give the run's limits, pinning, versions and label, a --reproduce command when one is passed, and a link to the run's raw data. Co-Authored-By: Claude Opus 5.5 --- analysis/greenbench_analysis/cli.py | 8 + analysis/greenbench_analysis/report.py | 259 ++++++++++++++++++++---- analysis/greenbench_analysis/summary.py | 14 +- analysis/tests/test_report.py | 79 +++++++- analysis/tests/test_summary.py | 19 ++ tests/integration/test_pipeline.py | 7 +- 6 files changed, 338 insertions(+), 48 deletions(-) diff --git a/analysis/greenbench_analysis/cli.py b/analysis/greenbench_analysis/cli.py index a3e0140..aa6f405 100644 --- a/analysis/greenbench_analysis/cli.py +++ b/analysis/greenbench_analysis/cli.py @@ -96,6 +96,12 @@ def build_parser() -> argparse.ArgumentParser: help="also rewrite the results block between the and lines " "of this file (README.md)", ) + report.add_argument( + "--reproduce", + metavar="COMMAND", + help="the command that reproduces the whole run, such as the script that ran the harness; shown with the " + "harness and report commands", + ) return parser @@ -131,6 +137,8 @@ def _report(args: argparse.Namespace, argv: list[str]) -> int: command="PYTHONPATH=analysis python3 -m greenbench_analysis " + shlex.join(argv), guide_link=guide, margin=args.equivalence_margin, + reproduce=args.reproduce, + raw_data=args.runs.parent if args.runs.parent != Path() else None, ) written = write_report(runs, meta, summary, args.out, args.img_dir, options) if args.summary_csv: diff --git a/analysis/greenbench_analysis/report.py b/analysis/greenbench_analysis/report.py index 12fd6a3..ec49e65 100644 --- a/analysis/greenbench_analysis/report.py +++ b/analysis/greenbench_analysis/report.py @@ -2,7 +2,9 @@ Every number in the report comes from the run's runs.csv and meta.json. With withhold_timings set, time and energy sections say "pending" instead of showing numbers: -used for runs on a busy machine, whose timings would mislead. +used for runs on a busy machine, whose timings would mislead. A run without energy (in a +container, say) gives time and operation counts, and marks every energy, power and carbon +section as pending a run on bare-metal Linux. """ from __future__ import annotations @@ -16,7 +18,7 @@ from . import charts from .carbon import GridIntensity, grams_co2e -from .load import ORDERS, DataError, Run +from .load import ALGORITHMS, ORDERS, DataError, Run from .stats import Estimate from .summary import CONFIDENCE, Comparison, Configuration, RankAgreement, Summary @@ -38,6 +40,14 @@ def _pending(options: ReportOptions) -> str: ) +def _energy_pending(reason: str, options: ReportOptions, what: str) -> str: + """The text for a section that needs energy, in a run that did not measure it.""" + return ( + f"**Pending.** {what} This run did not measure energy ({reason}); the study on bare-metal Linux will " + f"fill this section in ([how]({options.guide_link}))." + ) + + @dataclass(frozen=True) class ReportOptions: withhold_timings: bool @@ -46,6 +56,8 @@ class ReportOptions: command: str guide_link: str = "bare-metal.md" # docs/bare-metal.md, relative to the report margin: float = EQUIVALENCE_MARGIN + reproduce: str | None = None # the command that reproduces the whole run, such as a script + raw_data: Path | None = None # the directory holding runs.csv and meta.json def _scaled(value: float, units: list[tuple[float, str]]) -> tuple[float, str]: @@ -107,6 +119,16 @@ def format_ratio(estimate: Estimate | None) -> str: return f"{estimate.value:.3g} ({estimate.low:.3g} to {estimate.high:.3g})" +def format_time_ratio(comparison: Comparison) -> str: + """The time ratio and its interval, or "undefined" when a time per sort is at or below zero, + in the medians or in too many resamples (Comparison.time_reliable): that happens when the + startup cost subtracted from a process is as large as its sorting.""" + ratio = comparison.time_ratio + if ratio is None or ratio.low <= 0 or not comparison.time_reliable: + return "undefined" + return format_ratio(ratio) + + def _names(languages: Iterable[str]) -> str: """ "C", "C and Python", "C, Python and Node.js".""" names = [LANGUAGE_NAMES[lang] for lang in languages] @@ -172,19 +194,24 @@ def _loops_line(runs: list[Run], settings: dict[str, Any]) -> str | None: ) +def _versions(meta: dict[str, Any]) -> list[str]: + """ "C: gcc 14.2.0", "Python: 3.12.14", ... from meta.json.""" + versions = [] + for language, worker in meta.get("workers", {}).items(): + version = str(worker.get("version", "?")) + if version.startswith("Python "): + version = version[len("Python ") :] + versions.append(f"{LANGUAGE_NAMES.get(language, language)}: {version}") + return versions + + def _environment(meta: dict[str, Any], runs: list[Run], summary: Summary, options: ReportOptions) -> list[str]: host = meta.get("host", {}) settings = meta.get("settings", {}) energy = meta.get("energy", {}) - workers = meta.get("workers", {}) cpu = host.get("cpu_model") or "not recorded (no model name in /proc/cpuinfo)" warmup = settings.get("warmup", "?") - versions = [] - for language, worker in workers.items(): - version = str(worker.get("version", "?")) - if version.startswith("Python "): - version = version[len("Python ") :] - versions.append(f"{LANGUAGE_NAMES.get(language, language)}: {version}") + versions = _versions(meta) retries = meta.get("worker_retries", 0) label = f"; label: {meta['label']}" if meta.get("label") else "" lines = [ @@ -218,6 +245,8 @@ def _environment(meta: dict[str, Any], runs: list[Run], summary: Summary, option loops = _loops_line(runs, settings) if loops: lines.append(loops) + if options.reproduce: + lines.append(f"- Reproduce the whole run: `{options.reproduce}`") if meta.get("command"): lines.append(f"- Harness command: `{meta['command']}`") return lines @@ -257,7 +286,13 @@ def _banners(meta: dict[str, Any], summary: Summary, options: ReportOptions, rea "", ] elif not summary.energy_measured: - lines += [f"> **Energy was not measured in this run:** {reason}", ""] + lines += [ + "> **Energy results are pending.** This run measured time and operation counts only. Energy was not " + f"measured in this run: {reason}. Energy, power, carbon estimates and the answer to whether run time " + "is a good stand-in for energy need the study on bare-metal Linux, which reads the processor's " + f"energy counters ([how]({options.guide_link})).", + "", + ] return lines @@ -313,8 +348,6 @@ def _per_sort_rows(configurations: list[Configuration], energy: bool) -> list[li else format_estimate(c.energy_per_sort_uj, ENERGY_UNITS) ) row.append("undefined" if c.power_w is None else format_estimate(c.power_w, POWER_UNITS)) - else: - row.append("energy unavailable") rows.append(row) return rows @@ -344,6 +377,11 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str ) elif summary.energy_measured: power = " Power while sorting is energy per sort divided by time per sort." + else: + power = ( + " Energy per sort is pending: this run did not measure energy, so the tables give time only " + "(see the note at the top)." + ) lines += [ f"Median over the measured runs of each configuration, with a 95% bootstrap confidence interval in " f"brackets. Per-sort figures are the whole-process measurement {subtract}.{idle}{power}", @@ -352,11 +390,11 @@ def _per_sort_section(summary: Summary, options: ReportOptions, images: dict[str for key in ("time_per_sort", "energy_per_sort"): if images.get(key): lines += [f"![{key.replace('_', ' ').capitalize()}, largest input, random order]({images[key]})", ""] - header = ["Language", "Algorithm", "Order", "Size", "Sorts per process", "Time per sort", "Energy per sort"] - align = [":---", ":---", ":---", "---:", "---:", "---:", "---:"] + header = ["Language", "Algorithm", "Order", "Size", "Sorts per process", "Time per sort"] + align = [":---", ":---", ":---", "---:", "---:", "---:"] if summary.energy_measured: - header.append(_power_label(summary)) - align.append("---:") + header += ["Energy per sort", _power_label(summary)] + align += ["---:", "---:"] lines += [f"Inputs of {largest:,} elements:", ""] lines += _table( header, align, _per_sort_rows([c for c in summary.configurations if c.n == largest], summary.energy_measured) @@ -435,7 +473,7 @@ def _language_rows(comparisons: Iterable[Comparison], margin: float) -> list[lis order.replace("_", " "), f"{n:,}", f"{LANGUAGE_NAMES[language]} / {LANGUAGE_NAMES[c.reference[0]]}", - format_ratio(c.time_ratio), + format_time_ratio(c), format_ratio(c.energy_ratio), format_ratio(c.power_ratio), power_reading(c.power_ratio, LANGUAGE_NAMES[language], margin, reliable=c.power_reliable), @@ -455,7 +493,7 @@ def _order_rows(comparisons: Iterable[Comparison], margin: float) -> list[list[s algorithm, f"{n:,}", f"{name} / random", - format_ratio(c.time_ratio), + format_time_ratio(c), format_ratio(c.energy_ratio), format_ratio(c.power_ratio), power_reading(c.power_ratio, f"{name.capitalize()} input", margin, reliable=c.power_reliable), @@ -553,12 +591,87 @@ def _power_paragraph(summary: Summary) -> list[str]: ] +def _language_time_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: + rows = [] + for c in comparisons: + language, algorithm, order, n = c.subject + pair = f"{LANGUAGE_NAMES[language]} / {LANGUAGE_NAMES[c.reference[0]]}" + rows.append([algorithm, order.replace("_", " "), f"{n:,}", pair, format_time_ratio(c)]) + return rows + + +def _order_time_rows(comparisons: Iterable[Comparison]) -> list[list[str]]: + rows = [] + for c in comparisons: + language, algorithm, order, n = c.subject + name = order.replace("_", " ") + rows.append([LANGUAGE_NAMES[language], algorithm, f"{n:,}", f"{name} / random", format_time_ratio(c)]) + return rows + + +LANGUAGE_TIME_HEADER = ( + ["Algorithm", "Order", "Size", "Languages", "Time ratio"], + [":---", ":---", "---:", ":---", "---:"], +) +ORDER_TIME_HEADER = ( + ["Language", "Algorithm", "Size", "Orders", "Time ratio"], + [":---", ":---", "---:", ":---", "---:"], +) + + +def _time_ratio_section(summary: Summary) -> list[str]: + """Ratios of time per sort, for a run without energy, whose proxy section is pending. + + With energy, the same time ratios appear beside the energy and power ratios instead.""" + largest = max((c.n for c in summary.configurations), default=0) + lines = [ + "## Time ratios", + "", + "How many times one configuration's time per sort the other's is, with a 95% bootstrap interval in " + "brackets. They compare time alone: whether energy follows the same ratios is the question that needs " + "the energy study. A ratio is undefined when a time per sort is at or below zero, or is in more than " + f"{(1 - CONFIDENCE) / 2:.1%} of the resamples, because the startup cost subtracted from it is as large " + "as the sorting.", + "", + "### Between languages", + "", + "For the same algorithm and input, the first language's time per sort over the second's. Random input, " + "largest size:", + "", + ] + main = [c for c in summary.language_comparisons if c.subject[2] == "random" and c.subject[3] == largest] + lines += _table(*LANGUAGE_TIME_HEADER, _language_time_rows(main)) if main else ["No language pairs in this run."] + if summary.language_comparisons: + lines += ["", "
Every order and size", ""] + lines += _table(*LANGUAGE_TIME_HEADER, _language_time_rows(summary.language_comparisons)) + lines += ["", "
"] + lines += [ + "", + "### Between input orders", + "", + "For the same language and algorithm, each input order's time per sort over random input's. Sorted " + "against random, largest size:", + "", + ] + main = [c for c in summary.order_comparisons if c.subject[2] == "sorted" and c.subject[3] == largest] + lines += _table(*ORDER_TIME_HEADER, _order_time_rows(main)) if main else ["No sorted and random pairs in this run."] + if summary.order_comparisons: + lines += ["", "
Every order and size", ""] + lines += _table(*ORDER_TIME_HEADER, _order_time_rows(summary.order_comparisons)) + lines += ["", "
"] + return [*lines, ""] + + def _proxy_section(summary: Summary, options: ReportOptions, image: str | None, reason: str) -> list[str]: lines = ["## Is run time a good stand-in for energy?", ""] if options.withhold_timings: return [*lines, _pending(options), ""] if not summary.energy_measured: - return [*lines, f"Energy was not measured in this run ({reason}), so this question stays open.", ""] + what = ( + "The answer compares the power drawn while sorting across configurations, so it needs energy as well " + "as time." + ) + return [*lines, _energy_pending(reason, options, what), ""] largest = max((c.n for c in summary.configurations), default=0) margin = options.margin lower, upper = 1 / (1 + margin), 1 + margin @@ -681,22 +794,23 @@ def _startup_section(summary: Summary, options: ReportOptions) -> list[str]: "process; their median is subtracted from the sorting runs above. Random order shown.", "", ] - rows = [ - [ - LANGUAGE_NAMES[s.language], - f"{s.n:,}", - f"{s.launches:,}", - format_estimate(s.time_ns, TIME_UNITS), - "energy unavailable" if s.energy_uj is None else format_estimate(s.energy_uj, ENERGY_UNITS), - ] - for s in summary.startups - if s.order == "random" - ] - lines += _table( - ["Language", "Size", "Processes per batch", "Time per process", "Energy per process"], - [":---", "---:", "---:", "---:", "---:"], - rows, - ) + energy = summary.energy_measured + rows = [] + for s in summary.startups: + if s.order != "random": + continue + row = [LANGUAGE_NAMES[s.language], f"{s.n:,}", f"{s.launches:,}", format_estimate(s.time_ns, TIME_UNITS)] + if energy: + row.append("energy unavailable" if s.energy_uj is None else format_estimate(s.energy_uj, ENERGY_UNITS)) + rows.append(row) + header = ["Language", "Size", "Processes per batch", "Time per process"] + align = [":---", "---:", "---:", "---:"] + if energy: + header.append("Energy per process") + align.append("---:") + else: + lines += ["Energy per process is pending: this run did not measure energy.", ""] + lines += _table(header, align, rows) return [*lines, ""] @@ -729,7 +843,8 @@ def _carbon_section(summary: Summary, options: ReportOptions, reason: str) -> li if options.withhold_timings: return [*lines, _pending(options), ""] if not summary.energy_measured or summary.total_energy_uj is None: - return [*lines, f"Energy was not measured in this run ({reason}), so there is nothing to convert.", ""] + what = "Carbon estimates are converted from energy; the grid intensities they will use are below." + return [*lines, _energy_pending(reason, options, what), ""] regions = options.regions largest = max((c.n for c in summary.configurations), default=0) order = "random" if any(c.order == "random" for c in summary.configurations) else ORDERS[0] @@ -849,9 +964,18 @@ def draw(key: str, make: Callable[[Path], Path | None]) -> None: *_banners(meta, summary, options, reason), ] lines += ["## How these results were produced", "", *_environment(meta, runs, summary, options)] - lines += [f"- Report command: `{options.command}`", ""] + lines += [f"- Report command: `{options.command}`"] + if options.raw_data is not None and not options.withhold_timings: + shown = options.raw_data.as_posix().rstrip("/") + lines.append( + f"- Raw data: [{shown}/]({_relative(options.raw_data, out.parent)}/), with every measured run in " + "`runs.csv` and the settings, limits and versions in `meta.json`." + ) + lines += [""] lines += _operations_section(summary, images.get("operations")) lines += _per_sort_section(summary, options, images) + if not options.withhold_timings and not summary.energy_measured: + lines += _time_ratio_section(summary) lines += _proxy_section(summary, options, images.get("energy_vs_time"), reason) lines += _startup_section(summary, options) lines += _carbon_section(summary, options, reason) @@ -953,7 +1077,15 @@ def readme_block( lines += [f"> **Note:** {runs} measured runs per configuration; the study uses {STUDY_RUNS}.", ""] if not summary.energy_measured: reason = meta.get("energy", {}).get("reason") or "energy unavailable" - return [*lines, f"Energy was not measured in this run ({reason}); the report gives time and operation counts."] + return [ + *lines, + "> **Energy results are pending.** This run measured time and operation counts only; energy was not " + f"measured ({reason}). Whether run time is a good stand-in for energy needs the study on bare-metal " + f"Linux ([how]({guide_link})).", + "", + *_readme_time_tables(summary), + *_readme_provenance(meta, options, report_link), + ] margin = options.margin lines += [ f"Is run time a good stand-in for energy? Each power ratio (energy ratio over time ratio) read against " @@ -972,9 +1104,60 @@ def readme_block( [":---", ":---", "---:", ":---", *RATIO_ALIGN], _language_rows(main, margin), ) + lines += [""] + return [*lines, *_readme_provenance(meta, options, report_link)] + + +def _readme_time_tables(summary: Summary) -> list[str]: + """Time per sort by algorithm and language, and the time ratios between languages, for the + largest random input: the README's results for a run without energy.""" + largest = max((c.n for c in summary.configurations), default=0) + chosen = {(c.language, c.algorithm): c for c in summary.configurations if c.order == "random" and c.n == largest} + if not chosen: + return [] + languages = [lang for lang in LANGUAGE_NAMES if any(key[0] == lang for key in chosen)] + algorithms = [alg for alg in ALGORITHMS if any(key[1] == alg for key in chosen)] + rows = [ + [algorithm] + + [ + format_estimate(chosen[(lang, algorithm)].time_per_sort_ns, TIME_UNITS) + if (lang, algorithm) in chosen + else "" + for lang in languages + ] + for algorithm in algorithms + ] + lines = [ + f"Time per sort, random input of {largest:,} elements: the median of each configuration's measured runs, " + "with a 95% bootstrap interval in brackets.", + "", + *_table( + ["Algorithm", *(LANGUAGE_NAMES[lang] for lang in languages)], [":---"] + ["---:"] * len(languages), rows + ), + "", + ] + main = [c for c in summary.language_comparisons if c.subject[2] == "random" and c.subject[3] == largest] + if main: + lines += [f"Time ratios between languages, random input of {largest:,} elements:", ""] + lines += [*_table(*LANGUAGE_TIME_HEADER, _language_time_rows(main)), ""] return lines +def _readme_provenance(meta: dict[str, Any], options: ReportOptions, report_link: str) -> list[str]: + """One paragraph under the README's results: limits, pinning, versions, label and commands.""" + host = meta.get("host", {}) + parts = [ + f"Environment: {host.get('online_cpus', '?')} CPUs online{_limits(host)}; CPU pinning " + f"{_pinning(meta.get('settings', {}))}; " + "; ".join(_versions(meta)) + "." + ] + if meta.get("label"): + parts.append(f"Run label: {str(meta['label']).rstrip('.')}.") + if options.reproduce: + parts.append(f"Reproduce with `{options.reproduce}`.") + parts.append(f"Every setting and command: [{report_link}]({report_link}#how-these-results-were-produced).") + return [" ".join(parts)] + + def check_readme(path: Path) -> str: """The README's text, if it has exactly one pair of results markers in the right order.""" try: diff --git a/analysis/greenbench_analysis/summary.py b/analysis/greenbench_analysis/summary.py index f7ad724..986b5b8 100644 --- a/analysis/greenbench_analysis/summary.py +++ b/analysis/greenbench_analysis/summary.py @@ -27,6 +27,7 @@ from dataclasses import dataclass import numpy as np +import numpy.typing as npt from .load import ALGORITHMS, LANGUAGES, ORDERS, DataError, Run from .stats import Estimate, FloatArray, bootstrap_median, interval, median, ratio_draws, resample_medians, spearman @@ -98,6 +99,7 @@ class Comparison: power_reliable is False when a time or energy per sort of either configuration fell to zero or below in more than (1 - CONFIDENCE) / 2 of the resamples. The power ratio is undefined in those resamples, and an interval from the rest would look narrower than it is. + time_reliable is the same for the time ratio, from time per sort alone. """ subject: Key @@ -106,6 +108,7 @@ class Comparison: energy_ratio: Estimate | None power_ratio: Estimate | None power_reliable: bool = True + time_reliable: bool = True @dataclass(frozen=True) @@ -242,9 +245,15 @@ def _ratio(subject: float, reference: float, subject_draws: FloatArray, referenc return interval(subject / reference, ratio_draws(subject_draws, reference_draws)) +def _mostly_defined(defined: npt.NDArray[np.bool_]) -> bool: + """Whether the resamples in which a ratio is undefined are few enough to ignore.""" + return bool(np.mean(defined) >= 1 - (1 - CONFIDENCE) / 2) + + def _compare(a: Configuration, b: Configuration, draws: dict[Key, _Draws]) -> Comparison: da, db = draws[a.key], draws[b.key] time_ratio = _ratio(a.time_per_sort_ns.value, b.time_per_sort_ns.value, da.time, db.time) + time_reliable = _mostly_defined((da.time > 0) & (db.time > 0)) energy_ratio = power_ratio = None reliable = True if ( @@ -258,9 +267,8 @@ def _compare(a: Configuration, b: Configuration, draws: dict[Key, _Draws]) -> Co # (energy ratio) / (time ratio), resample by resample. power_draws = ratio_draws(ratio_draws(da.energy, db.energy), ratio_draws(da.time, db.time)) power_ratio = interval(energy_ratio.value / time_ratio.value, power_draws, confidence=CONFIDENCE) - defined = (da.time > 0) & (db.time > 0) & (da.energy > 0) & (db.energy > 0) - reliable = bool(np.mean(defined) >= 1 - (1 - CONFIDENCE) / 2) - return Comparison(a.key, b.key, time_ratio, energy_ratio, power_ratio, reliable) + reliable = _mostly_defined((da.time > 0) & (db.time > 0) & (da.energy > 0) & (db.energy > 0)) + return Comparison(a.key, b.key, time_ratio, energy_ratio, power_ratio, reliable, time_reliable) def _language_comparisons(configurations: list[Configuration], draws: dict[Key, _Draws]) -> list[Comparison]: diff --git a/analysis/tests/test_report.py b/analysis/tests/test_report.py index 184af63..8ec15be 100644 --- a/analysis/tests/test_report.py +++ b/analysis/tests/test_report.py @@ -20,11 +20,12 @@ format_estimate, format_grams, format_time, + format_time_ratio, power_reading, write_report, ) from greenbench_analysis.stats import Estimate -from greenbench_analysis.summary import summarise +from greenbench_analysis.summary import Comparison, summarise REPOSITORY = Path(__file__).resolve().parents[2] GRID = REPOSITORY / "data" / "grid_intensity.csv" @@ -154,6 +155,19 @@ def test_power_ratio_readings() -> None: assert classify_power_ratio(Estimate(1.0, 0.96, 1.04), 0.05) == SAME +def test_time_ratios_reaching_zero_are_undefined() -> None: + key, reference = ("python", "quick", "random", 100), ("c", "quick", "random", 100) + + def ratio(estimate: Estimate | None, *, reliable: bool = True) -> str: + return format_time_ratio(Comparison(key, reference, estimate, None, None, time_reliable=reliable)) + + assert ratio(Estimate(10.2, 9.87, 10.5)) == "10.2 (9.87 to 10.5)" + assert ratio(Estimate(0.00632, -0.103, 0.2)) == "undefined" + assert ratio(Estimate(0.5, 0.0, 0.9)) == "undefined" + assert ratio(None) == "undefined" + assert ratio(Estimate(10.2, 9.87, 10.5), reliable=False) == "undefined" + + def test_wide_intervals_are_inconclusive_not_the_same(tmp_path: Path) -> None: runs = synthetic.study(energy=True, sizes=(100, 200), energy_jitter=4_000.0) summary = summarise(runs, resamples=500) @@ -169,12 +183,38 @@ def test_report_warns_about_unfinished_and_short_runs(tmp_path: Path) -> None: assert "5 measured runs per configuration; the study uses 30" in text -def test_report_without_energy_says_so(tmp_path: Path) -> None: +def test_report_without_energy_gives_time_and_marks_energy_pending(tmp_path: Path) -> None: text = render(tmp_path, energy=False, withhold=False) - assert "Energy was not measured in this run" in text - assert "no RAPL package zone" in text - assert "energy unavailable" in text + assert "> **Energy results are pending.** This run measured time and operation counts only." in text + assert "Energy was not measured in this run: no RAPL package zone" in text + assert "need the study on bare-metal Linux" in text + # Time per sort with its interval, and no energy or power columns or figures. + assert "| C | quick | random | 200 | 10 | 1.00 ms (" in text + assert "Energy per sort is pending" in text + assert "| Time per sort |\n" in text # the last column of the per-sort table + assert "Energy per sort |" not in text + assert " mJ (" not in text + assert " W (" not in text + # Time ratios between languages and between orders, in their own section. + assert "## Time ratios" in text + assert "| Algorithm | Order | Size | Languages | Time ratio |" in text + assert "| quick | random | 200 | Python / C | 10 (" in text + assert "| Python | quick | 200 | sorted / random | 1 (" in text + # The sections that need energy say pending, and why. + assert text.count("**Pending.**") == 2 + assert "the study on bare-metal Linux will fill this section in ([how](bare-metal.md))" in text + assert "Energy per process is pending" in text + assert "| Python | 200 | 1 | 50.0 ms (" in text # startup: 5 ms for C, ten times that for Python + assert "### Grid intensity" in text + assert (tmp_path / "docs" / "img" / "time_per_sort.png").exists() assert not (tmp_path / "docs" / "img" / "energy_per_sort.png").exists() + assert not (tmp_path / "docs" / "img" / "energy_vs_time.png").exists() + + +def test_report_with_energy_has_no_separate_time_ratios(tmp_path: Path) -> None: + text = render(tmp_path, energy=True, withhold=False) + assert "## Time ratios" not in text # they sit beside the energy and power ratios instead + assert "Energy results are pending" not in text def test_withheld_report_shows_counts_but_no_timings(tmp_path: Path) -> None: @@ -294,6 +334,35 @@ def test_readme_block_gives_the_answer_after_a_measured_run(tmp_path: Path) -> N assert "Between languages, random input of 200 elements:" in text assert "| quick | random | 200 | Python / C | 10 (" in text assert "same power within 10%: time predicts energy" in text + assert "Environment: 8 CPUs online; CPU pinning off; C: gcc 14; Python: 3.12.0." in text + + +def test_readme_block_gives_time_and_marks_energy_pending_without_energy(tmp_path: Path) -> None: + host = {**synthetic.meta(energy=False)["host"], "allowed_cpus": 2} + label: dict[str, object] = {"label": "a container run.", "host": host} + args = _readme_run(tmp_path, energy=False, meta_changes=label) + reproduce = "./scripts/run-docker-timing.sh --cpus 2,4" + assert main([*args, "--readme", str(tmp_path / "README.md"), "--reproduce", reproduce]) == 0 + text = (tmp_path / "README.md").read_text(encoding="utf-8") + assert "**Results** from [docs/results.md](docs/results.md): Test CPU, Linux 6.8.0, 120 measured runs" in text + assert "> **Energy results are pending.** This run measured time and operation counts only" in text + assert "(no RAPL package zone under /sys/class/powercap)" in text + assert "bare-metal Linux ([how](" in text + assert "| Algorithm | C | Python |" in text + assert "| bubble | 1.00 ms (" in text + assert "| quick | 1.00 ms (" in text + assert "Time ratios between languages, random input of 200 elements:" in text + assert "| quick | random | 200 | Python / C | 10 (" in text + assert "Environment: 8 CPUs online, 2 of them allowed to the harness; CPU pinning off;" in text + assert "Run label: a container run. Reproduce with `./scripts/run-docker-timing.sh --cpus 2,4`." in text + assert "[docs/results.md](docs/results.md#how-these-results-were-produced)" in text + assert " mJ" not in text + assert text.endswith("\n\n## Rest\n") + + report_text = (tmp_path / "docs" / "results.md").read_text(encoding="utf-8") + assert f"- Reproduce the whole run: `{reproduce}`" in report_text + raw = Path(args[args.index("--runs") + 1]).parent + assert f"- Raw data: [{raw.as_posix()}/](../)" in report_text def test_readme_block_refuses_simulated_energy_and_missing_markers( diff --git a/analysis/tests/test_summary.py b/analysis/tests/test_summary.py index 0949b22..62ccfc1 100644 --- a/analysis/tests/test_summary.py +++ b/analysis/tests/test_summary.py @@ -138,6 +138,25 @@ def test_a_power_ratio_is_unreliable_when_energy_per_sort_reaches_zero_in_resamp assert all(c.power_reliable for c in python_orders) +def test_a_time_ratio_is_unreliable_when_time_per_sort_reaches_zero_in_resamples() -> None: + # C's processes take about as long as its startup batches, so its time per sort is at or + # below zero at two of five runs, which a resample's median lands on often. Python's is not. + runs = [] + for language, startup_ms, sorting_ms in (("c", 5.0, (4.9, 4.95, 5.1, 6.0, 7.0)), ("python", 50.0, (150.0,) * 5)): + for repetition, wall_ms in enumerate(sorting_ms, start=1): + runs.append(synthetic.make_run(language=language, algorithm="none", wall_ns=int(startup_ms * 1e6))) + runs.append( + synthetic.make_run(language=language, repetition=repetition, inner_reps=10, wall_ns=int(wall_ms * 1e6)) + ) + summary = summarise(runs, resamples=2_000) + c_time = configuration(summary, "c", "quick").time_per_sort_ns + assert c_time.value == pytest.approx(10_000) # (5.1 ms - 5 ms) / 10 sorts + python_c = comparison(summary.language_comparisons, "python", "c") + assert python_c.time_ratio is not None + assert not python_c.time_reliable + assert python_c.power_reliable # no energy, so nothing to judge + + def test_input_orders_are_compared_with_random_input() -> None: summary = summarise(synthetic.study(energy=True), resamples=200) keys = {(c.subject, c.reference) for c in summary.order_comparisons} diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index f4e92f3..d771964 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -73,10 +73,13 @@ def test_harness_then_report(tmp_path: Path) -> None: ) assert reported.returncode == 0, reported.stderr text = results.read_text(encoding="utf-8") - assert "Energy was not measured in this run" in text + assert "> **Energy results are pending.**" in text + assert "Energy was not measured in this run: turned off with --energy off." in text assert "C, Python and Node.js reported identical counts and identical input hashes" in text assert "| quick | sorted | 90 | 4,005 | 0 |" in text # n(n-1)/2 comparisons: sorted input is the worst case - assert "energy unavailable" in text + assert "## Time ratios" in text + assert "| quick | random | 90 | Node.js / C |" in text + assert "Energy per sort |" not in text assert (tmp_path / "img" / "operations.png").exists() assert (tmp_path / "img" / "time_per_sort.png").exists() From 9e55f7b99b293f732d3def4c40370d64bf5c2c1b Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:12:15 +0400 Subject: [PATCH 49/51] feat: measure time for the study profile in Docker, with energy off scripts/run-docker-timing.sh runs the study profile in a container on machines that cannot boot Linux directly, and writes the raw data to data/runs/-docker/, the report and the README's results block. - It builds the image from the Dockerfile and, in study mode, refuses uncommitted changes to tracked files, so the recorded commit is the code that ran. - The harness and workers run from the image's own file system, not a mounted folder, and the results are copied out afterwards. - The container gets the two CPUs that run-study.sh --show-cpus would choose, the harness pins to them, and it has a memory limit without swap and no network. - environment.txt records the host, Docker, the image and base-image digests, lscpu inside the container and the exact docker command. --rehearse runs the smoke profile under results/. Its option handling is tested; its runs need Docker. ADR 20 records the decision. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- README.md | 33 ++- docs/decisions.md | 42 ++++ scripts/run-docker-timing.sh | 199 ++++++++++++++++++ .../integration/test_docker_timing_script.py | 44 ++++ 5 files changed, 310 insertions(+), 10 deletions(-) create mode 100755 scripts/run-docker-timing.sh create mode 100644 tests/integration/test_docker_timing_script.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cb9d4c..d51694c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Lint the study script + - name: Lint the scripts run: make lint-shell actionlint: diff --git a/README.md b/README.md index 276b75c..37bd244 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,9 @@ compare?** The research question, method and threats to validity are in told otherwise, and says when a run has fewer than 30 runs per configuration or loops shorter than their target. - **Runs anywhere, measures where it can.** The same harness runs on WSL2, in Docker and in CI - with energy off; `scripts/run-study.sh` runs the full study on bare-metal Linux. + with energy off. `scripts/run-docker-timing.sh` measures time for the whole study profile in + a Docker container, and `scripts/run-study.sh` runs the full study, energy included, on + bare-metal Linux. ## Architecture @@ -143,6 +145,15 @@ docker run --rm -v "$PWD/results:/work/results" greenbench The base images are pinned by digest. If Docker Hub limits your pulls, the [Dockerfile](Dockerfile) gives a build command that takes the same images from Amazon's mirror. +To measure time and operation counts for the whole study profile without bare-metal Linux, +run `./scripts/run-docker-timing.sh` from bash (Git Bash on Windows) on a machine doing nothing +else. It builds the image, runs the harness with energy off in a container limited to two CPUs +and 2 GiB of memory, and writes the raw data to `data/runs/`, the report to +`docs/results.md` and the results block of this README +([ADR 20](docs/decisions.md#20-publish-time-from-a-quiet-docker-run-and-keep-energy-pending)). +`--rehearse` runs the smoke profile instead and writes under `results/`; `--help` lists the +other options. + ## Configuration `build/greenbench run` takes these options; the defaults are the study profile. @@ -173,7 +184,8 @@ The report is `python -m greenbench_analysis report --runs DIR/runs.csv` with [data/grid_intensity.csv](data/grid_intensity.csv)), `--intensity NAME=G_PER_KWH` for any other grid, `--equivalence-margin PERCENT` (10 by default; see [study.md](docs/study.md#what-would-count-as-an-answer)), `--withhold-timings` for runs on a -busy machine, `--allow-partial` to report on a run that did not finish, `--summary-csv`, and +busy machine, `--allow-partial` to report on a run that did not finish, `--summary-csv`, +`--reproduce COMMAND` to show the command that reproduces the whole run, and `--readme README.md`, which rewrites the results block at the top of this file. ## Running the tests @@ -184,8 +196,8 @@ make sanitize # the C tests and a harness run under AddressSanitiz .venv/bin/pip install --require-hashes -r requirements-dev.txt make check-python # ruff, mypy --strict, and the statistics and Python worker tests make check-node # ESLint, tsc --strict and the Node.js worker tests -make integration # cross-language parity, the harness, the study script and the report pipeline -make lint-shell # ShellCheck on the study script +make integration # cross-language parity, the harness, the scripts and the report pipeline +make lint-shell # ShellCheck on the scripts ``` The RAPL tests build fake powercap trees from the manifests in @@ -197,7 +209,8 @@ counters between readings to test wraparound. The integration tests also run With it, energy that is higher during runs than during idle windows goes from the harness through to every section of the report, and a counter wraps inside a measured interval, however busy the machine is. The real build must refuse counters that never move. Tests of -the study script check its choice of CPUs on fake CPU topologies. CI runs all of the above +the study script check its choice of CPUs on fake CPU topologies; the Docker timing script's +runs need Docker, so its tests check only its options. CI runs all of the above and lints the workflow with actionlint; the C build and tests run with both GCC and Clang, the statistics tests on Python 3.12 and 3.14, and the Node.js worker tests also on Node.js 18 and 20. @@ -211,8 +224,9 @@ workers/python/ Python worker (standard library only) and its tests workers/node/ Node.js worker (no runtime dependencies), tests, ESLint and tsc config analysis/ greenbench_analysis: loading, statistics, carbon, charts, report; tests tests/ C unit tests, fake powercap manifests, integration tests (pytest) -scripts/run-study.sh the one-command study for bare-metal Linux -data/ grid carbon intensity with sources; the study script writes runs to data/runs/ +scripts/ run-study.sh: the one-command study for bare-metal Linux; + run-docker-timing.sh: time and operation counts in Docker, energy off +data/ grid carbon intensity with sources; the scripts write runs to data/runs/ docs/ study.md, bare-metal.md, decisions.md, results.md (generated), img/ ``` @@ -223,8 +237,9 @@ language is measured as a whole process, how repetitions and startup batches are why the idle baseline follows every run, why missing or frozen energy is never written as zero, why the study pins to two cores and measures Node.js 24, why power ratios rather than correlation decide whether time is a good stand-in for energy, why an interval that -includes 1 is not enough to call two powers the same, and why the energy tests use a -simulated build of the harness. +includes 1 is not enough to call two powers the same, why the energy tests use a simulated +build of the harness, and why times measured in Docker are published while energy waits for +bare metal. ## Limitations and roadmap diff --git a/docs/decisions.md b/docs/decisions.md index d86c854..db44c3d 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -352,3 +352,45 @@ question "yes". A study run on a noisy machine may end with many inconclusive re is the honest outcome. The margin is a judgement, stated with its reason, and a reader can apply another one without measuring again. Tests cover every reading, the boundaries of the margin and the resampling rule. + +## 20. Publish time from a quiet Docker run, and keep energy pending + +**Context.** RAPL is not available in Docker Desktop, so the energy half of the study waits +for bare-metal Linux. The earlier runs in Docker were made on a machine busy with other work, +so their timings were withheld (ADR 12) and the report showed only operation counts. Time per +sort, and the time ratios between languages and input orders, are useful on their own, are +half of what the study compares, and can be measured in a container on a machine doing +nothing else. + +**Decision.** `scripts/run-docker-timing.sh` runs the study profile in a container with +`--energy off`: + +- It builds the image from the Dockerfile and refuses uncommitted changes to tracked files in + study mode, so that the recorded commit is the code that ran. +- The harness and workers run from the image's own file system, not from a mounted folder: + a mounted Windows folder slows every process start and once failed a run (ADR 14). The + results are copied out when the run ends. +- The container may use two CPUs, chosen as `run-study.sh --show-cpus` chooses them on bare + metal, and the harness pins itself and every worker to them (ADR 15). It has a memory limit + without swap and no network. +- `data/runs/-docker/` keeps `runs.csv`, `meta.json`, `summary.csv` and + `environment.txt`, which records the host, the Docker version, the image and base-image + digests, `lscpu` inside the container and the exact `docker run` command. +- The report is run with `--reproduce`, and with `--readme` in study mode. + +For a run without energy, the report gives time per sort, startup time per process and the +time ratios between languages and between input orders, with their intervals. Its note at +the top, and the sections on the proxy question and carbon, say that energy is pending a run +on bare-metal Linux. The README block gives a table of time per sort and a table of time +ratios between languages, with the same note. + +**Consequences.** The published times describe Linux in Docker Desktop's virtual machine, +not bare metal. The virtual machine's CPUs are scheduled by Windows on whichever physical +cores it chooses, so pinning inside it keeps the measured processes on two virtual CPUs but +not on two fixed physical cores, and Windows' own background work shares the processor. The +run's label and `environment.txt` record the host, its power source and the container's +limits. Time ratios between languages are less sensitive to this than absolute times, +because every configuration runs in the same conditions in a shuffled order (ADR 7). The +bare-metal study replaces `docs/results.md` and the README block with results that include +energy. A time ratio is undefined when a time per sort is at or below zero, or is in more than +2.5% of the resamples, as a power ratio is (ADR 19). diff --git a/scripts/run-docker-timing.sh b/scripts/run-docker-timing.sh new file mode 100755 index 0000000..d281d13 --- /dev/null +++ b/scripts/run-docker-timing.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# Measures time and operation counts for the study profile in a Docker container, with energy +# off: containers cannot read the RAPL energy counters. For machines that cannot boot Linux +# directly, such as Windows or macOS with Docker Desktop. +# +# ./scripts/run-docker-timing.sh the study profile; writes data/runs/-docker/, +# docs/results.md, docs/img/ and the results block +# at the top of README.md +# ./scripts/run-docker-timing.sh --rehearse the short smoke profile, written under results/ +# +# The energy study runs on bare-metal Linux (scripts/run-study.sh, docs/bare-metal.md), and its +# report replaces this one. Run this from bash (Git Bash on Windows) with Docker running, on a +# machine doing nothing else: every published time depends on it. +set -euo pipefail + +cd "$(dirname "$0")/.." + +MODE=study +CPUS=auto +MEMORY=2g +NOTE="" +MIRROR=0 +IMAGE=greenbench +CONTAINER=greenbench-timing + +usage() { + cat <<'EOF' +Usage: scripts/run-docker-timing.sh [--rehearse] [--cpus LIST] [--memory SIZE] [--note TEXT] [--mirror] + + --rehearse run the short smoke profile; results go to results/ and nothing else changes + --cpus LIST the CPUs the container may use, which the harness and workers are pinned to, + such as 2,4 (default: the two that scripts/run-study.sh --show-cpus chooses + inside the container) + --memory SIZE the container's memory limit, with no swap (default 2g) + --note TEXT added to the run's label, for example the host machine and its power source + --mirror build the image from Amazon's mirror of the Docker official images, for when + Docker Hub limits pulls (see the Dockerfile) +EOF +} + +say() { printf '%s\n' "$*"; } +fail() { + printf 'run-docker-timing: %s\n' "$*" >&2 + exit 1 +} + +# A word as a POSIX shell would need it written: as it is when that is safe, else in single quotes. +quote() { + if [[ "$1" =~ ^[A-Za-z0-9_./,:=+-]+$ ]]; then + printf '%s' "$1" + else + printf "'%s'" "${1//\'/\'\\\'\'}" + fi +} + +while (($# > 0)); do + case "$1" in + --rehearse) MODE=rehearse ;; + --cpus) + [[ $# -ge 2 && "$2" =~ ^[0-9]+([-,][0-9]+)*$ ]] || fail "--cpus needs a CPU list, such as 2,4 or 2-3" + CPUS="$2" + shift + ;; + --memory) + [[ $# -ge 2 && "$2" =~ ^[0-9]+[bkmg]?$ ]] || fail "--memory needs a size, such as 2g or 1536m" + MEMORY="$2" + shift + ;; + --note) + [[ $# -ge 2 && -n "$2" ]] || fail "--note needs a text" + NOTE="$2" + shift + ;; + --mirror) MIRROR=1 ;; + -h | --help) + usage + exit 0 + ;; + *) + usage >&2 + exit 2 + ;; + esac + shift +done + +command -v docker >/dev/null || fail "docker not found; install Docker Desktop or Docker Engine" +docker info >/dev/null 2>&1 || fail "cannot reach the Docker daemon; start Docker Desktop" + +# Git Bash on Windows rewrites arguments that look like paths, which would break the paths +# inside the container, and docker needs the Windows form of this folder's path to mount it. +export MSYS_NO_PATHCONV=1 +ROOT="$(pwd -W 2>/dev/null || pwd)" + +commit=unknown clean=unknown +if command -v git >/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then + commit="$(git rev-parse HEAD)" + if git diff --quiet HEAD; then clean=yes; else clean=no; fi +fi +# The image holds a copy of the working tree, so the results describe the files, not a commit. +if [[ "$MODE" == study && "$clean" != yes ]]; then + fail "commit or discard the changes to tracked files first, so that the results name the code they measured" +fi +if docker container inspect "$CONTAINER" >/dev/null 2>&1; then + fail "a container named $CONTAINER already exists; remove it with: docker rm -f $CONTAINER" +fi + +say "== Building the image ($IMAGE)" +build=(docker build) +if ((MIRROR)); then + build=(docker buildx build) + while read -r ref; do + build+=(--build-context "$ref=docker-image://public.ecr.aws/docker/library/$ref") + done < <(sed -n 's|^FROM \([^ ]*\).*|\1|p' Dockerfile) +fi +"${build[@]}" --quiet -t "$IMAGE" . >/dev/null + +if [[ "$CPUS" == auto ]]; then + # The same choice as on bare metal: two CPUs on different cores, never CPU 0 or its sibling. + choice="$(docker run --rm --network none "$IMAGE" bash scripts/run-study.sh --show-cpus)" + say "== $choice" + CPUS="$(sed -n 's/^CPUs for the study: \([0-9][0-9,-]*\) .*/\1/p' <<<"$choice")" + [[ -n "$CPUS" ]] || fail "could not choose two CPUs inside the container; give them with --cpus" +fi + +stamp="$(date -u +%Y%m%dT%H%M%SZ)" +if [[ "$MODE" == study ]]; then + out="data/runs/$stamp-docker" + profile=study + report=docs/results.md + images=docs/img +else + out="results/docker-rehearsal-$stamp" + profile=smoke + report="$out/results.md" + images="$out/img" +fi +[[ ! -e "$out" ]] || fail "$out already exists" +mkdir -p "$out" + +reproduce="./scripts/run-docker-timing.sh" +if [[ "$MODE" == rehearse ]]; then reproduce+=" --rehearse"; fi +reproduce+=" --cpus $CPUS --memory $MEMORY" +label="run-docker-timing.sh $MODE $stamp: a Docker container limited to CPUs $CPUS and $MEMORY of memory" +label+=" without swap; energy off, because containers cannot read RAPL${NOTE:+; $NOTE}" + +# The harness and workers run from the image's own file system, not from a mounted folder, and +# write inside the container; the results are copied out afterwards. The single-quoted script +# expands its arguments inside the container, not here. +# shellcheck disable=SC2016 +measure=(docker run --name "$CONTAINER" --init --network none + --cpuset-cpus "$CPUS" --memory "$MEMORY" --memory-swap "$MEMORY" "$IMAGE" + sh -c 'mkdir -p "$1" && { uname -srmo; echo; lscpu; } >"$1/container.txt" 2>&1; make -s all && shift && exec "$@"' + sh "$out" build/greenbench run --profile "$profile" --energy off --cpus "$CPUS" --label "$label" --out "$out") +trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1 || true' EXIT + +say "== Measuring (profile $profile) on CPUs $CPUS with $MEMORY of memory; leave the machine alone" +if ! "${measure[@]}"; then + docker cp "$CONTAINER:/work/$out/." "$out/" >/dev/null 2>&1 || true + fail "the harness did not finish; whatever it wrote is in $out/" +fi +docker cp "$CONTAINER:/work/$out/." "$out/" >/dev/null + +measured="" +for word in "${measure[@]}"; do measured+="$(quote "$word") "; done +{ + say "recorded_utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + say "git_commit: $commit" + say "git_worktree_clean: $clean" + say "host: $(uname -srm)" + say "note: ${NOTE:-none}" + say "docker: $(docker version --format '{{.Server.Platform.Name}}, engine {{.Server.Version}}')" + say "docker_host: $(docker info --format '{{.OperatingSystem}}, kernel {{.KernelVersion}}, {{.NCPU}} CPUs, {{.MemTotal}} bytes of memory')" + say "image: $IMAGE $(docker image inspect --format '{{.Id}}' "$IMAGE")" + say "base_images: $(sed -n 's|^FROM \([^ ]*\).*|\1|p' Dockerfile | tr '\n' ' ')" + say "reproduce: $reproduce" + say "measure_command: ${measured% }" + say + say "## Inside the container" + cat "$out/container.txt" +} >"$out/environment.txt" +rm -f "$out/container.txt" + +say "== Report" +report_args=(--runs "$out/runs.csv" --out "$report" --img-dir "$images" --summary-csv "$out/summary.csv" + --reproduce "$reproduce") +if [[ "$MODE" == study ]]; then report_args+=(--readme README.md); fi +# Files written into a mounted folder belong to the container's user; on Linux, use the caller's. +user=() +if [[ "$(uname -s)" == Linux ]]; then user=(--user "$(id -u):$(id -g)" -e HOME=/tmp); fi +docker run --rm --network none "${user[@]}" -v "$ROOT:/work" -w /work -e PYTHONPATH=analysis "$IMAGE" \ + python3 -m greenbench_analysis report "${report_args[@]}" + +say +say "Done. Raw data: $out/ Report: $report" +if [[ "$MODE" == study ]]; then + say "Next: read $report, then commit the results:" + say " git add README.md $report $images $out && git commit -m 'docs: add time results measured in Docker'" +fi diff --git a/tests/integration/test_docker_timing_script.py b/tests/integration/test_docker_timing_script.py new file mode 100644 index 0000000..0fe5440 --- /dev/null +++ b/tests/integration/test_docker_timing_script.py @@ -0,0 +1,44 @@ +"""The Docker timing script's options. Its runs need Docker, so CI checks only what it does +before reaching Docker: help, and refusing bad options.""" + +from __future__ import annotations + +import subprocess + +import pytest + +from workers import REPOSITORY + +SCRIPT = REPOSITORY / "scripts" / "run-docker-timing.sh" + + +def run(*options: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["bash", str(SCRIPT), *options], capture_output=True, text=True, check=False, timeout=60) + + +def test_help() -> None: + completed = run("--help") + assert completed.returncode == 0 + assert completed.stdout.startswith("Usage: scripts/run-docker-timing.sh") + assert "--memory SIZE" in completed.stdout + + +@pytest.mark.parametrize( + ("options", "message"), + [ + (("--cpus", "two"), "--cpus needs a CPU list"), + (("--cpus",), "--cpus needs a CPU list"), + (("--memory", "lots"), "--memory needs a size"), + (("--note", ""), "--note needs a text"), + ], +) +def test_bad_option_values_are_refused(options: tuple[str, ...], message: str) -> None: + completed = run(*options) + assert completed.returncode == 1 + assert message in completed.stderr + + +def test_unknown_options_are_a_usage_error() -> None: + completed = run("--energy", "require") + assert completed.returncode == 2 + assert completed.stderr.startswith("Usage:") From 59cccaed300df27967ab7f65ebe3ea57e2fb1e07 Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:45:59 +0400 Subject: [PATCH 50/51] docs: publish time results measured in Docker, with energy pending docs/results.md, its time chart and the README's results block now come from a full study-profile run in Docker Desktop on an otherwise idle Windows 11 laptop (AMD Ryzen 7 6800H, on mains power), with energy off. Run with ./scripts/run-docker-timing.sh at 9e55f7b: - 5,760 measured runs (4,320 sorting, 1,440 startup batches), status complete, no retries, every sort loop at least 200 ms; - a container on CPUs 2 and 4 of the Docker VM, pinned there, with 2 GiB of memory, no swap and no network; - gcc 14.2.0, Python 3.12.14 and Node.js 24.21.0. The raw data, the environment record and the summary are in data/runs/20261002T211256Z-docker/. Energy, power, the proxy question and carbon stay pending a run on bare-metal Linux. study.md's status box and ADR 12 now say so. Co-Authored-By: Claude Opus 5.5 --- README.md | 31 +- .../20261002T211256Z-docker/environment.txt | 59 + data/runs/20261002T211256Z-docker/meta.json | 253 + data/runs/20261002T211256Z-docker/runs.csv | 5761 +++++++++++++++++ data/runs/20261002T211256Z-docker/summary.csv | 145 + docs/decisions.md | 3 + docs/img/time_per_sort.png | Bin 0 -> 26827 bytes docs/results.md | 536 +- docs/study.md | 6 +- 9 files changed, 6781 insertions(+), 13 deletions(-) create mode 100644 data/runs/20261002T211256Z-docker/environment.txt create mode 100644 data/runs/20261002T211256Z-docker/meta.json create mode 100644 data/runs/20261002T211256Z-docker/runs.csv create mode 100644 data/runs/20261002T211256Z-docker/summary.csv create mode 100644 docs/img/time_per_sort.png diff --git a/README.md b/README.md index 37bd244..13dfeeb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,33 @@ written method. -> **Status: time and energy results are pending.** Everything is built and tested, but the energy study needs Linux running directly on hardware. [docs/results.md](docs/results.md) comes from a run that checked the pipeline, so it shows the exact operation counts and marks every time, energy and carbon figure as pending until the study runs on bare-metal Linux with one command ([how](docs/bare-metal.md)). +**Results** from [docs/results.md](docs/results.md): AMD Ryzen 7 6800H with Radeon Graphics, Linux 6.18.33.2-microsoft-standard-WSL2, 5,760 measured runs on 2026-10-02. + +> **Energy results are pending.** This run measured time and operation counts only; energy was not measured (turned off with --energy off). Whether run time is a good stand-in for energy needs the study on bare-metal Linux ([how](docs/bare-metal.md)). + +Time per sort, random input of 2,000 elements: the median of each configuration's measured runs, with a 95% bootstrap interval in brackets. + +| Algorithm | C | Python | Node.js | +| :--- | ---: | ---: | ---: | +| bubble | 6.05 ms (5.94 to 6.08) | 181 ms (179 to 183) | 2.47 ms (2.45 to 2.50) | +| insertion | 473 µs (468 to 477) | 102 ms (102 to 103) | 1.05 ms (1.03 to 1.06) | +| quick | 22.6 µs (22.4 to 22.7) | 2.75 ms (2.69 to 2.78) | 113 µs (111 to 114) | + +Time ratios between languages, random input of 2,000 elements: + +| Algorithm | Order | Size | Languages | Time ratio | +| :--- | :--- | ---: | :--- | ---: | +| bubble | random | 2,000 | Python / C | 30 (29.5 to 30.6) | +| bubble | random | 2,000 | Node.js / C | 0.408 (0.404 to 0.419) | +| bubble | random | 2,000 | Python / Node.js | 73.5 (71.9 to 74.3) | +| insertion | random | 2,000 | Python / C | 216 (214 to 220) | +| insertion | random | 2,000 | Node.js / C | 2.21 (2.17 to 2.25) | +| insertion | random | 2,000 | Python / Node.js | 97.7 (96.4 to 99.6) | +| quick | random | 2,000 | Python / C | 122 (119 to 124) | +| quick | random | 2,000 | Node.js / C | 4.99 (4.89 to 5.07) | +| quick | random | 2,000 | Python / Node.js | 24.5 (23.8 to 25) | + +Environment: 16 CPUs online, 2 of them allowed to the harness, a cgroup memory limit of 2.0 GiB; CPU pinning to CPUs 2, 4; C: gcc 14.2.0; Python: 3.12.14; Node.js: v24.21.0. Run label: run-docker-timing.sh study 20261002T211256Z: a Docker container limited to CPUs 2,4 and 2g of memory without swap; energy off, because containers cannot read RAPL; Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores. Reproduce with `./scripts/run-docker-timing.sh --cpus 2,4 --memory 2g`. Every setting and command: [docs/results.md](docs/results.md#how-these-results-were-produced). ## Sample output @@ -254,6 +280,9 @@ bare metal. in the study profile (100,000 at most). - **One setup per run.** Results describe the recorded compiler, Python and Node.js versions and the two pinned CPUs of one machine. Pinned and unpinned runs have not been compared. + GCC 14.2 at `-O2` compiles the C bubble sort's swap into code that stalls on every swap, + which made it slower than Node.js on random input in the time run in Docker; the + [threats to validity](docs/study.md#workload-and-implementations) give the measurements. Next: diff --git a/data/runs/20261002T211256Z-docker/environment.txt b/data/runs/20261002T211256Z-docker/environment.txt new file mode 100644 index 0000000..a046085 --- /dev/null +++ b/data/runs/20261002T211256Z-docker/environment.txt @@ -0,0 +1,59 @@ +recorded_utc: 2026-10-02T21:39:09Z +git_commit: 9e55f7b99b293f732d3def4c40370d64bf5c2c1b +git_worktree_clean: yes +host: MINGW64_NT-10.0-26200 3.6.9-b4195d69.x86_64 x86_64 +note: Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores +docker: Docker Desktop 4.93.0 (240920), engine 29.8.1 +docker_host: Docker Desktop, kernel 6.18.33.2-microsoft-standard-WSL2, 16 CPUs, 7926865920 bytes of memory +image: greenbench sha256:ef094901a804f86d93ed0aca648752631d93b03af04230dab1278e5018fa8230 +base_images: node:24-trixie-slim@sha256:8ec5d7557396cfe32d21c3f9c13072355ceab22b584578ca4bb28af31120cffe python:3.12-slim-trixie@sha256:f77ac9e44ae96ef2c90b8053ea08c31f8be030f824196b0ae4db6d462c84e51f +reproduce: ./scripts/run-docker-timing.sh --cpus 2,4 --memory 2g +measure_command: docker run --name greenbench-timing --init --network none --cpuset-cpus 2,4 --memory 2g --memory-swap 2g greenbench sh -c 'mkdir -p "$1" && { uname -srmo; echo; lscpu; } >"$1/container.txt" 2>&1; make -s all && shift && exec "$@"' sh data/runs/20261002T211256Z-docker build/greenbench run --profile study --energy off --cpus 2,4 --label 'run-docker-timing.sh study 20261002T211256Z: a Docker container limited to CPUs 2,4 and 2g of memory without swap; energy off, because containers cannot read RAPL; Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores' --out data/runs/20261002T211256Z-docker + +## Inside the container +Linux 6.18.33.2-microsoft-standard-WSL2 x86_64 GNU/Linux + +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Address sizes: 48 bits physical, 48 bits virtual +Byte Order: Little Endian +CPU(s): 16 +On-line CPU(s) list: 0-15 +Vendor ID: AuthenticAMD +Model name: AMD Ryzen 7 6800H with Radeon Graphics +CPU family: 25 +Model: 68 +Thread(s) per core: 2 +Core(s) per socket: 8 +Socket(s): 1 +Stepping: 1 +BogoMIPS: 6388.00 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload umip vaes vpclmulqdq rdpid +Virtualization: AMD-V +Hypervisor vendor: Microsoft +Virtualization type: full +L1d cache: 256 KiB (8 instances) +L1i cache: 256 KiB (8 instances) +L2 cache: 4 MiB (8 instances) +L3 cache: 16 MiB (1 instance) +NUMA node(s): 1 +NUMA node0 CPU(s): 0-15 +Vulnerability Gather data sampling: Not affected +Vulnerability Ghostwrite: Not affected +Vulnerability Indirect target selection: Not affected +Vulnerability Itlb multihit: Not affected +Vulnerability L1tf: Not affected +Vulnerability Mds: Not affected +Vulnerability Meltdown: Not affected +Vulnerability Mmio stale data: Not affected +Vulnerability Old microcode: Not affected +Vulnerability Reg file data sampling: Not affected +Vulnerability Retbleed: Not affected +Vulnerability Spec rstack overflow: Vulnerable: Safe RET, no microcode +Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl +Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization +Vulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected +Vulnerability Srbds: Not affected +Vulnerability Tsa: Vulnerable: No microcode +Vulnerability Tsx async abort: Not affected +Vulnerability Vmscape: Not affected diff --git a/data/runs/20261002T211256Z-docker/meta.json b/data/runs/20261002T211256Z-docker/meta.json new file mode 100644 index 0000000..3e17cc3 --- /dev/null +++ b/data/runs/20261002T211256Z-docker/meta.json @@ -0,0 +1,253 @@ +{ + "tool": "greenbench", + "version": "2.2.0", + "csv_schema": 2, + "status": "complete", + "started_utc": "2026-10-02T21:12:57Z", + "finished_utc": "2026-10-02T21:39:08Z", + "label": "run-docker-timing.sh study 20261002T211256Z: a Docker container limited to CPUs 2,4 and 2g of memory without swap; energy off, because containers cannot read RAPL; Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores", + "command": "build/greenbench run --profile study --energy off --cpus 2,4 --label 'run-docker-timing.sh study 20261002T211256Z: a Docker container limited to CPUs 2,4 and 2g of memory without swap; energy off, because containers cannot read RAPL; Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores' --out data/runs/20261002T211256Z-docker", + "measured_runs": 5760, + "worker_retries": 0, + "short_loops": 0, + "capped_loops": 0, + "clock": "CLOCK_MONOTONIC_RAW", + "host": { + "sysname": "Linux", + "release": "6.18.33.2-microsoft-standard-WSL2", + "machine": "x86_64", + "cpu_model": "AMD Ryzen 7 6800H with Radeon Graphics", + "online_cpus": 16, + "allowed_cpus": 2, + "cgroup_cpu_max": "max 100000", + "cgroup_memory_max": "2147483648", + "scaling_governor": null, + "intel_pstate_no_turbo": null, + "cpufreq_boost": null + }, + "energy": { + "status": "off", + "reason": "turned off with --energy off", + "powercap_root": "/sys/class/powercap", + "idle_baseline": false, + "counter_wraps": 0, + "simulated": false, + "domains": [] + }, + "settings": { + "profile": "study", + "languages": ["c", "python", "node"], + "algorithms": ["bubble", "insertion", "quick"], + "orders": ["random", "sorted", "reversed", "nearly_sorted"], + "sizes": [250, 500, 1000, 2000], + "runs": 30, + "warmup": 3, + "seed": 2463534242, + "order_seed": 1234567, + "min_loop_ns": 200000000, + "aim_loop_ns": 240000000, + "max_reps": 1000000, + "startup_trials": true, + "retries": 1, + "cpus": [2, 4] + }, + "workers": { + "c": {"path": "/work/build/greenbench-worker", "version": "gcc 14.2.0"}, + "python": {"path": "/work/workers/python/greenbench_worker.py", "version": "Python 3.12.14"}, + "node": {"path": "/work/workers/node/worker.js", "version": "v24.21.0"} + }, + "calibration": [ + {"language": "c", "algorithm": "bubble", "order": "random", "n": 250, "inner_reps": 2568, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "random", "n": 500, "inner_reps": 635, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "random", "n": 1000, "inner_reps": 154, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "random", "n": 2000, "inner_reps": 40, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "sorted", "n": 250, "inner_reps": 11386, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "sorted", "n": 500, "inner_reps": 2858, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "sorted", "n": 1000, "inner_reps": 702, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "sorted", "n": 2000, "inner_reps": 173, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "reversed", "n": 250, "inner_reps": 1384, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "reversed", "n": 500, "inner_reps": 346, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "reversed", "n": 1000, "inner_reps": 87, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "reversed", "n": 2000, "inner_reps": 22, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "nearly_sorted", "n": 250, "inner_reps": 7880, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "nearly_sorted", "n": 500, "inner_reps": 1998, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "nearly_sorted", "n": 1000, "inner_reps": 507, "launches": 1}, + {"language": "c", "algorithm": "bubble", "order": "nearly_sorted", "n": 2000, "inner_reps": 121, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "random", "n": 250, "inner_reps": 26020, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "random", "n": 500, "inner_reps": 6945, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "random", "n": 1000, "inner_reps": 1983, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "random", "n": 2000, "inner_reps": 492, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "sorted", "n": 250, "inner_reps": 786886, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "sorted", "n": 500, "inner_reps": 399335, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "sorted", "n": 1000, "inner_reps": 196883, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "sorted", "n": 2000, "inner_reps": 101352, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "reversed", "n": 250, "inner_reps": 15645, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "reversed", "n": 500, "inner_reps": 3967, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "reversed", "n": 1000, "inner_reps": 1033, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "reversed", "n": 2000, "inner_reps": 260, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "nearly_sorted", "n": 250, "inner_reps": 99421, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "nearly_sorted", "n": 500, "inner_reps": 32350, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "nearly_sorted", "n": 1000, "inner_reps": 11445, "launches": 1}, + {"language": "c", "algorithm": "insertion", "order": "nearly_sorted", "n": 2000, "inner_reps": 2962, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "random", "n": 250, "inner_reps": 136055, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "random", "n": 500, "inner_reps": 55776, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "random", "n": 1000, "inner_reps": 23484, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "random", "n": 2000, "inner_reps": 10587, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "sorted", "n": 250, "inner_reps": 10851, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "sorted", "n": 500, "inner_reps": 2762, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "sorted", "n": 1000, "inner_reps": 691, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "sorted", "n": 2000, "inner_reps": 174, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "reversed", "n": 250, "inner_reps": 12781, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "reversed", "n": 500, "inner_reps": 3240, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "reversed", "n": 1000, "inner_reps": 814, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "reversed", "n": 2000, "inner_reps": 209, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "nearly_sorted", "n": 250, "inner_reps": 77221, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "nearly_sorted", "n": 500, "inner_reps": 31784, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "nearly_sorted", "n": 1000, "inner_reps": 11273, "launches": 1}, + {"language": "c", "algorithm": "quick", "order": "nearly_sorted", "n": 2000, "inner_reps": 6145, "launches": 1}, + {"language": "c", "algorithm": "none", "order": "random", "n": 250, "inner_reps": 0, "launches": 429}, + {"language": "c", "algorithm": "none", "order": "random", "n": 500, "inner_reps": 0, "launches": 426}, + {"language": "c", "algorithm": "none", "order": "random", "n": 1000, "inner_reps": 0, "launches": 367}, + {"language": "c", "algorithm": "none", "order": "random", "n": 2000, "inner_reps": 0, "launches": 415}, + {"language": "c", "algorithm": "none", "order": "sorted", "n": 250, "inner_reps": 0, "launches": 429}, + {"language": "c", "algorithm": "none", "order": "sorted", "n": 500, "inner_reps": 0, "launches": 344}, + {"language": "c", "algorithm": "none", "order": "sorted", "n": 1000, "inner_reps": 0, "launches": 421}, + {"language": "c", "algorithm": "none", "order": "sorted", "n": 2000, "inner_reps": 0, "launches": 416}, + {"language": "c", "algorithm": "none", "order": "reversed", "n": 250, "inner_reps": 0, "launches": 427}, + {"language": "c", "algorithm": "none", "order": "reversed", "n": 500, "inner_reps": 0, "launches": 402}, + {"language": "c", "algorithm": "none", "order": "reversed", "n": 1000, "inner_reps": 0, "launches": 405}, + {"language": "c", "algorithm": "none", "order": "reversed", "n": 2000, "inner_reps": 0, "launches": 413}, + {"language": "c", "algorithm": "none", "order": "nearly_sorted", "n": 250, "inner_reps": 0, "launches": 388}, + {"language": "c", "algorithm": "none", "order": "nearly_sorted", "n": 500, "inner_reps": 0, "launches": 405}, + {"language": "c", "algorithm": "none", "order": "nearly_sorted", "n": 1000, "inner_reps": 0, "launches": 421}, + {"language": "c", "algorithm": "none", "order": "nearly_sorted", "n": 2000, "inner_reps": 0, "launches": 387}, + {"language": "python", "algorithm": "bubble", "order": "random", "n": 250, "inner_reps": 112, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "random", "n": 500, "inner_reps": 26, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "random", "n": 1000, "inner_reps": 6, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "random", "n": 2000, "inner_reps": 2, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "sorted", "n": 250, "inner_reps": 206, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "sorted", "n": 500, "inner_reps": 46, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "sorted", "n": 1000, "inner_reps": 11, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "sorted", "n": 2000, "inner_reps": 3, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "reversed", "n": 250, "inner_reps": 88, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "reversed", "n": 500, "inner_reps": 20, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "reversed", "n": 1000, "inner_reps": 5, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "reversed", "n": 2000, "inner_reps": 2, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "nearly_sorted", "n": 250, "inner_reps": 188, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "nearly_sorted", "n": 500, "inner_reps": 41, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "nearly_sorted", "n": 1000, "inner_reps": 9, "launches": 1}, + {"language": "python", "algorithm": "bubble", "order": "nearly_sorted", "n": 2000, "inner_reps": 3, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "random", "n": 250, "inner_reps": 206, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "random", "n": 500, "inner_reps": 44, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "random", "n": 1000, "inner_reps": 10, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "random", "n": 2000, "inner_reps": 3, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "sorted", "n": 250, "inner_reps": 18845, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "sorted", "n": 500, "inner_reps": 6624, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "sorted", "n": 1000, "inner_reps": 3178, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "sorted", "n": 2000, "inner_reps": 1523, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "reversed", "n": 250, "inner_reps": 102, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "reversed", "n": 500, "inner_reps": 22, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "reversed", "n": 1000, "inner_reps": 6, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "reversed", "n": 2000, "inner_reps": 2, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "nearly_sorted", "n": 250, "inner_reps": 1416, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "nearly_sorted", "n": 500, "inner_reps": 316, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "nearly_sorted", "n": 1000, "inner_reps": 76, "launches": 1}, + {"language": "python", "algorithm": "insertion", "order": "nearly_sorted", "n": 2000, "inner_reps": 20, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "random", "n": 250, "inner_reps": 1271, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "random", "n": 500, "inner_reps": 507, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "random", "n": 1000, "inner_reps": 209, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "random", "n": 2000, "inner_reps": 90, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "sorted", "n": 250, "inner_reps": 133, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "sorted", "n": 500, "inner_reps": 30, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "sorted", "n": 1000, "inner_reps": 8, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "sorted", "n": 2000, "inner_reps": 2, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "reversed", "n": 250, "inner_reps": 151, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "reversed", "n": 500, "inner_reps": 33, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "reversed", "n": 1000, "inner_reps": 8, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "reversed", "n": 2000, "inner_reps": 2, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "nearly_sorted", "n": 250, "inner_reps": 882, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "nearly_sorted", "n": 500, "inner_reps": 296, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "nearly_sorted", "n": 1000, "inner_reps": 110, "launches": 1}, + {"language": "python", "algorithm": "quick", "order": "nearly_sorted", "n": 2000, "inner_reps": 55, "launches": 1}, + {"language": "python", "algorithm": "none", "order": "random", "n": 250, "inner_reps": 0, "launches": 17}, + {"language": "python", "algorithm": "none", "order": "random", "n": 500, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "random", "n": 1000, "inner_reps": 0, "launches": 15}, + {"language": "python", "algorithm": "none", "order": "random", "n": 2000, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "sorted", "n": 250, "inner_reps": 0, "launches": 17}, + {"language": "python", "algorithm": "none", "order": "sorted", "n": 500, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "sorted", "n": 1000, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "sorted", "n": 2000, "inner_reps": 0, "launches": 15}, + {"language": "python", "algorithm": "none", "order": "reversed", "n": 250, "inner_reps": 0, "launches": 17}, + {"language": "python", "algorithm": "none", "order": "reversed", "n": 500, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "reversed", "n": 1000, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "reversed", "n": 2000, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "nearly_sorted", "n": 250, "inner_reps": 0, "launches": 17}, + {"language": "python", "algorithm": "none", "order": "nearly_sorted", "n": 500, "inner_reps": 0, "launches": 17}, + {"language": "python", "algorithm": "none", "order": "nearly_sorted", "n": 1000, "inner_reps": 0, "launches": 16}, + {"language": "python", "algorithm": "none", "order": "nearly_sorted", "n": 2000, "inner_reps": 0, "launches": 15}, + {"language": "node", "algorithm": "bubble", "order": "random", "n": 250, "inner_reps": 5439, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "random", "n": 500, "inner_reps": 1407, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "random", "n": 1000, "inner_reps": 362, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "random", "n": 2000, "inner_reps": 98, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "sorted", "n": 250, "inner_reps": 8081, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "sorted", "n": 500, "inner_reps": 1964, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "sorted", "n": 1000, "inner_reps": 374, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "sorted", "n": 2000, "inner_reps": 85, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "reversed", "n": 250, "inner_reps": 5861, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "reversed", "n": 500, "inner_reps": 1534, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "reversed", "n": 1000, "inner_reps": 378, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "reversed", "n": 2000, "inner_reps": 94, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "nearly_sorted", "n": 250, "inner_reps": 6161, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "nearly_sorted", "n": 500, "inner_reps": 1566, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "nearly_sorted", "n": 1000, "inner_reps": 413, "launches": 1}, + {"language": "node", "algorithm": "bubble", "order": "nearly_sorted", "n": 2000, "inner_reps": 110, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "random", "n": 250, "inner_reps": 15349, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "random", "n": 500, "inner_reps": 4059, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "random", "n": 1000, "inner_reps": 1091, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "random", "n": 2000, "inner_reps": 234, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "sorted", "n": 250, "inner_reps": 607595, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "sorted", "n": 500, "inner_reps": 324764, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "sorted", "n": 1000, "inner_reps": 160107, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "sorted", "n": 2000, "inner_reps": 89220, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "reversed", "n": 250, "inner_reps": 8539, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "reversed", "n": 500, "inner_reps": 2178, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "reversed", "n": 1000, "inner_reps": 554, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "reversed", "n": 2000, "inner_reps": 144, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "nearly_sorted", "n": 250, "inner_reps": 79550, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "nearly_sorted", "n": 500, "inner_reps": 26045, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "nearly_sorted", "n": 1000, "inner_reps": 7758, "launches": 1}, + {"language": "node", "algorithm": "insertion", "order": "nearly_sorted", "n": 2000, "inner_reps": 2100, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "random", "n": 250, "inner_reps": 36348, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "random", "n": 500, "inner_reps": 16148, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "random", "n": 1000, "inner_reps": 6513, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "random", "n": 2000, "inner_reps": 2126, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "sorted", "n": 250, "inner_reps": 4507, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "sorted", "n": 500, "inner_reps": 1164, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "sorted", "n": 1000, "inner_reps": 296, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "sorted", "n": 2000, "inner_reps": 74, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "reversed", "n": 250, "inner_reps": 4755, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "reversed", "n": 500, "inner_reps": 1194, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "reversed", "n": 1000, "inner_reps": 307, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "reversed", "n": 2000, "inner_reps": 80, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "nearly_sorted", "n": 250, "inner_reps": 25320, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "nearly_sorted", "n": 500, "inner_reps": 10954, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "nearly_sorted", "n": 1000, "inner_reps": 3752, "launches": 1}, + {"language": "node", "algorithm": "quick", "order": "nearly_sorted", "n": 2000, "inner_reps": 1909, "launches": 1}, + {"language": "node", "algorithm": "none", "order": "random", "n": 250, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "random", "n": 500, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "random", "n": 1000, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "random", "n": 2000, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "sorted", "n": 250, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "sorted", "n": 500, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "sorted", "n": 1000, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "sorted", "n": 2000, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "reversed", "n": 250, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "reversed", "n": 500, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "reversed", "n": 1000, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "reversed", "n": 2000, "inner_reps": 0, "launches": 9}, + {"language": "node", "algorithm": "none", "order": "nearly_sorted", "n": 250, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "nearly_sorted", "n": 500, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "nearly_sorted", "n": 1000, "inner_reps": 0, "launches": 10}, + {"language": "node", "algorithm": "none", "order": "nearly_sorted", "n": 2000, "inner_reps": 0, "launches": 9} + ] +} diff --git a/data/runs/20261002T211256Z-docker/runs.csv b/data/runs/20261002T211256Z-docker/runs.csv new file mode 100644 index 0000000..664fff1 --- /dev/null +++ b/data/runs/20261002T211256Z-docker/runs.csv @@ -0,0 +1,5761 @@ +sequence,language,algorithm,order,n,seed,repetition,inner_reps,launches,wall_ns,loop_ns,comparisons,writes,input_hash,energy_status,pkg_uj,dram_uj,idle_ns,idle_pkg_uj,idle_dram_uj,net_pkg_uj,net_dram_uj +1,c,bubble,sorted,500,2463534242,26,2858,1,245603209,244434167,124750,0,d9a24379,off,,,,,,, +2,c,none,nearly_sorted,1000,2463534242,2,0,421,256663447,90,0,0,1cfe4afd,off,,,,,,, +3,c,bubble,random,1000,2463534242,7,154,1,236344213,235581739,499500,496926,57561285,off,,,,,,, +4,python,bubble,reversed,250,2463534242,3,88,1,337228335,320610818,31125,62250,be175004,off,,,,,,, +5,node,bubble,random,250,2463534242,30,5439,1,285365957,252262285,31125,31196,b47adce4,off,,,,,,, +6,c,none,reversed,1000,2463534242,11,0,405,231133161,90,0,0,0ca43265,off,,,,,,, +7,c,insertion,sorted,1000,2463534242,23,196883,1,235752869,235053117,999,0,8e15cb75,off,,,,,,, +8,c,none,sorted,1000,2463534242,20,0,421,253352361,90,0,0,8e15cb75,off,,,,,,, +9,node,quick,nearly_sorted,2000,2463534242,19,1909,1,260106150,228597284,58002,11768,5835dc59,off,,,,,,, +10,python,quick,nearly_sorted,500,2463534242,14,296,1,247182527,230101529,10192,1850,482b4785,off,,,,,,, +11,node,quick,reversed,2000,2463534242,5,80,1,278665621,248891745,1999000,2000,ad35ba05,off,,,,,,, +12,c,insertion,nearly_sorted,2000,2463534242,20,2962,1,224483889,223387832,118277,118270,5835dc59,off,,,,,,, +13,python,insertion,random,250,2463534242,30,206,1,262043168,243987359,15844,15838,b47adce4,off,,,,,,, +14,python,none,nearly_sorted,500,2463534242,1,0,17,311020586,521,0,0,482b4785,off,,,,,,, +15,python,quick,sorted,500,2463534242,17,30,1,269900975,252795543,124750,0,d9a24379,off,,,,,,, +16,node,insertion,sorted,500,2463534242,29,324764,1,277858889,248006900,499,0,d9a24379,off,,,,,,, +17,node,bubble,random,500,2463534242,17,1407,1,286155507,253678225,124750,119688,11ae932d,off,,,,,,, +18,c,none,reversed,500,2463534242,25,0,402,252503050,90,0,0,d33e3559,off,,,,,,, +19,c,none,nearly_sorted,500,2463534242,22,0,405,239949844,90,0,0,482b4785,off,,,,,,, +20,c,quick,nearly_sorted,1000,2463534242,6,11273,1,241608623,240702785,28499,7892,1cfe4afd,off,,,,,,, +21,python,quick,random,500,2463534242,27,507,1,268360739,250969103,4578,4552,11ae932d,off,,,,,,, +22,c,bubble,sorted,500,2463534242,13,2858,1,242477981,241474693,124750,0,d9a24379,off,,,,,,, +23,python,none,random,2000,2463534242,25,0,16,278616709,611,0,0,41262855,off,,,,,,, +24,python,insertion,nearly_sorted,2000,2463534242,5,20,1,264113080,247258594,118277,118270,5835dc59,off,,,,,,, +25,python,quick,nearly_sorted,1000,2463534242,3,110,1,303944062,286732621,28499,7892,1cfe4afd,off,,,,,,, +26,c,none,reversed,500,2463534242,18,0,402,237089875,90,0,0,d33e3559,off,,,,,,, +27,python,insertion,reversed,1000,2463534242,25,6,1,304771875,287337755,499500,500499,0ca43265,off,,,,,,, +28,node,insertion,reversed,2000,2463534242,28,144,1,268629222,239940644,1999000,2000999,ad35ba05,off,,,,,,, +29,c,none,reversed,1000,2463534242,30,0,405,229250984,150,0,0,0ca43265,off,,,,,,, +30,node,insertion,reversed,2000,2463534242,20,144,1,265717645,234722737,1999000,2000999,ad35ba05,off,,,,,,, +31,c,quick,nearly_sorted,2000,2463534242,23,6145,1,246438665,245268503,58002,11768,5835dc59,off,,,,,,, +32,python,none,random,500,2463534242,29,0,16,235294891,752,0,0,11ae932d,off,,,,,,, +33,c,quick,sorted,2000,2463534242,10,174,1,247129029,246228387,1999000,0,69842aa5,off,,,,,,, +34,node,quick,reversed,2000,2463534242,12,80,1,275472706,243118878,1999000,2000,ad35ba05,off,,,,,,, +35,c,bubble,reversed,2000,2463534242,10,22,1,246596550,245537175,1999000,3998000,ad35ba05,off,,,,,,, +36,node,none,nearly_sorted,1000,2463534242,11,0,10,252456533,792,0,0,1cfe4afd,off,,,,,,, +37,c,none,reversed,2000,2463534242,30,0,413,235262409,90,0,0,ad35ba05,off,,,,,,, +38,c,insertion,random,2000,2463534242,30,492,1,235093924,234342024,996390,996389,41262855,off,,,,,,, +39,c,quick,nearly_sorted,250,2463534242,20,77221,1,245945120,244810804,4346,1008,21a19c14,off,,,,,,, +40,c,none,sorted,1000,2463534242,8,0,421,252917607,90,0,0,8e15cb75,off,,,,,,, +41,python,none,reversed,1000,2463534242,13,0,16,246613042,602,0,0,0ca43265,off,,,,,,, +42,python,none,nearly_sorted,500,2463534242,18,0,17,262299060,541,0,0,482b4785,off,,,,,,, +43,c,none,random,500,2463534242,12,0,426,241223249,81,0,0,11ae932d,off,,,,,,, +44,python,none,random,500,2463534242,1,0,16,243675436,701,0,0,11ae932d,off,,,,,,, +45,c,bubble,reversed,250,2463534242,28,1384,1,241518571,240505065,31125,62250,be175004,off,,,,,,, +46,c,none,nearly_sorted,250,2463534242,29,0,388,228058672,90,0,0,21a19c14,off,,,,,,, +47,python,bubble,nearly_sorted,500,2463534242,13,41,1,261670781,244742726,124750,14926,482b4785,off,,,,,,, +48,node,insertion,reversed,500,2463534242,17,2178,1,269001059,238500702,124750,125249,d33e3559,off,,,,,,, +49,node,none,sorted,500,2463534242,23,0,10,271183518,791,0,0,d9a24379,off,,,,,,, +50,python,none,reversed,250,2463534242,16,0,17,254725976,581,0,0,be175004,off,,,,,,, +51,c,none,random,250,2463534242,18,0,429,242338560,80,0,0,b47adce4,off,,,,,,, +52,node,bubble,reversed,1000,2463534242,12,378,1,268116912,239160260,499500,999000,0ca43265,off,,,,,,, +53,python,bubble,random,2000,2463534242,28,2,1,373545571,355567246,1999000,1988792,41262855,off,,,,,,, +54,python,bubble,nearly_sorted,1000,2463534242,1,9,1,263643968,246807453,499500,59780,1cfe4afd,off,,,,,,, +55,node,bubble,nearly_sorted,500,2463534242,22,1566,1,278631206,247220055,124750,14926,482b4785,off,,,,,,, +56,node,none,nearly_sorted,500,2463534242,19,0,10,291530861,762,0,0,482b4785,off,,,,,,, +57,c,bubble,sorted,250,2463534242,4,11386,1,242110033,241136528,31125,0,7fd8c1f4,off,,,,,,, +58,node,bubble,nearly_sorted,250,2463534242,27,6161,1,277235039,248435464,31125,4056,21a19c14,off,,,,,,, +59,node,insertion,nearly_sorted,1000,2463534242,25,7758,1,286306701,256666270,30889,30860,1cfe4afd,off,,,,,,, +60,python,insertion,sorted,500,2463534242,4,6624,1,262343992,245375074,499,0,d9a24379,off,,,,,,, +61,node,bubble,reversed,2000,2463534242,10,94,1,289277067,259397234,1999000,3998000,ad35ba05,off,,,,,,, +62,node,insertion,nearly_sorted,2000,2463534242,13,2100,1,286395427,245060139,118277,118270,5835dc59,off,,,,,,, +63,c,bubble,sorted,500,2463534242,16,2858,1,245063618,244174873,124750,0,d9a24379,off,,,,,,, +64,c,none,nearly_sorted,250,2463534242,8,0,388,223849380,91,0,0,21a19c14,off,,,,,,, +65,c,bubble,nearly_sorted,250,2463534242,22,7880,1,238978112,238240942,31125,4056,21a19c14,off,,,,,,, +66,node,bubble,nearly_sorted,250,2463534242,30,6161,1,282806140,254442351,31125,4056,21a19c14,off,,,,,,, +67,python,bubble,sorted,500,2463534242,22,46,1,271336447,255346131,124750,0,d9a24379,off,,,,,,, +68,node,insertion,random,500,2463534242,23,4059,1,278514866,250659652,60338,60339,11ae932d,off,,,,,,, +69,python,bubble,nearly_sorted,500,2463534242,17,41,1,254366141,239310392,124750,14926,482b4785,off,,,,,,, +70,c,insertion,sorted,500,2463534242,13,399335,1,243387907,242395247,499,0,d9a24379,off,,,,,,, +71,c,insertion,nearly_sorted,1000,2463534242,8,11445,1,242053907,241134655,30889,30860,1cfe4afd,off,,,,,,, +72,python,bubble,random,1000,2463534242,23,6,1,282761827,266567931,499500,496926,57561285,off,,,,,,, +73,python,quick,random,500,2463534242,11,507,1,288766450,271951604,4578,4552,11ae932d,off,,,,,,, +74,python,none,random,250,2463534242,15,0,17,245076111,531,0,0,b47adce4,off,,,,,,, +75,node,insertion,random,1000,2463534242,27,1091,1,269324435,239536422,249457,249457,57561285,off,,,,,,, +76,node,quick,random,250,2463534242,27,36348,1,250792795,223766090,1933,1940,b47adce4,off,,,,,,, +77,python,bubble,sorted,500,2463534242,8,46,1,256836782,241999160,124750,0,d9a24379,off,,,,,,, +78,node,insertion,random,250,2463534242,8,15349,1,264467973,235688319,15844,15838,b47adce4,off,,,,,,, +79,node,insertion,random,250,2463534242,10,15349,1,258674585,231311146,15844,15838,b47adce4,off,,,,,,, +80,python,bubble,nearly_sorted,1000,2463534242,24,9,1,248892764,233565383,499500,59780,1cfe4afd,off,,,,,,, +81,c,insertion,random,2000,2463534242,6,492,1,238566912,237433249,996390,996389,41262855,off,,,,,,, +82,node,insertion,reversed,2000,2463534242,23,144,1,267736478,238519065,1999000,2000999,ad35ba05,off,,,,,,, +83,c,none,reversed,500,2463534242,4,0,402,258752983,100,0,0,d33e3559,off,,,,,,, +84,node,quick,reversed,500,2463534242,3,1194,1,279580675,246813664,124750,500,d33e3559,off,,,,,,, +85,node,quick,nearly_sorted,2000,2463534242,22,1909,1,270397856,240280544,58002,11768,5835dc59,off,,,,,,, +86,node,quick,reversed,500,2463534242,30,1194,1,263765376,231452547,124750,500,d33e3559,off,,,,,,, +87,python,insertion,nearly_sorted,2000,2463534242,25,20,1,258415030,242020159,118277,118270,5835dc59,off,,,,,,, +88,python,none,sorted,2000,2463534242,27,0,15,245430535,761,0,0,69842aa5,off,,,,,,, +89,c,insertion,sorted,500,2463534242,24,399335,1,240470730,239587306,499,0,d9a24379,off,,,,,,, +90,node,insertion,reversed,500,2463534242,24,2178,1,262758239,234788671,124750,125249,d33e3559,off,,,,,,, +91,python,quick,random,250,2463534242,30,1271,1,252136673,236764666,1933,1940,b47adce4,off,,,,,,, +92,c,none,reversed,250,2463534242,1,0,427,239040159,100,0,0,be175004,off,,,,,,, +93,python,insertion,sorted,1000,2463534242,16,3178,1,271534016,252403754,999,0,8e15cb75,off,,,,,,, +94,c,insertion,nearly_sorted,1000,2463534242,7,11445,1,244375037,243230302,30889,30860,1cfe4afd,off,,,,,,, +95,node,insertion,nearly_sorted,500,2463534242,9,26045,1,248852989,219162454,7962,7926,482b4785,off,,,,,,, +96,c,bubble,random,500,2463534242,30,635,1,232013697,231012385,124750,119688,11ae932d,off,,,,,,, +97,python,insertion,nearly_sorted,250,2463534242,25,1416,1,263725234,247926218,2277,2251,21a19c14,off,,,,,,, +98,node,bubble,random,250,2463534242,28,5439,1,260428943,232808625,31125,31196,b47adce4,off,,,,,,, +99,node,insertion,random,1000,2463534242,16,1091,1,268807956,239178051,249457,249457,57561285,off,,,,,,, +100,c,none,nearly_sorted,500,2463534242,26,0,405,233327834,100,0,0,482b4785,off,,,,,,, +101,node,bubble,nearly_sorted,500,2463534242,14,1566,1,274703124,244070227,124750,14926,482b4785,off,,,,,,, +102,c,bubble,sorted,500,2463534242,5,2858,1,245023092,244105863,124750,0,d9a24379,off,,,,,,, +103,python,none,random,500,2463534242,9,0,16,247522786,521,0,0,11ae932d,off,,,,,,, +104,c,quick,reversed,2000,2463534242,12,209,1,239886716,238815989,1999000,2000,ad35ba05,off,,,,,,, +105,c,quick,random,1000,2463534242,25,23484,1,232676514,231614212,11196,10210,57561285,off,,,,,,, +106,c,none,reversed,500,2463534242,11,0,402,226191951,91,0,0,d33e3559,off,,,,,,, +107,node,none,reversed,2000,2463534242,19,0,9,235003264,2445,0,0,ad35ba05,off,,,,,,, +108,c,bubble,sorted,2000,2463534242,9,173,1,235491639,234491516,1999000,0,69842aa5,off,,,,,,, +109,c,quick,reversed,2000,2463534242,20,209,1,237117275,235973544,1999000,2000,ad35ba05,off,,,,,,, +110,c,bubble,sorted,2000,2463534242,25,173,1,233019137,231983184,1999000,0,69842aa5,off,,,,,,, +111,node,quick,nearly_sorted,1000,2463534242,16,3752,1,265940721,236968368,28499,7892,1cfe4afd,off,,,,,,, +112,node,insertion,reversed,250,2463534242,19,8539,1,282307787,256166037,31125,31374,be175004,off,,,,,,, +113,python,quick,random,2000,2463534242,30,90,1,256169800,238853901,25197,24544,41262855,off,,,,,,, +114,c,insertion,reversed,1000,2463534242,18,1033,1,240550330,239670871,499500,500499,0ca43265,off,,,,,,, +115,python,quick,random,500,2463534242,14,507,1,268263867,252953511,4578,4552,11ae932d,off,,,,,,, +116,node,none,reversed,2000,2463534242,25,0,9,242470327,1072,0,0,ad35ba05,off,,,,,,, +117,c,quick,reversed,2000,2463534242,28,209,1,246127072,245212497,1999000,2000,ad35ba05,off,,,,,,, +118,python,quick,reversed,500,2463534242,4,33,1,270693749,254723999,124750,500,d33e3559,off,,,,,,, +119,c,none,reversed,2000,2463534242,9,0,413,244371029,90,0,0,ad35ba05,off,,,,,,, +120,c,insertion,sorted,1000,2463534242,27,196883,1,239740913,238974116,999,0,8e15cb75,off,,,,,,, +121,python,quick,random,250,2463534242,12,1271,1,252769819,236700703,1933,1940,b47adce4,off,,,,,,, +122,node,insertion,nearly_sorted,250,2463534242,26,79550,1,249183258,222161284,2277,2251,21a19c14,off,,,,,,, +123,python,insertion,sorted,2000,2463534242,6,1523,1,264108179,246535537,1999,0,69842aa5,off,,,,,,, +124,node,none,reversed,1000,2463534242,11,0,9,234013269,1012,0,0,0ca43265,off,,,,,,, +125,c,insertion,random,500,2463534242,7,6945,1,217977456,216964729,60338,60339,11ae932d,off,,,,,,, +126,python,bubble,sorted,2000,2463534242,26,3,1,324492935,307523144,1999000,0,69842aa5,off,,,,,,, +127,c,quick,sorted,1000,2463534242,29,691,1,239327856,238306150,499500,0,8e15cb75,off,,,,,,, +128,c,insertion,nearly_sorted,2000,2463534242,8,2962,1,226625834,225448819,118277,118270,5835dc59,off,,,,,,, +129,python,bubble,nearly_sorted,250,2463534242,14,188,1,260208169,243575456,31125,4056,21a19c14,off,,,,,,, +130,node,bubble,reversed,250,2463534242,9,5861,1,256571213,229452801,31125,62250,be175004,off,,,,,,, +131,python,insertion,nearly_sorted,1000,2463534242,28,76,1,250848529,235250600,30889,30860,1cfe4afd,off,,,,,,, +132,python,quick,nearly_sorted,500,2463534242,2,296,1,261910060,245259840,10192,1850,482b4785,off,,,,,,, +133,c,quick,reversed,2000,2463534242,1,209,1,245274833,244350101,1999000,2000,ad35ba05,off,,,,,,, +134,python,none,reversed,500,2463534242,3,0,16,246663727,571,0,0,d33e3559,off,,,,,,, +135,node,none,nearly_sorted,250,2463534242,22,0,10,264723411,782,0,0,21a19c14,off,,,,,,, +136,node,quick,random,250,2463534242,19,36348,1,265188052,236999677,1933,1940,b47adce4,off,,,,,,, +137,node,quick,reversed,500,2463534242,9,1194,1,255834644,229663396,124750,500,d33e3559,off,,,,,,, +138,c,bubble,nearly_sorted,1000,2463534242,2,507,1,243981489,242883442,499500,59780,1cfe4afd,off,,,,,,, +139,node,quick,nearly_sorted,2000,2463534242,10,1909,1,261908427,232009771,58002,11768,5835dc59,off,,,,,,, +140,python,none,reversed,2000,2463534242,25,0,16,249976595,561,0,0,ad35ba05,off,,,,,,, +141,c,bubble,sorted,2000,2463534242,18,173,1,233521418,232555627,1999000,0,69842aa5,off,,,,,,, +142,c,insertion,nearly_sorted,1000,2463534242,24,11445,1,240272688,239291191,30889,30860,1cfe4afd,off,,,,,,, +143,c,none,reversed,250,2463534242,28,0,427,239760749,90,0,0,be175004,off,,,,,,, +144,node,quick,sorted,250,2463534242,1,4507,1,284170737,256982558,31125,0,7fd8c1f4,off,,,,,,, +145,node,none,reversed,500,2463534242,18,0,9,227770639,962,0,0,d33e3559,off,,,,,,, +146,c,none,random,500,2463534242,20,0,426,240476972,80,0,0,11ae932d,off,,,,,,, +147,node,insertion,random,2000,2463534242,6,234,1,282357721,252239943,996390,996389,41262855,off,,,,,,, +148,python,quick,nearly_sorted,250,2463534242,23,882,1,276994928,259369958,4346,1008,21a19c14,off,,,,,,, +149,c,insertion,reversed,2000,2463534242,9,260,1,246458742,245286997,1999000,2000999,ad35ba05,off,,,,,,, +150,node,insertion,nearly_sorted,2000,2463534242,11,2100,1,269958853,236146779,118277,118270,5835dc59,off,,,,,,, +151,c,insertion,random,1000,2463534242,8,1983,1,243124754,242228825,249457,249457,57561285,off,,,,,,, +152,c,bubble,nearly_sorted,2000,2463534242,29,121,1,241358073,240426968,1999000,232556,5835dc59,off,,,,,,, +153,c,bubble,sorted,1000,2463534242,27,702,1,240467714,239527402,499500,0,8e15cb75,off,,,,,,, +154,node,quick,random,250,2463534242,1,36348,1,254621580,226002272,1933,1940,b47adce4,off,,,,,,, +155,node,insertion,random,500,2463534242,21,4059,1,274282417,245390111,60338,60339,11ae932d,off,,,,,,, +156,python,quick,nearly_sorted,500,2463534242,11,296,1,238020277,222524332,10192,1850,482b4785,off,,,,,,, +157,c,insertion,nearly_sorted,250,2463534242,7,99421,1,240898693,240026023,2277,2251,21a19c14,off,,,,,,, +158,node,bubble,nearly_sorted,1000,2463534242,19,413,1,274599710,240846985,499500,59780,1cfe4afd,off,,,,,,, +159,python,bubble,random,2000,2463534242,26,2,1,393192923,375373148,1999000,1988792,41262855,off,,,,,,, +160,c,insertion,nearly_sorted,1000,2463534242,22,11445,1,409968873,409045814,30889,30860,1cfe4afd,off,,,,,,, +161,python,insertion,reversed,2000,2463534242,18,2,1,423167880,406591422,1999000,2000999,ad35ba05,off,,,,,,, +162,c,bubble,reversed,500,2463534242,7,346,1,243047548,241873922,124750,249500,d33e3559,off,,,,,,, +163,c,insertion,nearly_sorted,2000,2463534242,18,2962,1,225105340,224069604,118277,118270,5835dc59,off,,,,,,, +164,c,bubble,sorted,1000,2463534242,13,702,1,240865189,239902685,499500,0,8e15cb75,off,,,,,,, +165,python,bubble,random,2000,2463534242,23,2,1,370543971,353563643,1999000,1988792,41262855,off,,,,,,, +166,node,quick,reversed,500,2463534242,14,1194,1,266897406,237648653,124750,500,d33e3559,off,,,,,,, +167,c,none,random,250,2463534242,17,0,429,247303906,80,0,0,b47adce4,off,,,,,,, +168,c,quick,nearly_sorted,250,2463534242,16,77221,1,244150948,243433184,4346,1008,21a19c14,off,,,,,,, +169,c,insertion,reversed,1000,2463534242,12,1033,1,246234462,245295331,499500,500499,0ca43265,off,,,,,,, +170,python,bubble,random,500,2463534242,19,26,1,263689314,247507865,124750,119688,11ae932d,off,,,,,,, +171,node,none,nearly_sorted,1000,2463534242,29,0,10,271924797,792,0,0,1cfe4afd,off,,,,,,, +172,node,none,random,1000,2463534242,10,0,10,271456881,11983,0,0,57561285,off,,,,,,, +173,c,bubble,nearly_sorted,250,2463534242,3,7880,1,242511816,241532900,31125,4056,21a19c14,off,,,,,,, +174,node,none,reversed,2000,2463534242,13,0,9,261173810,852,0,0,ad35ba05,off,,,,,,, +175,node,quick,reversed,1000,2463534242,26,307,1,278055346,248583373,499500,1000,0ca43265,off,,,,,,, +176,python,insertion,nearly_sorted,2000,2463534242,2,20,1,285094438,268510010,118277,118270,5835dc59,off,,,,,,, +177,c,insertion,random,500,2463534242,21,6945,1,218928469,217841251,60338,60339,11ae932d,off,,,,,,, +178,python,none,random,250,2463534242,30,0,17,269339362,481,0,0,b47adce4,off,,,,,,, +179,python,none,reversed,2000,2463534242,3,0,16,242483373,611,0,0,ad35ba05,off,,,,,,, +180,c,none,sorted,250,2463534242,19,0,429,241089339,90,0,0,7fd8c1f4,off,,,,,,, +181,node,quick,reversed,1000,2463534242,2,307,1,271437164,242593646,499500,1000,0ca43265,off,,,,,,, +182,node,quick,sorted,250,2463534242,8,4507,1,265237695,237388175,31125,0,7fd8c1f4,off,,,,,,, +183,node,insertion,reversed,500,2463534242,23,2178,1,262493903,232908537,124750,125249,d33e3559,off,,,,,,, +184,c,quick,nearly_sorted,1000,2463534242,19,11273,1,237944177,237061958,28499,7892,1cfe4afd,off,,,,,,, +185,python,insertion,random,2000,2463534242,18,3,1,321687976,304959157,996390,996389,41262855,off,,,,,,, +186,c,quick,sorted,1000,2463534242,26,691,1,237029521,235905646,499500,0,8e15cb75,off,,,,,,, +187,c,quick,random,2000,2463534242,28,10587,1,231557055,230445843,25197,24544,41262855,off,,,,,,, +188,c,none,sorted,2000,2463534242,2,0,416,245181209,90,0,0,69842aa5,off,,,,,,, +189,c,insertion,random,500,2463534242,8,6945,1,216993041,216221066,60338,60339,11ae932d,off,,,,,,, +190,c,insertion,nearly_sorted,1000,2463534242,28,11445,1,245087002,243819067,30889,30860,1cfe4afd,off,,,,,,, +191,node,bubble,sorted,500,2463534242,9,1964,1,254862461,225270608,124750,0,d9a24379,off,,,,,,, +192,python,none,nearly_sorted,2000,2463534242,22,0,15,244821253,561,0,0,5835dc59,off,,,,,,, +193,python,insertion,nearly_sorted,1000,2463534242,9,76,1,253020118,237108344,30889,30860,1cfe4afd,off,,,,,,, +194,c,insertion,reversed,500,2463534242,11,3967,1,236828285,235941812,124750,125249,d33e3559,off,,,,,,, +195,node,bubble,reversed,1000,2463534242,19,378,1,277084377,247896233,499500,999000,0ca43265,off,,,,,,, +196,python,none,random,2000,2463534242,22,0,16,255478455,1272,0,0,41262855,off,,,,,,, +197,python,bubble,reversed,250,2463534242,9,88,1,272312374,257741801,31125,62250,be175004,off,,,,,,, +198,python,bubble,sorted,250,2463534242,7,206,1,256770547,241233721,31125,0,7fd8c1f4,off,,,,,,, +199,node,quick,sorted,2000,2463534242,22,74,1,271376771,240997254,1999000,0,69842aa5,off,,,,,,, +200,node,insertion,random,500,2463534242,6,4059,1,275085901,246416069,60338,60339,11ae932d,off,,,,,,, +201,node,quick,nearly_sorted,250,2463534242,24,25320,1,258873258,232312942,4346,1008,21a19c14,off,,,,,,, +202,python,none,sorted,250,2463534242,23,0,17,248359364,481,0,0,7fd8c1f4,off,,,,,,, +203,node,bubble,random,1000,2463534242,18,362,1,279829763,251589179,499500,496926,57561285,off,,,,,,, +204,python,bubble,sorted,2000,2463534242,17,3,1,320360100,303008675,1999000,0,69842aa5,off,,,,,,, +205,python,quick,reversed,1000,2463534242,8,8,1,283232259,267917752,499500,1000,0ca43265,off,,,,,,, +206,node,bubble,reversed,500,2463534242,11,1534,1,262068916,234729857,124750,249500,d33e3559,off,,,,,,, +207,python,bubble,sorted,500,2463534242,30,46,1,262316652,246729208,124750,0,d9a24379,off,,,,,,, +208,node,insertion,reversed,500,2463534242,4,2178,1,258254438,230484726,124750,125249,d33e3559,off,,,,,,, +209,c,none,sorted,250,2463534242,16,0,429,239821473,90,0,0,7fd8c1f4,off,,,,,,, +210,c,quick,nearly_sorted,250,2463534242,5,77221,1,250421449,249661420,4346,1008,21a19c14,off,,,,,,, +211,python,bubble,random,500,2463534242,18,26,1,272749445,256253468,124750,119688,11ae932d,off,,,,,,, +212,c,none,random,500,2463534242,15,0,426,258415068,90,0,0,11ae932d,off,,,,,,, +213,c,bubble,sorted,1000,2463534242,4,702,1,249882568,249001778,499500,0,8e15cb75,off,,,,,,, +214,python,bubble,sorted,1000,2463534242,7,11,1,283029600,266423970,499500,0,8e15cb75,off,,,,,,, +215,python,quick,random,2000,2463534242,7,90,1,258029788,240480373,25197,24544,41262855,off,,,,,,, +216,python,insertion,random,1000,2463534242,28,10,1,259420163,243653116,249457,249457,57561285,off,,,,,,, +217,node,none,nearly_sorted,500,2463534242,17,0,10,282687628,1052,0,0,482b4785,off,,,,,,, +218,node,insertion,random,250,2463534242,16,15349,1,267848749,237526386,15844,15838,b47adce4,off,,,,,,, +219,node,none,nearly_sorted,1000,2463534242,12,0,10,272338472,761,0,0,1cfe4afd,off,,,,,,, +220,node,bubble,sorted,500,2463534242,29,1964,1,262928407,232804362,124750,0,d9a24379,off,,,,,,, +221,node,none,nearly_sorted,2000,2463534242,12,0,9,234555214,612,0,0,5835dc59,off,,,,,,, +222,python,insertion,sorted,2000,2463534242,2,1523,1,287189525,270509388,1999,0,69842aa5,off,,,,,,, +223,c,bubble,nearly_sorted,250,2463534242,23,7880,1,240138438,238975810,31125,4056,21a19c14,off,,,,,,, +224,python,bubble,nearly_sorted,500,2463534242,23,41,1,251864704,236205209,124750,14926,482b4785,off,,,,,,, +225,c,quick,nearly_sorted,2000,2463534242,2,6145,1,253700374,252509884,58002,11768,5835dc59,off,,,,,,, +226,python,quick,nearly_sorted,500,2463534242,26,296,1,244666663,228261598,10192,1850,482b4785,off,,,,,,, +227,c,insertion,sorted,500,2463534242,4,399335,1,242788143,241661738,499,0,d9a24379,off,,,,,,, +228,python,none,random,2000,2463534242,9,0,16,260690073,782,0,0,41262855,off,,,,,,, +229,python,bubble,sorted,2000,2463534242,14,3,1,334885380,319247528,1999000,0,69842aa5,off,,,,,,, +230,node,quick,reversed,250,2463534242,27,4755,1,270640741,242523694,31125,250,be175004,off,,,,,,, +231,c,quick,random,500,2463534242,21,55776,1,241590518,240688980,4578,4552,11ae932d,off,,,,,,, +232,node,quick,reversed,2000,2463534242,13,80,1,274180445,244005583,1999000,2000,ad35ba05,off,,,,,,, +233,c,none,nearly_sorted,500,2463534242,19,0,405,239203655,90,0,0,482b4785,off,,,,,,, +234,node,bubble,reversed,250,2463534242,1,5861,1,268083669,237792866,31125,62250,be175004,off,,,,,,, +235,python,none,nearly_sorted,2000,2463534242,14,0,15,248977573,581,0,0,5835dc59,off,,,,,,, +236,node,bubble,reversed,250,2463534242,29,5861,1,266274077,236482379,31125,62250,be175004,off,,,,,,, +237,node,none,nearly_sorted,2000,2463534242,4,0,9,243090329,812,0,0,5835dc59,off,,,,,,, +238,node,none,sorted,2000,2463534242,17,0,10,275003117,1012,0,0,69842aa5,off,,,,,,, +239,c,quick,reversed,500,2463534242,8,3240,1,240837949,239863903,124750,500,d33e3559,off,,,,,,, +240,node,bubble,reversed,250,2463534242,3,5861,1,261790004,233103425,31125,62250,be175004,off,,,,,,, +241,node,none,sorted,250,2463534242,18,0,9,243841588,1182,0,0,7fd8c1f4,off,,,,,,, +242,node,bubble,sorted,1000,2463534242,9,374,1,260297828,229871741,499500,0,8e15cb75,off,,,,,,, +243,c,quick,nearly_sorted,250,2463534242,4,77221,1,242229966,241163139,4346,1008,21a19c14,off,,,,,,, +244,c,none,random,250,2463534242,11,0,429,244400354,90,0,0,b47adce4,off,,,,,,, +245,python,insertion,sorted,500,2463534242,15,6624,1,245614306,229040693,499,0,d9a24379,off,,,,,,, +246,c,none,random,1000,2463534242,28,0,367,213088267,80,0,0,57561285,off,,,,,,, +247,python,none,sorted,2000,2463534242,14,0,15,238808345,621,0,0,69842aa5,off,,,,,,, +248,node,insertion,nearly_sorted,250,2463534242,5,79550,1,244685008,216360915,2277,2251,21a19c14,off,,,,,,, +249,python,bubble,random,250,2463534242,12,112,1,253874666,238827011,31125,31196,b47adce4,off,,,,,,, +250,python,quick,reversed,1000,2463534242,28,8,1,269589024,252201689,499500,1000,0ca43265,off,,,,,,, +251,python,bubble,nearly_sorted,250,2463534242,21,188,1,261076419,244742454,31125,4056,21a19c14,off,,,,,,, +252,python,insertion,reversed,2000,2463534242,29,2,1,419598528,403325172,1999000,2000999,ad35ba05,off,,,,,,, +253,python,none,random,250,2463534242,1,0,17,240285279,541,0,0,b47adce4,off,,,,,,, +254,node,none,nearly_sorted,2000,2463534242,16,0,9,238277644,731,0,0,5835dc59,off,,,,,,, +255,c,quick,random,1000,2463534242,2,23484,1,236283803,235416658,11196,10210,57561285,off,,,,,,, +256,node,bubble,reversed,500,2463534242,26,1534,1,274027979,244368095,124750,249500,d33e3559,off,,,,,,, +257,c,insertion,reversed,1000,2463534242,9,1033,1,245591277,244697622,499500,500499,0ca43265,off,,,,,,, +258,c,insertion,random,250,2463534242,26,26020,1,239942190,238801482,15844,15838,b47adce4,off,,,,,,, +259,node,none,reversed,500,2463534242,9,0,9,282397293,862,0,0,d33e3559,off,,,,,,, +260,python,none,random,500,2463534242,21,0,16,256014530,611,0,0,11ae932d,off,,,,,,, +261,c,insertion,nearly_sorted,250,2463534242,3,99421,1,240985765,240067386,2277,2251,21a19c14,off,,,,,,, +262,node,none,random,1000,2463534242,25,0,10,269696682,942,0,0,57561285,off,,,,,,, +263,python,bubble,sorted,2000,2463534242,29,3,1,328818590,312314372,1999000,0,69842aa5,off,,,,,,, +264,python,insertion,sorted,2000,2463534242,24,1523,1,263815100,247472905,1999,0,69842aa5,off,,,,,,, +265,node,bubble,random,1000,2463534242,24,362,1,273054846,241882047,499500,496926,57561285,off,,,,,,, +266,python,quick,random,500,2463534242,20,507,1,266595430,251252291,4578,4552,11ae932d,off,,,,,,, +267,node,bubble,random,500,2463534242,24,1407,1,280545453,251513815,124750,119688,11ae932d,off,,,,,,, +268,python,none,random,250,2463534242,2,0,17,279927967,511,0,0,b47adce4,off,,,,,,, +269,python,insertion,random,1000,2463534242,19,10,1,264035494,247034868,249457,249457,57561285,off,,,,,,, +270,c,insertion,random,1000,2463534242,25,1983,1,241404910,240214769,249457,249457,57561285,off,,,,,,, +271,node,quick,random,2000,2463534242,28,2126,1,266229403,236418338,25197,24544,41262855,off,,,,,,, +272,python,quick,random,2000,2463534242,8,90,1,269681965,252706040,25197,24544,41262855,off,,,,,,, +273,python,bubble,reversed,1000,2463534242,14,5,1,298841042,281962110,499500,999000,0ca43265,off,,,,,,, +274,c,bubble,sorted,2000,2463534242,7,173,1,243861503,242832695,1999000,0,69842aa5,off,,,,,,, +275,python,quick,reversed,250,2463534242,27,151,1,262843147,245744130,31125,250,be175004,off,,,,,,, +276,c,bubble,nearly_sorted,250,2463534242,5,7880,1,238941697,237868483,31125,4056,21a19c14,off,,,,,,, +277,node,bubble,reversed,500,2463534242,25,1534,1,268532829,239469970,124750,249500,d33e3559,off,,,,,,, +278,python,insertion,reversed,500,2463534242,23,22,1,249692462,234989075,124750,125249,d33e3559,off,,,,,,, +279,python,insertion,random,250,2463534242,10,206,1,255410399,239135444,15844,15838,b47adce4,off,,,,,,, +280,node,insertion,nearly_sorted,500,2463534242,13,26045,1,252277136,222720754,7962,7926,482b4785,off,,,,,,, +281,c,quick,sorted,1000,2463534242,30,691,1,243386685,242411088,499500,0,8e15cb75,off,,,,,,, +282,c,none,random,2000,2463534242,4,0,415,238574186,131,0,0,41262855,off,,,,,,, +283,node,none,nearly_sorted,2000,2463534242,18,0,9,261898158,1753,0,0,5835dc59,off,,,,,,, +284,c,insertion,nearly_sorted,1000,2463534242,27,11445,1,244468050,243521948,30889,30860,1cfe4afd,off,,,,,,, +285,c,insertion,sorted,250,2463534242,25,786886,1,246572966,245663181,249,0,7fd8c1f4,off,,,,,,, +286,python,quick,nearly_sorted,2000,2463534242,18,55,1,261241226,244311552,58002,11768,5835dc59,off,,,,,,, +287,node,insertion,sorted,1000,2463534242,16,160107,1,257891428,228828263,999,0,8e15cb75,off,,,,,,, +288,python,quick,nearly_sorted,1000,2463534242,12,110,1,258469242,242212890,28499,7892,1cfe4afd,off,,,,,,, +289,node,insertion,nearly_sorted,2000,2463534242,3,2100,1,263802465,234896306,118277,118270,5835dc59,off,,,,,,, +290,python,quick,sorted,500,2463534242,14,30,1,247875928,232256853,124750,0,d9a24379,off,,,,,,, +291,c,bubble,sorted,500,2463534242,6,2858,1,259447594,258450817,124750,0,d9a24379,off,,,,,,, +292,node,quick,sorted,500,2463534242,18,1164,1,287936996,256896917,124750,0,d9a24379,off,,,,,,, +293,c,none,sorted,500,2463534242,6,0,344,204649960,90,0,0,d9a24379,off,,,,,,, +294,c,none,reversed,1000,2463534242,9,0,405,236168968,90,0,0,0ca43265,off,,,,,,, +295,node,insertion,random,2000,2463534242,20,234,1,252249685,222680859,996390,996389,41262855,off,,,,,,, +296,python,none,sorted,500,2463534242,20,0,16,225327460,621,0,0,d9a24379,off,,,,,,, +297,c,quick,random,500,2463534242,12,55776,1,246496513,245428091,4578,4552,11ae932d,off,,,,,,, +298,c,insertion,nearly_sorted,250,2463534242,21,99421,1,242313996,241395483,2277,2251,21a19c14,off,,,,,,, +299,c,none,nearly_sorted,1000,2463534242,14,0,421,237616802,90,0,0,1cfe4afd,off,,,,,,, +300,c,quick,sorted,500,2463534242,5,2762,1,244160124,243359114,124750,0,d9a24379,off,,,,,,, +301,node,bubble,sorted,250,2463534242,24,8081,1,262622926,235164523,31125,0,7fd8c1f4,off,,,,,,, +302,python,none,reversed,250,2463534242,17,0,17,271554364,541,0,0,be175004,off,,,,,,, +303,c,insertion,reversed,250,2463534242,2,15645,1,240926424,240029214,31125,31374,be175004,off,,,,,,, +304,python,quick,sorted,2000,2463534242,16,2,1,299105757,282828120,1999000,0,69842aa5,off,,,,,,, +305,node,none,random,1000,2463534242,21,0,10,260192761,741,0,0,57561285,off,,,,,,, +306,node,none,reversed,250,2463534242,29,0,9,257355513,711,0,0,be175004,off,,,,,,, +307,python,insertion,nearly_sorted,500,2463534242,5,316,1,246849544,230042994,7962,7926,482b4785,off,,,,,,, +308,node,quick,nearly_sorted,250,2463534242,27,25320,1,264238854,235721737,4346,1008,21a19c14,off,,,,,,, +309,c,insertion,random,500,2463534242,9,6945,1,212562259,211617759,60338,60339,11ae932d,off,,,,,,, +310,python,insertion,sorted,500,2463534242,9,6624,1,243136685,226980432,499,0,d9a24379,off,,,,,,, +311,node,none,sorted,1000,2463534242,18,0,9,231646924,931,0,0,8e15cb75,off,,,,,,, +312,node,bubble,random,500,2463534242,6,1407,1,272009897,242441645,124750,119688,11ae932d,off,,,,,,, +313,python,quick,sorted,250,2463534242,7,133,1,249461766,234540808,31125,0,7fd8c1f4,off,,,,,,, +314,c,bubble,random,1000,2463534242,12,154,1,235474593,234578227,499500,496926,57561285,off,,,,,,, +315,c,insertion,nearly_sorted,250,2463534242,22,99421,1,240236579,239325465,2277,2251,21a19c14,off,,,,,,, +316,python,insertion,nearly_sorted,500,2463534242,10,316,1,251286901,235633889,7962,7926,482b4785,off,,,,,,, +317,c,insertion,nearly_sorted,500,2463534242,30,32350,1,241322356,240414160,7962,7926,482b4785,off,,,,,,, +318,c,quick,random,500,2463534242,4,55776,1,242938424,241830339,4578,4552,11ae932d,off,,,,,,, +319,c,quick,nearly_sorted,500,2463534242,1,31784,1,241949263,240887868,10192,1850,482b4785,off,,,,,,, +320,node,bubble,random,250,2463534242,16,5439,1,258028633,229661148,31125,31196,b47adce4,off,,,,,,, +321,c,insertion,sorted,2000,2463534242,11,101352,1,238243176,237342288,1999,0,69842aa5,off,,,,,,, +322,python,none,sorted,1000,2463534242,4,0,16,247934577,621,0,0,8e15cb75,off,,,,,,, +323,node,insertion,sorted,250,2463534242,2,607595,1,263276780,233524242,249,0,7fd8c1f4,off,,,,,,, +324,node,none,random,1000,2463534242,9,0,10,265272882,892,0,0,57561285,off,,,,,,, +325,node,insertion,nearly_sorted,2000,2463534242,24,2100,1,281927273,248253144,118277,118270,5835dc59,off,,,,,,, +326,python,insertion,sorted,2000,2463534242,3,1523,1,265041156,245968311,1999,0,69842aa5,off,,,,,,, +327,c,none,random,1000,2463534242,14,0,367,217539536,91,0,0,57561285,off,,,,,,, +328,c,quick,nearly_sorted,1000,2463534242,17,11273,1,242688566,241746218,28499,7892,1cfe4afd,off,,,,,,, +329,c,quick,nearly_sorted,2000,2463534242,6,6145,1,243189646,242304076,58002,11768,5835dc59,off,,,,,,, +330,python,none,reversed,1000,2463534242,4,0,16,253503194,621,0,0,0ca43265,off,,,,,,, +331,c,quick,sorted,1000,2463534242,25,691,1,240692567,239809914,499500,0,8e15cb75,off,,,,,,, +332,node,quick,reversed,1000,2463534242,9,307,1,268358584,238586935,499500,1000,0ca43265,off,,,,,,, +333,c,none,random,250,2463534242,29,0,429,242875207,90,0,0,b47adce4,off,,,,,,, +334,python,quick,random,1000,2463534242,15,209,1,266725724,250310685,11196,10210,57561285,off,,,,,,, +335,node,insertion,nearly_sorted,500,2463534242,5,26045,1,263418897,235709924,7962,7926,482b4785,off,,,,,,, +336,c,quick,reversed,250,2463534242,29,12781,1,235902579,235033520,31125,250,be175004,off,,,,,,, +337,c,none,nearly_sorted,500,2463534242,28,0,405,227678978,90,0,0,482b4785,off,,,,,,, +338,node,quick,random,500,2463534242,18,16148,1,268823084,240207950,4578,4552,11ae932d,off,,,,,,, +339,node,quick,sorted,250,2463534242,27,4507,1,262543667,236074459,31125,0,7fd8c1f4,off,,,,,,, +340,c,insertion,sorted,250,2463534242,6,786886,1,252442697,251541269,249,0,7fd8c1f4,off,,,,,,, +341,node,quick,random,2000,2463534242,22,2126,1,250671067,221370359,25197,24544,41262855,off,,,,,,, +342,python,none,random,2000,2463534242,11,0,16,252595302,911,0,0,41262855,off,,,,,,, +343,node,insertion,reversed,500,2463534242,7,2178,1,265109125,237774805,124750,125249,d33e3559,off,,,,,,, +344,c,none,nearly_sorted,2000,2463534242,2,0,387,220036354,80,0,0,5835dc59,off,,,,,,, +345,c,insertion,nearly_sorted,2000,2463534242,23,2962,1,221538299,220782132,118277,118270,5835dc59,off,,,,,,, +346,c,insertion,random,500,2463534242,1,6945,1,214245693,212943061,60338,60339,11ae932d,off,,,,,,, +347,node,quick,reversed,1000,2463534242,6,307,1,272924040,245357741,499500,1000,0ca43265,off,,,,,,, +348,node,none,random,250,2463534242,14,0,10,261281551,681,0,0,b47adce4,off,,,,,,, +349,node,bubble,random,1000,2463534242,5,362,1,261910119,233363502,499500,496926,57561285,off,,,,,,, +350,node,quick,reversed,1000,2463534242,10,307,1,267819835,238274906,499500,1000,0ca43265,off,,,,,,, +351,python,bubble,reversed,1000,2463534242,9,5,1,286471861,270851384,499500,999000,0ca43265,off,,,,,,, +352,c,bubble,reversed,2000,2463534242,23,22,1,246959500,245752328,1999000,3998000,ad35ba05,off,,,,,,, +353,python,none,sorted,1000,2463534242,17,0,16,244290156,591,0,0,8e15cb75,off,,,,,,, +354,python,none,nearly_sorted,2000,2463534242,6,0,15,235749592,581,0,0,5835dc59,off,,,,,,, +355,c,none,random,2000,2463534242,28,0,415,239772782,90,0,0,41262855,off,,,,,,, +356,node,quick,random,2000,2463534242,6,2126,1,263129054,233123688,25197,24544,41262855,off,,,,,,, +357,c,none,random,250,2463534242,16,0,429,239203385,90,0,0,b47adce4,off,,,,,,, +358,c,none,nearly_sorted,250,2463534242,13,0,388,217961115,90,0,0,21a19c14,off,,,,,,, +359,node,insertion,random,500,2463534242,17,4059,1,274305810,243758969,60338,60339,11ae932d,off,,,,,,, +360,node,none,random,1000,2463534242,19,0,10,266153913,882,0,0,57561285,off,,,,,,, +361,c,insertion,sorted,1000,2463534242,16,196883,1,235331940,234365649,999,0,8e15cb75,off,,,,,,, +362,python,bubble,random,1000,2463534242,13,6,1,270815759,255138231,499500,496926,57561285,off,,,,,,, +363,python,quick,sorted,2000,2463534242,1,2,1,311680463,295254902,1999000,0,69842aa5,off,,,,,,, +364,python,none,reversed,500,2463534242,23,0,16,247870428,661,0,0,d33e3559,off,,,,,,, +365,c,quick,sorted,250,2463534242,25,10851,1,247717061,246621377,31125,0,7fd8c1f4,off,,,,,,, +366,python,none,random,2000,2463534242,13,0,16,260873587,812,0,0,41262855,off,,,,,,, +367,c,none,nearly_sorted,500,2463534242,17,0,405,228139200,80,0,0,482b4785,off,,,,,,, +368,node,insertion,sorted,2000,2463534242,25,89220,1,321577922,289605189,1999,0,69842aa5,off,,,,,,, +369,node,bubble,sorted,1000,2463534242,3,374,1,318200169,281837069,499500,0,8e15cb75,off,,,,,,, +370,node,none,nearly_sorted,250,2463534242,26,0,10,267687486,621,0,0,21a19c14,off,,,,,,, +371,python,bubble,nearly_sorted,2000,2463534242,6,3,1,353381343,337120067,1999000,232556,5835dc59,off,,,,,,, +372,node,none,reversed,1000,2463534242,17,0,9,232597185,831,0,0,0ca43265,off,,,,,,, +373,python,quick,reversed,250,2463534242,28,151,1,265169099,248633778,31125,250,be175004,off,,,,,,, +374,c,none,sorted,2000,2463534242,29,0,416,249308202,90,0,0,69842aa5,off,,,,,,, +375,python,insertion,reversed,500,2463534242,24,22,1,249730142,234087877,124750,125249,d33e3559,off,,,,,,, +376,node,none,reversed,500,2463534242,16,0,9,235664554,762,0,0,d33e3559,off,,,,,,, +377,c,bubble,nearly_sorted,500,2463534242,29,1998,1,239112064,237963180,124750,14926,482b4785,off,,,,,,, +378,node,bubble,random,500,2463534242,1,1407,1,272810958,244045263,124750,119688,11ae932d,off,,,,,,, +379,c,quick,sorted,1000,2463534242,9,691,1,244863673,243968977,499500,0,8e15cb75,off,,,,,,, +380,node,bubble,sorted,500,2463534242,5,1964,1,277388207,237336960,124750,0,d9a24379,off,,,,,,, +381,c,none,reversed,500,2463534242,24,0,402,247555247,81,0,0,d33e3559,off,,,,,,, +382,node,bubble,nearly_sorted,250,2463534242,5,6161,1,295765877,260501397,31125,4056,21a19c14,off,,,,,,, +383,python,insertion,random,250,2463534242,13,206,1,272973844,254355079,15844,15838,b47adce4,off,,,,,,, +384,node,bubble,reversed,500,2463534242,15,1534,1,270468107,242900405,124750,249500,d33e3559,off,,,,,,, +385,python,insertion,random,2000,2463534242,26,3,1,324693670,307670043,996390,996389,41262855,off,,,,,,, +386,python,none,sorted,250,2463534242,25,0,17,252846946,651,0,0,7fd8c1f4,off,,,,,,, +387,python,insertion,random,1000,2463534242,18,10,1,266909887,251436511,249457,249457,57561285,off,,,,,,, +388,c,insertion,random,2000,2463534242,1,492,1,236788661,235879435,996390,996389,41262855,off,,,,,,, +389,c,insertion,random,2000,2463534242,25,492,1,232349029,231410969,996390,996389,41262855,off,,,,,,, +390,c,insertion,random,500,2463534242,2,6945,1,216241182,215127737,60338,60339,11ae932d,off,,,,,,, +391,python,none,sorted,250,2463534242,12,0,17,237052404,491,0,0,7fd8c1f4,off,,,,,,, +392,python,insertion,sorted,500,2463534242,11,6624,1,243099648,228141985,499,0,d9a24379,off,,,,,,, +393,c,insertion,nearly_sorted,250,2463534242,4,99421,1,246004960,244761261,2277,2251,21a19c14,off,,,,,,, +394,node,none,sorted,1000,2463534242,15,0,9,232362725,791,0,0,8e15cb75,off,,,,,,, +395,c,insertion,reversed,250,2463534242,17,15645,1,243312585,242308316,31125,31374,be175004,off,,,,,,, +396,python,bubble,sorted,2000,2463534242,23,3,1,328301922,311682056,1999000,0,69842aa5,off,,,,,,, +397,python,insertion,random,1000,2463534242,30,10,1,262197839,245923835,249457,249457,57561285,off,,,,,,, +398,python,bubble,random,500,2463534242,8,26,1,253483017,238020948,124750,119688,11ae932d,off,,,,,,, +399,python,bubble,random,250,2463534242,15,112,1,240733763,225938532,31125,31196,b47adce4,off,,,,,,, +400,c,bubble,nearly_sorted,2000,2463534242,11,121,1,236254027,235202758,1999000,232556,5835dc59,off,,,,,,, +401,c,insertion,sorted,250,2463534242,28,786886,1,242192458,241137982,249,0,7fd8c1f4,off,,,,,,, +402,c,quick,reversed,1000,2463534242,22,814,1,233687999,232511486,499500,1000,0ca43265,off,,,,,,, +403,c,bubble,reversed,2000,2463534242,13,22,1,248161784,247044186,1999000,3998000,ad35ba05,off,,,,,,, +404,python,insertion,reversed,250,2463534242,25,102,1,392739313,377906964,31125,31374,be175004,off,,,,,,, +405,node,quick,nearly_sorted,1000,2463534242,27,3752,1,262719176,234001173,28499,7892,1cfe4afd,off,,,,,,, +406,python,quick,nearly_sorted,500,2463534242,12,296,1,238295604,221853456,10192,1850,482b4785,off,,,,,,, +407,c,none,nearly_sorted,1000,2463534242,16,0,421,241894319,91,0,0,1cfe4afd,off,,,,,,, +408,python,insertion,reversed,250,2463534242,14,102,1,258664627,241122370,31125,31374,be175004,off,,,,,,, +409,node,quick,nearly_sorted,250,2463534242,30,25320,1,279441444,250352155,4346,1008,21a19c14,off,,,,,,, +410,node,none,sorted,250,2463534242,3,0,9,238361558,791,0,0,7fd8c1f4,off,,,,,,, +411,c,insertion,sorted,500,2463534242,19,399335,1,263189647,262301934,499,0,d9a24379,off,,,,,,, +412,c,insertion,reversed,2000,2463534242,14,260,1,239524929,238394110,1999000,2000999,ad35ba05,off,,,,,,, +413,node,bubble,nearly_sorted,1000,2463534242,12,413,1,263259506,236026046,499500,59780,1cfe4afd,off,,,,,,, +414,node,none,nearly_sorted,1000,2463534242,23,0,10,260841677,631,0,0,1cfe4afd,off,,,,,,, +415,c,quick,reversed,500,2463534242,21,3240,1,235278440,234359289,124750,500,d33e3559,off,,,,,,, +416,python,quick,random,500,2463534242,22,507,1,269058957,252993219,4578,4552,11ae932d,off,,,,,,, +417,python,bubble,random,250,2463534242,25,112,1,244450148,229690781,31125,31196,b47adce4,off,,,,,,, +418,python,bubble,reversed,500,2463534242,2,20,1,252976137,237498243,124750,249500,d33e3559,off,,,,,,, +419,python,bubble,sorted,2000,2463534242,27,3,1,328767004,312438211,1999000,0,69842aa5,off,,,,,,, +420,node,insertion,nearly_sorted,1000,2463534242,13,7758,1,262169926,233934138,30889,30860,1cfe4afd,off,,,,,,, +421,c,quick,random,1000,2463534242,14,23484,1,227109029,226199405,11196,10210,57561285,off,,,,,,, +422,python,insertion,reversed,2000,2463534242,28,2,1,408928914,392295217,1999000,2000999,ad35ba05,off,,,,,,, +423,python,insertion,nearly_sorted,500,2463534242,25,316,1,311615261,296585675,7962,7926,482b4785,off,,,,,,, +424,node,none,random,1000,2463534242,30,0,10,271034089,722,0,0,57561285,off,,,,,,, +425,c,quick,random,2000,2463534242,25,10587,1,244967609,243874320,25197,24544,41262855,off,,,,,,, +426,c,insertion,reversed,500,2463534242,13,3967,1,239679577,238219842,124750,125249,d33e3559,off,,,,,,, +427,c,insertion,nearly_sorted,1000,2463534242,6,11445,1,245374440,244400827,30889,30860,1cfe4afd,off,,,,,,, +428,node,bubble,random,250,2463534242,18,5439,1,258836499,230177997,31125,31196,b47adce4,off,,,,,,, +429,python,insertion,sorted,2000,2463534242,23,1523,1,262433952,245541417,1999,0,69842aa5,off,,,,,,, +430,c,insertion,sorted,2000,2463534242,20,101352,1,236774372,235661838,1999,0,69842aa5,off,,,,,,, +431,node,quick,reversed,2000,2463534242,17,80,1,275194204,245711703,1999000,2000,ad35ba05,off,,,,,,, +432,node,bubble,random,500,2463534242,8,1407,1,275095901,246499304,124750,119688,11ae932d,off,,,,,,, +433,python,none,sorted,500,2463534242,21,0,16,233825879,581,0,0,d9a24379,off,,,,,,, +434,python,none,nearly_sorted,250,2463534242,19,0,17,259095674,541,0,0,21a19c14,off,,,,,,, +435,c,none,nearly_sorted,250,2463534242,23,0,388,216084368,100,0,0,21a19c14,off,,,,,,, +436,c,quick,sorted,2000,2463534242,12,174,1,239830500,238903965,1999000,0,69842aa5,off,,,,,,, +437,python,quick,sorted,1000,2463534242,4,8,1,293497266,277205242,499500,0,8e15cb75,off,,,,,,, +438,python,quick,sorted,500,2463534242,29,30,1,247260457,231607392,124750,0,d9a24379,off,,,,,,, +439,c,none,nearly_sorted,2000,2463534242,26,0,387,218429391,100,0,0,5835dc59,off,,,,,,, +440,node,quick,reversed,1000,2463534242,3,307,1,271964031,241070016,499500,1000,0ca43265,off,,,,,,, +441,node,insertion,random,1000,2463534242,30,1091,1,279009516,247527899,249457,249457,57561285,off,,,,,,, +442,node,quick,reversed,2000,2463534242,30,80,1,294361986,264117419,1999000,2000,ad35ba05,off,,,,,,, +443,python,none,random,250,2463534242,14,0,17,253684905,12874,0,0,b47adce4,off,,,,,,, +444,node,none,reversed,500,2463534242,25,0,9,239733278,711,0,0,d33e3559,off,,,,,,, +445,node,quick,sorted,500,2463534242,29,1164,1,272034884,242159878,124750,0,d9a24379,off,,,,,,, +446,node,none,reversed,1000,2463534242,28,0,9,234079292,742,0,0,0ca43265,off,,,,,,, +447,c,none,reversed,1000,2463534242,20,0,405,237645316,100,0,0,0ca43265,off,,,,,,, +448,c,none,random,1000,2463534242,12,0,367,214043303,91,0,0,57561285,off,,,,,,, +449,c,bubble,nearly_sorted,2000,2463534242,12,121,1,240212907,239475807,1999000,232556,5835dc59,off,,,,,,, +450,c,bubble,sorted,2000,2463534242,29,173,1,245430936,244455216,1999000,0,69842aa5,off,,,,,,, +451,node,insertion,random,2000,2463534242,22,234,1,273595108,244424043,996390,996389,41262855,off,,,,,,, +452,node,bubble,random,1000,2463534242,29,362,1,265044864,237675834,499500,496926,57561285,off,,,,,,, +453,python,quick,sorted,2000,2463534242,26,2,1,304637484,287532008,1999000,0,69842aa5,off,,,,,,, +454,python,none,sorted,250,2463534242,28,0,17,246094359,691,0,0,7fd8c1f4,off,,,,,,, +455,python,none,sorted,2000,2463534242,12,0,15,234791707,652,0,0,69842aa5,off,,,,,,, +456,python,none,random,500,2463534242,18,0,16,233190507,621,0,0,11ae932d,off,,,,,,, +457,node,quick,reversed,2000,2463534242,19,80,1,276214357,245970779,1999000,2000,ad35ba05,off,,,,,,, +458,node,quick,reversed,2000,2463534242,15,80,1,274705388,246151716,1999000,2000,ad35ba05,off,,,,,,, +459,c,bubble,reversed,500,2463534242,1,346,1,240933529,239970274,124750,249500,d33e3559,off,,,,,,, +460,node,none,reversed,250,2463534242,1,0,9,234458601,692,0,0,be175004,off,,,,,,, +461,node,quick,nearly_sorted,500,2463534242,28,10954,1,263307478,234154391,10192,1850,482b4785,off,,,,,,, +462,c,none,nearly_sorted,2000,2463534242,12,0,387,223036617,90,0,0,5835dc59,off,,,,,,, +463,node,bubble,reversed,2000,2463534242,28,94,1,272277960,242929408,1999000,3998000,ad35ba05,off,,,,,,, +464,python,none,reversed,1000,2463534242,8,0,16,231529824,601,0,0,0ca43265,off,,,,,,, +465,c,bubble,random,1000,2463534242,16,154,1,231091423,230104914,499500,496926,57561285,off,,,,,,, +466,python,insertion,sorted,2000,2463534242,18,1523,1,264356595,248073090,1999,0,69842aa5,off,,,,,,, +467,c,none,sorted,2000,2463534242,28,0,416,241619984,90,0,0,69842aa5,off,,,,,,, +468,node,quick,nearly_sorted,500,2463534242,21,10954,1,266837163,237474798,10192,1850,482b4785,off,,,,,,, +469,python,bubble,random,1000,2463534242,7,6,1,264866681,248122868,499500,496926,57561285,off,,,,,,, +470,c,quick,nearly_sorted,2000,2463534242,26,6145,1,247859075,246616598,58002,11768,5835dc59,off,,,,,,, +471,node,none,sorted,1000,2463534242,14,0,9,248826550,912,0,0,8e15cb75,off,,,,,,, +472,python,none,nearly_sorted,2000,2463534242,24,0,15,245553065,552,0,0,5835dc59,off,,,,,,, +473,python,insertion,random,2000,2463534242,10,3,1,320343538,303702290,996390,996389,41262855,off,,,,,,, +474,c,insertion,sorted,500,2463534242,25,399335,1,241865224,240700781,499,0,d9a24379,off,,,,,,, +475,python,bubble,nearly_sorted,2000,2463534242,11,3,1,351214803,334684154,1999000,232556,5835dc59,off,,,,,,, +476,node,none,nearly_sorted,250,2463534242,15,0,10,287262852,591,0,0,21a19c14,off,,,,,,, +477,node,insertion,nearly_sorted,1000,2463534242,17,7758,1,263823145,235457829,30889,30860,1cfe4afd,off,,,,,,, +478,c,quick,sorted,250,2463534242,23,10851,1,244442103,243536476,31125,0,7fd8c1f4,off,,,,,,, +479,node,none,sorted,2000,2463534242,16,0,10,267116536,651,0,0,69842aa5,off,,,,,,, +480,python,bubble,reversed,1000,2463534242,5,5,1,294987459,277123658,499500,999000,0ca43265,off,,,,,,, +481,node,none,random,500,2463534242,16,0,9,243964237,802,0,0,11ae932d,off,,,,,,, +482,python,none,random,1000,2463534242,30,0,15,231214343,592,0,0,57561285,off,,,,,,, +483,python,none,sorted,2000,2463534242,30,0,15,231218531,601,0,0,69842aa5,off,,,,,,, +484,c,none,sorted,250,2463534242,17,0,429,248823174,100,0,0,7fd8c1f4,off,,,,,,, +485,node,none,random,2000,2463534242,30,0,9,246921900,1443,0,0,41262855,off,,,,,,, +486,c,bubble,reversed,2000,2463534242,3,22,1,248930837,247825444,1999000,3998000,ad35ba05,off,,,,,,, +487,node,insertion,sorted,2000,2463534242,7,89220,1,266911051,238827459,1999,0,69842aa5,off,,,,,,, +488,python,quick,nearly_sorted,2000,2463534242,30,55,1,248682150,231676456,58002,11768,5835dc59,off,,,,,,, +489,node,quick,sorted,2000,2463534242,8,74,1,268588945,238877974,1999000,0,69842aa5,off,,,,,,, +490,python,quick,nearly_sorted,250,2463534242,25,882,1,255206182,240256128,4346,1008,21a19c14,off,,,,,,, +491,python,bubble,reversed,500,2463534242,24,20,1,262669626,247357160,124750,249500,d33e3559,off,,,,,,, +492,c,insertion,random,2000,2463534242,26,492,1,227493941,226545773,996390,996389,41262855,off,,,,,,, +493,node,insertion,random,2000,2463534242,28,234,1,251857160,221791449,996390,996389,41262855,off,,,,,,, +494,node,none,random,1000,2463534242,14,0,10,255678210,681,0,0,57561285,off,,,,,,, +495,node,none,random,250,2463534242,27,0,10,259796479,2545,0,0,b47adce4,off,,,,,,, +496,c,none,nearly_sorted,500,2463534242,11,0,405,229914287,80,0,0,482b4785,off,,,,,,, +497,c,quick,sorted,2000,2463534242,14,174,1,236459473,235735567,1999000,0,69842aa5,off,,,,,,, +498,node,quick,sorted,500,2463534242,7,1164,1,265577031,237913287,124750,0,d9a24379,off,,,,,,, +499,python,none,sorted,250,2463534242,19,0,17,238803466,571,0,0,7fd8c1f4,off,,,,,,, +500,python,insertion,nearly_sorted,250,2463534242,22,1416,1,255310322,240995541,2277,2251,21a19c14,off,,,,,,, +501,python,none,sorted,2000,2463534242,29,0,15,232094030,641,0,0,69842aa5,off,,,,,,, +502,node,none,sorted,500,2463534242,6,0,10,260080140,812,0,0,d9a24379,off,,,,,,, +503,node,insertion,nearly_sorted,2000,2463534242,5,2100,1,258265328,230165853,118277,118270,5835dc59,off,,,,,,, +504,python,none,reversed,500,2463534242,24,0,16,226981430,551,0,0,d33e3559,off,,,,,,, +505,node,quick,random,500,2463534242,22,16148,1,272664214,244267161,4578,4552,11ae932d,off,,,,,,, +506,c,quick,random,2000,2463534242,27,10587,1,238195787,237207256,25197,24544,41262855,off,,,,,,, +507,c,insertion,reversed,250,2463534242,16,15645,1,242479104,241457551,31125,31374,be175004,off,,,,,,, +508,node,none,nearly_sorted,1000,2463534242,3,0,10,259461841,711,0,0,1cfe4afd,off,,,,,,, +509,python,quick,random,250,2463534242,13,1271,1,253360166,237029838,1933,1940,b47adce4,off,,,,,,, +510,node,quick,sorted,1000,2463534242,16,296,1,268207963,240677430,499500,0,8e15cb75,off,,,,,,, +511,c,none,reversed,250,2463534242,20,0,427,237212323,90,0,0,be175004,off,,,,,,, +512,c,bubble,random,250,2463534242,18,2568,1,241069242,240318471,31125,31196,b47adce4,off,,,,,,, +513,node,bubble,nearly_sorted,1000,2463534242,7,413,1,267072204,235183241,499500,59780,1cfe4afd,off,,,,,,, +514,python,insertion,reversed,1000,2463534242,4,6,1,292644137,277555306,499500,500499,0ca43265,off,,,,,,, +515,c,bubble,nearly_sorted,2000,2463534242,30,121,1,239269509,238144241,1999000,232556,5835dc59,off,,,,,,, +516,c,quick,random,2000,2463534242,7,10587,1,239293404,238164027,25197,24544,41262855,off,,,,,,, +517,node,bubble,sorted,1000,2463534242,19,374,1,253597561,224912528,499500,0,8e15cb75,off,,,,,,, +518,c,bubble,sorted,2000,2463534242,30,173,1,231276750,230363278,1999000,0,69842aa5,off,,,,,,, +519,node,insertion,reversed,2000,2463534242,14,144,1,301841344,271825004,1999000,2000999,ad35ba05,off,,,,,,, +520,node,quick,reversed,250,2463534242,8,4755,1,269527305,241189056,31125,250,be175004,off,,,,,,, +521,node,insertion,sorted,500,2463534242,23,324764,1,260593582,232517306,499,0,d9a24379,off,,,,,,, +522,python,none,random,2000,2463534242,19,0,16,263870213,721,0,0,41262855,off,,,,,,, +523,node,insertion,random,2000,2463534242,27,234,1,272003544,244685036,996390,996389,41262855,off,,,,,,, +524,python,insertion,sorted,250,2463534242,16,18845,1,260570409,245314261,249,0,7fd8c1f4,off,,,,,,, +525,node,quick,sorted,250,2463534242,12,4507,1,272406671,244860915,31125,0,7fd8c1f4,off,,,,,,, +526,c,none,sorted,500,2463534242,29,0,344,206105046,140,0,0,d9a24379,off,,,,,,, +527,node,insertion,reversed,250,2463534242,8,8539,1,275706966,246460810,31125,31374,be175004,off,,,,,,, +528,c,insertion,reversed,1000,2463534242,11,1033,1,244741534,243609022,499500,500499,0ca43265,off,,,,,,, +529,python,none,sorted,250,2463534242,22,0,17,251950334,521,0,0,7fd8c1f4,off,,,,,,, +530,c,insertion,nearly_sorted,1000,2463534242,25,11445,1,243011102,242110854,30889,30860,1cfe4afd,off,,,,,,, +531,python,none,nearly_sorted,250,2463534242,13,0,17,255492411,511,0,0,21a19c14,off,,,,,,, +532,python,bubble,reversed,500,2463534242,27,20,1,257805969,242276943,124750,249500,d33e3559,off,,,,,,, +533,node,bubble,random,2000,2463534242,4,98,1,272543416,243381792,1999000,1988792,41262855,off,,,,,,, +534,python,none,random,250,2463534242,11,0,17,241472618,601,0,0,b47adce4,off,,,,,,, +535,node,bubble,reversed,500,2463534242,13,1534,1,267103613,237676220,124750,249500,d33e3559,off,,,,,,, +536,c,insertion,random,500,2463534242,19,6945,1,215434089,214549742,60338,60339,11ae932d,off,,,,,,, +537,node,quick,random,250,2463534242,29,36348,1,261609746,231438748,1933,1940,b47adce4,off,,,,,,, +538,node,none,nearly_sorted,1000,2463534242,28,0,10,250727362,831,0,0,1cfe4afd,off,,,,,,, +539,c,bubble,sorted,500,2463534242,23,2858,1,244723570,243695534,124750,0,d9a24379,off,,,,,,, +540,node,insertion,sorted,500,2463534242,22,324764,1,262982288,233710476,499,0,d9a24379,off,,,,,,, +541,python,quick,random,250,2463534242,23,1271,1,252463605,236463244,1933,1940,b47adce4,off,,,,,,, +542,node,bubble,random,2000,2463534242,22,98,1,270317446,241522715,1999000,1988792,41262855,off,,,,,,, +543,node,bubble,random,2000,2463534242,25,98,1,266443374,238064001,1999000,1988792,41262855,off,,,,,,, +544,c,quick,nearly_sorted,500,2463534242,15,31784,1,242417709,241385335,10192,1850,482b4785,off,,,,,,, +545,c,bubble,nearly_sorted,2000,2463534242,10,121,1,241018937,240136273,1999000,232556,5835dc59,off,,,,,,, +546,python,none,sorted,250,2463534242,11,0,17,235640588,521,0,0,7fd8c1f4,off,,,,,,, +547,node,bubble,reversed,2000,2463534242,16,94,1,273920277,245132529,1999000,3998000,ad35ba05,off,,,,,,, +548,c,insertion,nearly_sorted,250,2463534242,24,99421,1,239956036,238912542,2277,2251,21a19c14,off,,,,,,, +549,c,insertion,sorted,2000,2463534242,28,101352,1,236758643,235566660,1999,0,69842aa5,off,,,,,,, +550,python,insertion,nearly_sorted,250,2463534242,29,1416,1,253383039,237761789,2277,2251,21a19c14,off,,,,,,, +551,python,quick,reversed,500,2463534242,7,33,1,251873980,236607945,124750,500,d33e3559,off,,,,,,, +552,node,bubble,reversed,2000,2463534242,23,94,1,270997221,240050986,1999000,3998000,ad35ba05,off,,,,,,, +553,python,insertion,nearly_sorted,250,2463534242,4,1416,1,256246754,240028169,2277,2251,21a19c14,off,,,,,,, +554,python,bubble,sorted,2000,2463534242,13,3,1,329372650,313260458,1999000,0,69842aa5,off,,,,,,, +555,python,none,sorted,1000,2463534242,20,0,16,233328204,601,0,0,8e15cb75,off,,,,,,, +556,node,quick,reversed,500,2463534242,17,1194,1,264672035,233995576,124750,500,d33e3559,off,,,,,,, +557,node,insertion,reversed,2000,2463534242,12,144,1,265912650,238060597,1999000,2000999,ad35ba05,off,,,,,,, +558,node,bubble,sorted,500,2463534242,27,1964,1,254725084,226109370,124750,0,d9a24379,off,,,,,,, +559,c,none,sorted,2000,2463534242,10,0,416,237152101,81,0,0,69842aa5,off,,,,,,, +560,node,insertion,random,1000,2463534242,18,1091,1,266692101,237893756,249457,249457,57561285,off,,,,,,, +561,python,none,random,250,2463534242,5,0,17,247235067,612,0,0,b47adce4,off,,,,,,, +562,c,quick,nearly_sorted,500,2463534242,24,31784,1,242320226,241389423,10192,1850,482b4785,off,,,,,,, +563,node,bubble,sorted,2000,2463534242,5,85,1,262713856,234056335,1999000,0,69842aa5,off,,,,,,, +564,node,none,random,1000,2463534242,1,0,10,260550070,832,0,0,57561285,off,,,,,,, +565,python,bubble,nearly_sorted,2000,2463534242,12,3,1,350145489,334067248,1999000,232556,5835dc59,off,,,,,,, +566,node,none,random,1000,2463534242,6,0,10,266549194,942,0,0,57561285,off,,,,,,, +567,c,bubble,reversed,250,2463534242,8,1384,1,239433677,238508403,31125,62250,be175004,off,,,,,,, +568,node,quick,random,250,2463534242,15,36348,1,255848439,228245170,1933,1940,b47adce4,off,,,,,,, +569,c,insertion,sorted,1000,2463534242,11,196883,1,230475598,229615136,999,0,8e15cb75,off,,,,,,, +570,python,none,random,2000,2463534242,12,0,16,250592089,902,0,0,41262855,off,,,,,,, +571,node,bubble,random,500,2463534242,2,1407,1,279878343,251092895,124750,119688,11ae932d,off,,,,,,, +572,node,bubble,nearly_sorted,500,2463534242,13,1566,1,266441732,238800402,124750,14926,482b4785,off,,,,,,, +573,python,quick,random,250,2463534242,4,1271,1,259499903,244190946,1933,1940,b47adce4,off,,,,,,, +574,python,none,sorted,250,2463534242,2,0,17,250691584,501,0,0,7fd8c1f4,off,,,,,,, +575,node,insertion,nearly_sorted,500,2463534242,15,26045,1,250297325,222523155,7962,7926,482b4785,off,,,,,,, +576,node,quick,random,500,2463534242,20,16148,1,271141691,242815045,4578,4552,11ae932d,off,,,,,,, +577,python,quick,sorted,500,2463534242,13,30,1,250138388,234681670,124750,0,d9a24379,off,,,,,,, +578,python,none,random,2000,2463534242,4,0,16,252283648,752,0,0,41262855,off,,,,,,, +579,python,none,reversed,1000,2463534242,11,0,16,232527033,782,0,0,0ca43265,off,,,,,,, +580,node,bubble,nearly_sorted,1000,2463534242,6,413,1,267358130,239921996,499500,59780,1cfe4afd,off,,,,,,, +581,c,bubble,reversed,1000,2463534242,11,87,1,243283401,242351056,499500,999000,0ca43265,off,,,,,,, +582,c,none,nearly_sorted,2000,2463534242,16,0,387,220973161,80,0,0,5835dc59,off,,,,,,, +583,node,none,nearly_sorted,250,2463534242,10,0,10,276113377,741,0,0,21a19c14,off,,,,,,, +584,python,insertion,reversed,2000,2463534242,24,2,1,415401734,398209434,1999000,2000999,ad35ba05,off,,,,,,, +585,node,bubble,sorted,250,2463534242,2,8081,1,260351308,232139002,31125,0,7fd8c1f4,off,,,,,,, +586,node,none,sorted,500,2463534242,1,0,10,260395150,1212,0,0,d9a24379,off,,,,,,, +587,node,none,sorted,250,2463534242,22,0,9,235817008,912,0,0,7fd8c1f4,off,,,,,,, +588,python,none,nearly_sorted,250,2463534242,15,0,17,253288542,521,0,0,21a19c14,off,,,,,,, +589,node,none,nearly_sorted,500,2463534242,22,0,10,266621419,672,0,0,482b4785,off,,,,,,, +590,python,quick,reversed,500,2463534242,24,33,1,255884096,240276526,124750,500,d33e3559,off,,,,,,, +591,python,quick,nearly_sorted,1000,2463534242,23,110,1,257397603,241652664,28499,7892,1cfe4afd,off,,,,,,, +592,python,none,random,500,2463534242,10,0,16,250112374,611,0,0,11ae932d,off,,,,,,, +593,node,quick,random,250,2463534242,3,36348,1,261827869,234277154,1933,1940,b47adce4,off,,,,,,, +594,node,bubble,reversed,1000,2463534242,18,378,1,269546780,239607780,499500,999000,0ca43265,off,,,,,,, +595,c,insertion,nearly_sorted,500,2463534242,12,32350,1,240466783,239600369,7962,7926,482b4785,off,,,,,,, +596,c,quick,reversed,500,2463534242,11,3240,1,239690267,238801202,124750,500,d33e3559,off,,,,,,, +597,node,bubble,sorted,1000,2463534242,8,374,1,261209106,232380649,499500,0,8e15cb75,off,,,,,,, +598,c,bubble,reversed,250,2463534242,22,1384,1,254881818,253806474,31125,62250,be175004,off,,,,,,, +599,node,insertion,sorted,500,2463534242,18,324764,1,274373126,242317795,499,0,d9a24379,off,,,,,,, +600,c,insertion,sorted,250,2463534242,12,786886,1,254215929,253302648,249,0,7fd8c1f4,off,,,,,,, +601,python,bubble,random,250,2463534242,16,112,1,252524680,236315434,31125,31196,b47adce4,off,,,,,,, +602,c,none,sorted,1000,2463534242,19,0,421,248674445,90,0,0,8e15cb75,off,,,,,,, +603,python,quick,random,2000,2463534242,10,90,1,267547012,249674497,25197,24544,41262855,off,,,,,,, +604,python,bubble,reversed,1000,2463534242,16,5,1,290662514,274425710,499500,999000,0ca43265,off,,,,,,, +605,c,bubble,random,250,2463534242,17,2568,1,244284708,243230241,31125,31196,b47adce4,off,,,,,,, +606,c,bubble,nearly_sorted,2000,2463534242,25,121,1,240265516,239314553,1999000,232556,5835dc59,off,,,,,,, +607,node,quick,nearly_sorted,500,2463534242,9,10954,1,291836284,262238189,10192,1850,482b4785,off,,,,,,, +608,python,quick,reversed,500,2463534242,17,33,1,267249306,247741361,124750,500,d33e3559,off,,,,,,, +609,c,insertion,nearly_sorted,500,2463534242,19,32350,1,243968854,242947360,7962,7926,482b4785,off,,,,,,, +610,node,quick,sorted,500,2463534242,1,1164,1,267979364,240344149,124750,0,d9a24379,off,,,,,,, +611,c,quick,reversed,2000,2463534242,16,209,1,240143226,239300577,1999000,2000,ad35ba05,off,,,,,,, +612,python,quick,sorted,250,2463534242,17,133,1,253888567,237738618,31125,0,7fd8c1f4,off,,,,,,, +613,node,none,nearly_sorted,500,2463534242,14,0,10,260339266,872,0,0,482b4785,off,,,,,,, +614,python,quick,reversed,2000,2463534242,25,2,1,285090922,268123347,1999000,2000,ad35ba05,off,,,,,,, +615,c,bubble,nearly_sorted,2000,2463534242,8,121,1,240036356,239106723,1999000,232556,5835dc59,off,,,,,,, +616,c,bubble,nearly_sorted,250,2463534242,6,7880,1,237140970,236091163,31125,4056,21a19c14,off,,,,,,, +617,python,quick,reversed,1000,2463534242,17,8,1,272056485,255884796,499500,1000,0ca43265,off,,,,,,, +618,c,quick,sorted,250,2463534242,17,10851,1,242825082,241951185,31125,0,7fd8c1f4,off,,,,,,, +619,c,insertion,sorted,250,2463534242,24,786886,1,241406150,240518420,249,0,7fd8c1f4,off,,,,,,, +620,c,none,sorted,250,2463534242,7,0,429,246487159,91,0,0,7fd8c1f4,off,,,,,,, +621,c,none,reversed,500,2463534242,14,0,402,229451239,80,0,0,d33e3559,off,,,,,,, +622,c,bubble,nearly_sorted,2000,2463534242,19,121,1,251634382,250833110,1999000,232556,5835dc59,off,,,,,,, +623,c,bubble,nearly_sorted,2000,2463534242,18,121,1,240589803,239683685,1999000,232556,5835dc59,off,,,,,,, +624,node,bubble,reversed,250,2463534242,5,5861,1,266983739,238526782,31125,62250,be175004,off,,,,,,, +625,node,quick,random,500,2463534242,6,16148,1,270826988,242213195,4578,4552,11ae932d,off,,,,,,, +626,node,insertion,reversed,1000,2463534242,12,554,1,271754208,243192300,499500,500499,0ca43265,off,,,,,,, +627,python,insertion,random,1000,2463534242,23,10,1,267781392,251502712,249457,249457,57561285,off,,,,,,, +628,python,none,nearly_sorted,250,2463534242,30,0,17,274648001,541,0,0,21a19c14,off,,,,,,, +629,c,none,sorted,2000,2463534242,6,0,416,251925788,90,0,0,69842aa5,off,,,,,,, +630,python,none,reversed,2000,2463534242,14,0,16,269545789,632,0,0,ad35ba05,off,,,,,,, +631,node,none,nearly_sorted,250,2463534242,25,0,10,273784262,602,0,0,21a19c14,off,,,,,,, +632,node,insertion,nearly_sorted,500,2463534242,12,26045,1,259523657,229635971,7962,7926,482b4785,off,,,,,,, +633,c,quick,random,2000,2463534242,10,10587,1,265887454,264924179,25197,24544,41262855,off,,,,,,, +634,python,none,nearly_sorted,1000,2463534242,30,0,16,252867383,531,0,0,1cfe4afd,off,,,,,,, +635,node,quick,sorted,500,2463534242,23,1164,1,270328145,241953124,124750,0,d9a24379,off,,,,,,, +636,node,none,reversed,500,2463534242,22,0,9,271182176,942,0,0,d33e3559,off,,,,,,, +637,node,quick,sorted,2000,2463534242,24,74,1,300911320,264097511,1999000,0,69842aa5,off,,,,,,, +638,c,none,nearly_sorted,1000,2463534242,4,0,421,245120434,90,0,0,1cfe4afd,off,,,,,,, +639,node,quick,nearly_sorted,500,2463534242,19,10954,1,263072048,235384052,10192,1850,482b4785,off,,,,,,, +640,c,bubble,random,1000,2463534242,10,154,1,232698213,231539595,499500,496926,57561285,off,,,,,,, +641,python,bubble,sorted,500,2463534242,24,46,1,261642428,245549263,124750,0,d9a24379,off,,,,,,, +642,python,insertion,nearly_sorted,500,2463534242,20,316,1,284653552,266803326,7962,7926,482b4785,off,,,,,,, +643,c,none,nearly_sorted,250,2463534242,18,0,388,230714216,100,0,0,21a19c14,off,,,,,,, +644,python,insertion,nearly_sorted,500,2463534242,13,316,1,265629320,249663762,7962,7926,482b4785,off,,,,,,, +645,c,quick,sorted,1000,2463534242,16,691,1,244726365,243557666,499500,0,8e15cb75,off,,,,,,, +646,node,none,reversed,500,2463534242,20,0,9,235408804,882,0,0,d33e3559,off,,,,,,, +647,node,insertion,nearly_sorted,500,2463534242,2,26045,1,258917069,227883820,7962,7926,482b4785,off,,,,,,, +648,node,quick,nearly_sorted,250,2463534242,4,25320,1,264043046,236762393,4346,1008,21a19c14,off,,,,,,, +649,c,none,sorted,1000,2463534242,17,0,421,238418835,80,0,0,8e15cb75,off,,,,,,, +650,c,quick,nearly_sorted,2000,2463534242,28,6145,1,244740583,243812865,58002,11768,5835dc59,off,,,,,,, +651,c,bubble,sorted,1000,2463534242,29,702,1,234226980,233002025,499500,0,8e15cb75,off,,,,,,, +652,node,none,sorted,1000,2463534242,2,0,9,232971285,781,0,0,8e15cb75,off,,,,,,, +653,python,quick,sorted,2000,2463534242,21,2,1,307563987,291547624,1999000,0,69842aa5,off,,,,,,, +654,c,none,sorted,250,2463534242,6,0,429,238968455,90,0,0,7fd8c1f4,off,,,,,,, +655,node,quick,sorted,1000,2463534242,3,296,1,277505116,248453095,499500,0,8e15cb75,off,,,,,,, +656,python,insertion,reversed,1000,2463534242,2,6,1,305612020,288269745,499500,500499,0ca43265,off,,,,,,, +657,node,bubble,reversed,500,2463534242,2,1534,1,273157297,243354232,124750,249500,d33e3559,off,,,,,,, +658,c,insertion,random,1000,2463534242,18,1983,1,239352625,238263654,249457,249457,57561285,off,,,,,,, +659,python,bubble,random,1000,2463534242,24,6,1,273569031,255906469,499500,496926,57561285,off,,,,,,, +660,c,none,sorted,500,2463534242,12,0,344,194849150,80,0,0,d9a24379,off,,,,,,, +661,c,insertion,nearly_sorted,2000,2463534242,13,2962,1,229620533,228881531,118277,118270,5835dc59,off,,,,,,, +662,c,none,random,1000,2463534242,7,0,367,231057172,90,0,0,57561285,off,,,,,,, +663,c,bubble,random,1000,2463534242,28,154,1,235488123,234669428,499500,496926,57561285,off,,,,,,, +664,node,quick,random,2000,2463534242,12,2126,1,270485991,242070840,25197,24544,41262855,off,,,,,,, +665,c,insertion,reversed,2000,2463534242,10,260,1,239408069,238289752,1999000,2000999,ad35ba05,off,,,,,,, +666,c,none,sorted,500,2463534242,19,0,344,192635904,101,0,0,d9a24379,off,,,,,,, +667,c,insertion,reversed,2000,2463534242,16,260,1,245754081,244954313,1999000,2000999,ad35ba05,off,,,,,,, +668,python,quick,sorted,1000,2463534242,25,8,1,282552885,266525687,499500,0,8e15cb75,off,,,,,,, +669,c,bubble,nearly_sorted,1000,2463534242,30,507,1,250198036,249166586,499500,59780,1cfe4afd,off,,,,,,, +670,python,none,nearly_sorted,500,2463534242,16,0,17,249096289,561,0,0,482b4785,off,,,,,,, +671,python,quick,reversed,2000,2463534242,4,2,1,268929725,251808285,1999000,2000,ad35ba05,off,,,,,,, +672,node,none,random,250,2463534242,9,0,10,257409494,692,0,0,b47adce4,off,,,,,,, +673,c,insertion,sorted,500,2463534242,22,399335,1,239857361,238679525,499,0,d9a24379,off,,,,,,, +674,node,quick,nearly_sorted,250,2463534242,17,25320,1,264310328,235031089,4346,1008,21a19c14,off,,,,,,, +675,c,insertion,random,250,2463534242,11,26020,1,235880149,234790465,15844,15838,b47adce4,off,,,,,,, +676,node,quick,sorted,500,2463534242,28,1164,1,273278683,246393859,124750,0,d9a24379,off,,,,,,, +677,python,insertion,random,500,2463534242,28,44,1,258150946,241707528,60338,60339,11ae932d,off,,,,,,, +678,python,bubble,reversed,1000,2463534242,3,5,1,296012069,279801308,499500,999000,0ca43265,off,,,,,,, +679,python,insertion,sorted,250,2463534242,3,18845,1,264837112,249657078,249,0,7fd8c1f4,off,,,,,,, +680,python,none,nearly_sorted,250,2463534242,14,0,17,254048903,511,0,0,21a19c14,off,,,,,,, +681,node,none,random,1000,2463534242,15,0,10,275155916,1153,0,0,57561285,off,,,,,,, +682,python,insertion,reversed,1000,2463534242,23,6,1,289348601,272994347,499500,500499,0ca43265,off,,,,,,, +683,node,none,nearly_sorted,2000,2463534242,21,0,9,249833087,842,0,0,5835dc59,off,,,,,,, +684,node,bubble,reversed,250,2463534242,16,5861,1,259050941,231676676,31125,62250,be175004,off,,,,,,, +685,python,quick,sorted,2000,2463534242,29,2,1,300630334,284046544,1999000,0,69842aa5,off,,,,,,, +686,c,insertion,reversed,1000,2463534242,23,1033,1,243479609,242372575,499500,500499,0ca43265,off,,,,,,, +687,c,quick,sorted,2000,2463534242,15,174,1,242776843,241788468,1999000,0,69842aa5,off,,,,,,, +688,node,insertion,sorted,250,2463534242,23,607595,1,262669252,233619274,249,0,7fd8c1f4,off,,,,,,, +689,python,insertion,nearly_sorted,1000,2463534242,14,76,1,252498201,235310351,30889,30860,1cfe4afd,off,,,,,,, +690,c,none,nearly_sorted,2000,2463534242,25,0,387,222120391,81,0,0,5835dc59,off,,,,,,, +691,python,quick,nearly_sorted,1000,2463534242,1,110,1,265530364,249804210,28499,7892,1cfe4afd,off,,,,,,, +692,python,insertion,reversed,500,2463534242,19,22,1,250262390,234991993,124750,125249,d33e3559,off,,,,,,, +693,node,quick,random,2000,2463534242,16,2126,1,272136244,242530284,25197,24544,41262855,off,,,,,,, +694,c,bubble,sorted,2000,2463534242,21,173,1,236467809,235532716,1999000,0,69842aa5,off,,,,,,, +695,python,insertion,nearly_sorted,500,2463534242,23,316,1,252826777,237042257,7962,7926,482b4785,off,,,,,,, +696,node,insertion,random,1000,2463534242,13,1091,1,271649682,242686018,249457,249457,57561285,off,,,,,,, +697,c,insertion,random,1000,2463534242,29,1983,1,241529053,240604771,249457,249457,57561285,off,,,,,,, +698,c,none,random,1000,2463534242,15,0,367,216749315,81,0,0,57561285,off,,,,,,, +699,c,quick,nearly_sorted,250,2463534242,3,77221,1,244544765,243820668,4346,1008,21a19c14,off,,,,,,, +700,c,quick,random,500,2463534242,25,55776,1,248983674,248061675,4578,4552,11ae932d,off,,,,,,, +701,node,bubble,nearly_sorted,500,2463534242,18,1566,1,288724821,259808902,124750,14926,482b4785,off,,,,,,, +702,node,insertion,sorted,500,2463534242,16,324764,1,269611072,239205711,499,0,d9a24379,off,,,,,,, +703,python,bubble,sorted,2000,2463534242,19,3,1,334606227,316471981,1999000,0,69842aa5,off,,,,,,, +704,node,bubble,nearly_sorted,250,2463534242,21,6161,1,276362755,246700689,31125,4056,21a19c14,off,,,,,,, +705,c,none,sorted,1000,2463534242,30,0,421,242798603,90,0,0,8e15cb75,off,,,,,,, +706,node,quick,sorted,1000,2463534242,29,296,1,273464173,243859769,499500,0,8e15cb75,off,,,,,,, +707,python,bubble,nearly_sorted,2000,2463534242,24,3,1,362689839,345534287,1999000,232556,5835dc59,off,,,,,,, +708,node,insertion,reversed,250,2463534242,27,8539,1,267996490,241489510,31125,31374,be175004,off,,,,,,, +709,python,insertion,nearly_sorted,500,2463534242,15,316,1,260163651,244240328,7962,7926,482b4785,off,,,,,,, +710,node,bubble,random,2000,2463534242,23,98,1,267700971,238632194,1999000,1988792,41262855,off,,,,,,, +711,c,bubble,random,1000,2463534242,18,154,1,234849425,233973654,499500,496926,57561285,off,,,,,,, +712,c,insertion,nearly_sorted,500,2463534242,21,32350,1,243129823,242227780,7962,7926,482b4785,off,,,,,,, +713,python,quick,random,1000,2463534242,2,209,1,259788463,243756341,11196,10210,57561285,off,,,,,,, +714,node,quick,sorted,1000,2463534242,30,296,1,270411633,242132677,499500,0,8e15cb75,off,,,,,,, +715,python,none,sorted,2000,2463534242,23,0,15,234680497,661,0,0,69842aa5,off,,,,,,, +716,node,quick,reversed,2000,2463534242,16,80,1,276867501,245688194,1999000,2000,ad35ba05,off,,,,,,, +717,c,none,nearly_sorted,500,2463534242,10,0,405,237122636,80,0,0,482b4785,off,,,,,,, +718,node,quick,nearly_sorted,250,2463534242,2,25320,1,278272523,249523546,4346,1008,21a19c14,off,,,,,,, +719,c,none,sorted,1000,2463534242,27,0,421,246537219,90,0,0,8e15cb75,off,,,,,,, +720,c,quick,sorted,2000,2463534242,5,174,1,254103962,253362480,1999000,0,69842aa5,off,,,,,,, +721,python,none,nearly_sorted,2000,2463534242,7,0,15,242865727,591,0,0,5835dc59,off,,,,,,, +722,c,bubble,nearly_sorted,2000,2463534242,6,121,1,242216492,241058212,1999000,232556,5835dc59,off,,,,,,, +723,python,insertion,random,500,2463534242,14,44,1,255496419,238920612,60338,60339,11ae932d,off,,,,,,, +724,node,insertion,sorted,250,2463534242,8,607595,1,253686157,225816690,249,0,7fd8c1f4,off,,,,,,, +725,python,quick,random,1000,2463534242,27,209,1,255694971,239787174,11196,10210,57561285,off,,,,,,, +726,node,quick,nearly_sorted,250,2463534242,1,25320,1,266661604,237288594,4346,1008,21a19c14,off,,,,,,, +727,node,none,sorted,2000,2463534242,15,0,10,265538760,651,0,0,69842aa5,off,,,,,,, +728,python,bubble,reversed,1000,2463534242,29,5,1,292202409,276612443,499500,999000,0ca43265,off,,,,,,, +729,python,none,reversed,250,2463534242,6,0,17,241732916,511,0,0,be175004,off,,,,,,, +730,node,insertion,sorted,250,2463534242,30,607595,1,253607469,224170866,249,0,7fd8c1f4,off,,,,,,, +731,node,none,random,1000,2463534242,22,0,10,271244533,872,0,0,57561285,off,,,,,,, +732,c,insertion,reversed,2000,2463534242,18,260,1,242238845,241290142,1999000,2000999,ad35ba05,off,,,,,,, +733,c,none,random,2000,2463534242,26,0,415,242485404,90,0,0,41262855,off,,,,,,, +734,python,none,nearly_sorted,1000,2463534242,6,0,16,241095742,591,0,0,1cfe4afd,off,,,,,,, +735,python,insertion,random,1000,2463534242,10,10,1,290922181,274939597,249457,249457,57561285,off,,,,,,, +736,node,insertion,nearly_sorted,1000,2463534242,27,7758,1,271693113,242973202,30889,30860,1cfe4afd,off,,,,,,, +737,node,none,sorted,500,2463534242,3,0,10,269464075,752,0,0,d9a24379,off,,,,,,, +738,python,none,nearly_sorted,2000,2463534242,30,0,15,296327009,551,0,0,5835dc59,off,,,,,,, +739,python,none,nearly_sorted,1000,2463534242,25,0,16,251801916,542,0,0,1cfe4afd,off,,,,,,, +740,node,bubble,random,1000,2463534242,2,362,1,269621010,240293653,499500,496926,57561285,off,,,,,,, +741,c,bubble,nearly_sorted,250,2463534242,12,7880,1,239372682,238226962,31125,4056,21a19c14,off,,,,,,, +742,c,bubble,reversed,500,2463534242,9,346,1,245649746,244415925,124750,249500,d33e3559,off,,,,,,, +743,python,none,nearly_sorted,500,2463534242,19,0,17,264924408,481,0,0,482b4785,off,,,,,,, +744,node,bubble,nearly_sorted,500,2463534242,16,1566,1,281733200,248954007,124750,14926,482b4785,off,,,,,,, +745,node,insertion,sorted,250,2463534242,28,607595,1,262071983,230688643,249,0,7fd8c1f4,off,,,,,,, +746,node,bubble,reversed,1000,2463534242,27,378,1,276179993,244210674,499500,999000,0ca43265,off,,,,,,, +747,c,insertion,reversed,500,2463534242,7,3967,1,241823345,240876762,124750,125249,d33e3559,off,,,,,,, +748,node,quick,nearly_sorted,1000,2463534242,11,3752,1,265478146,234598233,28499,7892,1cfe4afd,off,,,,,,, +749,python,insertion,sorted,1000,2463534242,6,3178,1,272099596,254907477,999,0,8e15cb75,off,,,,,,, +750,c,quick,sorted,1000,2463534242,8,691,1,245825746,244865196,499500,0,8e15cb75,off,,,,,,, +751,c,none,nearly_sorted,1000,2463534242,29,0,421,251374695,101,0,0,1cfe4afd,off,,,,,,, +752,node,quick,random,500,2463534242,24,16148,1,283856488,250098708,4578,4552,11ae932d,off,,,,,,, +753,node,bubble,random,250,2463534242,17,5439,1,269664131,235978838,31125,31196,b47adce4,off,,,,,,, +754,python,bubble,reversed,1000,2463534242,28,5,1,299926204,282353001,499500,999000,0ca43265,off,,,,,,, +755,c,insertion,sorted,250,2463534242,29,786886,1,250155682,248970107,249,0,7fd8c1f4,off,,,,,,, +756,node,insertion,reversed,250,2463534242,20,8539,1,279082029,247734901,31125,31374,be175004,off,,,,,,, +757,python,bubble,sorted,250,2463534242,24,206,1,334461596,315653375,31125,0,7fd8c1f4,off,,,,,,, +758,node,insertion,reversed,2000,2463534242,18,144,1,346968725,304898165,1999000,2000999,ad35ba05,off,,,,,,, +759,node,insertion,sorted,2000,2463534242,17,89220,1,343180086,310428944,1999,0,69842aa5,off,,,,,,, +760,c,bubble,sorted,500,2463534242,28,2858,1,248318143,247451755,124750,0,d9a24379,off,,,,,,, +761,c,bubble,random,500,2463534242,22,635,1,235100831,234193868,124750,119688,11ae932d,off,,,,,,, +762,python,insertion,random,250,2463534242,14,206,1,263755263,245914622,15844,15838,b47adce4,off,,,,,,, +763,node,quick,random,2000,2463534242,23,2126,1,264689833,234807631,25197,24544,41262855,off,,,,,,, +764,c,insertion,reversed,500,2463534242,18,3967,1,239402078,238504867,124750,125249,d33e3559,off,,,,,,, +765,c,insertion,nearly_sorted,500,2463534242,26,32350,1,243707608,242528188,7962,7926,482b4785,off,,,,,,, +766,node,insertion,sorted,250,2463534242,15,607595,1,264071108,233559674,249,0,7fd8c1f4,off,,,,,,, +767,node,insertion,reversed,500,2463534242,12,2178,1,265697817,235160687,124750,125249,d33e3559,off,,,,,,, +768,c,insertion,random,1000,2463534242,20,1983,1,243659106,242783715,249457,249457,57561285,off,,,,,,, +769,python,insertion,nearly_sorted,500,2463534242,28,316,1,263846789,247558054,7962,7926,482b4785,off,,,,,,, +770,c,quick,nearly_sorted,2000,2463534242,30,6145,1,246431151,245383618,58002,11768,5835dc59,off,,,,,,, +771,python,insertion,reversed,500,2463534242,25,22,1,251222441,235704515,124750,125249,d33e3559,off,,,,,,, +772,python,bubble,reversed,500,2463534242,17,20,1,272344813,256669183,124750,249500,d33e3559,off,,,,,,, +773,python,insertion,nearly_sorted,250,2463534242,24,1416,1,263684096,247799962,2277,2251,21a19c14,off,,,,,,, +774,python,quick,sorted,1000,2463534242,15,8,1,308253048,291754978,499500,0,8e15cb75,off,,,,,,, +775,c,none,sorted,500,2463534242,1,0,344,205265352,90,0,0,d9a24379,off,,,,,,, +776,c,bubble,reversed,500,2463534242,27,346,1,245850523,245142676,124750,249500,d33e3559,off,,,,,,, +777,c,none,sorted,500,2463534242,30,0,344,199818124,101,0,0,d9a24379,off,,,,,,, +778,node,insertion,random,1000,2463534242,28,1091,1,269114450,239045031,249457,249457,57561285,off,,,,,,, +779,python,none,sorted,1000,2463534242,13,0,16,267377748,982,0,0,8e15cb75,off,,,,,,, +780,c,none,sorted,500,2463534242,16,0,344,198640997,100,0,0,d9a24379,off,,,,,,, +781,c,none,sorted,2000,2463534242,8,0,416,241702529,91,0,0,69842aa5,off,,,,,,, +782,python,insertion,sorted,2000,2463534242,8,1523,1,259732560,242632361,1999,0,69842aa5,off,,,,,,, +783,node,none,nearly_sorted,250,2463534242,2,0,10,266542299,651,0,0,21a19c14,off,,,,,,, +784,c,quick,nearly_sorted,1000,2463534242,7,11273,1,235913812,234823176,28499,7892,1cfe4afd,off,,,,,,, +785,c,bubble,reversed,2000,2463534242,20,22,1,246203421,245196536,1999000,3998000,ad35ba05,off,,,,,,, +786,c,bubble,nearly_sorted,500,2463534242,13,1998,1,237555998,236648177,124750,14926,482b4785,off,,,,,,, +787,python,quick,reversed,250,2463534242,21,151,1,276868252,261044776,31125,250,be175004,off,,,,,,, +788,node,quick,random,1000,2463534242,23,6513,1,268617168,239780525,11196,10210,57561285,off,,,,,,, +789,c,none,sorted,500,2463534242,4,0,344,203734704,90,0,0,d9a24379,off,,,,,,, +790,python,bubble,random,500,2463534242,9,26,1,257643824,239944832,124750,119688,11ae932d,off,,,,,,, +791,node,none,reversed,2000,2463534242,4,0,9,257775921,902,0,0,ad35ba05,off,,,,,,, +792,python,quick,sorted,1000,2463534242,26,8,1,288497245,271145156,499500,0,8e15cb75,off,,,,,,, +793,python,bubble,nearly_sorted,1000,2463534242,27,9,1,254983285,238068452,499500,59780,1cfe4afd,off,,,,,,, +794,node,insertion,reversed,250,2463534242,14,8539,1,270395685,240202758,31125,31374,be175004,off,,,,,,, +795,python,insertion,random,2000,2463534242,13,3,1,318207916,301275242,996390,996389,41262855,off,,,,,,, +796,python,bubble,random,1000,2463534242,27,6,1,277207889,260536590,499500,496926,57561285,off,,,,,,, +797,node,none,nearly_sorted,2000,2463534242,24,0,9,247577127,762,0,0,5835dc59,off,,,,,,, +798,c,quick,sorted,500,2463534242,18,2762,1,248232867,247041134,124750,0,d9a24379,off,,,,,,, +799,c,insertion,random,2000,2463534242,13,492,1,236840978,235793696,996390,996389,41262855,off,,,,,,, +800,c,none,nearly_sorted,1000,2463534242,1,0,421,250659515,100,0,0,1cfe4afd,off,,,,,,, +801,node,bubble,nearly_sorted,2000,2463534242,19,110,1,269474666,238209156,1999000,232556,5835dc59,off,,,,,,, +802,c,quick,nearly_sorted,500,2463534242,19,31784,1,242697804,241670360,10192,1850,482b4785,off,,,,,,, +803,node,insertion,sorted,1000,2463534242,28,160107,1,273134425,241848363,999,0,8e15cb75,off,,,,,,, +804,node,insertion,nearly_sorted,500,2463534242,26,26045,1,254277465,226127714,7962,7926,482b4785,off,,,,,,, +805,c,quick,reversed,2000,2463534242,26,209,1,239149795,238144070,1999000,2000,ad35ba05,off,,,,,,, +806,c,none,sorted,500,2463534242,9,0,344,197636055,90,0,0,d9a24379,off,,,,,,, +807,node,none,random,250,2463534242,28,0,10,265059271,832,0,0,b47adce4,off,,,,,,, +808,python,quick,nearly_sorted,250,2463534242,22,882,1,268751720,252728077,4346,1008,21a19c14,off,,,,,,, +809,node,quick,sorted,250,2463534242,17,4507,1,270298790,240707910,31125,0,7fd8c1f4,off,,,,,,, +810,c,bubble,random,250,2463534242,28,2568,1,240761034,239833626,31125,31196,b47adce4,off,,,,,,, +811,python,quick,nearly_sorted,1000,2463534242,30,110,1,257281736,241414601,28499,7892,1cfe4afd,off,,,,,,, +812,c,insertion,random,500,2463534242,23,6945,1,212463042,211500149,60338,60339,11ae932d,off,,,,,,, +813,c,none,sorted,250,2463534242,22,0,429,240416529,90,0,0,7fd8c1f4,off,,,,,,, +814,node,insertion,sorted,250,2463534242,5,607595,1,268278574,238308539,249,0,7fd8c1f4,off,,,,,,, +815,c,insertion,reversed,2000,2463534242,26,260,1,253054694,252146803,1999000,2000999,ad35ba05,off,,,,,,, +816,node,bubble,nearly_sorted,2000,2463534242,7,110,1,270151454,239833158,1999000,232556,5835dc59,off,,,,,,, +817,node,none,nearly_sorted,2000,2463534242,14,0,9,254050731,1433,0,0,5835dc59,off,,,,,,, +818,node,quick,sorted,250,2463534242,11,4507,1,267072194,238199466,31125,0,7fd8c1f4,off,,,,,,, +819,python,none,reversed,500,2463534242,10,0,16,251924836,591,0,0,d33e3559,off,,,,,,, +820,python,insertion,nearly_sorted,1000,2463534242,7,76,1,248853751,232744513,30889,30860,1cfe4afd,off,,,,,,, +821,node,quick,nearly_sorted,500,2463534242,23,10954,1,339725822,311530960,10192,1850,482b4785,off,,,,,,, +822,python,quick,random,2000,2463534242,26,90,1,266335834,247824233,25197,24544,41262855,off,,,,,,, +823,c,quick,random,500,2463534242,19,55776,1,238744394,237808010,4578,4552,11ae932d,off,,,,,,, +824,python,quick,reversed,2000,2463534242,27,2,1,276823709,258302844,1999000,2000,ad35ba05,off,,,,,,, +825,node,quick,reversed,500,2463534242,8,1194,1,268381709,239832277,124750,500,d33e3559,off,,,,,,, +826,python,bubble,nearly_sorted,2000,2463534242,14,3,1,376535596,359136084,1999000,232556,5835dc59,off,,,,,,, +827,c,insertion,reversed,250,2463534242,13,15645,1,246307550,245367717,31125,31374,be175004,off,,,,,,, +828,c,insertion,reversed,1000,2463534242,20,1033,1,250026769,248747343,499500,500499,0ca43265,off,,,,,,, +829,python,none,reversed,1000,2463534242,25,0,16,267683425,571,0,0,0ca43265,off,,,,,,, +830,python,bubble,sorted,250,2463534242,21,206,1,272622876,256925653,31125,0,7fd8c1f4,off,,,,,,, +831,python,none,nearly_sorted,1000,2463534242,19,0,16,258022117,521,0,0,1cfe4afd,off,,,,,,, +832,node,insertion,reversed,500,2463534242,2,2178,1,273979137,244349897,124750,125249,d33e3559,off,,,,,,, +833,c,bubble,reversed,2000,2463534242,8,22,1,246941717,246021845,1999000,3998000,ad35ba05,off,,,,,,, +834,python,quick,random,2000,2463534242,19,90,1,275156995,256812634,25197,24544,41262855,off,,,,,,, +835,c,insertion,reversed,1000,2463534242,27,1033,1,245008464,244076007,499500,500499,0ca43265,off,,,,,,, +836,node,insertion,nearly_sorted,250,2463534242,17,79550,1,248709190,220390111,2277,2251,21a19c14,off,,,,,,, +837,c,insertion,reversed,2000,2463534242,24,260,1,245239217,244149145,1999000,2000999,ad35ba05,off,,,,,,, +838,c,none,reversed,500,2463534242,29,0,402,232701701,90,0,0,d33e3559,off,,,,,,, +839,python,insertion,reversed,500,2463534242,8,22,1,251211068,234784295,124750,125249,d33e3559,off,,,,,,, +840,python,none,reversed,500,2463534242,20,0,16,253793688,481,0,0,d33e3559,off,,,,,,, +841,node,none,random,250,2463534242,18,0,10,269699948,891,0,0,b47adce4,off,,,,,,, +842,python,insertion,nearly_sorted,500,2463534242,3,316,1,250074027,234170102,7962,7926,482b4785,off,,,,,,, +843,c,none,random,250,2463534242,24,0,429,248931597,110,0,0,b47adce4,off,,,,,,, +844,python,quick,nearly_sorted,500,2463534242,15,296,1,239078183,222281620,10192,1850,482b4785,off,,,,,,, +845,python,quick,sorted,1000,2463534242,28,8,1,284355181,268679728,499500,0,8e15cb75,off,,,,,,, +846,node,quick,sorted,250,2463534242,3,4507,1,270663914,242867513,31125,0,7fd8c1f4,off,,,,,,, +847,python,insertion,reversed,250,2463534242,29,102,1,247705759,231307383,31125,31374,be175004,off,,,,,,, +848,node,none,sorted,250,2463534242,17,0,9,242288667,641,0,0,7fd8c1f4,off,,,,,,, +849,node,none,random,250,2463534242,22,0,10,274222985,802,0,0,b47adce4,off,,,,,,, +850,python,bubble,sorted,1000,2463534242,8,11,1,289293998,271835905,499500,0,8e15cb75,off,,,,,,, +851,python,insertion,random,500,2463534242,6,44,1,254490346,238781729,60338,60339,11ae932d,off,,,,,,, +852,node,quick,nearly_sorted,1000,2463534242,15,3752,1,266599445,236893896,28499,7892,1cfe4afd,off,,,,,,, +853,python,bubble,nearly_sorted,250,2463534242,30,188,1,274227273,258153626,31125,4056,21a19c14,off,,,,,,, +854,python,bubble,reversed,250,2463534242,23,88,1,250150160,234005104,31125,62250,be175004,off,,,,,,, +855,c,none,random,250,2463534242,4,0,429,252914571,80,0,0,b47adce4,off,,,,,,, +856,c,none,random,2000,2463534242,9,0,415,248991579,90,0,0,41262855,off,,,,,,, +857,python,none,reversed,1000,2463534242,14,0,16,263904597,611,0,0,0ca43265,off,,,,,,, +858,python,insertion,nearly_sorted,250,2463534242,23,1416,1,254129518,237264059,2277,2251,21a19c14,off,,,,,,, +859,python,none,nearly_sorted,1000,2463534242,16,0,16,260536747,601,0,0,1cfe4afd,off,,,,,,, +860,node,none,sorted,250,2463534242,27,0,9,248402615,842,0,0,7fd8c1f4,off,,,,,,, +861,node,none,sorted,1000,2463534242,28,0,9,254141090,1543,0,0,8e15cb75,off,,,,,,, +862,c,insertion,random,1000,2463534242,5,1983,1,245331269,244403302,249457,249457,57561285,off,,,,,,, +863,node,bubble,sorted,2000,2463534242,20,85,1,264875627,235030993,1999000,0,69842aa5,off,,,,,,, +864,node,none,nearly_sorted,2000,2463534242,10,0,9,255771195,2936,0,0,5835dc59,off,,,,,,, +865,node,none,sorted,250,2463534242,11,0,9,252903630,741,0,0,7fd8c1f4,off,,,,,,, +866,c,none,nearly_sorted,500,2463534242,9,0,405,238754353,80,0,0,482b4785,off,,,,,,, +867,c,insertion,nearly_sorted,2000,2463534242,28,2962,1,225682997,224913015,118277,118270,5835dc59,off,,,,,,, +868,python,quick,nearly_sorted,1000,2463534242,17,110,1,267613808,249239401,28499,7892,1cfe4afd,off,,,,,,, +869,node,none,sorted,2000,2463534242,30,0,10,283346804,1212,0,0,69842aa5,off,,,,,,, +870,c,insertion,random,1000,2463534242,24,1983,1,243886353,242931332,249457,249457,57561285,off,,,,,,, +871,c,quick,reversed,2000,2463534242,21,209,1,244816694,243806532,1999000,2000,ad35ba05,off,,,,,,, +872,python,bubble,nearly_sorted,250,2463534242,24,188,1,262939158,246045105,31125,4056,21a19c14,off,,,,,,, +873,c,none,nearly_sorted,250,2463534242,19,0,388,240285063,90,0,0,21a19c14,off,,,,,,, +874,node,none,reversed,500,2463534242,29,0,9,267524421,982,0,0,d33e3559,off,,,,,,, +875,node,quick,reversed,1000,2463534242,11,307,1,292171592,255757702,499500,1000,0ca43265,off,,,,,,, +876,python,insertion,nearly_sorted,500,2463534242,22,316,1,269995502,252366571,7962,7926,482b4785,off,,,,,,, +877,node,none,sorted,500,2463534242,8,0,10,273007196,1113,0,0,d9a24379,off,,,,,,, +878,node,none,sorted,250,2463534242,13,0,9,240589513,1032,0,0,7fd8c1f4,off,,,,,,, +879,python,none,random,1000,2463534242,16,0,15,253955982,792,0,0,57561285,off,,,,,,, +880,python,insertion,nearly_sorted,250,2463534242,8,1416,1,254835632,239297908,2277,2251,21a19c14,off,,,,,,, +881,node,bubble,nearly_sorted,2000,2463534242,3,110,1,271956087,243058338,1999000,232556,5835dc59,off,,,,,,, +882,c,insertion,sorted,500,2463534242,10,399335,1,238959889,238080842,499,0,d9a24379,off,,,,,,, +883,node,quick,nearly_sorted,250,2463534242,3,25320,1,270433793,239754160,4346,1008,21a19c14,off,,,,,,, +884,python,bubble,reversed,2000,2463534242,11,2,1,481030116,464151576,1999000,3998000,ad35ba05,off,,,,,,, +885,c,insertion,sorted,1000,2463534242,29,196883,1,233253295,232111895,999,0,8e15cb75,off,,,,,,, +886,node,bubble,sorted,1000,2463534242,26,374,1,260546965,229333000,499500,0,8e15cb75,off,,,,,,, +887,python,insertion,random,2000,2463534242,27,3,1,362063366,344805899,996390,996389,41262855,off,,,,,,, +888,python,none,nearly_sorted,1000,2463534242,20,0,16,247456421,501,0,0,1cfe4afd,off,,,,,,, +889,c,bubble,sorted,250,2463534242,20,11386,1,241908364,240953927,31125,0,7fd8c1f4,off,,,,,,, +890,python,bubble,reversed,1000,2463534242,23,5,1,288177706,272292804,499500,999000,0ca43265,off,,,,,,, +891,c,none,nearly_sorted,250,2463534242,26,0,388,228085039,80,0,0,21a19c14,off,,,,,,, +892,python,bubble,random,2000,2463534242,6,2,1,398359806,379206889,1999000,1988792,41262855,off,,,,,,, +893,node,bubble,random,500,2463534242,5,1407,1,275424286,243947129,124750,119688,11ae932d,off,,,,,,, +894,python,quick,sorted,250,2463534242,19,133,1,255899307,238177801,31125,0,7fd8c1f4,off,,,,,,, +895,python,none,reversed,1000,2463534242,24,0,16,263070362,591,0,0,0ca43265,off,,,,,,, +896,c,insertion,random,2000,2463534242,15,492,1,235283559,234227972,996390,996389,41262855,off,,,,,,, +897,c,quick,sorted,500,2463534242,11,2762,1,242639395,241600818,124750,0,d9a24379,off,,,,,,, +898,c,none,nearly_sorted,1000,2463534242,6,0,421,252134986,91,0,0,1cfe4afd,off,,,,,,, +899,node,quick,sorted,1000,2463534242,11,296,1,279499677,250102851,499500,0,8e15cb75,off,,,,,,, +900,python,none,sorted,500,2463534242,5,0,16,260076283,591,0,0,d9a24379,off,,,,,,, +901,node,quick,random,250,2463534242,10,36348,1,267475289,237628597,1933,1940,b47adce4,off,,,,,,, +902,c,quick,nearly_sorted,2000,2463534242,10,6145,1,243534137,242562500,58002,11768,5835dc59,off,,,,,,, +903,c,quick,reversed,1000,2463534242,10,814,1,234326239,233386585,499500,1000,0ca43265,off,,,,,,, +904,c,bubble,reversed,1000,2463534242,29,87,1,247999600,246845648,499500,999000,0ca43265,off,,,,,,, +905,node,insertion,reversed,250,2463534242,5,8539,1,265723165,236384120,31125,31374,be175004,off,,,,,,, +906,c,none,sorted,500,2463534242,28,0,344,202693633,90,0,0,d9a24379,off,,,,,,, +907,node,bubble,reversed,2000,2463534242,8,94,1,276852122,246240947,1999000,3998000,ad35ba05,off,,,,,,, +908,python,bubble,random,500,2463534242,17,26,1,258363944,242421875,124750,119688,11ae932d,off,,,,,,, +909,c,quick,sorted,500,2463534242,2,2762,1,243394590,242437754,124750,0,d9a24379,off,,,,,,, +910,c,insertion,reversed,250,2463534242,14,15645,1,247671361,246544164,31125,31374,be175004,off,,,,,,, +911,python,quick,reversed,500,2463534242,10,33,1,257392706,240599758,124750,500,d33e3559,off,,,,,,, +912,node,bubble,reversed,2000,2463534242,30,94,1,272537165,242902259,1999000,3998000,ad35ba05,off,,,,,,, +913,python,insertion,random,250,2463534242,5,206,1,263875713,248441332,15844,15838,b47adce4,off,,,,,,, +914,node,insertion,sorted,250,2463534242,29,607595,1,323747528,290117567,249,0,7fd8c1f4,off,,,,,,, +915,python,bubble,nearly_sorted,250,2463534242,4,188,1,266069154,248385135,31125,4056,21a19c14,off,,,,,,, +916,node,none,sorted,2000,2463534242,25,0,10,269012088,691,0,0,69842aa5,off,,,,,,, +917,c,quick,random,250,2463534242,26,136055,1,238847338,237735344,1933,1940,b47adce4,off,,,,,,, +918,python,insertion,sorted,250,2463534242,8,18845,1,257391771,241044402,249,0,7fd8c1f4,off,,,,,,, +919,python,none,sorted,2000,2463534242,1,0,15,242460359,622,0,0,69842aa5,off,,,,,,, +920,node,none,random,2000,2463534242,22,0,9,244559042,791,0,0,41262855,off,,,,,,, +921,c,quick,reversed,1000,2463534242,2,814,1,239235016,238295464,499500,1000,0ca43265,off,,,,,,, +922,c,bubble,sorted,1000,2463534242,17,702,1,236956432,235799282,499500,0,8e15cb75,off,,,,,,, +923,c,bubble,reversed,1000,2463534242,25,87,1,239660352,238493825,499500,999000,0ca43265,off,,,,,,, +924,node,quick,nearly_sorted,500,2463534242,22,10954,1,280643086,248656844,10192,1850,482b4785,off,,,,,,, +925,c,insertion,nearly_sorted,250,2463534242,8,99421,1,243513363,242564925,2277,2251,21a19c14,off,,,,,,, +926,python,bubble,reversed,1000,2463534242,20,5,1,291472982,274957638,499500,999000,0ca43265,off,,,,,,, +927,c,bubble,reversed,250,2463534242,27,1384,1,241284555,240161943,31125,62250,be175004,off,,,,,,, +928,python,quick,nearly_sorted,250,2463534242,6,882,1,266516722,250391176,4346,1008,21a19c14,off,,,,,,, +929,node,quick,random,2000,2463534242,29,2126,1,261428998,230409001,25197,24544,41262855,off,,,,,,, +930,node,insertion,reversed,250,2463534242,26,8539,1,265724006,236962239,31125,31374,be175004,off,,,,,,, +931,python,none,nearly_sorted,500,2463534242,27,0,17,246536318,531,0,0,482b4785,off,,,,,,, +932,node,quick,nearly_sorted,500,2463534242,14,10954,1,269322081,240443427,10192,1850,482b4785,off,,,,,,, +933,node,insertion,nearly_sorted,500,2463534242,25,26045,1,249321758,221456325,7962,7926,482b4785,off,,,,,,, +934,node,none,reversed,250,2463534242,9,0,9,229916331,682,0,0,be175004,off,,,,,,, +935,node,bubble,random,1000,2463534242,22,362,1,264885015,237548865,499500,496926,57561285,off,,,,,,, +936,c,bubble,sorted,250,2463534242,15,11386,1,242607415,241748086,31125,0,7fd8c1f4,off,,,,,,, +937,node,none,reversed,500,2463534242,26,0,9,236036250,781,0,0,d33e3559,off,,,,,,, +938,c,bubble,sorted,250,2463534242,29,11386,1,243275006,240790992,31125,0,7fd8c1f4,off,,,,,,, +939,python,bubble,random,500,2463534242,20,26,1,269431095,253887994,124750,119688,11ae932d,off,,,,,,, +940,node,quick,reversed,500,2463534242,24,1194,1,267188932,239521054,124750,500,d33e3559,off,,,,,,, +941,c,none,sorted,2000,2463534242,25,0,416,238972843,80,0,0,69842aa5,off,,,,,,, +942,python,bubble,reversed,250,2463534242,22,88,1,262569712,247186431,31125,62250,be175004,off,,,,,,, +943,c,quick,random,500,2463534242,30,55776,1,247845958,246737807,4578,4552,11ae932d,off,,,,,,, +944,python,none,sorted,1000,2463534242,1,0,16,235994792,581,0,0,8e15cb75,off,,,,,,, +945,node,insertion,reversed,1000,2463534242,20,554,1,270213549,241856470,499500,500499,0ca43265,off,,,,,,, +946,python,quick,random,2000,2463534242,14,90,1,272087944,255354319,25197,24544,41262855,off,,,,,,, +947,c,quick,sorted,250,2463534242,29,10851,1,242163002,241028382,31125,0,7fd8c1f4,off,,,,,,, +948,python,bubble,sorted,1000,2463534242,11,11,1,281727588,265216300,499500,0,8e15cb75,off,,,,,,, +949,c,quick,random,1000,2463534242,7,23484,1,251340681,250145483,11196,10210,57561285,off,,,,,,, +950,node,insertion,random,2000,2463534242,12,234,1,276240706,247651173,996390,996389,41262855,off,,,,,,, +951,node,insertion,random,2000,2463534242,23,234,1,276395857,247526704,996390,996389,41262855,off,,,,,,, +952,c,quick,random,1000,2463534242,29,23484,1,230410807,229475606,11196,10210,57561285,off,,,,,,, +953,c,insertion,sorted,250,2463534242,16,786886,1,243364554,242241509,249,0,7fd8c1f4,off,,,,,,, +954,python,insertion,reversed,2000,2463534242,23,2,1,423810705,407586766,1999000,2000999,ad35ba05,off,,,,,,, +955,node,insertion,random,1000,2463534242,8,1091,1,267469406,240166204,249457,249457,57561285,off,,,,,,, +956,python,bubble,sorted,250,2463534242,30,206,1,260544570,245301417,31125,0,7fd8c1f4,off,,,,,,, +957,python,insertion,random,1000,2463534242,21,10,1,254989381,239739062,249457,249457,57561285,off,,,,,,, +958,node,quick,sorted,250,2463534242,23,4507,1,267219889,236666252,31125,0,7fd8c1f4,off,,,,,,, +959,python,quick,random,250,2463534242,7,1271,1,250829570,234016546,1933,1940,b47adce4,off,,,,,,, +960,python,quick,reversed,2000,2463534242,18,2,1,270878681,254829993,1999000,2000,ad35ba05,off,,,,,,, +961,node,bubble,sorted,2000,2463534242,27,85,1,263426951,235646067,1999000,0,69842aa5,off,,,,,,, +962,node,none,sorted,1000,2463534242,6,0,9,239401807,882,0,0,8e15cb75,off,,,,,,, +963,python,none,sorted,500,2463534242,3,0,16,257457314,551,0,0,d9a24379,off,,,,,,, +964,node,quick,sorted,250,2463534242,9,4507,1,267483093,236288069,31125,0,7fd8c1f4,off,,,,,,, +965,node,none,nearly_sorted,1000,2463534242,25,0,10,262826467,732,0,0,1cfe4afd,off,,,,,,, +966,python,bubble,sorted,2000,2463534242,8,3,1,330108889,313968426,1999000,0,69842aa5,off,,,,,,, +967,c,quick,sorted,1000,2463534242,1,691,1,248929650,247819362,499500,0,8e15cb75,off,,,,,,, +968,python,none,sorted,500,2463534242,6,0,16,230475903,631,0,0,d9a24379,off,,,,,,, +969,c,none,reversed,500,2463534242,1,0,402,226494818,80,0,0,d33e3559,off,,,,,,, +970,node,quick,random,1000,2463534242,7,6513,1,261845459,233150107,11196,10210,57561285,off,,,,,,, +971,python,bubble,sorted,2000,2463534242,21,3,1,325888840,309750860,1999000,0,69842aa5,off,,,,,,, +972,python,none,random,1000,2463534242,3,0,15,235604962,761,0,0,57561285,off,,,,,,, +973,python,quick,reversed,1000,2463534242,3,8,1,268136437,252258437,499500,1000,0ca43265,off,,,,,,, +974,python,none,sorted,500,2463534242,10,0,16,228719658,521,0,0,d9a24379,off,,,,,,, +975,c,bubble,sorted,500,2463534242,20,2858,1,242860638,241955543,124750,0,d9a24379,off,,,,,,, +976,node,none,reversed,250,2463534242,10,0,9,223420728,952,0,0,be175004,off,,,,,,, +977,c,bubble,random,2000,2463534242,21,40,1,244574261,243308908,1999000,1988792,41262855,off,,,,,,, +978,c,quick,nearly_sorted,250,2463534242,19,77221,1,243898506,242748578,4346,1008,21a19c14,off,,,,,,, +979,c,quick,random,1000,2463534242,21,23484,1,232092447,230795759,11196,10210,57561285,off,,,,,,, +980,node,quick,random,500,2463534242,7,16148,1,269527556,241204115,4578,4552,11ae932d,off,,,,,,, +981,c,quick,random,1000,2463534242,3,23484,1,235852004,234966314,11196,10210,57561285,off,,,,,,, +982,node,insertion,nearly_sorted,500,2463534242,23,26045,1,253203953,224497590,7962,7926,482b4785,off,,,,,,, +983,python,quick,random,500,2463534242,26,507,1,260597289,244095248,4578,4552,11ae932d,off,,,,,,, +984,node,bubble,sorted,2000,2463534242,30,85,1,264283878,235483556,1999000,0,69842aa5,off,,,,,,, +985,c,bubble,sorted,250,2463534242,14,11386,1,241492105,240550049,31125,0,7fd8c1f4,off,,,,,,, +986,c,insertion,random,1000,2463534242,15,1983,1,235579504,234434679,249457,249457,57561285,off,,,,,,, +987,node,quick,reversed,1000,2463534242,13,307,1,273762613,245568417,499500,1000,0ca43265,off,,,,,,, +988,python,insertion,nearly_sorted,1000,2463534242,5,76,1,266852940,251177611,30889,30860,1cfe4afd,off,,,,,,, +989,node,insertion,nearly_sorted,1000,2463534242,24,7758,1,274278249,245884997,30889,30860,1cfe4afd,off,,,,,,, +990,python,bubble,random,500,2463534242,2,26,1,266855177,250301940,124750,119688,11ae932d,off,,,,,,, +991,c,none,random,2000,2463534242,3,0,415,245514032,90,0,0,41262855,off,,,,,,, +992,node,quick,random,500,2463534242,26,16148,1,287699961,259154831,4578,4552,11ae932d,off,,,,,,, +993,c,bubble,reversed,1000,2463534242,12,87,1,251035118,250107248,499500,999000,0ca43265,off,,,,,,, +994,c,quick,sorted,500,2463534242,3,2762,1,239472770,238590106,124750,0,d9a24379,off,,,,,,, +995,c,quick,sorted,500,2463534242,20,2762,1,243272491,242400357,124750,0,d9a24379,off,,,,,,, +996,node,none,nearly_sorted,500,2463534242,3,0,10,261255081,672,0,0,482b4785,off,,,,,,, +997,python,bubble,reversed,500,2463534242,1,20,1,269290888,253597505,124750,249500,d33e3559,off,,,,,,, +998,python,quick,sorted,2000,2463534242,28,2,1,302419962,285959041,1999000,0,69842aa5,off,,,,,,, +999,node,insertion,nearly_sorted,2000,2463534242,30,2100,1,259530800,231370456,118277,118270,5835dc59,off,,,,,,, +1000,python,insertion,nearly_sorted,1000,2463534242,29,76,1,253811872,238476029,30889,30860,1cfe4afd,off,,,,,,, +1001,python,none,sorted,1000,2463534242,15,0,16,252495516,571,0,0,8e15cb75,off,,,,,,, +1002,c,quick,nearly_sorted,1000,2463534242,12,11273,1,234697560,233766116,28499,7892,1cfe4afd,off,,,,,,, +1003,node,none,nearly_sorted,500,2463534242,28,0,10,259141581,701,0,0,482b4785,off,,,,,,, +1004,node,insertion,sorted,1000,2463534242,20,160107,1,256609446,228846984,999,0,8e15cb75,off,,,,,,, +1005,c,bubble,reversed,1000,2463534242,7,87,1,244262366,243199334,499500,999000,0ca43265,off,,,,,,, +1006,c,none,nearly_sorted,2000,2463534242,8,0,387,220343951,120,0,0,5835dc59,off,,,,,,, +1007,node,bubble,reversed,1000,2463534242,4,378,1,279142955,250529051,499500,999000,0ca43265,off,,,,,,, +1008,node,none,nearly_sorted,2000,2463534242,13,0,9,240190385,671,0,0,5835dc59,off,,,,,,, +1009,node,insertion,sorted,500,2463534242,19,324764,1,258883628,230489724,499,0,d9a24379,off,,,,,,, +1010,python,none,reversed,250,2463534242,10,0,17,252297574,521,0,0,be175004,off,,,,,,, +1011,c,bubble,reversed,500,2463534242,16,346,1,244035141,243016119,124750,249500,d33e3559,off,,,,,,, +1012,python,insertion,reversed,1000,2463534242,29,6,1,295146545,278738718,499500,500499,0ca43265,off,,,,,,, +1013,node,bubble,sorted,500,2463534242,22,1964,1,247207616,218568170,124750,0,d9a24379,off,,,,,,, +1014,python,quick,nearly_sorted,2000,2463534242,11,55,1,248560661,232439662,58002,11768,5835dc59,off,,,,,,, +1015,c,insertion,sorted,500,2463534242,3,399335,1,241795644,240888663,499,0,d9a24379,off,,,,,,, +1016,node,none,reversed,2000,2463534242,22,0,9,242741385,832,0,0,ad35ba05,off,,,,,,, +1017,node,none,random,500,2463534242,1,0,9,241521290,872,0,0,11ae932d,off,,,,,,, +1018,c,none,sorted,1000,2463534242,7,0,421,243936906,90,0,0,8e15cb75,off,,,,,,, +1019,c,insertion,nearly_sorted,500,2463534242,23,32350,1,243060404,242330707,7962,7926,482b4785,off,,,,,,, +1020,c,insertion,random,2000,2463534242,7,492,1,235927185,234966697,996390,996389,41262855,off,,,,,,, +1021,c,none,nearly_sorted,2000,2463534242,24,0,387,223104865,80,0,0,5835dc59,off,,,,,,, +1022,c,quick,sorted,2000,2463534242,27,174,1,239715314,238957504,1999000,0,69842aa5,off,,,,,,, +1023,c,bubble,sorted,2000,2463534242,15,173,1,237269530,236038400,1999000,0,69842aa5,off,,,,,,, +1024,python,bubble,random,2000,2463534242,19,2,1,383580559,365506746,1999000,1988792,41262855,off,,,,,,, +1025,c,none,sorted,500,2463534242,7,0,344,208793255,90,0,0,d9a24379,off,,,,,,, +1026,c,insertion,reversed,500,2463534242,23,3967,1,237080679,236266010,124750,125249,d33e3559,off,,,,,,, +1027,python,none,reversed,500,2463534242,14,0,16,248005952,591,0,0,d33e3559,off,,,,,,, +1028,node,quick,random,250,2463534242,9,36348,1,258472765,228230466,1933,1940,b47adce4,off,,,,,,, +1029,c,insertion,random,250,2463534242,22,26020,1,232209538,231117902,15844,15838,b47adce4,off,,,,,,, +1030,node,insertion,nearly_sorted,1000,2463534242,4,7758,1,262405838,234851401,30889,30860,1cfe4afd,off,,,,,,, +1031,c,bubble,nearly_sorted,1000,2463534242,5,507,1,243255128,242290983,499500,59780,1cfe4afd,off,,,,,,, +1032,python,quick,nearly_sorted,250,2463534242,26,882,1,261872148,246186775,4346,1008,21a19c14,off,,,,,,, +1033,node,insertion,reversed,250,2463534242,24,8539,1,270615333,242614521,31125,31374,be175004,off,,,,,,, +1034,node,none,reversed,250,2463534242,2,0,9,231048703,842,0,0,be175004,off,,,,,,, +1035,python,quick,nearly_sorted,2000,2463534242,27,55,1,252415235,234991300,58002,11768,5835dc59,off,,,,,,, +1036,python,quick,sorted,500,2463534242,5,30,1,243204344,228057324,124750,0,d9a24379,off,,,,,,, +1037,python,none,random,1000,2463534242,10,0,15,224726053,601,0,0,57561285,off,,,,,,, +1038,python,insertion,nearly_sorted,500,2463534242,19,316,1,244204529,229347062,7962,7926,482b4785,off,,,,,,, +1039,c,quick,random,500,2463534242,3,55776,1,237225728,236311926,4578,4552,11ae932d,off,,,,,,, +1040,python,bubble,sorted,500,2463534242,10,46,1,258870543,243076419,124750,0,d9a24379,off,,,,,,, +1041,python,none,nearly_sorted,250,2463534242,8,0,17,240559917,511,0,0,21a19c14,off,,,,,,, +1042,python,quick,reversed,500,2463534242,11,33,1,249795665,235115247,124750,500,d33e3559,off,,,,,,, +1043,c,bubble,nearly_sorted,500,2463534242,15,1998,1,235636361,234509889,124750,14926,482b4785,off,,,,,,, +1044,c,bubble,reversed,250,2463534242,30,1384,1,236628089,235745515,31125,62250,be175004,off,,,,,,, +1045,c,none,reversed,2000,2463534242,8,0,413,234732126,80,0,0,ad35ba05,off,,,,,,, +1046,python,quick,reversed,500,2463534242,19,33,1,248095249,232758184,124750,500,d33e3559,off,,,,,,, +1047,python,insertion,nearly_sorted,2000,2463534242,15,20,1,255059501,239088533,118277,118270,5835dc59,off,,,,,,, +1048,python,bubble,random,2000,2463534242,22,2,1,373648165,357083744,1999000,1988792,41262855,off,,,,,,, +1049,c,none,reversed,500,2463534242,2,0,402,228883716,90,0,0,d33e3559,off,,,,,,, +1050,node,none,random,2000,2463534242,8,0,9,244564271,771,0,0,41262855,off,,,,,,, +1051,python,none,sorted,1000,2463534242,23,0,16,270139432,551,0,0,8e15cb75,off,,,,,,, +1052,node,insertion,nearly_sorted,2000,2463534242,9,2100,1,264149506,235166327,118277,118270,5835dc59,off,,,,,,, +1053,c,none,random,1000,2463534242,29,0,367,210046583,81,0,0,57561285,off,,,,,,, +1054,python,insertion,random,2000,2463534242,25,3,1,325754299,308984536,996390,996389,41262855,off,,,,,,, +1055,python,quick,nearly_sorted,500,2463534242,13,296,1,246254789,231060647,10192,1850,482b4785,off,,,,,,, +1056,python,quick,sorted,500,2463534242,18,30,1,266127072,248818765,124750,0,d9a24379,off,,,,,,, +1057,python,quick,sorted,500,2463534242,9,30,1,251108907,235255209,124750,0,d9a24379,off,,,,,,, +1058,python,bubble,random,250,2463534242,22,112,1,246929906,231957100,31125,31196,b47adce4,off,,,,,,, +1059,c,none,reversed,1000,2463534242,5,0,405,230340425,80,0,0,0ca43265,off,,,,,,, +1060,node,none,random,2000,2463534242,17,0,9,241510459,761,0,0,41262855,off,,,,,,, +1061,c,insertion,nearly_sorted,500,2463534242,2,32350,1,240423692,239323811,7962,7926,482b4785,off,,,,,,, +1062,python,none,sorted,2000,2463534242,18,0,15,228364488,612,0,0,69842aa5,off,,,,,,, +1063,python,bubble,nearly_sorted,500,2463534242,12,41,1,252419188,236001272,124750,14926,482b4785,off,,,,,,, +1064,c,none,reversed,500,2463534242,8,0,402,225181566,90,0,0,d33e3559,off,,,,,,, +1065,c,bubble,sorted,2000,2463534242,28,173,1,233209863,232477981,1999000,0,69842aa5,off,,,,,,, +1066,python,insertion,reversed,1000,2463534242,5,6,1,301536703,284987364,499500,500499,0ca43265,off,,,,,,, +1067,c,bubble,sorted,1000,2463534242,8,702,1,238017994,237036133,499500,0,8e15cb75,off,,,,,,, +1068,python,quick,reversed,500,2463534242,1,33,1,266553091,250082691,124750,500,d33e3559,off,,,,,,, +1069,c,none,sorted,1000,2463534242,9,0,421,240957543,90,0,0,8e15cb75,off,,,,,,, +1070,python,none,sorted,250,2463534242,3,0,17,256304504,582,0,0,7fd8c1f4,off,,,,,,, +1071,node,insertion,random,500,2463534242,30,4059,1,270953587,241953059,60338,60339,11ae932d,off,,,,,,, +1072,c,insertion,reversed,2000,2463534242,7,260,1,243856101,242983846,1999000,2000999,ad35ba05,off,,,,,,, +1073,node,insertion,reversed,500,2463534242,10,2178,1,265853394,235366974,124750,125249,d33e3559,off,,,,,,, +1074,node,quick,nearly_sorted,1000,2463534242,14,3752,1,293640354,261696879,28499,7892,1cfe4afd,off,,,,,,, +1075,python,none,nearly_sorted,2000,2463534242,2,0,15,293593215,772,0,0,5835dc59,off,,,,,,, +1076,python,bubble,nearly_sorted,1000,2463534242,25,9,1,283547670,264066223,499500,59780,1cfe4afd,off,,,,,,, +1077,c,none,sorted,250,2463534242,30,0,429,262995726,90,0,0,7fd8c1f4,off,,,,,,, +1078,node,quick,reversed,500,2463534242,16,1194,1,288537840,257330496,124750,500,d33e3559,off,,,,,,, +1079,python,insertion,sorted,500,2463534242,21,6624,1,241258176,225542409,499,0,d9a24379,off,,,,,,, +1080,c,bubble,nearly_sorted,250,2463534242,9,7880,1,243160230,242266335,31125,4056,21a19c14,off,,,,,,, +1081,c,none,sorted,250,2463534242,23,0,429,247728032,90,0,0,7fd8c1f4,off,,,,,,, +1082,python,quick,sorted,250,2463534242,27,133,1,253617118,237008181,31125,0,7fd8c1f4,off,,,,,,, +1083,c,bubble,nearly_sorted,500,2463534242,28,1998,1,240667108,239501654,124750,14926,482b4785,off,,,,,,, +1084,node,none,nearly_sorted,2000,2463534242,9,0,9,242721379,642,0,0,5835dc59,off,,,,,,, +1085,python,quick,sorted,500,2463534242,24,30,1,243117131,227962586,124750,0,d9a24379,off,,,,,,, +1086,c,quick,sorted,1000,2463534242,17,691,1,245407151,244525103,499500,0,8e15cb75,off,,,,,,, +1087,node,insertion,random,1000,2463534242,1,1091,1,264617553,237541597,249457,249457,57561285,off,,,,,,, +1088,c,none,sorted,1000,2463534242,12,0,421,241395864,90,0,0,8e15cb75,off,,,,,,, +1089,node,quick,nearly_sorted,1000,2463534242,9,3752,1,265129943,235836820,28499,7892,1cfe4afd,off,,,,,,, +1090,python,bubble,reversed,1000,2463534242,15,5,1,293238541,277590550,499500,999000,0ca43265,off,,,,,,, +1091,python,bubble,nearly_sorted,500,2463534242,6,41,1,262586196,246163035,124750,14926,482b4785,off,,,,,,, +1092,node,insertion,random,250,2463534242,9,15349,1,259906084,232899849,15844,15838,b47adce4,off,,,,,,, +1093,node,insertion,nearly_sorted,2000,2463534242,16,2100,1,264326127,234916079,118277,118270,5835dc59,off,,,,,,, +1094,python,bubble,reversed,2000,2463534242,4,2,1,482963701,466687462,1999000,3998000,ad35ba05,off,,,,,,, +1095,c,none,nearly_sorted,250,2463534242,30,0,388,215979632,80,0,0,21a19c14,off,,,,,,, +1096,python,quick,sorted,250,2463534242,21,133,1,253971201,238293956,31125,0,7fd8c1f4,off,,,,,,, +1097,python,none,nearly_sorted,1000,2463534242,8,0,16,254232250,581,0,0,1cfe4afd,off,,,,,,, +1098,node,none,random,2000,2463534242,15,0,9,239558039,721,0,0,41262855,off,,,,,,, +1099,node,insertion,sorted,500,2463534242,14,324764,1,265243988,238191969,499,0,d9a24379,off,,,,,,, +1100,node,quick,nearly_sorted,250,2463534242,7,25320,1,287781493,257749942,4346,1008,21a19c14,off,,,,,,, +1101,python,insertion,sorted,250,2463534242,11,18845,1,259243784,243310496,249,0,7fd8c1f4,off,,,,,,, +1102,c,quick,nearly_sorted,500,2463534242,4,31784,1,247708364,246585700,10192,1850,482b4785,off,,,,,,, +1103,node,none,nearly_sorted,500,2463534242,21,0,10,277908049,601,0,0,482b4785,off,,,,,,, +1104,python,bubble,nearly_sorted,250,2463534242,28,188,1,268724141,252967097,31125,4056,21a19c14,off,,,,,,, +1105,python,quick,random,2000,2463534242,2,90,1,282126334,264997849,25197,24544,41262855,off,,,,,,, +1106,c,insertion,sorted,1000,2463534242,6,196883,1,238218361,237278480,999,0,8e15cb75,off,,,,,,, +1107,node,none,random,2000,2463534242,29,0,9,248378579,921,0,0,41262855,off,,,,,,, +1108,node,none,random,2000,2463534242,18,0,9,260163085,2214,0,0,41262855,off,,,,,,, +1109,c,none,random,250,2463534242,7,0,429,250699220,90,0,0,b47adce4,off,,,,,,, +1110,python,none,nearly_sorted,2000,2463534242,16,0,15,263843104,601,0,0,5835dc59,off,,,,,,, +1111,python,insertion,random,250,2463534242,29,206,1,271403499,250377244,15844,15838,b47adce4,off,,,,,,, +1112,c,quick,nearly_sorted,2000,2463534242,21,6145,1,250723595,249767243,58002,11768,5835dc59,off,,,,,,, +1113,node,insertion,sorted,2000,2463534242,23,89220,1,274982036,243246567,1999,0,69842aa5,off,,,,,,, +1114,c,bubble,random,250,2463534242,4,2568,1,252125905,251258368,31125,31196,b47adce4,off,,,,,,, +1115,python,bubble,nearly_sorted,250,2463534242,25,188,1,267086208,250906352,31125,4056,21a19c14,off,,,,,,, +1116,c,bubble,random,2000,2463534242,10,40,1,243597441,242616263,1999000,1988792,41262855,off,,,,,,, +1117,python,bubble,reversed,2000,2463534242,25,2,1,504546006,487638344,1999000,3998000,ad35ba05,off,,,,,,, +1118,c,insertion,nearly_sorted,500,2463534242,20,32350,1,243760966,242866742,7962,7926,482b4785,off,,,,,,, +1119,c,quick,sorted,1000,2463534242,3,691,1,246242126,245312555,499500,0,8e15cb75,off,,,,,,, +1120,python,bubble,random,250,2463534242,9,112,1,252562661,236853145,31125,31196,b47adce4,off,,,,,,, +1121,python,quick,random,250,2463534242,24,1271,1,258122531,241918665,1933,1940,b47adce4,off,,,,,,, +1122,node,bubble,nearly_sorted,500,2463534242,17,1566,1,281337899,252135446,124750,14926,482b4785,off,,,,,,, +1123,c,bubble,sorted,2000,2463534242,12,173,1,248372098,247193280,1999000,0,69842aa5,off,,,,,,, +1124,node,none,reversed,2000,2463534242,18,0,9,255783434,892,0,0,ad35ba05,off,,,,,,, +1125,python,insertion,nearly_sorted,250,2463534242,30,1416,1,267622749,250904919,2277,2251,21a19c14,off,,,,,,, +1126,c,quick,reversed,500,2463534242,10,3240,1,243224192,242304859,124750,500,d33e3559,off,,,,,,, +1127,node,none,sorted,2000,2463534242,5,0,10,281175753,1002,0,0,69842aa5,off,,,,,,, +1128,c,none,sorted,500,2463534242,24,0,344,200147247,100,0,0,d9a24379,off,,,,,,, +1129,node,none,random,500,2463534242,30,0,9,255560624,752,0,0,11ae932d,off,,,,,,, +1130,c,bubble,sorted,500,2463534242,9,2858,1,249124087,248085091,124750,0,d9a24379,off,,,,,,, +1131,c,insertion,nearly_sorted,250,2463534242,28,99421,1,242437707,241322077,2277,2251,21a19c14,off,,,,,,, +1132,python,insertion,reversed,500,2463534242,7,22,1,261749608,245439482,124750,125249,d33e3559,off,,,,,,, +1133,node,quick,sorted,1000,2463534242,13,296,1,301309674,273113766,499500,0,8e15cb75,off,,,,,,, +1134,c,bubble,random,500,2463534242,4,635,1,235519044,234467721,124750,119688,11ae932d,off,,,,,,, +1135,c,quick,reversed,500,2463534242,5,3240,1,243113192,241976752,124750,500,d33e3559,off,,,,,,, +1136,python,bubble,nearly_sorted,2000,2463534242,9,3,1,363803075,346944777,1999000,232556,5835dc59,off,,,,,,, +1137,c,insertion,nearly_sorted,2000,2463534242,4,2962,1,230650186,229684485,118277,118270,5835dc59,off,,,,,,, +1138,python,bubble,nearly_sorted,1000,2463534242,12,9,1,252264739,235820345,499500,59780,1cfe4afd,off,,,,,,, +1139,python,insertion,random,250,2463534242,27,206,1,261475909,245495162,15844,15838,b47adce4,off,,,,,,, +1140,python,bubble,sorted,2000,2463534242,3,3,1,339009586,321756071,1999000,0,69842aa5,off,,,,,,, +1141,node,none,sorted,250,2463534242,7,0,9,249016048,1102,0,0,7fd8c1f4,off,,,,,,, +1142,c,bubble,sorted,1000,2463534242,16,702,1,246691018,245524320,499500,0,8e15cb75,off,,,,,,, +1143,node,none,random,250,2463534242,5,0,10,273491524,772,0,0,b47adce4,off,,,,,,, +1144,c,insertion,sorted,2000,2463534242,25,101352,1,245578843,244480125,1999,0,69842aa5,off,,,,,,, +1145,node,insertion,sorted,500,2463534242,28,324764,1,270428063,241045530,499,0,d9a24379,off,,,,,,, +1146,c,quick,sorted,250,2463534242,5,10851,1,248391224,247541007,31125,0,7fd8c1f4,off,,,,,,, +1147,python,quick,reversed,500,2463534242,14,33,1,259784615,242955026,124750,500,d33e3559,off,,,,,,, +1148,node,bubble,sorted,1000,2463534242,28,374,1,261438345,233155578,499500,0,8e15cb75,off,,,,,,, +1149,python,insertion,random,250,2463534242,23,206,1,268205317,252066672,15844,15838,b47adce4,off,,,,,,, +1150,node,quick,reversed,2000,2463534242,8,80,1,285835838,256618213,1999000,2000,ad35ba05,off,,,,,,, +1151,python,none,nearly_sorted,1000,2463534242,27,0,16,263108976,571,0,0,1cfe4afd,off,,,,,,, +1152,python,quick,nearly_sorted,1000,2463534242,5,110,1,270614082,252298207,28499,7892,1cfe4afd,off,,,,,,, +1153,node,quick,nearly_sorted,500,2463534242,15,10954,1,272118070,242727049,10192,1850,482b4785,off,,,,,,, +1154,c,quick,reversed,2000,2463534242,9,209,1,244968129,244073872,1999000,2000,ad35ba05,off,,,,,,, +1155,node,bubble,reversed,500,2463534242,28,1534,1,280239139,249130567,124750,249500,d33e3559,off,,,,,,, +1156,node,bubble,random,250,2463534242,2,5439,1,267061883,235988028,31125,31196,b47adce4,off,,,,,,, +1157,node,insertion,sorted,2000,2463534242,19,89220,1,282027742,252847181,1999,0,69842aa5,off,,,,,,, +1158,python,insertion,nearly_sorted,500,2463534242,2,316,1,257539709,242208162,7962,7926,482b4785,off,,,,,,, +1159,c,none,nearly_sorted,250,2463534242,7,0,388,222181675,80,0,0,21a19c14,off,,,,,,, +1160,c,bubble,nearly_sorted,250,2463534242,18,7880,1,240039261,239300547,31125,4056,21a19c14,off,,,,,,, +1161,python,quick,reversed,2000,2463534242,6,2,1,281947353,264280343,1999000,2000,ad35ba05,off,,,,,,, +1162,c,quick,nearly_sorted,500,2463534242,12,31784,1,241620914,240635350,10192,1850,482b4785,off,,,,,,, +1163,python,quick,random,250,2463534242,11,1271,1,266408589,249390583,1933,1940,b47adce4,off,,,,,,, +1164,python,quick,reversed,1000,2463534242,21,8,1,276723130,259461473,499500,1000,0ca43265,off,,,,,,, +1165,python,insertion,random,250,2463534242,15,206,1,262581708,246962265,15844,15838,b47adce4,off,,,,,,, +1166,c,quick,sorted,2000,2463534242,24,174,1,252978030,252094486,1999000,0,69842aa5,off,,,,,,, +1167,c,quick,random,250,2463534242,4,136055,1,241089740,239949463,1933,1940,b47adce4,off,,,,,,, +1168,c,none,random,2000,2463534242,8,0,415,244642388,90,0,0,41262855,off,,,,,,, +1169,c,none,sorted,500,2463534242,18,0,344,202369666,80,0,0,d9a24379,off,,,,,,, +1170,node,insertion,sorted,2000,2463534242,27,89220,1,276925689,246550997,1999,0,69842aa5,off,,,,,,, +1171,node,insertion,random,250,2463534242,1,15349,1,277379129,249096836,15844,15838,b47adce4,off,,,,,,, +1172,node,none,sorted,1000,2463534242,20,0,9,243799068,942,0,0,8e15cb75,off,,,,,,, +1173,node,bubble,random,250,2463534242,8,5439,1,264820274,232763601,31125,31196,b47adce4,off,,,,,,, +1174,c,insertion,sorted,1000,2463534242,15,196883,1,238440323,237329734,999,0,8e15cb75,off,,,,,,, +1175,node,bubble,sorted,500,2463534242,17,1964,1,264550678,235654189,124750,0,d9a24379,off,,,,,,, +1176,python,insertion,random,500,2463534242,21,44,1,261034059,244703689,60338,60339,11ae932d,off,,,,,,, +1177,node,bubble,sorted,2000,2463534242,26,85,1,275790111,245324005,1999000,0,69842aa5,off,,,,,,, +1178,node,none,nearly_sorted,2000,2463534242,3,0,9,258672510,1473,0,0,5835dc59,off,,,,,,, +1179,python,none,reversed,2000,2463534242,26,0,16,280483750,632,0,0,ad35ba05,off,,,,,,, +1180,node,insertion,sorted,250,2463534242,11,607595,1,264152660,233760002,249,0,7fd8c1f4,off,,,,,,, +1181,python,bubble,nearly_sorted,500,2463534242,4,41,1,258700274,241970896,124750,14926,482b4785,off,,,,,,, +1182,python,none,sorted,250,2463534242,6,0,17,261027836,501,0,0,7fd8c1f4,off,,,,,,, +1183,c,bubble,random,1000,2463534242,19,154,1,237441142,236550365,499500,496926,57561285,off,,,,,,, +1184,python,insertion,sorted,1000,2463534242,19,3178,1,263157728,246927527,999,0,8e15cb75,off,,,,,,, +1185,c,none,reversed,500,2463534242,10,0,402,231182042,81,0,0,d33e3559,off,,,,,,, +1186,c,insertion,sorted,500,2463534242,1,399335,1,247778777,247073877,499,0,d9a24379,off,,,,,,, +1187,node,bubble,random,500,2463534242,23,1407,1,281060306,251739982,124750,119688,11ae932d,off,,,,,,, +1188,node,quick,reversed,500,2463534242,13,1194,1,266603986,236201533,124750,500,d33e3559,off,,,,,,, +1189,python,insertion,random,500,2463534242,19,44,1,258232307,241910790,60338,60339,11ae932d,off,,,,,,, +1190,node,insertion,nearly_sorted,500,2463534242,30,26045,1,255240079,226011265,7962,7926,482b4785,off,,,,,,, +1191,python,none,sorted,250,2463534242,17,0,17,264882601,521,0,0,7fd8c1f4,off,,,,,,, +1192,node,insertion,nearly_sorted,2000,2463534242,10,2100,1,264264039,235027773,118277,118270,5835dc59,off,,,,,,, +1193,python,none,reversed,250,2463534242,14,0,17,261762643,551,0,0,be175004,off,,,,,,, +1194,node,none,sorted,500,2463534242,26,0,10,271879393,882,0,0,d9a24379,off,,,,,,, +1195,python,quick,nearly_sorted,1000,2463534242,4,110,1,268066984,251592046,28499,7892,1cfe4afd,off,,,,,,, +1196,c,insertion,nearly_sorted,500,2463534242,9,32350,1,243374736,242277532,7962,7926,482b4785,off,,,,,,, +1197,python,bubble,nearly_sorted,250,2463534242,23,188,1,268006054,252082541,31125,4056,21a19c14,off,,,,,,, +1198,c,bubble,sorted,1000,2463534242,5,702,1,240995083,240041196,499500,0,8e15cb75,off,,,,,,, +1199,node,bubble,reversed,2000,2463534242,9,94,1,281397550,249892311,1999000,3998000,ad35ba05,off,,,,,,, +1200,python,quick,sorted,1000,2463534242,1,8,1,295666662,277979319,499500,0,8e15cb75,off,,,,,,, +1201,python,none,reversed,2000,2463534242,17,0,16,267031457,671,0,0,ad35ba05,off,,,,,,, +1202,node,insertion,reversed,1000,2463534242,14,554,1,273977454,242965259,499500,500499,0ca43265,off,,,,,,, +1203,python,bubble,reversed,2000,2463534242,2,2,1,483302896,467018163,1999000,3998000,ad35ba05,off,,,,,,, +1204,node,insertion,random,250,2463534242,27,15349,1,261577346,233878773,15844,15838,b47adce4,off,,,,,,, +1205,python,insertion,nearly_sorted,500,2463534242,11,316,1,251046781,234764901,7962,7926,482b4785,off,,,,,,, +1206,node,insertion,reversed,250,2463534242,22,8539,1,274111776,244807047,31125,31374,be175004,off,,,,,,, +1207,c,insertion,random,500,2463534242,27,6945,1,219538010,218493282,60338,60339,11ae932d,off,,,,,,, +1208,c,quick,random,500,2463534242,29,55776,1,246074092,244846492,4578,4552,11ae932d,off,,,,,,, +1209,node,quick,random,500,2463534242,15,16148,1,277768178,247320511,4578,4552,11ae932d,off,,,,,,, +1210,python,insertion,nearly_sorted,2000,2463534242,26,20,1,255016800,237614187,118277,118270,5835dc59,off,,,,,,, +1211,c,quick,reversed,250,2463534242,23,12781,1,241208023,240304009,31125,250,be175004,off,,,,,,, +1212,c,none,sorted,2000,2463534242,20,0,416,240706311,100,0,0,69842aa5,off,,,,,,, +1213,python,bubble,nearly_sorted,500,2463534242,10,41,1,257440744,241311399,124750,14926,482b4785,off,,,,,,, +1214,c,insertion,random,250,2463534242,18,26020,1,230169655,229061669,15844,15838,b47adce4,off,,,,,,, +1215,c,quick,random,2000,2463534242,3,10587,1,241242767,240281075,25197,24544,41262855,off,,,,,,, +1216,python,insertion,reversed,2000,2463534242,17,2,1,411376561,393955826,1999000,2000999,ad35ba05,off,,,,,,, +1217,python,quick,sorted,500,2463534242,26,30,1,263342574,247301224,124750,0,d9a24379,off,,,,,,, +1218,python,bubble,nearly_sorted,1000,2463534242,19,9,1,269881044,253326384,499500,59780,1cfe4afd,off,,,,,,, +1219,c,insertion,nearly_sorted,250,2463534242,19,99421,1,249798746,248848772,2277,2251,21a19c14,off,,,,,,, +1220,c,insertion,nearly_sorted,2000,2463534242,10,2962,1,232888572,231898755,118277,118270,5835dc59,off,,,,,,, +1221,node,quick,sorted,500,2463534242,13,1164,1,275247804,245183968,124750,0,d9a24379,off,,,,,,, +1222,node,none,nearly_sorted,2000,2463534242,8,0,9,257073695,802,0,0,5835dc59,off,,,,,,, +1223,python,none,nearly_sorted,500,2463534242,5,0,17,264578890,531,0,0,482b4785,off,,,,,,, +1224,c,bubble,reversed,2000,2463534242,12,22,1,248629050,247751626,1999000,3998000,ad35ba05,off,,,,,,, +1225,node,insertion,sorted,1000,2463534242,14,160107,1,261922792,231933981,999,0,8e15cb75,off,,,,,,, +1226,c,none,nearly_sorted,500,2463534242,21,0,405,233553367,90,0,0,482b4785,off,,,,,,, +1227,python,none,sorted,500,2463534242,2,0,16,252269502,591,0,0,d9a24379,off,,,,,,, +1228,c,insertion,random,1000,2463534242,12,1983,1,243349966,242324885,249457,249457,57561285,off,,,,,,, +1229,python,quick,random,1000,2463534242,3,209,1,270694542,254080991,11196,10210,57561285,off,,,,,,, +1230,python,quick,sorted,500,2463534242,1,30,1,261233932,245561390,124750,0,d9a24379,off,,,,,,, +1231,node,insertion,random,500,2463534242,13,4059,1,279817790,247503141,60338,60339,11ae932d,off,,,,,,, +1232,python,quick,nearly_sorted,500,2463534242,17,296,1,244570114,228382615,10192,1850,482b4785,off,,,,,,, +1233,python,bubble,random,250,2463534242,4,112,1,262113117,246753194,31125,31196,b47adce4,off,,,,,,, +1234,node,bubble,reversed,2000,2463534242,25,94,1,269864647,241406531,1999000,3998000,ad35ba05,off,,,,,,, +1235,python,quick,sorted,500,2463534242,2,30,1,245050403,229212746,124750,0,d9a24379,off,,,,,,, +1236,node,insertion,nearly_sorted,500,2463534242,1,26045,1,257750634,228685698,7962,7926,482b4785,off,,,,,,, +1237,python,quick,reversed,1000,2463534242,16,8,1,281973862,266562037,499500,1000,0ca43265,off,,,,,,, +1238,python,none,reversed,1000,2463534242,16,0,16,249096875,581,0,0,0ca43265,off,,,,,,, +1239,c,insertion,reversed,250,2463534242,26,15645,1,241956284,241102795,31125,31374,be175004,off,,,,,,, +1240,node,none,reversed,250,2463534242,23,0,9,249129017,861,0,0,be175004,off,,,,,,, +1241,c,none,sorted,2000,2463534242,26,0,416,250423203,90,0,0,69842aa5,off,,,,,,, +1242,python,bubble,nearly_sorted,1000,2463534242,13,9,1,264109010,245357220,499500,59780,1cfe4afd,off,,,,,,, +1243,python,none,reversed,250,2463534242,28,0,17,256692040,531,0,0,be175004,off,,,,,,, +1244,node,bubble,reversed,2000,2463534242,13,94,1,272840243,243652258,1999000,3998000,ad35ba05,off,,,,,,, +1245,c,quick,reversed,250,2463534242,15,12781,1,243474669,242566819,31125,250,be175004,off,,,,,,, +1246,node,quick,sorted,500,2463534242,2,1164,1,269855739,240863488,124750,0,d9a24379,off,,,,,,, +1247,c,none,random,500,2463534242,1,0,426,246655300,90,0,0,11ae932d,off,,,,,,, +1248,c,quick,nearly_sorted,500,2463534242,6,31784,1,245234899,244111193,10192,1850,482b4785,off,,,,,,, +1249,node,bubble,sorted,500,2463534242,12,1964,1,255418734,226048717,124750,0,d9a24379,off,,,,,,, +1250,python,bubble,random,1000,2463534242,16,6,1,273250782,257034605,499500,496926,57561285,off,,,,,,, +1251,python,quick,nearly_sorted,2000,2463534242,28,55,1,253446097,236580220,58002,11768,5835dc59,off,,,,,,, +1252,node,bubble,random,250,2463534242,12,5439,1,273589948,242900005,31125,31196,b47adce4,off,,,,,,, +1253,python,insertion,sorted,500,2463534242,13,6624,1,243202039,226879813,499,0,d9a24379,off,,,,,,, +1254,python,insertion,reversed,1000,2463534242,30,6,1,324720069,308134104,499500,500499,0ca43265,off,,,,,,, +1255,node,none,reversed,250,2463534242,15,0,9,236131819,741,0,0,be175004,off,,,,,,, +1256,python,insertion,sorted,250,2463534242,15,18845,1,269984303,251909168,249,0,7fd8c1f4,off,,,,,,, +1257,python,quick,nearly_sorted,2000,2463534242,17,55,1,250897049,234591250,58002,11768,5835dc59,off,,,,,,, +1258,c,quick,sorted,500,2463534242,24,2762,1,243470291,242388324,124750,0,d9a24379,off,,,,,,, +1259,python,bubble,random,500,2463534242,28,26,1,270506389,253399204,124750,119688,11ae932d,off,,,,,,, +1260,c,bubble,nearly_sorted,2000,2463534242,14,121,1,246035440,244947450,1999000,232556,5835dc59,off,,,,,,, +1261,node,insertion,reversed,1000,2463534242,7,554,1,278687913,246471913,499500,500499,0ca43265,off,,,,,,, +1262,node,none,reversed,2000,2463534242,14,0,9,248300183,1032,0,0,ad35ba05,off,,,,,,, +1263,c,none,sorted,1000,2463534242,25,0,421,257184964,80,0,0,8e15cb75,off,,,,,,, +1264,node,quick,sorted,250,2463534242,20,4507,1,270385322,237824458,31125,0,7fd8c1f4,off,,,,,,, +1265,node,none,reversed,500,2463534242,28,0,9,254347516,872,0,0,d33e3559,off,,,,,,, +1266,c,bubble,random,500,2463534242,6,635,1,238435077,237524530,124750,119688,11ae932d,off,,,,,,, +1267,node,bubble,reversed,1000,2463534242,14,378,1,281142861,251508927,499500,999000,0ca43265,off,,,,,,, +1268,python,none,random,500,2463534242,20,0,16,256877698,702,0,0,11ae932d,off,,,,,,, +1269,c,quick,nearly_sorted,1000,2463534242,26,11273,1,243959177,242978610,28499,7892,1cfe4afd,off,,,,,,, +1270,python,quick,reversed,2000,2463534242,10,2,1,279441896,262092326,1999000,2000,ad35ba05,off,,,,,,, +1271,node,bubble,reversed,2000,2463534242,3,94,1,278199857,247153474,1999000,3998000,ad35ba05,off,,,,,,, +1272,node,bubble,reversed,1000,2463534242,13,378,1,276201632,247868639,499500,999000,0ca43265,off,,,,,,, +1273,python,quick,nearly_sorted,2000,2463534242,26,55,1,254959063,237092003,58002,11768,5835dc59,off,,,,,,, +1274,node,quick,random,1000,2463534242,5,6513,1,272105957,242476519,11196,10210,57561285,off,,,,,,, +1275,c,none,reversed,250,2463534242,9,0,427,249226821,171,0,0,be175004,off,,,,,,, +1276,python,bubble,nearly_sorted,250,2463534242,12,188,1,274787231,257917245,31125,4056,21a19c14,off,,,,,,, +1277,node,bubble,nearly_sorted,250,2463534242,10,6161,1,273931298,243920819,31125,4056,21a19c14,off,,,,,,, +1278,python,quick,reversed,1000,2463534242,6,8,1,270045616,254138301,499500,1000,0ca43265,off,,,,,,, +1279,node,quick,reversed,1000,2463534242,15,307,1,271591724,241318888,499500,1000,0ca43265,off,,,,,,, +1280,python,bubble,random,500,2463534242,24,26,1,270364074,254549075,124750,119688,11ae932d,off,,,,,,, +1281,node,quick,random,2000,2463534242,17,2126,1,275788427,248102348,25197,24544,41262855,off,,,,,,, +1282,node,quick,reversed,250,2463534242,24,4755,1,269719154,240495677,31125,250,be175004,off,,,,,,, +1283,node,none,reversed,2000,2463534242,9,0,9,245354784,812,0,0,ad35ba05,off,,,,,,, +1284,node,bubble,sorted,2000,2463534242,29,85,1,272043851,241246069,1999000,0,69842aa5,off,,,,,,, +1285,python,insertion,sorted,1000,2463534242,4,3178,1,262028282,245815250,999,0,8e15cb75,off,,,,,,, +1286,node,insertion,reversed,250,2463534242,25,8539,1,267493912,239949030,31125,31374,be175004,off,,,,,,, +1287,c,quick,sorted,500,2463534242,23,2762,1,255840434,254938534,124750,0,d9a24379,off,,,,,,, +1288,node,quick,random,1000,2463534242,20,6513,1,297152035,266898489,11196,10210,57561285,off,,,,,,, +1289,node,bubble,nearly_sorted,1000,2463534242,18,413,1,270205205,241706490,499500,59780,1cfe4afd,off,,,,,,, +1290,python,bubble,random,1000,2463534242,12,6,1,276468513,260216924,499500,496926,57561285,off,,,,,,, +1291,c,bubble,sorted,250,2463534242,6,11386,1,242201294,241232717,31125,0,7fd8c1f4,off,,,,,,, +1292,python,insertion,sorted,250,2463534242,5,18845,1,262464609,246000866,249,0,7fd8c1f4,off,,,,,,, +1293,python,none,reversed,2000,2463534242,23,0,16,267000319,591,0,0,ad35ba05,off,,,,,,, +1294,c,none,random,2000,2463534242,14,0,415,248141917,90,0,0,41262855,off,,,,,,, +1295,node,quick,sorted,500,2463534242,20,1164,1,291084875,260762909,124750,0,d9a24379,off,,,,,,, +1296,node,bubble,reversed,1000,2463534242,25,378,1,273671423,245342149,499500,999000,0ca43265,off,,,,,,, +1297,c,quick,sorted,2000,2463534242,2,174,1,243786384,242544689,1999000,0,69842aa5,off,,,,,,, +1298,python,none,reversed,250,2463534242,24,0,17,262857904,581,0,0,be175004,off,,,,,,, +1299,node,bubble,nearly_sorted,500,2463534242,19,1566,1,275435948,244673791,124750,14926,482b4785,off,,,,,,, +1300,node,bubble,sorted,250,2463534242,16,8081,1,265675207,236602294,31125,0,7fd8c1f4,off,,,,,,, +1301,c,bubble,random,2000,2463534242,6,40,1,243042449,241937420,1999000,1988792,41262855,off,,,,,,, +1302,c,insertion,random,1000,2463534242,14,1983,1,243146816,241897315,249457,249457,57561285,off,,,,,,, +1303,node,none,sorted,2000,2463534242,20,0,10,280721193,1543,0,0,69842aa5,off,,,,,,, +1304,node,insertion,nearly_sorted,2000,2463534242,28,2100,1,268035629,236384162,118277,118270,5835dc59,off,,,,,,, +1305,node,insertion,reversed,1000,2463534242,4,554,1,270275998,242186393,499500,500499,0ca43265,off,,,,,,, +1306,c,none,reversed,2000,2463534242,1,0,413,243996588,161,0,0,ad35ba05,off,,,,,,, +1307,python,insertion,nearly_sorted,2000,2463534242,12,20,1,279104172,261862243,118277,118270,5835dc59,off,,,,,,, +1308,python,quick,random,2000,2463534242,23,90,1,265537136,247648184,25197,24544,41262855,off,,,,,,, +1309,c,none,sorted,2000,2463534242,1,0,416,245969846,90,0,0,69842aa5,off,,,,,,, +1310,node,quick,reversed,250,2463534242,12,4755,1,271563229,242897333,31125,250,be175004,off,,,,,,, +1311,python,none,random,500,2463534242,4,0,16,250367699,782,0,0,11ae932d,off,,,,,,, +1312,node,none,sorted,500,2463534242,30,0,10,271084694,841,0,0,d9a24379,off,,,,,,, +1313,node,bubble,reversed,2000,2463534242,24,94,1,287159087,256312611,1999000,3998000,ad35ba05,off,,,,,,, +1314,node,bubble,reversed,250,2463534242,23,5861,1,269555127,240078582,31125,62250,be175004,off,,,,,,, +1315,python,insertion,random,500,2463534242,20,44,1,248814128,232979438,60338,60339,11ae932d,off,,,,,,, +1316,node,bubble,sorted,2000,2463534242,28,85,1,264419831,234478201,1999000,0,69842aa5,off,,,,,,, +1317,c,bubble,sorted,1000,2463534242,6,702,1,242583760,241638911,499500,0,8e15cb75,off,,,,,,, +1318,python,bubble,random,250,2463534242,23,112,1,252272588,236107625,31125,31196,b47adce4,off,,,,,,, +1319,c,none,nearly_sorted,500,2463534242,6,0,405,232065538,100,0,0,482b4785,off,,,,,,, +1320,node,insertion,sorted,1000,2463534242,8,160107,1,259635406,228999398,999,0,8e15cb75,off,,,,,,, +1321,c,insertion,sorted,1000,2463534242,8,196883,1,238911598,237913033,999,0,8e15cb75,off,,,,,,, +1322,node,insertion,sorted,500,2463534242,3,324764,1,268775355,239472621,499,0,d9a24379,off,,,,,,, +1323,python,bubble,nearly_sorted,250,2463534242,19,188,1,259456142,241994907,31125,4056,21a19c14,off,,,,,,, +1324,c,bubble,random,500,2463534242,10,635,1,233808975,232843081,124750,119688,11ae932d,off,,,,,,, +1325,c,none,nearly_sorted,500,2463534242,18,0,405,234404020,170,0,0,482b4785,off,,,,,,, +1326,node,insertion,reversed,1000,2463534242,18,554,1,263845626,233727296,499500,500499,0ca43265,off,,,,,,, +1327,python,quick,nearly_sorted,250,2463534242,16,882,1,266172708,250807076,4346,1008,21a19c14,off,,,,,,, +1328,c,none,sorted,1000,2463534242,3,0,421,245970778,90,0,0,8e15cb75,off,,,,,,, +1329,node,insertion,reversed,250,2463534242,15,8539,1,270365676,239811825,31125,31374,be175004,off,,,,,,, +1330,c,none,sorted,1000,2463534242,15,0,421,241906892,90,0,0,8e15cb75,off,,,,,,, +1331,node,bubble,sorted,250,2463534242,20,8081,1,262497820,231583169,31125,0,7fd8c1f4,off,,,,,,, +1332,c,quick,reversed,1000,2463534242,26,814,1,240291275,239319803,499500,1000,0ca43265,off,,,,,,, +1333,c,insertion,nearly_sorted,2000,2463534242,15,2962,1,243029605,241663883,118277,118270,5835dc59,off,,,,,,, +1334,c,quick,reversed,1000,2463534242,23,814,1,259938675,258973369,499500,1000,0ca43265,off,,,,,,, +1335,python,bubble,nearly_sorted,2000,2463534242,30,3,1,380447307,361193318,1999000,232556,5835dc59,off,,,,,,, +1336,node,none,reversed,250,2463534242,24,0,9,250763821,862,0,0,be175004,off,,,,,,, +1337,c,none,sorted,250,2463534242,5,0,429,257327792,90,0,0,7fd8c1f4,off,,,,,,, +1338,c,bubble,reversed,250,2463534242,2,1384,1,241800612,241104398,31125,62250,be175004,off,,,,,,, +1339,node,quick,nearly_sorted,250,2463534242,10,25320,1,268575211,237277679,4346,1008,21a19c14,off,,,,,,, +1340,c,quick,reversed,2000,2463534242,23,209,1,243446437,242306721,1999000,2000,ad35ba05,off,,,,,,, +1341,node,insertion,reversed,1000,2463534242,24,554,1,271272536,241661444,499500,500499,0ca43265,off,,,,,,, +1342,c,none,random,500,2463534242,6,0,426,254047335,90,0,0,11ae932d,off,,,,,,, +1343,node,quick,reversed,2000,2463534242,3,80,1,278110909,245596318,1999000,2000,ad35ba05,off,,,,,,, +1344,c,quick,reversed,250,2463534242,5,12781,1,244466519,243566963,31125,250,be175004,off,,,,,,, +1345,node,bubble,reversed,2000,2463534242,12,94,1,277501880,246333065,1999000,3998000,ad35ba05,off,,,,,,, +1346,c,bubble,nearly_sorted,500,2463534242,6,1998,1,236954300,236016713,124750,14926,482b4785,off,,,,,,, +1347,node,insertion,reversed,2000,2463534242,3,144,1,265483516,235646629,1999000,2000999,ad35ba05,off,,,,,,, +1348,node,bubble,reversed,1000,2463534242,6,378,1,270494467,242595982,499500,999000,0ca43265,off,,,,,,, +1349,c,quick,nearly_sorted,250,2463534242,28,77221,1,241111382,240168154,4346,1008,21a19c14,off,,,,,,, +1350,python,none,sorted,250,2463534242,27,0,17,264712510,501,0,0,7fd8c1f4,off,,,,,,, +1351,node,quick,nearly_sorted,500,2463534242,13,10954,1,261156246,233945143,10192,1850,482b4785,off,,,,,,, +1352,c,quick,reversed,1000,2463534242,24,814,1,252529189,251609787,499500,1000,0ca43265,off,,,,,,, +1353,c,insertion,sorted,1000,2463534242,20,196883,1,235842406,234827404,999,0,8e15cb75,off,,,,,,, +1354,c,bubble,reversed,250,2463534242,11,1384,1,240819385,239804502,31125,62250,be175004,off,,,,,,, +1355,node,quick,reversed,250,2463534242,26,4755,1,271560404,243236634,31125,250,be175004,off,,,,,,, +1356,python,quick,sorted,1000,2463534242,17,8,1,291971638,276287413,499500,0,8e15cb75,off,,,,,,, +1357,node,bubble,nearly_sorted,2000,2463534242,4,110,1,267297614,238817562,1999000,232556,5835dc59,off,,,,,,, +1358,python,quick,sorted,2000,2463534242,5,2,1,304393135,287911690,1999000,0,69842aa5,off,,,,,,, +1359,node,bubble,random,250,2463534242,21,5439,1,265119865,236858037,31125,31196,b47adce4,off,,,,,,, +1360,node,quick,nearly_sorted,500,2463534242,1,10954,1,263037091,235028931,10192,1850,482b4785,off,,,,,,, +1361,node,none,sorted,250,2463534242,28,0,9,240821929,841,0,0,7fd8c1f4,off,,,,,,, +1362,c,none,sorted,2000,2463534242,4,0,416,249779416,91,0,0,69842aa5,off,,,,,,, +1363,python,none,reversed,500,2463534242,22,0,16,243522149,592,0,0,d33e3559,off,,,,,,, +1364,c,bubble,nearly_sorted,2000,2463534242,3,121,1,237688497,236700484,1999000,232556,5835dc59,off,,,,,,, +1365,python,insertion,nearly_sorted,1000,2463534242,4,76,1,249551818,233610025,30889,30860,1cfe4afd,off,,,,,,, +1366,node,bubble,random,250,2463534242,26,5439,1,253977523,226739501,31125,31196,b47adce4,off,,,,,,, +1367,python,insertion,nearly_sorted,2000,2463534242,11,20,1,261336464,244150207,118277,118270,5835dc59,off,,,,,,, +1368,python,quick,random,500,2463534242,25,507,1,261554629,244934888,4578,4552,11ae932d,off,,,,,,, +1369,c,none,sorted,2000,2463534242,15,0,416,247167855,90,0,0,69842aa5,off,,,,,,, +1370,c,none,nearly_sorted,250,2463534242,3,0,388,222902296,91,0,0,21a19c14,off,,,,,,, +1371,python,insertion,nearly_sorted,250,2463534242,21,1416,1,264667747,248625201,2277,2251,21a19c14,off,,,,,,, +1372,node,insertion,random,250,2463534242,17,15349,1,269928687,240719548,15844,15838,b47adce4,off,,,,,,, +1373,c,insertion,random,2000,2463534242,5,492,1,236979537,235874436,996390,996389,41262855,off,,,,,,, +1374,node,quick,sorted,500,2463534242,15,1164,1,268085683,239534993,124750,0,d9a24379,off,,,,,,, +1375,node,bubble,reversed,500,2463534242,17,1534,1,269877832,241892724,124750,249500,d33e3559,off,,,,,,, +1376,node,quick,nearly_sorted,2000,2463534242,17,1909,1,254048656,225363377,58002,11768,5835dc59,off,,,,,,, +1377,node,quick,sorted,2000,2463534242,6,74,1,272107059,241990059,1999000,0,69842aa5,off,,,,,,, +1378,node,quick,random,250,2463534242,11,36348,1,257410275,228342047,1933,1940,b47adce4,off,,,,,,, +1379,python,quick,sorted,1000,2463534242,11,8,1,289601745,273579447,499500,0,8e15cb75,off,,,,,,, +1380,node,bubble,sorted,2000,2463534242,12,85,1,268169921,237824714,1999000,0,69842aa5,off,,,,,,, +1381,python,none,reversed,2000,2463534242,20,0,16,264828779,641,0,0,ad35ba05,off,,,,,,, +1382,c,insertion,reversed,1000,2463534242,22,1033,1,240722313,239835620,499500,500499,0ca43265,off,,,,,,, +1383,c,none,reversed,250,2463534242,5,0,427,244319922,91,0,0,be175004,off,,,,,,, +1384,node,none,nearly_sorted,250,2463534242,1,0,10,267867003,852,0,0,21a19c14,off,,,,,,, +1385,python,insertion,reversed,1000,2463534242,28,6,1,295088037,279319503,499500,500499,0ca43265,off,,,,,,, +1386,c,bubble,reversed,250,2463534242,21,1384,1,241960993,240874537,31125,62250,be175004,off,,,,,,, +1387,c,none,random,1000,2463534242,16,0,367,215748058,100,0,0,57561285,off,,,,,,, +1388,c,quick,reversed,1000,2463534242,14,814,1,242861597,242087941,499500,1000,0ca43265,off,,,,,,, +1389,node,none,sorted,1000,2463534242,7,0,9,248698162,932,0,0,8e15cb75,off,,,,,,, +1390,python,none,reversed,500,2463534242,27,0,16,254857873,561,0,0,d33e3559,off,,,,,,, +1391,c,bubble,random,1000,2463534242,5,154,1,236912722,235642843,499500,496926,57561285,off,,,,,,, +1392,node,insertion,sorted,1000,2463534242,2,160107,1,261157048,230412664,999,0,8e15cb75,off,,,,,,, +1393,node,insertion,sorted,500,2463534242,17,324764,1,267003535,237555797,499,0,d9a24379,off,,,,,,, +1394,c,none,reversed,250,2463534242,27,0,427,246707108,90,0,0,be175004,off,,,,,,, +1395,python,quick,nearly_sorted,500,2463534242,9,296,1,254923556,237871055,10192,1850,482b4785,off,,,,,,, +1396,python,quick,nearly_sorted,2000,2463534242,21,55,1,269138015,251628010,58002,11768,5835dc59,off,,,,,,, +1397,python,insertion,sorted,2000,2463534242,28,1523,1,266939956,249435173,1999,0,69842aa5,off,,,,,,, +1398,python,insertion,random,500,2463534242,24,44,1,252233095,236121420,60338,60339,11ae932d,off,,,,,,, +1399,node,none,random,500,2463534242,6,0,9,247134668,922,0,0,11ae932d,off,,,,,,, +1400,c,insertion,nearly_sorted,1000,2463534242,21,11445,1,246162918,245231684,30889,30860,1cfe4afd,off,,,,,,, +1401,node,none,reversed,250,2463534242,8,0,9,240835504,1042,0,0,be175004,off,,,,,,, +1402,python,insertion,random,2000,2463534242,14,3,1,331355897,313179445,996390,996389,41262855,off,,,,,,, +1403,c,bubble,sorted,1000,2463534242,7,702,1,240546092,239379565,499500,0,8e15cb75,off,,,,,,, +1404,python,quick,sorted,1000,2463534242,5,8,1,288554452,272183927,499500,0,8e15cb75,off,,,,,,, +1405,c,none,sorted,1000,2463534242,16,0,421,247224707,100,0,0,8e15cb75,off,,,,,,, +1406,node,none,reversed,250,2463534242,25,0,9,242537754,802,0,0,be175004,off,,,,,,, +1407,python,insertion,random,2000,2463534242,2,3,1,333339384,314268559,996390,996389,41262855,off,,,,,,, +1408,python,none,sorted,500,2463534242,28,0,16,253333466,491,0,0,d9a24379,off,,,,,,, +1409,c,none,sorted,2000,2463534242,7,0,416,242459438,130,0,0,69842aa5,off,,,,,,, +1410,c,none,reversed,1000,2463534242,1,0,405,238428032,81,0,0,0ca43265,off,,,,,,, +1411,node,quick,reversed,250,2463534242,25,4755,1,276564563,246077399,31125,250,be175004,off,,,,,,, +1412,c,bubble,reversed,500,2463534242,29,346,1,243247514,242384106,124750,249500,d33e3559,off,,,,,,, +1413,python,quick,nearly_sorted,1000,2463534242,8,110,1,262704429,245853029,28499,7892,1cfe4afd,off,,,,,,, +1414,node,none,random,250,2463534242,29,0,10,274687814,852,0,0,b47adce4,off,,,,,,, +1415,node,none,nearly_sorted,1000,2463534242,17,0,10,275707877,742,0,0,1cfe4afd,off,,,,,,, +1416,c,quick,reversed,500,2463534242,16,3240,1,243722715,242550628,124750,500,d33e3559,off,,,,,,, +1417,python,none,reversed,500,2463534242,16,0,16,251239732,541,0,0,d33e3559,off,,,,,,, +1418,node,bubble,nearly_sorted,500,2463534242,9,1566,1,276122294,248203167,124750,14926,482b4785,off,,,,,,, +1419,python,none,sorted,500,2463534242,12,0,16,246920187,641,0,0,d9a24379,off,,,,,,, +1420,python,insertion,sorted,500,2463534242,29,6624,1,239937863,224145000,499,0,d9a24379,off,,,,,,, +1421,node,insertion,nearly_sorted,2000,2463534242,12,2100,1,266232959,235524355,118277,118270,5835dc59,off,,,,,,, +1422,python,none,reversed,1000,2463534242,17,0,16,254700337,591,0,0,0ca43265,off,,,,,,, +1423,node,insertion,nearly_sorted,2000,2463534242,7,2100,1,266510581,236344542,118277,118270,5835dc59,off,,,,,,, +1424,c,none,nearly_sorted,1000,2463534242,19,0,421,244684507,81,0,0,1cfe4afd,off,,,,,,, +1425,python,bubble,random,500,2463534242,21,26,1,263661863,247167013,124750,119688,11ae932d,off,,,,,,, +1426,c,bubble,nearly_sorted,1000,2463534242,11,507,1,244607829,243722554,499500,59780,1cfe4afd,off,,,,,,, +1427,python,bubble,nearly_sorted,500,2463534242,7,41,1,251154035,235200071,124750,14926,482b4785,off,,,,,,, +1428,node,quick,reversed,2000,2463534242,23,80,1,274859607,245028593,1999000,2000,ad35ba05,off,,,,,,, +1429,python,bubble,sorted,500,2463534242,14,46,1,273993676,258013373,124750,0,d9a24379,off,,,,,,, +1430,node,none,random,1000,2463534242,16,0,10,281980001,872,0,0,57561285,off,,,,,,, +1431,c,bubble,random,2000,2463534242,15,40,1,247109772,245983873,1999000,1988792,41262855,off,,,,,,, +1432,python,none,reversed,250,2463534242,26,0,17,261155766,531,0,0,be175004,off,,,,,,, +1433,c,quick,sorted,1000,2463534242,11,691,1,243229071,242126583,499500,0,8e15cb75,off,,,,,,, +1434,c,quick,nearly_sorted,500,2463534242,20,31784,1,248283672,247212876,10192,1850,482b4785,off,,,,,,, +1435,python,quick,reversed,250,2463534242,13,151,1,260467424,244051973,31125,250,be175004,off,,,,,,, +1436,node,insertion,sorted,250,2463534242,17,607595,1,280585037,250657647,249,0,7fd8c1f4,off,,,,,,, +1437,node,bubble,sorted,500,2463534242,16,1964,1,268400813,239365542,124750,0,d9a24379,off,,,,,,, +1438,node,bubble,reversed,500,2463534242,30,1534,1,277575177,245882244,124750,249500,d33e3559,off,,,,,,, +1439,c,bubble,nearly_sorted,500,2463534242,18,1998,1,243510347,242487339,124750,14926,482b4785,off,,,,,,, +1440,c,bubble,sorted,2000,2463534242,10,173,1,242561408,241628640,1999000,0,69842aa5,off,,,,,,, +1441,python,insertion,reversed,250,2463534242,24,102,1,276492868,259813214,31125,31374,be175004,off,,,,,,, +1442,c,quick,nearly_sorted,2000,2463534242,22,6145,1,244205069,243178776,58002,11768,5835dc59,off,,,,,,, +1443,c,none,sorted,2000,2463534242,13,0,416,250503793,100,0,0,69842aa5,off,,,,,,, +1444,python,none,nearly_sorted,250,2463534242,25,0,17,256383592,481,0,0,21a19c14,off,,,,,,, +1445,c,quick,random,2000,2463534242,26,10587,1,247121604,246102456,25197,24544,41262855,off,,,,,,, +1446,node,quick,nearly_sorted,500,2463534242,6,10954,1,265589815,234147421,10192,1850,482b4785,off,,,,,,, +1447,node,quick,random,500,2463534242,10,16148,1,270938690,242668988,4578,4552,11ae932d,off,,,,,,, +1448,python,none,sorted,500,2463534242,15,0,16,281143124,872,0,0,d9a24379,off,,,,,,, +1449,node,none,nearly_sorted,1000,2463534242,13,0,10,307304782,912,0,0,1cfe4afd,off,,,,,,, +1450,python,insertion,sorted,2000,2463534242,4,1523,1,302174107,282971675,1999,0,69842aa5,off,,,,,,, +1451,c,bubble,reversed,2000,2463534242,11,22,1,254103510,253139923,1999000,3998000,ad35ba05,off,,,,,,, +1452,c,none,random,500,2463534242,4,0,426,266200399,80,0,0,11ae932d,off,,,,,,, +1453,c,bubble,reversed,1000,2463534242,15,87,1,250366896,249571417,499500,999000,0ca43265,off,,,,,,, +1454,c,quick,sorted,500,2463534242,30,2762,1,245581358,244649681,124750,0,d9a24379,off,,,,,,, +1455,node,none,sorted,1000,2463534242,24,0,9,249021726,831,0,0,8e15cb75,off,,,,,,, +1456,python,none,random,2000,2463534242,26,0,16,288997573,611,0,0,41262855,off,,,,,,, +1457,c,none,reversed,2000,2463534242,12,0,413,260928821,90,0,0,ad35ba05,off,,,,,,, +1458,c,quick,random,500,2463534242,26,55776,1,241249368,240498412,4578,4552,11ae932d,off,,,,,,, +1459,node,quick,random,250,2463534242,17,36348,1,276307952,244198390,1933,1940,b47adce4,off,,,,,,, +1460,node,insertion,nearly_sorted,1000,2463534242,30,7758,1,277276397,244936379,30889,30860,1cfe4afd,off,,,,,,, +1461,c,bubble,nearly_sorted,1000,2463534242,10,507,1,245947354,244994209,499500,59780,1cfe4afd,off,,,,,,, +1462,node,bubble,random,500,2463534242,15,1407,1,280549181,249901204,124750,119688,11ae932d,off,,,,,,, +1463,node,quick,random,1000,2463534242,30,6513,1,274757144,246833990,11196,10210,57561285,off,,,,,,, +1464,node,insertion,nearly_sorted,2000,2463534242,23,2100,1,270945253,241623999,118277,118270,5835dc59,off,,,,,,, +1465,python,quick,random,2000,2463534242,3,90,1,258300785,239976079,25197,24544,41262855,off,,,,,,, +1466,node,quick,random,500,2463534242,30,16148,1,284390228,251917136,4578,4552,11ae932d,off,,,,,,, +1467,node,insertion,nearly_sorted,1000,2463534242,14,7758,1,280988854,251522767,30889,30860,1cfe4afd,off,,,,,,, +1468,c,quick,reversed,500,2463534242,30,3240,1,248894658,247983099,124750,500,d33e3559,off,,,,,,, +1469,node,quick,sorted,250,2463534242,26,4507,1,270383298,242000864,31125,0,7fd8c1f4,off,,,,,,, +1470,c,quick,sorted,1000,2463534242,27,691,1,242572740,241597282,499500,0,8e15cb75,off,,,,,,, +1471,python,quick,nearly_sorted,250,2463534242,27,882,1,275617768,259068397,4346,1008,21a19c14,off,,,,,,, +1472,python,quick,sorted,500,2463534242,25,30,1,264076039,246619827,124750,0,d9a24379,off,,,,,,, +1473,node,insertion,nearly_sorted,250,2463534242,25,79550,1,244189870,214754024,2277,2251,21a19c14,off,,,,,,, +1474,node,insertion,sorted,250,2463534242,21,607595,1,253706696,225430889,249,0,7fd8c1f4,off,,,,,,, +1475,c,none,nearly_sorted,250,2463534242,9,0,388,218500136,80,0,0,21a19c14,off,,,,,,, +1476,c,none,nearly_sorted,250,2463534242,24,0,388,218097401,100,0,0,21a19c14,off,,,,,,, +1477,python,none,random,250,2463534242,27,0,17,242421626,521,0,0,b47adce4,off,,,,,,, +1478,python,bubble,nearly_sorted,500,2463534242,2,41,1,277485549,263009405,124750,14926,482b4785,off,,,,,,, +1479,c,bubble,reversed,2000,2463534242,27,22,1,247383966,246474452,1999000,3998000,ad35ba05,off,,,,,,, +1480,python,none,random,500,2463534242,22,0,16,228751267,601,0,0,11ae932d,off,,,,,,, +1481,c,none,reversed,2000,2463534242,6,0,413,231410470,91,0,0,ad35ba05,off,,,,,,, +1482,node,insertion,reversed,250,2463534242,16,8539,1,268014601,241214111,31125,31374,be175004,off,,,,,,, +1483,c,quick,random,250,2463534242,18,136055,1,240969820,239941218,1933,1940,b47adce4,off,,,,,,, +1484,c,quick,nearly_sorted,2000,2463534242,13,6145,1,249751347,248581882,58002,11768,5835dc59,off,,,,,,, +1485,node,insertion,reversed,1000,2463534242,17,554,1,270616536,240205310,499500,500499,0ca43265,off,,,,,,, +1486,python,quick,sorted,2000,2463534242,30,2,1,305715314,289535744,1999000,0,69842aa5,off,,,,,,, +1487,c,bubble,random,250,2463534242,14,2568,1,238559289,237545107,31125,31196,b47adce4,off,,,,,,, +1488,c,none,random,1000,2463534242,13,0,367,207210198,80,0,0,57561285,off,,,,,,, +1489,c,insertion,sorted,500,2463534242,9,399335,1,255124843,254361352,499,0,d9a24379,off,,,,,,, +1490,python,bubble,reversed,500,2463534242,6,20,1,284227454,267513584,124750,249500,d33e3559,off,,,,,,, +1491,c,quick,random,1000,2463534242,30,23484,1,228036347,227112596,11196,10210,57561285,off,,,,,,, +1492,node,insertion,reversed,250,2463534242,11,8539,1,269132355,239863292,31125,31374,be175004,off,,,,,,, +1493,python,quick,random,250,2463534242,18,1271,1,251837373,235781114,1933,1940,b47adce4,off,,,,,,, +1494,c,bubble,sorted,500,2463534242,29,2858,1,241645352,240728824,124750,0,d9a24379,off,,,,,,, +1495,c,quick,reversed,250,2463534242,30,12781,1,242003314,240862446,31125,250,be175004,off,,,,,,, +1496,node,insertion,random,2000,2463534242,4,234,1,272077723,241432841,996390,996389,41262855,off,,,,,,, +1497,node,insertion,random,1000,2463534242,11,1091,1,276339782,244155906,249457,249457,57561285,off,,,,,,, +1498,python,none,reversed,500,2463534242,19,0,16,238067172,551,0,0,d33e3559,off,,,,,,, +1499,python,insertion,sorted,500,2463534242,3,6624,1,238679047,223658366,499,0,d9a24379,off,,,,,,, +1500,node,insertion,reversed,2000,2463534242,21,144,1,296129057,267850121,1999000,2000999,ad35ba05,off,,,,,,, +1501,node,quick,sorted,500,2463534242,19,1164,1,270989815,243306131,124750,0,d9a24379,off,,,,,,, +1502,node,quick,random,500,2463534242,17,16148,1,270510246,242756746,4578,4552,11ae932d,off,,,,,,, +1503,c,bubble,nearly_sorted,500,2463534242,27,1998,1,241168067,240268221,124750,14926,482b4785,off,,,,,,, +1504,node,none,random,1000,2463534242,2,0,10,303283297,922,0,0,57561285,off,,,,,,, +1505,c,bubble,random,250,2463534242,8,2568,1,252113010,250883832,31125,31196,b47adce4,off,,,,,,, +1506,c,quick,reversed,250,2463534242,17,12781,1,252329514,251038445,31125,250,be175004,off,,,,,,, +1507,c,quick,sorted,1000,2463534242,4,691,1,244583167,243697056,499500,0,8e15cb75,off,,,,,,, +1508,c,insertion,nearly_sorted,500,2463534242,17,32350,1,242785768,241890551,7962,7926,482b4785,off,,,,,,, +1509,c,quick,sorted,2000,2463534242,25,174,1,235745553,234748718,1999000,0,69842aa5,off,,,,,,, +1510,python,bubble,nearly_sorted,500,2463534242,18,41,1,248007815,232773461,124750,14926,482b4785,off,,,,,,, +1511,c,quick,nearly_sorted,250,2463534242,1,77221,1,245323696,244160557,4346,1008,21a19c14,off,,,,,,, +1512,python,none,sorted,1000,2463534242,5,0,16,233493674,571,0,0,8e15cb75,off,,,,,,, +1513,node,none,nearly_sorted,250,2463534242,19,0,10,258193815,682,0,0,21a19c14,off,,,,,,, +1514,c,none,sorted,2000,2463534242,9,0,416,233899225,90,0,0,69842aa5,off,,,,,,, +1515,node,bubble,nearly_sorted,2000,2463534242,17,110,1,265035617,237097223,1999000,232556,5835dc59,off,,,,,,, +1516,python,none,sorted,1000,2463534242,29,0,16,321066804,621,0,0,8e15cb75,off,,,,,,, +1517,python,insertion,reversed,500,2463534242,13,22,1,252027348,235638230,124750,125249,d33e3559,off,,,,,,, +1518,python,bubble,reversed,2000,2463534242,8,2,1,542354653,520903855,1999000,3998000,ad35ba05,off,,,,,,, +1519,c,insertion,sorted,2000,2463534242,9,101352,1,236672923,235539338,1999,0,69842aa5,off,,,,,,, +1520,node,none,random,250,2463534242,1,0,10,257305419,3006,0,0,b47adce4,off,,,,,,, +1521,node,quick,random,500,2463534242,4,16148,1,264109471,235279471,4578,4552,11ae932d,off,,,,,,, +1522,python,insertion,sorted,1000,2463534242,1,3178,1,261167028,245381445,999,0,8e15cb75,off,,,,,,, +1523,c,quick,reversed,250,2463534242,26,12781,1,243018935,242137925,31125,250,be175004,off,,,,,,, +1524,node,insertion,nearly_sorted,1000,2463534242,3,7758,1,264143174,234872847,30889,30860,1cfe4afd,off,,,,,,, +1525,python,bubble,reversed,500,2463534242,13,20,1,259742867,244900560,124750,249500,d33e3559,off,,,,,,, +1526,node,quick,random,2000,2463534242,3,2126,1,260968514,233346714,25197,24544,41262855,off,,,,,,, +1527,c,bubble,reversed,1000,2463534242,17,87,1,239686161,238553557,499500,999000,0ca43265,off,,,,,,, +1528,python,none,nearly_sorted,1000,2463534242,18,0,16,236660368,511,0,0,1cfe4afd,off,,,,,,, +1529,node,none,reversed,2000,2463534242,6,0,9,236184027,761,0,0,ad35ba05,off,,,,,,, +1530,node,quick,sorted,500,2463534242,24,1164,1,268488909,242239463,124750,0,d9a24379,off,,,,,,, +1531,node,bubble,sorted,2000,2463534242,11,85,1,269264272,235839185,1999000,0,69842aa5,off,,,,,,, +1532,python,quick,nearly_sorted,500,2463534242,18,296,1,237293986,222276547,10192,1850,482b4785,off,,,,,,, +1533,node,bubble,sorted,1000,2463534242,18,374,1,237612754,210938765,499500,0,8e15cb75,off,,,,,,, +1534,c,none,sorted,1000,2463534242,18,0,421,240129060,90,0,0,8e15cb75,off,,,,,,, +1535,node,insertion,sorted,500,2463534242,11,324764,1,260243435,233150937,499,0,d9a24379,off,,,,,,, +1536,c,none,sorted,500,2463534242,20,0,344,194140293,90,0,0,d9a24379,off,,,,,,, +1537,node,insertion,random,500,2463534242,24,4059,1,248498004,219477322,60338,60339,11ae932d,off,,,,,,, +1538,python,insertion,random,2000,2463534242,22,3,1,319144811,302296604,996390,996389,41262855,off,,,,,,, +1539,python,quick,reversed,1000,2463534242,5,8,1,264647971,249147622,499500,1000,0ca43265,off,,,,,,, +1540,c,quick,random,250,2463534242,20,136055,1,233852438,232750724,1933,1940,b47adce4,off,,,,,,, +1541,node,none,random,500,2463534242,13,0,9,234924657,822,0,0,11ae932d,off,,,,,,, +1542,c,insertion,nearly_sorted,500,2463534242,6,32350,1,239422816,238292989,7962,7926,482b4785,off,,,,,,, +1543,node,insertion,reversed,2000,2463534242,7,144,1,261942190,233877035,1999000,2000999,ad35ba05,off,,,,,,, +1544,c,none,reversed,250,2463534242,3,0,427,236972304,140,0,0,be175004,off,,,,,,, +1545,c,none,nearly_sorted,2000,2463534242,17,0,387,260859410,80,0,0,5835dc59,off,,,,,,, +1546,node,insertion,reversed,500,2463534242,6,2178,1,274393865,242853649,124750,125249,d33e3559,off,,,,,,, +1547,c,quick,nearly_sorted,2000,2463534242,1,6145,1,248747974,247907848,58002,11768,5835dc59,off,,,,,,, +1548,c,quick,random,250,2463534242,10,136055,1,244039018,243171163,1933,1940,b47adce4,off,,,,,,, +1549,c,insertion,nearly_sorted,250,2463534242,13,99421,1,239631648,238692367,2277,2251,21a19c14,off,,,,,,, +1550,python,none,reversed,500,2463534242,9,0,16,260527627,651,0,0,d33e3559,off,,,,,,, +1551,python,quick,reversed,2000,2463534242,8,2,1,334722645,316249006,1999000,2000,ad35ba05,off,,,,,,, +1552,node,none,reversed,500,2463534242,3,0,9,252438920,942,0,0,d33e3559,off,,,,,,, +1553,python,bubble,random,2000,2463534242,5,2,1,390042419,371797384,1999000,1988792,41262855,off,,,,,,, +1554,node,insertion,random,500,2463534242,5,4059,1,284341728,255350292,60338,60339,11ae932d,off,,,,,,, +1555,python,none,reversed,2000,2463534242,2,0,16,259248972,591,0,0,ad35ba05,off,,,,,,, +1556,c,quick,reversed,1000,2463534242,3,814,1,243238439,242357563,499500,1000,0ca43265,off,,,,,,, +1557,node,insertion,reversed,500,2463534242,9,2178,1,260241210,232343946,124750,125249,d33e3559,off,,,,,,, +1558,c,none,reversed,2000,2463534242,21,0,413,253527430,90,0,0,ad35ba05,off,,,,,,, +1559,node,none,random,250,2463534242,30,0,10,268568187,821,0,0,b47adce4,off,,,,,,, +1560,c,quick,sorted,250,2463534242,4,10851,1,246435348,245517157,31125,0,7fd8c1f4,off,,,,,,, +1561,c,quick,sorted,2000,2463534242,26,174,1,253099137,251912916,1999000,0,69842aa5,off,,,,,,, +1562,c,quick,nearly_sorted,2000,2463534242,3,6145,1,250243982,249178319,58002,11768,5835dc59,off,,,,,,, +1563,c,none,nearly_sorted,500,2463534242,14,0,405,232860582,90,0,0,482b4785,off,,,,,,, +1564,python,insertion,sorted,500,2463534242,19,6624,1,250069348,233988421,499,0,d9a24379,off,,,,,,, +1565,node,insertion,nearly_sorted,500,2463534242,14,26045,1,255665266,227759909,7962,7926,482b4785,off,,,,,,, +1566,node,bubble,nearly_sorted,2000,2463534242,24,110,1,288470916,254829419,1999000,232556,5835dc59,off,,,,,,, +1567,c,insertion,reversed,250,2463534242,15,15645,1,244565575,243679425,31125,31374,be175004,off,,,,,,, +1568,c,none,sorted,1000,2463534242,22,0,421,241186421,141,0,0,8e15cb75,off,,,,,,, +1569,node,quick,sorted,500,2463534242,26,1164,1,272132097,241392345,124750,0,d9a24379,off,,,,,,, +1570,c,bubble,reversed,250,2463534242,19,1384,1,240145029,239255813,31125,62250,be175004,off,,,,,,, +1571,c,quick,sorted,250,2463534242,14,10851,1,245649085,244532284,31125,0,7fd8c1f4,off,,,,,,, +1572,python,quick,nearly_sorted,2000,2463534242,3,55,1,247605070,231264953,58002,11768,5835dc59,off,,,,,,, +1573,c,bubble,random,1000,2463534242,26,154,1,235397713,234257406,499500,496926,57561285,off,,,,,,, +1574,python,insertion,nearly_sorted,500,2463534242,6,316,1,256822866,241326547,7962,7926,482b4785,off,,,,,,, +1575,c,bubble,reversed,1000,2463534242,30,87,1,245198802,244257727,499500,999000,0ca43265,off,,,,,,, +1576,c,quick,nearly_sorted,2000,2463534242,15,6145,1,249350950,248361599,58002,11768,5835dc59,off,,,,,,, +1577,node,insertion,sorted,500,2463534242,2,324764,1,264511134,236779054,499,0,d9a24379,off,,,,,,, +1578,python,bubble,nearly_sorted,1000,2463534242,29,9,1,247173080,231546135,499500,59780,1cfe4afd,off,,,,,,, +1579,c,none,nearly_sorted,1000,2463534242,21,0,421,237520893,90,0,0,1cfe4afd,off,,,,,,, +1580,c,bubble,nearly_sorted,2000,2463534242,2,121,1,237763116,236876486,1999000,232556,5835dc59,off,,,,,,, +1581,python,quick,random,1000,2463534242,9,209,1,257347969,241120732,11196,10210,57561285,off,,,,,,, +1582,node,quick,random,2000,2463534242,24,2126,1,262517458,234545732,25197,24544,41262855,off,,,,,,, +1583,node,bubble,nearly_sorted,1000,2463534242,23,413,1,270229551,242260644,499500,59780,1cfe4afd,off,,,,,,, +1584,python,insertion,random,500,2463534242,15,44,1,262768919,246348031,60338,60339,11ae932d,off,,,,,,, +1585,c,bubble,sorted,250,2463534242,16,11386,1,240900786,239752194,31125,0,7fd8c1f4,off,,,,,,, +1586,node,quick,reversed,250,2463534242,2,4755,1,268819709,240896210,31125,250,be175004,off,,,,,,, +1587,python,none,random,500,2463534242,27,0,16,233987339,581,0,0,11ae932d,off,,,,,,, +1588,c,quick,reversed,1000,2463534242,30,814,1,238308240,237289971,499500,1000,0ca43265,off,,,,,,, +1589,c,bubble,sorted,1000,2463534242,20,702,1,244043493,242928387,499500,0,8e15cb75,off,,,,,,, +1590,c,none,reversed,1000,2463534242,23,0,405,231629963,90,0,0,0ca43265,off,,,,,,, +1591,c,bubble,sorted,2000,2463534242,5,173,1,233497582,232735393,1999000,0,69842aa5,off,,,,,,, +1592,node,bubble,sorted,500,2463534242,4,1964,1,265764002,238032199,124750,0,d9a24379,off,,,,,,, +1593,node,bubble,sorted,1000,2463534242,29,374,1,239808959,212587568,499500,0,8e15cb75,off,,,,,,, +1594,node,insertion,random,1000,2463534242,12,1091,1,269611422,240985989,249457,249457,57561285,off,,,,,,, +1595,python,bubble,reversed,250,2463534242,4,88,1,249014422,234432655,31125,62250,be175004,off,,,,,,, +1596,node,insertion,reversed,500,2463534242,28,2178,1,261138483,233356495,124750,125249,d33e3559,off,,,,,,, +1597,node,quick,sorted,2000,2463534242,17,74,1,271598205,242121805,1999000,0,69842aa5,off,,,,,,, +1598,c,insertion,reversed,1000,2463534242,16,1033,1,238077225,236860064,499500,500499,0ca43265,off,,,,,,, +1599,python,bubble,sorted,500,2463534242,15,46,1,266910992,250418846,124750,0,d9a24379,off,,,,,,, +1600,c,quick,sorted,2000,2463534242,1,174,1,238584475,237298797,1999000,0,69842aa5,off,,,,,,, +1601,node,none,nearly_sorted,250,2463534242,4,0,10,251290197,542,0,0,21a19c14,off,,,,,,, +1602,python,quick,reversed,250,2463534242,19,151,1,251283133,236317478,31125,250,be175004,off,,,,,,, +1603,c,insertion,nearly_sorted,1000,2463534242,11,11445,1,240231873,239298950,30889,30860,1cfe4afd,off,,,,,,, +1604,python,quick,reversed,1000,2463534242,11,8,1,271034720,255827171,499500,1000,0ca43265,off,,,,,,, +1605,python,bubble,sorted,1000,2463534242,23,11,1,275542426,260629696,499500,0,8e15cb75,off,,,,,,, +1606,node,quick,nearly_sorted,250,2463534242,9,25320,1,251922963,225256106,4346,1008,21a19c14,off,,,,,,, +1607,c,none,sorted,2000,2463534242,14,0,416,236533281,90,0,0,69842aa5,off,,,,,,, +1608,python,bubble,sorted,1000,2463534242,19,11,1,282593773,266060295,499500,0,8e15cb75,off,,,,,,, +1609,c,quick,sorted,1000,2463534242,20,691,1,240532917,239619215,499500,0,8e15cb75,off,,,,,,, +1610,python,quick,random,500,2463534242,28,507,1,258317898,242745096,4578,4552,11ae932d,off,,,,,,, +1611,node,insertion,nearly_sorted,500,2463534242,16,26045,1,249291571,220871524,7962,7926,482b4785,off,,,,,,, +1612,c,none,nearly_sorted,500,2463534242,29,0,405,229643539,90,0,0,482b4785,off,,,,,,, +1613,node,quick,nearly_sorted,500,2463534242,27,10954,1,271991703,243168949,10192,1850,482b4785,off,,,,,,, +1614,c,quick,sorted,250,2463534242,27,10851,1,240503963,239666339,31125,0,7fd8c1f4,off,,,,,,, +1615,c,none,nearly_sorted,500,2463534242,24,0,405,225743140,80,0,0,482b4785,off,,,,,,, +1616,node,bubble,random,500,2463534242,7,1407,1,267952765,237730075,124750,119688,11ae932d,off,,,,,,, +1617,node,quick,sorted,250,2463534242,4,4507,1,267418131,240076992,31125,0,7fd8c1f4,off,,,,,,, +1618,python,bubble,nearly_sorted,1000,2463534242,6,9,1,250372467,234886437,499500,59780,1cfe4afd,off,,,,,,, +1619,python,none,sorted,500,2463534242,19,0,16,238660258,561,0,0,d9a24379,off,,,,,,, +1620,python,quick,reversed,2000,2463534242,17,2,1,293044848,276471169,1999000,2000,ad35ba05,off,,,,,,, +1621,c,quick,reversed,500,2463534242,22,3240,1,234956416,233802503,124750,500,d33e3559,off,,,,,,, +1622,python,none,reversed,500,2463534242,2,0,16,234226014,561,0,0,d33e3559,off,,,,,,, +1623,node,bubble,reversed,2000,2463534242,5,94,1,274348704,242178739,1999000,3998000,ad35ba05,off,,,,,,, +1624,c,none,random,1000,2463534242,19,0,367,208623697,100,0,0,57561285,off,,,,,,, +1625,node,bubble,random,250,2463534242,9,5439,1,255776945,226275512,31125,31196,b47adce4,off,,,,,,, +1626,python,bubble,nearly_sorted,250,2463534242,18,188,1,262202429,246532544,31125,4056,21a19c14,off,,,,,,, +1627,c,insertion,sorted,250,2463534242,11,786886,1,243717354,242573672,249,0,7fd8c1f4,off,,,,,,, +1628,c,quick,nearly_sorted,500,2463534242,29,31784,1,242919429,241737825,10192,1850,482b4785,off,,,,,,, +1629,node,bubble,nearly_sorted,500,2463534242,10,1566,1,271365439,240597524,124750,14926,482b4785,off,,,,,,, +1630,node,none,sorted,2000,2463534242,26,0,10,262386292,831,0,0,69842aa5,off,,,,,,, +1631,python,insertion,reversed,2000,2463534242,6,2,1,429255988,410923146,1999000,2000999,ad35ba05,off,,,,,,, +1632,python,insertion,random,1000,2463534242,20,10,1,263035508,246010856,249457,249457,57561285,off,,,,,,, +1633,python,quick,random,250,2463534242,22,1271,1,253834926,238798241,1933,1940,b47adce4,off,,,,,,, +1634,python,bubble,sorted,500,2463534242,16,46,1,257265094,241776062,124750,0,d9a24379,off,,,,,,, +1635,python,quick,random,2000,2463534242,9,90,1,261452201,244859625,25197,24544,41262855,off,,,,,,, +1636,c,quick,nearly_sorted,1000,2463534242,14,11273,1,246900953,245955990,28499,7892,1cfe4afd,off,,,,,,, +1637,python,none,random,1000,2463534242,6,0,15,242191924,612,0,0,57561285,off,,,,,,, +1638,node,insertion,reversed,500,2463534242,1,2178,1,261041552,234003456,124750,125249,d33e3559,off,,,,,,, +1639,node,bubble,nearly_sorted,2000,2463534242,26,110,1,263770826,234985034,1999000,232556,5835dc59,off,,,,,,, +1640,python,none,nearly_sorted,2000,2463534242,19,0,15,244751323,561,0,0,5835dc59,off,,,,,,, +1641,node,bubble,nearly_sorted,1000,2463534242,13,413,1,271263148,243051545,499500,59780,1cfe4afd,off,,,,,,, +1642,python,quick,random,250,2463534242,5,1271,1,251110921,235229176,1933,1940,b47adce4,off,,,,,,, +1643,python,none,sorted,1000,2463534242,26,0,16,248668844,551,0,0,8e15cb75,off,,,,,,, +1644,python,none,reversed,1000,2463534242,22,0,16,239973107,701,0,0,0ca43265,off,,,,,,, +1645,python,insertion,random,1000,2463534242,2,10,1,279072854,261313535,249457,249457,57561285,off,,,,,,, +1646,node,bubble,sorted,1000,2463534242,14,374,1,272863697,241399989,499500,0,8e15cb75,off,,,,,,, +1647,node,insertion,sorted,250,2463534242,24,607595,1,252551640,225187623,249,0,7fd8c1f4,off,,,,,,, +1648,node,insertion,random,2000,2463534242,14,234,1,276297593,247839780,996390,996389,41262855,off,,,,,,, +1649,python,none,reversed,250,2463534242,8,0,17,255112651,531,0,0,be175004,off,,,,,,, +1650,python,bubble,random,2000,2463534242,1,2,1,377791096,361126799,1999000,1988792,41262855,off,,,,,,, +1651,node,none,sorted,2000,2463534242,29,0,10,265405193,912,0,0,69842aa5,off,,,,,,, +1652,node,quick,sorted,2000,2463534242,4,74,1,276994448,245777730,1999000,0,69842aa5,off,,,,,,, +1653,c,quick,random,250,2463534242,2,136055,1,262958874,262095046,1933,1940,b47adce4,off,,,,,,, +1654,c,insertion,sorted,250,2463534242,13,786886,1,246197954,245088024,249,0,7fd8c1f4,off,,,,,,, +1655,node,none,random,500,2463534242,25,0,9,242382142,1002,0,0,11ae932d,off,,,,,,, +1656,c,insertion,random,1000,2463534242,22,1983,1,244248651,243172970,249457,249457,57561285,off,,,,,,, +1657,c,quick,sorted,1000,2463534242,12,691,1,245051592,243969949,499500,0,8e15cb75,off,,,,,,, +1658,c,quick,sorted,2000,2463534242,22,174,1,239137736,238080823,1999000,0,69842aa5,off,,,,,,, +1659,python,insertion,random,1000,2463534242,4,10,1,255847327,239043313,249457,249457,57561285,off,,,,,,, +1660,python,quick,nearly_sorted,2000,2463534242,25,55,1,248617548,232265812,58002,11768,5835dc59,off,,,,,,, +1661,python,bubble,random,250,2463534242,10,112,1,254893949,239333883,31125,31196,b47adce4,off,,,,,,, +1662,c,insertion,sorted,2000,2463534242,21,101352,1,248589697,247545811,1999,0,69842aa5,off,,,,,,, +1663,node,insertion,nearly_sorted,2000,2463534242,14,2100,1,261888427,233099351,118277,118270,5835dc59,off,,,,,,, +1664,node,bubble,nearly_sorted,2000,2463534242,21,110,1,267044503,239231070,1999000,232556,5835dc59,off,,,,,,, +1665,node,bubble,sorted,250,2463534242,12,8081,1,265743482,234972146,31125,0,7fd8c1f4,off,,,,,,, +1666,python,none,random,250,2463534242,19,0,17,260687177,591,0,0,b47adce4,off,,,,,,, +1667,c,quick,reversed,250,2463534242,11,12781,1,238502802,237654663,31125,250,be175004,off,,,,,,, +1668,python,bubble,sorted,1000,2463534242,28,11,1,279526032,263906294,499500,0,8e15cb75,off,,,,,,, +1669,node,bubble,random,500,2463534242,14,1407,1,276437996,248079273,124750,119688,11ae932d,off,,,,,,, +1670,python,insertion,sorted,2000,2463534242,27,1523,1,262194733,244764281,1999,0,69842aa5,off,,,,,,, +1671,python,none,reversed,2000,2463534242,15,0,16,242838527,552,0,0,ad35ba05,off,,,,,,, +1672,python,bubble,sorted,250,2463534242,3,206,1,247464717,232578359,31125,0,7fd8c1f4,off,,,,,,, +1673,c,bubble,sorted,500,2463534242,15,2858,1,240645798,239765499,124750,0,d9a24379,off,,,,,,, +1674,python,quick,random,250,2463534242,10,1271,1,250736098,235059025,1933,1940,b47adce4,off,,,,,,, +1675,python,quick,sorted,500,2463534242,19,30,1,242847535,227800135,124750,0,d9a24379,off,,,,,,, +1676,node,insertion,sorted,250,2463534242,6,607595,1,251075404,223402525,249,0,7fd8c1f4,off,,,,,,, +1677,python,bubble,random,2000,2463534242,12,2,1,360358259,342990262,1999000,1988792,41262855,off,,,,,,, +1678,node,quick,sorted,2000,2463534242,19,74,1,268499948,241200187,1999000,0,69842aa5,off,,,,,,, +1679,node,bubble,random,1000,2463534242,3,362,1,268761740,241274421,499500,496926,57561285,off,,,,,,, +1680,python,quick,nearly_sorted,500,2463534242,4,296,1,245540211,229253870,10192,1850,482b4785,off,,,,,,, +1681,c,quick,nearly_sorted,1000,2463534242,13,11273,1,246771058,245802441,28499,7892,1cfe4afd,off,,,,,,, +1682,python,insertion,reversed,2000,2463534242,25,2,1,428445341,412056450,1999000,2000999,ad35ba05,off,,,,,,, +1683,python,quick,reversed,2000,2463534242,24,2,1,282128860,266031964,1999000,2000,ad35ba05,off,,,,,,, +1684,c,bubble,sorted,2000,2463534242,26,173,1,238607055,237707933,1999000,0,69842aa5,off,,,,,,, +1685,c,quick,sorted,500,2463534242,28,2762,1,246068291,244938553,124750,0,d9a24379,off,,,,,,, +1686,python,none,nearly_sorted,500,2463534242,20,0,17,252095859,521,0,0,482b4785,off,,,,,,, +1687,node,none,sorted,250,2463534242,1,0,9,245393635,682,0,0,7fd8c1f4,off,,,,,,, +1688,python,none,reversed,1000,2463534242,29,0,16,256044787,571,0,0,0ca43265,off,,,,,,, +1689,python,insertion,nearly_sorted,1000,2463534242,1,76,1,255959668,240647290,30889,30860,1cfe4afd,off,,,,,,, +1690,node,insertion,nearly_sorted,2000,2463534242,27,2100,1,256904508,227936038,118277,118270,5835dc59,off,,,,,,, +1691,c,quick,random,2000,2463534242,4,10587,1,238863759,237948269,25197,24544,41262855,off,,,,,,, +1692,python,bubble,nearly_sorted,1000,2463534242,21,9,1,268286269,252401660,499500,59780,1cfe4afd,off,,,,,,, +1693,python,none,nearly_sorted,2000,2463534242,15,0,15,236759886,561,0,0,5835dc59,off,,,,,,, +1694,node,bubble,random,2000,2463534242,15,98,1,266479694,237703962,1999000,1988792,41262855,off,,,,,,, +1695,c,bubble,sorted,2000,2463534242,22,173,1,234800110,233894726,1999000,0,69842aa5,off,,,,,,, +1696,node,quick,nearly_sorted,1000,2463534242,19,3752,1,254791362,228399184,28499,7892,1cfe4afd,off,,,,,,, +1697,node,bubble,random,250,2463534242,5,5439,1,253676018,225622440,31125,31196,b47adce4,off,,,,,,, +1698,c,none,reversed,1000,2463534242,7,0,405,232289869,90,0,0,0ca43265,off,,,,,,, +1699,node,insertion,reversed,250,2463534242,29,8539,1,270669925,241759315,31125,31374,be175004,off,,,,,,, +1700,c,quick,random,500,2463534242,28,55776,1,254200040,253306676,4578,4552,11ae932d,off,,,,,,, +1701,node,bubble,nearly_sorted,2000,2463534242,23,110,1,263599817,233708218,1999000,232556,5835dc59,off,,,,,,, +1702,node,none,sorted,250,2463534242,5,0,9,231410080,721,0,0,7fd8c1f4,off,,,,,,, +1703,node,none,sorted,250,2463534242,30,0,9,239208786,781,0,0,7fd8c1f4,off,,,,,,, +1704,node,quick,reversed,2000,2463534242,25,80,1,274227192,245760176,1999000,2000,ad35ba05,off,,,,,,, +1705,python,insertion,nearly_sorted,250,2463534242,27,1416,1,256755499,241124734,2277,2251,21a19c14,off,,,,,,, +1706,c,none,reversed,1000,2463534242,27,0,405,231508344,90,0,0,0ca43265,off,,,,,,, +1707,node,insertion,sorted,2000,2463534242,24,89220,1,271977005,242709270,1999,0,69842aa5,off,,,,,,, +1708,c,quick,random,500,2463534242,1,55776,1,237993397,237117284,4578,4552,11ae932d,off,,,,,,, +1709,node,bubble,sorted,1000,2463534242,24,374,1,258335370,230057475,499500,0,8e15cb75,off,,,,,,, +1710,node,quick,nearly_sorted,1000,2463534242,13,3752,1,297838552,267295978,28499,7892,1cfe4afd,off,,,,,,, +1711,node,insertion,reversed,500,2463534242,19,2178,1,266746934,236538489,124750,125249,d33e3559,off,,,,,,, +1712,node,insertion,sorted,1000,2463534242,19,160107,1,261959642,232931158,999,0,8e15cb75,off,,,,,,, +1713,python,insertion,reversed,500,2463534242,12,22,1,246115289,230979949,124750,125249,d33e3559,off,,,,,,, +1714,node,bubble,sorted,500,2463534242,30,1964,1,255061023,227492871,124750,0,d9a24379,off,,,,,,, +1715,node,insertion,random,250,2463534242,14,15349,1,262404446,235069495,15844,15838,b47adce4,off,,,,,,, +1716,node,quick,nearly_sorted,250,2463534242,19,25320,1,262252891,234339236,4346,1008,21a19c14,off,,,,,,, +1717,python,quick,random,500,2463534242,18,507,1,256667955,241163868,4578,4552,11ae932d,off,,,,,,, +1718,c,none,random,2000,2463534242,30,0,415,238457437,80,0,0,41262855,off,,,,,,, +1719,node,bubble,nearly_sorted,250,2463534242,4,6161,1,267249536,236702877,31125,4056,21a19c14,off,,,,,,, +1720,python,quick,random,500,2463534242,19,507,1,263383652,247809703,4578,4552,11ae932d,off,,,,,,, +1721,node,quick,reversed,1000,2463534242,8,307,1,280583904,246426842,499500,1000,0ca43265,off,,,,,,, +1722,c,insertion,random,500,2463534242,17,6945,1,229864624,228867597,60338,60339,11ae932d,off,,,,,,, +1723,node,quick,reversed,250,2463534242,30,4755,1,291211122,258002666,31125,250,be175004,off,,,,,,, +1724,node,quick,nearly_sorted,250,2463534242,28,25320,1,272318595,242047907,4346,1008,21a19c14,off,,,,,,, +1725,c,quick,sorted,500,2463534242,12,2762,1,243898345,242918958,124750,0,d9a24379,off,,,,,,, +1726,python,insertion,sorted,500,2463534242,28,6624,1,246625854,230308540,499,0,d9a24379,off,,,,,,, +1727,python,bubble,reversed,250,2463534242,8,88,1,258022633,242434954,31125,62250,be175004,off,,,,,,, +1728,node,quick,reversed,2000,2463534242,29,80,1,270438873,242148317,1999000,2000,ad35ba05,off,,,,,,, +1729,node,none,reversed,500,2463534242,27,0,9,234348597,842,0,0,d33e3559,off,,,,,,, +1730,python,insertion,random,1000,2463534242,14,10,1,257825003,241947923,249457,249457,57561285,off,,,,,,, +1731,c,quick,nearly_sorted,2000,2463534242,4,6145,1,252118308,251174801,58002,11768,5835dc59,off,,,,,,, +1732,python,quick,sorted,250,2463534242,9,133,1,257316952,241686662,31125,0,7fd8c1f4,off,,,,,,, +1733,python,insertion,sorted,500,2463534242,10,6624,1,245866111,229909644,499,0,d9a24379,off,,,,,,, +1734,c,none,sorted,500,2463534242,27,0,344,200262096,90,0,0,d9a24379,off,,,,,,, +1735,python,insertion,nearly_sorted,1000,2463534242,22,76,1,252266748,235771742,30889,30860,1cfe4afd,off,,,,,,, +1736,node,quick,reversed,250,2463534242,7,4755,1,266817496,236747601,31125,250,be175004,off,,,,,,, +1737,python,quick,reversed,2000,2463534242,3,2,1,273534244,257442643,1999000,2000,ad35ba05,off,,,,,,, +1738,node,none,nearly_sorted,1000,2463534242,10,0,10,273310384,861,0,0,1cfe4afd,off,,,,,,, +1739,node,quick,sorted,500,2463534242,8,1164,1,277870341,249471800,124750,0,d9a24379,off,,,,,,, +1740,python,insertion,reversed,1000,2463534242,6,6,1,293172896,277426365,499500,500499,0ca43265,off,,,,,,, +1741,c,bubble,random,1000,2463534242,17,154,1,234142791,233223578,499500,496926,57561285,off,,,,,,, +1742,c,insertion,reversed,250,2463534242,21,15645,1,244494401,243560681,31125,31374,be175004,off,,,,,,, +1743,c,insertion,reversed,500,2463534242,6,3967,1,239674898,238720381,124750,125249,d33e3559,off,,,,,,, +1744,python,insertion,nearly_sorted,250,2463534242,19,1416,1,254339030,239143333,2277,2251,21a19c14,off,,,,,,, +1745,c,none,reversed,250,2463534242,17,0,427,237903018,91,0,0,be175004,off,,,,,,, +1746,node,bubble,random,2000,2463534242,16,98,1,267688508,238421239,1999000,1988792,41262855,off,,,,,,, +1747,python,bubble,reversed,250,2463534242,26,88,1,255555941,239923760,31125,62250,be175004,off,,,,,,, +1748,python,quick,random,2000,2463534242,28,90,1,256173349,239386106,25197,24544,41262855,off,,,,,,, +1749,python,insertion,nearly_sorted,500,2463534242,8,316,1,247929378,233201851,7962,7926,482b4785,off,,,,,,, +1750,python,none,nearly_sorted,250,2463534242,26,0,17,238215554,521,0,0,21a19c14,off,,,,,,, +1751,c,insertion,random,2000,2463534242,21,492,1,233847888,232857414,996390,996389,41262855,off,,,,,,, +1752,python,bubble,nearly_sorted,1000,2463534242,15,9,1,256363394,240356049,499500,59780,1cfe4afd,off,,,,,,, +1753,c,insertion,sorted,500,2463534242,7,399335,1,242394366,241468790,499,0,d9a24379,off,,,,,,, +1754,c,quick,random,2000,2463534242,16,10587,1,231066125,229834889,25197,24544,41262855,off,,,,,,, +1755,c,bubble,sorted,250,2463534242,10,11386,1,243707987,242564334,31125,0,7fd8c1f4,off,,,,,,, +1756,c,bubble,nearly_sorted,2000,2463534242,22,121,1,240900146,239847432,1999000,232556,5835dc59,off,,,,,,, +1757,node,none,sorted,1000,2463534242,12,0,9,229038404,802,0,0,8e15cb75,off,,,,,,, +1758,c,quick,nearly_sorted,1000,2463534242,9,11273,1,239102626,238019268,28499,7892,1cfe4afd,off,,,,,,, +1759,node,insertion,random,250,2463534242,13,15349,1,256185701,228859528,15844,15838,b47adce4,off,,,,,,, +1760,python,quick,sorted,250,2463534242,13,133,1,248340599,233504721,31125,0,7fd8c1f4,off,,,,,,, +1761,c,none,random,1000,2463534242,21,0,367,209858070,100,0,0,57561285,off,,,,,,, +1762,python,none,sorted,500,2463534242,17,0,16,239366762,480,0,0,d9a24379,off,,,,,,, +1763,c,insertion,random,500,2463534242,6,6945,1,212843866,211829777,60338,60339,11ae932d,off,,,,,,, +1764,c,bubble,sorted,2000,2463534242,14,173,1,237291793,235981299,1999000,0,69842aa5,off,,,,,,, +1765,python,bubble,reversed,1000,2463534242,19,5,1,294916586,273387863,499500,999000,0ca43265,off,,,,,,, +1766,node,bubble,sorted,500,2463534242,3,1964,1,248935884,222002739,124750,0,d9a24379,off,,,,,,, +1767,python,insertion,reversed,500,2463534242,14,22,1,253919654,238854743,124750,125249,d33e3559,off,,,,,,, +1768,python,quick,random,1000,2463534242,30,209,1,258430007,242375385,11196,10210,57561285,off,,,,,,, +1769,c,insertion,sorted,2000,2463534242,18,101352,1,240818112,239884351,1999,0,69842aa5,off,,,,,,, +1770,python,bubble,sorted,1000,2463534242,20,11,1,276014292,260243321,499500,0,8e15cb75,off,,,,,,, +1771,python,none,sorted,1000,2463534242,12,0,16,236442221,661,0,0,8e15cb75,off,,,,,,, +1772,c,bubble,reversed,250,2463534242,4,1384,1,240713585,239758886,31125,62250,be175004,off,,,,,,, +1773,python,none,reversed,250,2463534242,27,0,17,242908759,551,0,0,be175004,off,,,,,,, +1774,c,insertion,sorted,250,2463534242,7,786886,1,243551845,242701612,249,0,7fd8c1f4,off,,,,,,, +1775,node,quick,random,2000,2463534242,25,2126,1,253980188,225252030,25197,24544,41262855,off,,,,,,, +1776,node,insertion,sorted,2000,2463534242,11,89220,1,264688386,237729930,1999,0,69842aa5,off,,,,,,, +1777,c,insertion,sorted,500,2463534242,21,399335,1,240692345,239803740,499,0,d9a24379,off,,,,,,, +1778,python,bubble,random,250,2463534242,27,112,1,254248812,237589498,31125,31196,b47adce4,off,,,,,,, +1779,python,quick,random,500,2463534242,29,507,1,268479551,252871794,4578,4552,11ae932d,off,,,,,,, +1780,python,bubble,reversed,500,2463534242,4,20,1,257197778,241765086,124750,249500,d33e3559,off,,,,,,, +1781,c,quick,nearly_sorted,250,2463534242,6,77221,1,242247500,241077810,4346,1008,21a19c14,off,,,,,,, +1782,python,bubble,random,250,2463534242,19,112,1,261270570,246347633,31125,31196,b47adce4,off,,,,,,, +1783,python,quick,reversed,2000,2463534242,9,2,1,280103000,263842739,1999000,2000,ad35ba05,off,,,,,,, +1784,c,quick,random,2000,2463534242,23,10587,1,237199072,236043505,25197,24544,41262855,off,,,,,,, +1785,python,bubble,reversed,2000,2463534242,23,2,1,512965657,496727213,1999000,3998000,ad35ba05,off,,,,,,, +1786,python,none,sorted,1000,2463534242,19,0,16,237355732,792,0,0,8e15cb75,off,,,,,,, +1787,node,none,random,2000,2463534242,7,0,9,232507817,822,0,0,41262855,off,,,,,,, +1788,python,insertion,nearly_sorted,250,2463534242,14,1416,1,258480031,241890724,2277,2251,21a19c14,off,,,,,,, +1789,c,none,reversed,500,2463534242,26,0,402,232698875,80,0,0,d33e3559,off,,,,,,, +1790,node,none,reversed,1000,2463534242,22,0,9,235055852,691,0,0,0ca43265,off,,,,,,, +1791,c,insertion,sorted,250,2463534242,17,786886,1,246095882,245203329,249,0,7fd8c1f4,off,,,,,,, +1792,node,insertion,reversed,500,2463534242,25,2178,1,281888340,251943692,124750,125249,d33e3559,off,,,,,,, +1793,node,insertion,nearly_sorted,500,2463534242,18,26045,1,270006532,238460616,7962,7926,482b4785,off,,,,,,, +1794,node,quick,random,250,2463534242,14,36348,1,260105177,230581630,1933,1940,b47adce4,off,,,,,,, +1795,node,none,reversed,2000,2463534242,30,0,9,233008305,782,0,0,ad35ba05,off,,,,,,, +1796,node,none,reversed,250,2463534242,21,0,9,230314937,652,0,0,be175004,off,,,,,,, +1797,node,quick,nearly_sorted,250,2463534242,18,25320,1,267465621,238103723,4346,1008,21a19c14,off,,,,,,, +1798,python,insertion,sorted,250,2463534242,4,18845,1,284075499,268299949,249,0,7fd8c1f4,off,,,,,,, +1799,c,quick,random,2000,2463534242,29,10587,1,255764622,254836855,25197,24544,41262855,off,,,,,,, +1800,c,bubble,random,2000,2463534242,28,40,1,246478119,245282447,1999000,1988792,41262855,off,,,,,,, +1801,node,insertion,reversed,2000,2463534242,24,144,1,268096694,236503979,1999000,2000999,ad35ba05,off,,,,,,, +1802,c,insertion,nearly_sorted,1000,2463534242,4,11445,1,243325511,242312842,30889,30860,1cfe4afd,off,,,,,,, +1803,node,bubble,random,500,2463534242,13,1407,1,265681307,238472914,124750,119688,11ae932d,off,,,,,,, +1804,node,insertion,sorted,2000,2463534242,8,89220,1,261863813,233538919,1999,0,69842aa5,off,,,,,,, +1805,c,quick,random,1000,2463534242,5,23484,1,230607806,229510680,11196,10210,57561285,off,,,,,,, +1806,c,none,sorted,1000,2463534242,26,0,421,239373715,91,0,0,8e15cb75,off,,,,,,, +1807,c,insertion,reversed,2000,2463534242,3,260,1,242262678,241520959,1999000,2000999,ad35ba05,off,,,,,,, +1808,node,quick,nearly_sorted,250,2463534242,11,25320,1,258287570,229833840,4346,1008,21a19c14,off,,,,,,, +1809,node,insertion,nearly_sorted,500,2463534242,4,26045,1,251169581,223475801,7962,7926,482b4785,off,,,,,,, +1810,python,none,random,1000,2463534242,22,0,15,227815435,842,0,0,57561285,off,,,,,,, +1811,c,none,random,1000,2463534242,6,0,367,209524373,81,0,0,57561285,off,,,,,,, +1812,c,bubble,reversed,1000,2463534242,8,87,1,244271573,243494499,499500,999000,0ca43265,off,,,,,,, +1813,python,none,reversed,2000,2463534242,12,0,16,251396836,641,0,0,ad35ba05,off,,,,,,, +1814,python,insertion,reversed,500,2463534242,28,22,1,245950130,230759596,124750,125249,d33e3559,off,,,,,,, +1815,node,insertion,random,500,2463534242,10,4059,1,269792852,242195101,60338,60339,11ae932d,off,,,,,,, +1816,c,none,nearly_sorted,2000,2463534242,15,0,387,218772856,90,0,0,5835dc59,off,,,,,,, +1817,node,quick,reversed,500,2463534242,5,1194,1,266010705,236971086,124750,500,d33e3559,off,,,,,,, +1818,python,bubble,nearly_sorted,1000,2463534242,23,9,1,256093639,240734389,499500,59780,1cfe4afd,off,,,,,,, +1819,python,none,random,250,2463534242,16,0,17,246434617,511,0,0,b47adce4,off,,,,,,, +1820,python,bubble,random,2000,2463534242,3,2,1,376578206,360162698,1999000,1988792,41262855,off,,,,,,, +1821,python,bubble,reversed,1000,2463534242,24,5,1,300605778,284532025,499500,999000,0ca43265,off,,,,,,, +1822,c,insertion,sorted,2000,2463534242,5,101352,1,236106671,234976363,1999,0,69842aa5,off,,,,,,, +1823,python,insertion,random,1000,2463534242,16,10,1,255445915,239281801,249457,249457,57561285,off,,,,,,, +1824,c,quick,random,1000,2463534242,19,23484,1,235939614,234785908,11196,10210,57561285,off,,,,,,, +1825,node,bubble,nearly_sorted,1000,2463534242,2,413,1,265763845,237563367,499500,59780,1cfe4afd,off,,,,,,, +1826,node,none,sorted,500,2463534242,5,0,10,257825284,781,0,0,d9a24379,off,,,,,,, +1827,c,quick,reversed,2000,2463534242,27,209,1,237639905,236743244,1999000,2000,ad35ba05,off,,,,,,, +1828,c,quick,random,250,2463534242,19,136055,1,243868147,242548444,1933,1940,b47adce4,off,,,,,,, +1829,node,insertion,nearly_sorted,500,2463534242,19,26045,1,245228667,217243865,7962,7926,482b4785,off,,,,,,, +1830,node,insertion,nearly_sorted,500,2463534242,3,26045,1,255011631,227743774,7962,7926,482b4785,off,,,,,,, +1831,node,bubble,random,500,2463534242,21,1407,1,276865296,248669459,124750,119688,11ae932d,off,,,,,,, +1832,node,insertion,reversed,500,2463534242,29,2178,1,260957403,233388945,124750,125249,d33e3559,off,,,,,,, +1833,python,insertion,sorted,250,2463534242,1,18845,1,274052004,252491967,249,0,7fd8c1f4,off,,,,,,, +1834,node,insertion,sorted,1000,2463534242,29,160107,1,264028600,233614796,999,0,8e15cb75,off,,,,,,, +1835,python,quick,nearly_sorted,1000,2463534242,14,110,1,260982640,244166514,28499,7892,1cfe4afd,off,,,,,,, +1836,node,insertion,sorted,500,2463534242,12,324764,1,260195627,230837004,499,0,d9a24379,off,,,,,,, +1837,node,bubble,random,500,2463534242,27,1407,1,267949257,240216241,124750,119688,11ae932d,off,,,,,,, +1838,node,quick,reversed,1000,2463534242,7,307,1,267092482,239026343,499500,1000,0ca43265,off,,,,,,, +1839,c,quick,sorted,1000,2463534242,13,691,1,236825178,235947563,499500,0,8e15cb75,off,,,,,,, +1840,c,insertion,random,250,2463534242,16,26020,1,229105390,228023513,15844,15838,b47adce4,off,,,,,,, +1841,c,quick,nearly_sorted,250,2463534242,25,77221,1,248647635,247150940,4346,1008,21a19c14,off,,,,,,, +1842,c,quick,random,1000,2463534242,18,23484,1,236399791,235220862,11196,10210,57561285,off,,,,,,, +1843,c,insertion,nearly_sorted,2000,2463534242,16,2962,1,226897153,225772396,118277,118270,5835dc59,off,,,,,,, +1844,python,none,random,2000,2463534242,3,0,16,279544548,912,0,0,41262855,off,,,,,,, +1845,c,none,nearly_sorted,2000,2463534242,5,0,387,224407046,90,0,0,5835dc59,off,,,,,,, +1846,node,none,sorted,500,2463534242,19,0,10,259254132,771,0,0,d9a24379,off,,,,,,, +1847,c,quick,reversed,2000,2463534242,3,209,1,252606294,251681496,1999000,2000,ad35ba05,off,,,,,,, +1848,python,bubble,random,500,2463534242,11,26,1,257482932,241674163,124750,119688,11ae932d,off,,,,,,, +1849,python,none,sorted,250,2463534242,21,0,17,240945560,531,0,0,7fd8c1f4,off,,,,,,, +1850,python,none,random,1000,2463534242,27,0,15,227512956,622,0,0,57561285,off,,,,,,, +1851,python,quick,reversed,2000,2463534242,19,2,1,272297526,256352671,1999000,2000,ad35ba05,off,,,,,,, +1852,python,quick,nearly_sorted,2000,2463534242,24,55,1,250407753,234058015,58002,11768,5835dc59,off,,,,,,, +1853,node,bubble,random,1000,2463534242,8,362,1,265903794,237195558,499500,496926,57561285,off,,,,,,, +1854,c,none,nearly_sorted,250,2463534242,10,0,388,216304802,90,0,0,21a19c14,off,,,,,,, +1855,python,insertion,reversed,250,2463534242,9,102,1,254527925,238397961,31125,31374,be175004,off,,,,,,, +1856,python,insertion,sorted,1000,2463534242,10,3178,1,257174541,241072272,999,0,8e15cb75,off,,,,,,, +1857,python,quick,random,2000,2463534242,4,90,1,275898268,257996672,25197,24544,41262855,off,,,,,,, +1858,node,bubble,nearly_sorted,500,2463534242,4,1566,1,273731894,244080797,124750,14926,482b4785,off,,,,,,, +1859,node,quick,nearly_sorted,1000,2463534242,20,3752,1,265864600,236985074,28499,7892,1cfe4afd,off,,,,,,, +1860,python,quick,sorted,1000,2463534242,24,8,1,292386394,275508830,499500,0,8e15cb75,off,,,,,,, +1861,node,bubble,reversed,1000,2463534242,11,378,1,266323169,238390035,499500,999000,0ca43265,off,,,,,,, +1862,c,insertion,sorted,1000,2463534242,12,196883,1,234882687,234020071,999,0,8e15cb75,off,,,,,,, +1863,node,insertion,reversed,250,2463534242,7,8539,1,262913319,235925957,31125,31374,be175004,off,,,,,,, +1864,c,bubble,reversed,2000,2463534242,19,22,1,245943176,245003046,1999000,3998000,ad35ba05,off,,,,,,, +1865,node,quick,reversed,500,2463534242,21,1194,1,261845618,233941140,124750,500,d33e3559,off,,,,,,, +1866,c,bubble,random,1000,2463534242,8,154,1,233773449,232925351,499500,496926,57561285,off,,,,,,, +1867,python,quick,nearly_sorted,1000,2463534242,28,110,1,263050348,247650859,28499,7892,1cfe4afd,off,,,,,,, +1868,python,none,random,1000,2463534242,4,0,15,226368440,661,0,0,57561285,off,,,,,,, +1869,node,bubble,reversed,500,2463534242,10,1534,1,262953164,235641429,124750,249500,d33e3559,off,,,,,,, +1870,c,none,reversed,500,2463534242,20,0,402,227996232,90,0,0,d33e3559,off,,,,,,, +1871,python,none,sorted,2000,2463534242,13,0,15,226727875,581,0,0,69842aa5,off,,,,,,, +1872,node,insertion,nearly_sorted,1000,2463534242,28,7758,1,264349912,236084292,30889,30860,1cfe4afd,off,,,,,,, +1873,c,bubble,sorted,1000,2463534242,2,702,1,238996156,238090399,499500,0,8e15cb75,off,,,,,,, +1874,python,insertion,random,250,2463534242,16,206,1,262831236,247578763,15844,15838,b47adce4,off,,,,,,, +1875,python,none,nearly_sorted,250,2463534242,16,0,17,250378720,521,0,0,21a19c14,off,,,,,,, +1876,python,none,sorted,250,2463534242,14,0,17,252207174,521,0,0,7fd8c1f4,off,,,,,,, +1877,node,insertion,nearly_sorted,250,2463534242,4,79550,1,238610254,210185357,2277,2251,21a19c14,off,,,,,,, +1878,python,quick,nearly_sorted,250,2463534242,13,882,1,261893778,246551848,4346,1008,21a19c14,off,,,,,,, +1879,python,quick,sorted,1000,2463534242,12,8,1,286891156,270934155,499500,0,8e15cb75,off,,,,,,, +1880,c,quick,nearly_sorted,500,2463534242,10,31784,1,237000016,235618799,10192,1850,482b4785,off,,,,,,, +1881,c,bubble,sorted,1000,2463534242,11,702,1,242981797,241508427,499500,0,8e15cb75,off,,,,,,, +1882,c,bubble,nearly_sorted,250,2463534242,17,7880,1,235744863,234440620,31125,4056,21a19c14,off,,,,,,, +1883,python,bubble,reversed,500,2463534242,12,20,1,259348098,244022005,124750,249500,d33e3559,off,,,,,,, +1884,node,quick,reversed,500,2463534242,10,1194,1,257892911,230164079,124750,500,d33e3559,off,,,,,,, +1885,node,none,sorted,250,2463534242,2,0,9,219639431,701,0,0,7fd8c1f4,off,,,,,,, +1886,python,none,random,500,2463534242,13,0,16,230785229,541,0,0,11ae932d,off,,,,,,, +1887,python,none,nearly_sorted,250,2463534242,3,0,17,236989997,511,0,0,21a19c14,off,,,,,,, +1888,python,bubble,sorted,2000,2463534242,28,3,1,343004066,326798217,1999000,0,69842aa5,off,,,,,,, +1889,c,none,nearly_sorted,2000,2463534242,7,0,387,226473918,100,0,0,5835dc59,off,,,,,,, +1890,c,insertion,random,2000,2463534242,17,492,1,232437745,231688852,996390,996389,41262855,off,,,,,,, +1891,c,quick,reversed,250,2463534242,7,12781,1,242457504,241169780,31125,250,be175004,off,,,,,,, +1892,c,bubble,random,500,2463534242,28,635,1,231118733,229762093,124750,119688,11ae932d,off,,,,,,, +1893,node,bubble,random,500,2463534242,22,1407,1,267601767,239426869,124750,119688,11ae932d,off,,,,,,, +1894,node,insertion,nearly_sorted,250,2463534242,2,79550,1,238482845,210530981,2277,2251,21a19c14,off,,,,,,, +1895,python,none,nearly_sorted,500,2463534242,17,0,17,248187813,542,0,0,482b4785,off,,,,,,, +1896,python,none,nearly_sorted,250,2463534242,23,0,17,249361352,471,0,0,21a19c14,off,,,,,,, +1897,node,none,sorted,500,2463534242,11,0,10,262973081,802,0,0,d9a24379,off,,,,,,, +1898,c,bubble,sorted,250,2463534242,5,11386,1,243391925,242442681,31125,0,7fd8c1f4,off,,,,,,, +1899,python,bubble,random,2000,2463534242,9,2,1,380285343,363533040,1999000,1988792,41262855,off,,,,,,, +1900,c,none,sorted,250,2463534242,18,0,429,245859461,90,0,0,7fd8c1f4,off,,,,,,, +1901,node,quick,nearly_sorted,250,2463534242,29,25320,1,267572940,235785397,4346,1008,21a19c14,off,,,,,,, +1902,node,bubble,random,2000,2463534242,11,98,1,267226153,238184009,1999000,1988792,41262855,off,,,,,,, +1903,python,bubble,random,2000,2463534242,11,2,1,359957768,343269644,1999000,1988792,41262855,off,,,,,,, +1904,c,none,nearly_sorted,2000,2463534242,10,0,387,222108528,161,0,0,5835dc59,off,,,,,,, +1905,node,quick,sorted,250,2463534242,21,4507,1,268028836,236788685,31125,0,7fd8c1f4,off,,,,,,, +1906,node,none,random,2000,2463534242,23,0,9,232345624,791,0,0,41262855,off,,,,,,, +1907,python,insertion,reversed,500,2463534242,9,22,1,247089124,232176680,124750,125249,d33e3559,off,,,,,,, +1908,python,bubble,sorted,1000,2463534242,26,11,1,279251618,264055199,499500,0,8e15cb75,off,,,,,,, +1909,python,bubble,nearly_sorted,1000,2463534242,7,9,1,247305189,231993359,499500,59780,1cfe4afd,off,,,,,,, +1910,node,none,reversed,250,2463534242,30,0,9,227300057,852,0,0,be175004,off,,,,,,, +1911,python,bubble,reversed,2000,2463534242,16,2,1,471230201,453436538,1999000,3998000,ad35ba05,off,,,,,,, +1912,c,quick,sorted,500,2463534242,1,2762,1,241063982,240001167,124750,0,d9a24379,off,,,,,,, +1913,python,insertion,reversed,1000,2463534242,24,6,1,294427430,278044377,499500,500499,0ca43265,off,,,,,,, +1914,node,insertion,sorted,250,2463534242,9,607595,1,255448478,225608595,249,0,7fd8c1f4,off,,,,,,, +1915,node,insertion,nearly_sorted,500,2463534242,27,26045,1,267218087,238606676,7962,7926,482b4785,off,,,,,,, +1916,python,insertion,nearly_sorted,500,2463534242,21,316,1,251837163,234592369,7962,7926,482b4785,off,,,,,,, +1917,node,bubble,sorted,1000,2463534242,4,374,1,259399362,231139699,499500,0,8e15cb75,off,,,,,,, +1918,c,bubble,nearly_sorted,500,2463534242,17,1998,1,238624441,237573207,124750,14926,482b4785,off,,,,,,, +1919,python,insertion,sorted,500,2463534242,1,6624,1,234260321,218718058,499,0,d9a24379,off,,,,,,, +1920,node,insertion,reversed,2000,2463534242,29,144,1,267748721,239231007,1999000,2000999,ad35ba05,off,,,,,,, +1921,node,none,sorted,250,2463534242,24,0,9,226681017,671,0,0,7fd8c1f4,off,,,,,,, +1922,c,bubble,nearly_sorted,500,2463534242,23,1998,1,240024384,237945798,124750,14926,482b4785,off,,,,,,, +1923,python,none,sorted,1000,2463534242,21,0,16,237174873,581,0,0,8e15cb75,off,,,,,,, +1924,c,none,reversed,2000,2463534242,16,0,413,233740668,90,0,0,ad35ba05,off,,,,,,, +1925,c,quick,reversed,500,2463534242,26,3240,1,239001407,238267661,124750,500,d33e3559,off,,,,,,, +1926,c,bubble,nearly_sorted,2000,2463534242,20,121,1,240754111,239761571,1999000,232556,5835dc59,off,,,,,,, +1927,python,quick,nearly_sorted,500,2463534242,8,296,1,241637326,226179373,10192,1850,482b4785,off,,,,,,, +1928,python,bubble,sorted,250,2463534242,20,206,1,270053310,255152920,31125,0,7fd8c1f4,off,,,,,,, +1929,node,insertion,reversed,2000,2463534242,4,144,1,266289276,235565399,1999000,2000999,ad35ba05,off,,,,,,, +1930,c,insertion,random,500,2463534242,26,6945,1,212463883,211412343,60338,60339,11ae932d,off,,,,,,, +1931,python,insertion,random,2000,2463534242,7,3,1,318278739,301788915,996390,996389,41262855,off,,,,,,, +1932,c,bubble,nearly_sorted,2000,2463534242,23,121,1,237455526,236155434,1999000,232556,5835dc59,off,,,,,,, +1933,python,none,reversed,2000,2463534242,10,0,16,242504806,571,0,0,ad35ba05,off,,,,,,, +1934,c,insertion,sorted,250,2463534242,27,786886,1,240523991,239602843,249,0,7fd8c1f4,off,,,,,,, +1935,c,bubble,reversed,500,2463534242,8,346,1,242530308,241584477,124750,249500,d33e3559,off,,,,,,, +1936,c,bubble,random,250,2463534242,12,2568,1,240255607,239374667,31125,31196,b47adce4,off,,,,,,, +1937,c,insertion,sorted,1000,2463534242,22,196883,1,237853245,236894368,999,0,8e15cb75,off,,,,,,, +1938,python,none,reversed,2000,2463534242,27,0,16,247585324,571,0,0,ad35ba05,off,,,,,,, +1939,c,insertion,reversed,1000,2463534242,26,1033,1,243648145,242750384,499500,500499,0ca43265,off,,,,,,, +1940,c,bubble,nearly_sorted,250,2463534242,27,7880,1,240173353,239027978,31125,4056,21a19c14,off,,,,,,, +1941,python,none,random,2000,2463534242,30,0,16,261883289,1703,0,0,41262855,off,,,,,,, +1942,node,quick,sorted,1000,2463534242,6,296,1,271191603,241658393,499500,0,8e15cb75,off,,,,,,, +1943,c,insertion,random,2000,2463534242,11,492,1,235319988,234455477,996390,996389,41262855,off,,,,,,, +1944,python,bubble,sorted,1000,2463534242,29,11,1,286481518,269781529,499500,0,8e15cb75,off,,,,,,, +1945,node,quick,reversed,250,2463534242,3,4755,1,263415801,237123792,31125,250,be175004,off,,,,,,, +1946,node,none,random,500,2463534242,29,0,9,228019526,731,0,0,11ae932d,off,,,,,,, +1947,python,bubble,random,250,2463534242,21,112,1,247700770,232507570,31125,31196,b47adce4,off,,,,,,, +1948,node,insertion,nearly_sorted,1000,2463534242,16,7758,1,259760159,233046294,30889,30860,1cfe4afd,off,,,,,,, +1949,c,none,nearly_sorted,1000,2463534242,20,0,421,239121001,131,0,0,1cfe4afd,off,,,,,,, +1950,node,quick,reversed,250,2463534242,18,4755,1,264377393,232750589,31125,250,be175004,off,,,,,,, +1951,node,none,random,250,2463534242,3,0,10,260535663,612,0,0,b47adce4,off,,,,,,, +1952,c,none,random,500,2463534242,13,0,426,241730311,90,0,0,11ae932d,off,,,,,,, +1953,python,insertion,reversed,500,2463534242,6,22,1,251611148,236470537,124750,125249,d33e3559,off,,,,,,, +1954,python,bubble,sorted,500,2463534242,5,46,1,276782452,261439782,124750,0,d9a24379,off,,,,,,, +1955,python,none,sorted,500,2463534242,14,0,16,235183131,531,0,0,d9a24379,off,,,,,,, +1956,python,quick,reversed,500,2463534242,18,33,1,251269618,236131254,124750,500,d33e3559,off,,,,,,, +1957,python,quick,nearly_sorted,2000,2463534242,6,55,1,247679379,231702762,58002,11768,5835dc59,off,,,,,,, +1958,node,quick,random,500,2463534242,28,16148,1,265669425,236643048,4578,4552,11ae932d,off,,,,,,, +1959,python,insertion,sorted,2000,2463534242,26,1523,1,263626345,247017681,1999,0,69842aa5,off,,,,,,, +1960,python,bubble,nearly_sorted,250,2463534242,26,188,1,252198769,237653853,31125,4056,21a19c14,off,,,,,,, +1961,python,bubble,sorted,250,2463534242,16,206,1,263555193,248587738,31125,0,7fd8c1f4,off,,,,,,, +1962,c,insertion,nearly_sorted,2000,2463534242,17,2962,1,224251344,223099585,118277,118270,5835dc59,off,,,,,,, +1963,python,none,nearly_sorted,1000,2463534242,10,0,16,233508933,591,0,0,1cfe4afd,off,,,,,,, +1964,python,bubble,sorted,500,2463534242,4,46,1,252967612,238203830,124750,0,d9a24379,off,,,,,,, +1965,c,bubble,random,250,2463534242,2,2568,1,241065955,239991432,31125,31196,b47adce4,off,,,,,,, +1966,c,quick,nearly_sorted,500,2463534242,30,31784,1,239568060,238443502,10192,1850,482b4785,off,,,,,,, +1967,python,bubble,random,1000,2463534242,30,6,1,267027508,249826210,499500,496926,57561285,off,,,,,,, +1968,python,bubble,nearly_sorted,2000,2463534242,1,3,1,354739820,338331331,1999000,232556,5835dc59,off,,,,,,, +1969,node,quick,sorted,2000,2463534242,15,74,1,271458354,242089811,1999000,0,69842aa5,off,,,,,,, +1970,python,bubble,random,250,2463534242,3,112,1,246069263,230954144,31125,31196,b47adce4,off,,,,,,, +1971,node,insertion,sorted,250,2463534242,14,607595,1,252233834,223897390,249,0,7fd8c1f4,off,,,,,,, +1972,c,quick,reversed,1000,2463534242,4,814,1,240139649,239088490,499500,1000,0ca43265,off,,,,,,, +1973,node,bubble,random,2000,2463534242,12,98,1,268481705,238357364,1999000,1988792,41262855,off,,,,,,, +1974,c,quick,random,250,2463534242,5,136055,1,240573923,239703172,1933,1940,b47adce4,off,,,,,,, +1975,node,none,sorted,2000,2463534242,1,0,10,266706668,1062,0,0,69842aa5,off,,,,,,, +1976,c,insertion,sorted,500,2463534242,30,399335,1,243698569,242755302,499,0,d9a24379,off,,,,,,, +1977,c,bubble,random,500,2463534242,25,635,1,229343487,228264567,124750,119688,11ae932d,off,,,,,,, +1978,c,quick,random,500,2463534242,16,55776,1,234833646,233920115,4578,4552,11ae932d,off,,,,,,, +1979,c,none,sorted,1000,2463534242,4,0,421,240202057,90,0,0,8e15cb75,off,,,,,,, +1980,python,none,reversed,1000,2463534242,5,0,16,244095233,641,0,0,0ca43265,off,,,,,,, +1981,c,none,nearly_sorted,500,2463534242,4,0,405,235827338,90,0,0,482b4785,off,,,,,,, +1982,python,bubble,sorted,500,2463534242,29,46,1,256588507,240256404,124750,0,d9a24379,off,,,,,,, +1983,python,bubble,sorted,500,2463534242,1,46,1,257372595,242390543,124750,0,d9a24379,off,,,,,,, +1984,python,bubble,reversed,250,2463534242,27,88,1,254216532,236780282,31125,62250,be175004,off,,,,,,, +1985,c,insertion,sorted,500,2463534242,17,399335,1,241515296,240387104,499,0,d9a24379,off,,,,,,, +1986,node,bubble,reversed,250,2463534242,7,5861,1,259557230,232562753,31125,62250,be175004,off,,,,,,, +1987,node,quick,reversed,500,2463534242,23,1194,1,263746531,234425074,124750,500,d33e3559,off,,,,,,, +1988,c,quick,reversed,2000,2463534242,8,209,1,241095683,240024584,1999000,2000,ad35ba05,off,,,,,,, +1989,node,insertion,sorted,250,2463534242,7,607595,1,260925893,230466456,249,0,7fd8c1f4,off,,,,,,, +1990,c,insertion,random,2000,2463534242,22,492,1,233097571,232150858,996390,996389,41262855,off,,,,,,, +1991,node,none,nearly_sorted,1000,2463534242,6,0,10,263545946,641,0,0,1cfe4afd,off,,,,,,, +1992,node,quick,reversed,500,2463534242,29,1194,1,265742130,235512545,124750,500,d33e3559,off,,,,,,, +1993,node,none,random,500,2463534242,28,0,9,232282105,841,0,0,11ae932d,off,,,,,,, +1994,python,bubble,sorted,1000,2463534242,30,11,1,276106064,260820637,499500,0,8e15cb75,off,,,,,,, +1995,c,insertion,random,500,2463534242,10,6945,1,214763402,213667469,60338,60339,11ae932d,off,,,,,,, +1996,python,insertion,reversed,2000,2463534242,14,2,1,415422362,399634904,1999000,2000999,ad35ba05,off,,,,,,, +1997,node,bubble,reversed,250,2463534242,18,5861,1,258423585,230389166,31125,62250,be175004,off,,,,,,, +1998,node,insertion,random,1000,2463534242,5,1091,1,265247624,237199428,249457,249457,57561285,off,,,,,,, +1999,python,quick,nearly_sorted,1000,2463534242,13,110,1,261802648,245187713,28499,7892,1cfe4afd,off,,,,,,, +2000,c,none,sorted,2000,2463534242,12,0,416,241853362,90,0,0,69842aa5,off,,,,,,, +2001,python,quick,random,250,2463534242,14,1271,1,250219921,234207696,1933,1940,b47adce4,off,,,,,,, +2002,c,insertion,nearly_sorted,500,2463534242,28,32350,1,240141694,239229565,7962,7926,482b4785,off,,,,,,, +2003,node,quick,reversed,500,2463534242,15,1194,1,265481522,237398591,124750,500,d33e3559,off,,,,,,, +2004,node,quick,reversed,500,2463534242,27,1194,1,274095005,244838895,124750,500,d33e3559,off,,,,,,, +2005,python,none,reversed,500,2463534242,6,0,16,240673300,952,0,0,d33e3559,off,,,,,,, +2006,python,none,nearly_sorted,2000,2463534242,27,0,15,254977927,611,0,0,5835dc59,off,,,,,,, +2007,python,insertion,reversed,2000,2463534242,19,2,1,410206645,393881058,1999000,2000999,ad35ba05,off,,,,,,, +2008,node,none,random,250,2463534242,12,0,10,291289732,871,0,0,b47adce4,off,,,,,,, +2009,python,quick,nearly_sorted,1000,2463534242,29,110,1,278214836,260045653,28499,7892,1cfe4afd,off,,,,,,, +2010,node,bubble,nearly_sorted,2000,2463534242,5,110,1,271457442,240424634,1999000,232556,5835dc59,off,,,,,,, +2011,python,none,random,500,2463534242,11,0,16,245631301,601,0,0,11ae932d,off,,,,,,, +2012,node,quick,sorted,1000,2463534242,22,296,1,274548035,246415287,499500,0,8e15cb75,off,,,,,,, +2013,node,quick,nearly_sorted,500,2463534242,4,10954,1,271566505,241195229,10192,1850,482b4785,off,,,,,,, +2014,node,bubble,reversed,1000,2463534242,26,378,1,270801612,243005030,499500,999000,0ca43265,off,,,,,,, +2015,node,none,nearly_sorted,500,2463534242,26,0,10,258280818,611,0,0,482b4785,off,,,,,,, +2016,python,quick,nearly_sorted,2000,2463534242,1,55,1,249945097,231989954,58002,11768,5835dc59,off,,,,,,, +2017,node,bubble,nearly_sorted,1000,2463534242,15,413,1,267378968,239528762,499500,59780,1cfe4afd,off,,,,,,, +2018,c,quick,sorted,250,2463534242,20,10851,1,247936261,246979390,31125,0,7fd8c1f4,off,,,,,,, +2019,node,quick,reversed,2000,2463534242,18,80,1,274202506,244201440,1999000,2000,ad35ba05,off,,,,,,, +2020,node,quick,random,250,2463534242,5,36348,1,298425412,270111614,1933,1940,b47adce4,off,,,,,,, +2021,python,quick,nearly_sorted,500,2463534242,23,296,1,250861383,233318377,10192,1850,482b4785,off,,,,,,, +2022,python,quick,reversed,1000,2463534242,12,8,1,273917073,257265929,499500,1000,0ca43265,off,,,,,,, +2023,python,none,sorted,2000,2463534242,8,0,15,242367914,962,0,0,69842aa5,off,,,,,,, +2024,python,bubble,nearly_sorted,500,2463534242,16,41,1,252945519,236542704,124750,14926,482b4785,off,,,,,,, +2025,python,bubble,sorted,250,2463534242,15,206,1,256341583,240906227,31125,0,7fd8c1f4,off,,,,,,, +2026,node,insertion,random,250,2463534242,2,15349,1,267545801,238153223,15844,15838,b47adce4,off,,,,,,, +2027,python,insertion,sorted,250,2463534242,6,18845,1,261708031,246238215,249,0,7fd8c1f4,off,,,,,,, +2028,node,none,reversed,500,2463534242,19,0,9,236559360,881,0,0,d33e3559,off,,,,,,, +2029,python,none,random,1000,2463534242,24,0,15,258874841,661,0,0,57561285,off,,,,,,, +2030,c,insertion,sorted,2000,2463534242,23,101352,1,240808483,239857432,1999,0,69842aa5,off,,,,,,, +2031,python,none,nearly_sorted,1000,2463534242,1,0,16,255424505,561,0,0,1cfe4afd,off,,,,,,, +2032,c,quick,random,500,2463534242,27,55776,1,265721692,264772534,4578,4552,11ae932d,off,,,,,,, +2033,node,none,reversed,500,2463534242,7,0,9,260872667,1072,0,0,d33e3559,off,,,,,,, +2034,c,quick,random,250,2463534242,1,136055,1,240668589,239591875,1933,1940,b47adce4,off,,,,,,, +2035,c,bubble,nearly_sorted,2000,2463534242,13,121,1,242438760,241292390,1999000,232556,5835dc59,off,,,,,,, +2036,node,insertion,reversed,250,2463534242,13,8539,1,264336365,236233252,31125,31374,be175004,off,,,,,,, +2037,node,insertion,reversed,250,2463534242,17,8539,1,276180433,247732252,31125,31374,be175004,off,,,,,,, +2038,c,quick,sorted,500,2463534242,21,2762,1,245082543,244181295,124750,0,d9a24379,off,,,,,,, +2039,c,insertion,reversed,2000,2463534242,5,260,1,240616303,239524838,1999000,2000999,ad35ba05,off,,,,,,, +2040,c,bubble,sorted,500,2463534242,3,2858,1,245954797,244825142,124750,0,d9a24379,off,,,,,,, +2041,c,bubble,random,250,2463534242,29,2568,1,248602500,247398453,31125,31196,b47adce4,off,,,,,,, +2042,python,bubble,reversed,2000,2463534242,18,2,1,500082633,476162820,1999000,3998000,ad35ba05,off,,,,,,, +2043,python,quick,nearly_sorted,500,2463534242,20,296,1,242599830,226359036,10192,1850,482b4785,off,,,,,,, +2044,node,insertion,sorted,500,2463534242,13,324764,1,263282723,233796763,499,0,d9a24379,off,,,,,,, +2045,c,quick,reversed,250,2463534242,1,12781,1,251771193,250908263,31125,250,be175004,off,,,,,,, +2046,node,bubble,sorted,2000,2463534242,2,85,1,272333036,239870612,1999000,0,69842aa5,off,,,,,,, +2047,node,quick,nearly_sorted,2000,2463534242,27,1909,1,255057908,226278923,58002,11768,5835dc59,off,,,,,,, +2048,c,none,sorted,250,2463534242,11,0,429,261995350,100,0,0,7fd8c1f4,off,,,,,,, +2049,python,bubble,nearly_sorted,1000,2463534242,10,9,1,259171138,241401027,499500,59780,1cfe4afd,off,,,,,,, +2050,node,none,reversed,500,2463534242,21,0,9,245697204,942,0,0,d33e3559,off,,,,,,, +2051,c,bubble,sorted,2000,2463534242,23,173,1,241538151,240552213,1999000,0,69842aa5,off,,,,,,, +2052,node,quick,nearly_sorted,2000,2463534242,13,1909,1,259487389,229825435,58002,11768,5835dc59,off,,,,,,, +2053,python,none,random,2000,2463534242,16,0,16,275735469,631,0,0,41262855,off,,,,,,, +2054,python,quick,nearly_sorted,2000,2463534242,20,55,1,249062493,233190618,58002,11768,5835dc59,off,,,,,,, +2055,python,bubble,reversed,250,2463534242,2,88,1,252319894,237169151,31125,62250,be175004,off,,,,,,, +2056,node,none,reversed,500,2463534242,2,0,9,257195996,1252,0,0,d33e3559,off,,,,,,, +2057,c,insertion,nearly_sorted,2000,2463534242,14,2962,1,227365500,226446498,118277,118270,5835dc59,off,,,,,,, +2058,c,none,random,500,2463534242,24,0,426,252514350,90,0,0,11ae932d,off,,,,,,, +2059,c,none,random,1000,2463534242,11,0,367,246560102,90,0,0,57561285,off,,,,,,, +2060,c,bubble,nearly_sorted,500,2463534242,9,1998,1,263221838,262461343,124750,14926,482b4785,off,,,,,,, +2061,node,none,sorted,250,2463534242,19,0,9,246186842,972,0,0,7fd8c1f4,off,,,,,,, +2062,node,insertion,random,1000,2463534242,20,1091,1,268551416,238753290,249457,249457,57561285,off,,,,,,, +2063,python,quick,sorted,500,2463534242,16,30,1,242930810,227904671,124750,0,d9a24379,off,,,,,,, +2064,node,quick,nearly_sorted,1000,2463534242,26,3752,1,261565904,232920742,28499,7892,1cfe4afd,off,,,,,,, +2065,c,quick,reversed,250,2463534242,14,12781,1,249819681,248963807,31125,250,be175004,off,,,,,,, +2066,node,quick,sorted,2000,2463534242,16,74,1,268579689,239759511,1999000,0,69842aa5,off,,,,,,, +2067,node,bubble,reversed,1000,2463534242,24,378,1,265704660,238614424,499500,999000,0ca43265,off,,,,,,, +2068,node,none,nearly_sorted,1000,2463534242,18,0,10,255320299,571,0,0,1cfe4afd,off,,,,,,, +2069,python,bubble,reversed,1000,2463534242,11,5,1,290982092,275881780,499500,999000,0ca43265,off,,,,,,, +2070,python,insertion,sorted,2000,2463534242,12,1523,1,256521090,240154776,1999,0,69842aa5,off,,,,,,, +2071,node,quick,random,1000,2463534242,4,6513,1,266423699,239363958,11196,10210,57561285,off,,,,,,, +2072,node,quick,random,1000,2463534242,16,6513,1,256371570,229635754,11196,10210,57561285,off,,,,,,, +2073,python,none,sorted,2000,2463534242,4,0,15,228709129,571,0,0,69842aa5,off,,,,,,, +2074,node,insertion,sorted,500,2463534242,25,324764,1,259831584,232207654,499,0,d9a24379,off,,,,,,, +2075,python,quick,random,1000,2463534242,23,209,1,272399677,256833356,11196,10210,57561285,off,,,,,,, +2076,python,quick,sorted,250,2463534242,3,133,1,248115617,232524979,31125,0,7fd8c1f4,off,,,,,,, +2077,c,quick,nearly_sorted,250,2463534242,12,77221,1,244096034,243009931,4346,1008,21a19c14,off,,,,,,, +2078,c,insertion,reversed,2000,2463534242,17,260,1,235430855,234392320,1999000,2000999,ad35ba05,off,,,,,,, +2079,c,quick,nearly_sorted,2000,2463534242,18,6145,1,240015297,238858158,58002,11768,5835dc59,off,,,,,,, +2080,node,bubble,sorted,500,2463534242,18,1964,1,268825349,240025918,124750,0,d9a24379,off,,,,,,, +2081,python,quick,nearly_sorted,2000,2463534242,16,55,1,252217767,235792247,58002,11768,5835dc59,off,,,,,,, +2082,node,insertion,reversed,1000,2463534242,27,554,1,254602173,225358121,499500,500499,0ca43265,off,,,,,,, +2083,python,bubble,reversed,500,2463534242,23,20,1,255456094,240292062,124750,249500,d33e3559,off,,,,,,, +2084,python,none,random,250,2463534242,9,0,17,241252245,581,0,0,b47adce4,off,,,,,,, +2085,node,quick,nearly_sorted,2000,2463534242,4,1909,1,255201026,226121497,58002,11768,5835dc59,off,,,,,,, +2086,node,bubble,nearly_sorted,250,2463534242,26,6161,1,271883349,245335746,31125,4056,21a19c14,off,,,,,,, +2087,node,bubble,reversed,250,2463534242,27,5861,1,256274398,229568386,31125,62250,be175004,off,,,,,,, +2088,node,bubble,sorted,250,2463534242,23,8081,1,257014735,226957270,31125,0,7fd8c1f4,off,,,,,,, +2089,node,none,reversed,2000,2463534242,23,0,9,235278721,892,0,0,ad35ba05,off,,,,,,, +2090,node,quick,sorted,500,2463534242,12,1164,1,271506183,242378458,124750,0,d9a24379,off,,,,,,, +2091,python,none,random,500,2463534242,23,0,16,243658484,1052,0,0,11ae932d,off,,,,,,, +2092,node,none,sorted,500,2463534242,16,0,10,270006803,851,0,0,d9a24379,off,,,,,,, +2093,python,none,reversed,250,2463534242,20,0,17,248849573,631,0,0,be175004,off,,,,,,, +2094,c,quick,sorted,500,2463534242,14,2762,1,258283723,257171540,124750,0,d9a24379,off,,,,,,, +2095,python,quick,sorted,250,2463534242,22,133,1,249638141,233400223,31125,0,7fd8c1f4,off,,,,,,, +2096,node,none,reversed,1000,2463534242,2,0,9,231284835,1092,0,0,0ca43265,off,,,,,,, +2097,python,insertion,reversed,250,2463534242,8,102,1,252339964,236553699,31125,31374,be175004,off,,,,,,, +2098,c,insertion,random,1000,2463534242,28,1983,1,241454791,240250658,249457,249457,57561285,off,,,,,,, +2099,c,quick,reversed,250,2463534242,10,12781,1,241726137,240529447,31125,250,be175004,off,,,,,,, +2100,python,insertion,nearly_sorted,500,2463534242,17,316,1,248969269,233391352,7962,7926,482b4785,off,,,,,,, +2101,python,none,reversed,250,2463534242,13,0,17,246282371,561,0,0,be175004,off,,,,,,, +2102,c,bubble,reversed,250,2463534242,26,1384,1,240019574,239082179,31125,62250,be175004,off,,,,,,, +2103,node,bubble,nearly_sorted,1000,2463534242,29,413,1,264299196,234014639,499500,59780,1cfe4afd,off,,,,,,, +2104,node,quick,random,2000,2463534242,1,2126,1,256144705,228203247,25197,24544,41262855,off,,,,,,, +2105,node,quick,sorted,500,2463534242,22,1164,1,266347295,240103844,124750,0,d9a24379,off,,,,,,, +2106,node,bubble,sorted,500,2463534242,20,1964,1,255023333,227329449,124750,0,d9a24379,off,,,,,,, +2107,c,bubble,sorted,1000,2463534242,15,702,1,234541578,233427320,499500,0,8e15cb75,off,,,,,,, +2108,c,none,nearly_sorted,250,2463534242,22,0,388,219273515,90,0,0,21a19c14,off,,,,,,, +2109,python,insertion,sorted,250,2463534242,14,18845,1,260817090,244905955,249,0,7fd8c1f4,off,,,,,,, +2110,node,bubble,nearly_sorted,500,2463534242,21,1566,1,267272559,240126438,124750,14926,482b4785,off,,,,,,, +2111,python,quick,nearly_sorted,2000,2463534242,7,55,1,241903726,226059909,58002,11768,5835dc59,off,,,,,,, +2112,c,quick,random,1000,2463534242,13,23484,1,235818692,234607653,11196,10210,57561285,off,,,,,,, +2113,node,bubble,random,500,2463534242,20,1407,1,272636060,244424371,124750,119688,11ae932d,off,,,,,,, +2114,python,insertion,random,2000,2463534242,23,3,1,326100407,308710315,996390,996389,41262855,off,,,,,,, +2115,node,bubble,sorted,500,2463534242,15,1964,1,267696884,240806876,124750,0,d9a24379,off,,,,,,, +2116,node,insertion,random,500,2463534242,2,4059,1,268769244,241145817,60338,60339,11ae932d,off,,,,,,, +2117,python,insertion,nearly_sorted,2000,2463534242,22,20,1,255059952,237958156,118277,118270,5835dc59,off,,,,,,, +2118,c,insertion,reversed,250,2463534242,10,15645,1,244807037,243704670,31125,31374,be175004,off,,,,,,, +2119,python,bubble,nearly_sorted,250,2463534242,13,188,1,257614579,242180829,31125,4056,21a19c14,off,,,,,,, +2120,node,none,nearly_sorted,250,2463534242,5,0,10,252515654,561,0,0,21a19c14,off,,,,,,, +2121,c,quick,nearly_sorted,1000,2463534242,27,11273,1,232719774,231721299,28499,7892,1cfe4afd,off,,,,,,, +2122,node,quick,sorted,1000,2463534242,14,296,1,272360394,242311076,499500,0,8e15cb75,off,,,,,,, +2123,node,bubble,reversed,2000,2463534242,18,94,1,274592026,246957780,1999000,3998000,ad35ba05,off,,,,,,, +2124,c,insertion,random,250,2463534242,30,26020,1,231950382,230926173,15844,15838,b47adce4,off,,,,,,, +2125,node,quick,sorted,2000,2463534242,26,74,1,273315233,243041181,1999000,0,69842aa5,off,,,,,,, +2126,node,bubble,reversed,2000,2463534242,29,94,1,268649940,240841331,1999000,3998000,ad35ba05,off,,,,,,, +2127,python,bubble,reversed,1000,2463534242,12,5,1,298565494,282453681,499500,999000,0ca43265,off,,,,,,, +2128,c,insertion,random,250,2463534242,12,26020,1,233022502,232056813,15844,15838,b47adce4,off,,,,,,, +2129,c,none,reversed,250,2463534242,6,0,427,244248640,90,0,0,be175004,off,,,,,,, +2130,python,quick,reversed,2000,2463534242,21,2,1,273756029,257913953,1999000,2000,ad35ba05,off,,,,,,, +2131,c,bubble,nearly_sorted,250,2463534242,14,7880,1,242169334,240934211,31125,4056,21a19c14,off,,,,,,, +2132,c,none,nearly_sorted,1000,2463534242,13,0,421,238864931,80,0,0,1cfe4afd,off,,,,,,, +2133,python,bubble,sorted,250,2463534242,4,206,1,278976814,263839578,31125,0,7fd8c1f4,off,,,,,,, +2134,node,bubble,random,500,2463534242,11,1407,1,268363834,240511726,124750,119688,11ae932d,off,,,,,,, +2135,node,insertion,random,2000,2463534242,8,234,1,267765001,239365051,996390,996389,41262855,off,,,,,,, +2136,c,quick,nearly_sorted,500,2463534242,26,31784,1,239044608,238126257,10192,1850,482b4785,off,,,,,,, +2137,c,none,reversed,500,2463534242,13,0,402,225861942,90,0,0,d33e3559,off,,,,,,, +2138,node,quick,reversed,1000,2463534242,30,307,1,264930129,237141154,499500,1000,0ca43265,off,,,,,,, +2139,c,none,nearly_sorted,2000,2463534242,6,0,387,220435803,90,0,0,5835dc59,off,,,,,,, +2140,node,insertion,reversed,1000,2463534242,3,554,1,271441031,243555451,499500,500499,0ca43265,off,,,,,,, +2141,node,bubble,reversed,1000,2463534242,16,378,1,265525575,236078591,499500,999000,0ca43265,off,,,,,,, +2142,c,quick,random,1000,2463534242,17,23484,1,232185786,231093112,11196,10210,57561285,off,,,,,,, +2143,c,none,nearly_sorted,250,2463534242,1,0,388,218563713,90,0,0,21a19c14,off,,,,,,, +2144,python,insertion,random,500,2463534242,22,44,1,255863268,239206788,60338,60339,11ae932d,off,,,,,,, +2145,c,insertion,random,2000,2463534242,29,492,1,233830416,232871800,996390,996389,41262855,off,,,,,,, +2146,node,insertion,sorted,250,2463534242,4,607595,1,256866638,227741909,249,0,7fd8c1f4,off,,,,,,, +2147,node,insertion,sorted,1000,2463534242,27,160107,1,257722952,228662620,999,0,8e15cb75,off,,,,,,, +2148,c,quick,random,250,2463534242,11,136055,1,248164940,247099649,1933,1940,b47adce4,off,,,,,,, +2149,python,bubble,reversed,500,2463534242,9,20,1,278879821,262074653,124750,249500,d33e3559,off,,,,,,, +2150,node,quick,nearly_sorted,500,2463534242,29,10954,1,292525394,264661829,10192,1850,482b4785,off,,,,,,, +2151,node,insertion,nearly_sorted,1000,2463534242,26,7758,1,267217115,238296317,30889,30860,1cfe4afd,off,,,,,,, +2152,c,bubble,reversed,2000,2463534242,2,22,1,243621587,242662970,1999000,3998000,ad35ba05,off,,,,,,, +2153,python,insertion,nearly_sorted,500,2463534242,14,316,1,248054270,232535627,7962,7926,482b4785,off,,,,,,, +2154,c,bubble,random,250,2463534242,19,2568,1,237408241,236502610,31125,31196,b47adce4,off,,,,,,, +2155,python,bubble,nearly_sorted,1000,2463534242,26,9,1,245989614,229957841,499500,59780,1cfe4afd,off,,,,,,, +2156,c,quick,random,1000,2463534242,8,23484,1,229348867,228093063,11196,10210,57561285,off,,,,,,, +2157,python,insertion,nearly_sorted,250,2463534242,15,1416,1,251822655,235125779,2277,2251,21a19c14,off,,,,,,, +2158,node,insertion,sorted,1000,2463534242,6,160107,1,251960373,223641202,999,0,8e15cb75,off,,,,,,, +2159,node,none,reversed,1000,2463534242,5,0,9,229965893,892,0,0,0ca43265,off,,,,,,, +2160,c,quick,reversed,250,2463534242,13,12781,1,237680722,236798017,31125,250,be175004,off,,,,,,, +2161,c,insertion,random,2000,2463534242,12,492,1,226669375,225573472,996390,996389,41262855,off,,,,,,, +2162,python,none,random,250,2463534242,23,0,17,240713124,491,0,0,b47adce4,off,,,,,,, +2163,node,none,random,500,2463534242,4,0,9,228736039,921,0,0,11ae932d,off,,,,,,, +2164,c,insertion,nearly_sorted,250,2463534242,5,99421,1,237302072,236255453,2277,2251,21a19c14,off,,,,,,, +2165,python,insertion,sorted,1000,2463534242,3,3178,1,249983299,234721086,999,0,8e15cb75,off,,,,,,, +2166,node,insertion,sorted,1000,2463534242,1,160107,1,257414603,229299491,999,0,8e15cb75,off,,,,,,, +2167,c,bubble,reversed,500,2463534242,11,346,1,240612166,239658718,124750,249500,d33e3559,off,,,,,,, +2168,python,quick,reversed,500,2463534242,2,33,1,248471293,232828416,124750,500,d33e3559,off,,,,,,, +2169,python,insertion,random,1000,2463534242,7,10,1,259749080,242782606,249457,249457,57561285,off,,,,,,, +2170,node,insertion,random,1000,2463534242,9,1091,1,263619943,236034101,249457,249457,57561285,off,,,,,,, +2171,node,quick,nearly_sorted,2000,2463534242,21,1909,1,253822733,224692432,58002,11768,5835dc59,off,,,,,,, +2172,python,insertion,sorted,1000,2463534242,8,3178,1,258669105,243482145,999,0,8e15cb75,off,,,,,,, +2173,node,bubble,reversed,2000,2463534242,22,94,1,271460308,242435271,1999000,3998000,ad35ba05,off,,,,,,, +2174,python,bubble,sorted,500,2463534242,11,46,1,284111346,268992290,124750,0,d9a24379,off,,,,,,, +2175,c,quick,random,2000,2463534242,21,10587,1,241252886,240224249,25197,24544,41262855,off,,,,,,, +2176,node,quick,random,1000,2463534242,14,6513,1,282174898,253939874,11196,10210,57561285,off,,,,,,, +2177,node,none,nearly_sorted,250,2463534242,30,0,10,262920333,841,0,0,21a19c14,off,,,,,,, +2178,node,quick,nearly_sorted,500,2463534242,20,10954,1,259726086,232095638,10192,1850,482b4785,off,,,,,,, +2179,c,none,reversed,250,2463534242,2,0,427,242610391,90,0,0,be175004,off,,,,,,, +2180,c,bubble,sorted,500,2463534242,11,2858,1,244327099,243547598,124750,0,d9a24379,off,,,,,,, +2181,c,insertion,random,2000,2463534242,4,492,1,233481129,232315977,996390,996389,41262855,off,,,,,,, +2182,node,bubble,nearly_sorted,250,2463534242,15,6161,1,282499686,255029381,31125,4056,21a19c14,off,,,,,,, +2183,python,quick,nearly_sorted,500,2463534242,22,296,1,252861632,237861922,10192,1850,482b4785,off,,,,,,, +2184,node,insertion,sorted,500,2463534242,21,324764,1,266406186,236593818,499,0,d9a24379,off,,,,,,, +2185,c,quick,sorted,2000,2463534242,30,174,1,238408084,237469346,1999000,0,69842aa5,off,,,,,,, +2186,python,insertion,sorted,500,2463534242,16,6624,1,234274559,218896817,499,0,d9a24379,off,,,,,,, +2187,c,insertion,reversed,500,2463534242,29,3967,1,233213596,231998402,124750,125249,d33e3559,off,,,,,,, +2188,python,none,reversed,250,2463534242,4,0,17,237584094,541,0,0,be175004,off,,,,,,, +2189,node,bubble,random,1000,2463534242,25,362,1,263712988,235759500,499500,496926,57561285,off,,,,,,, +2190,c,none,nearly_sorted,250,2463534242,11,0,388,216867526,140,0,0,21a19c14,off,,,,,,, +2191,node,none,reversed,500,2463534242,15,0,9,230981727,902,0,0,d33e3559,off,,,,,,, +2192,c,none,nearly_sorted,2000,2463534242,28,0,387,219117853,90,0,0,5835dc59,off,,,,,,, +2193,node,bubble,sorted,250,2463534242,21,8081,1,260659235,232750027,31125,0,7fd8c1f4,off,,,,,,, +2194,node,insertion,sorted,2000,2463534242,12,89220,1,255317604,226468024,1999,0,69842aa5,off,,,,,,, +2195,node,none,sorted,500,2463534242,13,0,10,257832988,832,0,0,d9a24379,off,,,,,,, +2196,c,quick,nearly_sorted,500,2463534242,27,31784,1,241050216,239988777,10192,1850,482b4785,off,,,,,,, +2197,c,none,sorted,250,2463534242,29,0,429,235661768,91,0,0,7fd8c1f4,off,,,,,,, +2198,python,bubble,sorted,1000,2463534242,21,11,1,276195711,261208860,499500,0,8e15cb75,off,,,,,,, +2199,python,none,reversed,500,2463534242,21,0,16,229086095,581,0,0,d33e3559,off,,,,,,, +2200,python,quick,sorted,500,2463534242,28,30,1,247537795,232656650,124750,0,d9a24379,off,,,,,,, +2201,c,none,random,250,2463534242,9,0,429,237870146,90,0,0,b47adce4,off,,,,,,, +2202,node,bubble,nearly_sorted,1000,2463534242,4,413,1,264673449,236353666,499500,59780,1cfe4afd,off,,,,,,, +2203,c,quick,reversed,500,2463534242,23,3240,1,233029314,231901992,124750,500,d33e3559,off,,,,,,, +2204,python,none,nearly_sorted,1000,2463534242,15,0,16,228928359,521,0,0,1cfe4afd,off,,,,,,, +2205,python,bubble,reversed,250,2463534242,15,88,1,248867918,234242307,31125,62250,be175004,off,,,,,,, +2206,node,bubble,reversed,2000,2463534242,11,94,1,265069550,236395356,1999000,3998000,ad35ba05,off,,,,,,, +2207,python,insertion,random,1000,2463534242,29,10,1,258612510,242605103,249457,249457,57561285,off,,,,,,, +2208,node,bubble,sorted,250,2463534242,9,8081,1,261187766,233600103,31125,0,7fd8c1f4,off,,,,,,, +2209,node,bubble,reversed,500,2463534242,23,1534,1,267982609,241309422,124750,249500,d33e3559,off,,,,,,, +2210,python,quick,random,2000,2463534242,15,90,1,271422286,254239843,25197,24544,41262855,off,,,,,,, +2211,python,insertion,random,500,2463534242,25,44,1,250503993,235187786,60338,60339,11ae932d,off,,,,,,, +2212,python,none,sorted,500,2463534242,27,0,16,229701809,531,0,0,d9a24379,off,,,,,,, +2213,python,quick,random,500,2463534242,17,507,1,258574828,241679244,4578,4552,11ae932d,off,,,,,,, +2214,python,insertion,sorted,250,2463534242,2,18845,1,255052948,240240176,249,0,7fd8c1f4,off,,,,,,, +2215,node,quick,random,250,2463534242,23,36348,1,253473941,226571543,1933,1940,b47adce4,off,,,,,,, +2216,c,insertion,sorted,500,2463534242,12,399335,1,236680134,235568453,499,0,d9a24379,off,,,,,,, +2217,c,quick,random,2000,2463534242,9,10587,1,230089375,228946774,25197,24544,41262855,off,,,,,,, +2218,c,insertion,sorted,2000,2463534242,8,101352,1,235476020,234481506,1999,0,69842aa5,off,,,,,,, +2219,python,none,random,2000,2463534242,18,0,16,254828357,661,0,0,41262855,off,,,,,,, +2220,node,none,sorted,250,2463534242,10,0,9,227632350,762,0,0,7fd8c1f4,off,,,,,,, +2221,python,quick,nearly_sorted,500,2463534242,27,296,1,237520482,222563879,10192,1850,482b4785,off,,,,,,, +2222,python,bubble,random,1000,2463534242,15,6,1,265014155,249721211,499500,496926,57561285,off,,,,,,, +2223,c,bubble,random,2000,2463534242,11,40,1,242625268,241546999,1999000,1988792,41262855,off,,,,,,, +2224,node,insertion,reversed,250,2463534242,3,8539,1,262584503,236310384,31125,31374,be175004,off,,,,,,, +2225,c,quick,nearly_sorted,250,2463534242,22,77221,1,242041514,241176809,4346,1008,21a19c14,off,,,,,,, +2226,node,bubble,sorted,1000,2463534242,20,374,1,239571004,211546011,499500,0,8e15cb75,off,,,,,,, +2227,c,bubble,random,500,2463534242,14,635,1,230497349,229650012,124750,119688,11ae932d,off,,,,,,, +2228,python,quick,sorted,250,2463534242,2,133,1,246694243,231409958,31125,0,7fd8c1f4,off,,,,,,, +2229,node,none,nearly_sorted,500,2463534242,7,0,10,254591834,672,0,0,482b4785,off,,,,,,, +2230,python,quick,sorted,250,2463534242,20,133,1,254898940,239174149,31125,0,7fd8c1f4,off,,,,,,, +2231,c,quick,sorted,250,2463534242,12,10851,1,242358137,241500150,31125,0,7fd8c1f4,off,,,,,,, +2232,c,none,reversed,1000,2463534242,6,0,405,226228890,80,0,0,0ca43265,off,,,,,,, +2233,python,quick,random,500,2463534242,9,507,1,259314365,243798203,4578,4552,11ae932d,off,,,,,,, +2234,python,insertion,random,250,2463534242,1,206,1,252460390,237533534,15844,15838,b47adce4,off,,,,,,, +2235,node,none,random,500,2463534242,2,0,9,225533798,752,0,0,11ae932d,off,,,,,,, +2236,c,insertion,sorted,1000,2463534242,25,196883,1,235425876,234347695,999,0,8e15cb75,off,,,,,,, +2237,python,insertion,reversed,1000,2463534242,3,6,1,284831686,269747710,499500,500499,0ca43265,off,,,,,,, +2238,python,none,nearly_sorted,500,2463534242,9,0,17,235461102,521,0,0,482b4785,off,,,,,,, +2239,node,insertion,nearly_sorted,2000,2463534242,15,2100,1,250828010,223540362,118277,118270,5835dc59,off,,,,,,, +2240,c,insertion,nearly_sorted,250,2463534242,18,99421,1,234056099,232899331,2277,2251,21a19c14,off,,,,,,, +2241,python,insertion,nearly_sorted,250,2463534242,26,1416,1,258865513,243419934,2277,2251,21a19c14,off,,,,,,, +2242,python,insertion,sorted,500,2463534242,2,6624,1,237738790,223141525,499,0,d9a24379,off,,,,,,, +2243,python,quick,sorted,2000,2463534242,12,2,1,306792683,290790167,1999000,0,69842aa5,off,,,,,,, +2244,python,quick,reversed,1000,2463534242,24,8,1,265926046,250596854,499500,1000,0ca43265,off,,,,,,, +2245,node,quick,nearly_sorted,1000,2463534242,2,3752,1,259808080,232215657,28499,7892,1cfe4afd,off,,,,,,, +2246,node,bubble,reversed,1000,2463534242,5,378,1,264276604,237129743,499500,999000,0ca43265,off,,,,,,, +2247,c,insertion,sorted,500,2463534242,6,399335,1,245367387,244379437,499,0,d9a24379,off,,,,,,, +2248,node,insertion,nearly_sorted,250,2463534242,7,79550,1,240261048,211920515,2277,2251,21a19c14,off,,,,,,, +2249,python,insertion,nearly_sorted,2000,2463534242,8,20,1,262575857,245358852,118277,118270,5835dc59,off,,,,,,, +2250,node,bubble,sorted,1000,2463534242,17,374,1,256569621,227732285,499500,0,8e15cb75,off,,,,,,, +2251,node,insertion,nearly_sorted,2000,2463534242,18,2100,1,261860897,234550864,118277,118270,5835dc59,off,,,,,,, +2252,node,insertion,nearly_sorted,500,2463534242,7,26045,1,240453809,213914280,7962,7926,482b4785,off,,,,,,, +2253,python,quick,sorted,1000,2463534242,9,8,1,286490025,270598895,499500,0,8e15cb75,off,,,,,,, +2254,node,none,random,500,2463534242,9,0,9,228225530,812,0,0,11ae932d,off,,,,,,, +2255,node,quick,reversed,500,2463534242,4,1194,1,257761435,230646745,124750,500,d33e3559,off,,,,,,, +2256,python,none,nearly_sorted,1000,2463534242,24,0,16,231343646,591,0,0,1cfe4afd,off,,,,,,, +2257,python,quick,reversed,250,2463534242,11,151,1,290659448,275895303,31125,250,be175004,off,,,,,,, +2258,python,quick,reversed,1000,2463534242,15,8,1,272481721,256593103,499500,1000,0ca43265,off,,,,,,, +2259,node,none,random,1000,2463534242,29,0,10,261767572,821,0,0,57561285,off,,,,,,, +2260,python,bubble,nearly_sorted,2000,2463534242,13,3,1,353037329,336443208,1999000,232556,5835dc59,off,,,,,,, +2261,python,none,sorted,1000,2463534242,7,0,16,233111909,642,0,0,8e15cb75,off,,,,,,, +2262,python,bubble,nearly_sorted,500,2463534242,29,41,1,249928535,235520216,124750,14926,482b4785,off,,,,,,, +2263,node,none,reversed,250,2463534242,13,0,9,233038823,701,0,0,be175004,off,,,,,,, +2264,c,none,random,250,2463534242,3,0,429,241653466,90,0,0,b47adce4,off,,,,,,, +2265,python,insertion,nearly_sorted,1000,2463534242,18,76,1,248648216,232292990,30889,30860,1cfe4afd,off,,,,,,, +2266,python,insertion,reversed,250,2463534242,6,102,1,249939926,234247197,31125,31374,be175004,off,,,,,,, +2267,c,none,random,500,2463534242,26,0,426,238155723,90,0,0,11ae932d,off,,,,,,, +2268,c,insertion,nearly_sorted,500,2463534242,8,32350,1,245749674,244977274,7962,7926,482b4785,off,,,,,,, +2269,node,bubble,sorted,2000,2463534242,15,85,1,281611188,251097592,1999000,0,69842aa5,off,,,,,,, +2270,python,quick,random,1000,2463534242,13,209,1,291395768,275163484,11196,10210,57561285,off,,,,,,, +2271,node,none,random,250,2463534242,15,0,10,254034099,781,0,0,b47adce4,off,,,,,,, +2272,node,quick,random,1000,2463534242,25,6513,1,267721660,239953187,11196,10210,57561285,off,,,,,,, +2273,node,insertion,sorted,500,2463534242,6,324764,1,257329896,228959979,499,0,d9a24379,off,,,,,,, +2274,node,none,sorted,500,2463534242,24,0,10,249399012,1683,0,0,d9a24379,off,,,,,,, +2275,node,quick,nearly_sorted,250,2463534242,13,25320,1,263903928,234343176,4346,1008,21a19c14,off,,,,,,, +2276,python,bubble,sorted,1000,2463534242,3,11,1,275541845,259686152,499500,0,8e15cb75,off,,,,,,, +2277,c,quick,nearly_sorted,250,2463534242,7,77221,1,237228193,236135797,4346,1008,21a19c14,off,,,,,,, +2278,python,quick,reversed,2000,2463534242,26,2,1,276645425,260766808,1999000,2000,ad35ba05,off,,,,,,, +2279,node,quick,sorted,2000,2463534242,30,74,1,271512455,243700070,1999000,0,69842aa5,off,,,,,,, +2280,node,quick,random,2000,2463534242,13,2126,1,254980923,228124437,25197,24544,41262855,off,,,,,,, +2281,c,none,random,1000,2463534242,30,0,367,209277342,90,0,0,57561285,off,,,,,,, +2282,c,none,nearly_sorted,2000,2463534242,1,0,387,221796354,90,0,0,5835dc59,off,,,,,,, +2283,c,none,sorted,500,2463534242,13,0,344,191415137,90,0,0,d9a24379,off,,,,,,, +2284,node,bubble,random,500,2463534242,4,1407,1,264178409,236579030,124750,119688,11ae932d,off,,,,,,, +2285,python,quick,nearly_sorted,2000,2463534242,23,55,1,249385768,233381294,58002,11768,5835dc59,off,,,,,,, +2286,c,quick,reversed,1000,2463534242,15,814,1,237469125,236557586,499500,1000,0ca43265,off,,,,,,, +2287,node,none,reversed,250,2463534242,26,0,9,232055148,712,0,0,be175004,off,,,,,,, +2288,python,bubble,nearly_sorted,1000,2463534242,28,9,1,297758934,281922268,499500,59780,1cfe4afd,off,,,,,,, +2289,node,insertion,sorted,1000,2463534242,12,160107,1,255467233,226432662,999,0,8e15cb75,off,,,,,,, +2290,node,bubble,reversed,1000,2463534242,30,378,1,262761975,235827151,499500,999000,0ca43265,off,,,,,,, +2291,node,none,random,250,2463534242,25,0,10,257189503,881,0,0,b47adce4,off,,,,,,, +2292,c,quick,reversed,500,2463534242,13,3240,1,240227565,239021895,124750,500,d33e3559,off,,,,,,, +2293,c,insertion,nearly_sorted,1000,2463534242,16,11445,1,245249445,244179310,30889,30860,1cfe4afd,off,,,,,,, +2294,c,none,reversed,500,2463534242,28,0,402,226120778,120,0,0,d33e3559,off,,,,,,, +2295,c,bubble,random,1000,2463534242,30,154,1,231132650,230381652,499500,496926,57561285,off,,,,,,, +2296,node,bubble,sorted,1000,2463534242,25,374,1,256834106,227755867,499500,0,8e15cb75,off,,,,,,, +2297,python,insertion,random,500,2463534242,18,44,1,245288565,229479292,60338,60339,11ae932d,off,,,,,,, +2298,c,insertion,reversed,500,2463534242,5,3967,1,231967689,231057060,124750,125249,d33e3559,off,,,,,,, +2299,c,none,reversed,1000,2463534242,24,0,405,226844815,90,0,0,0ca43265,off,,,,,,, +2300,c,bubble,reversed,1000,2463534242,19,87,1,241531945,240770893,499500,999000,0ca43265,off,,,,,,, +2301,node,insertion,nearly_sorted,1000,2463534242,10,7758,1,260475134,232979192,30889,30860,1cfe4afd,off,,,,,,, +2302,python,none,reversed,250,2463534242,18,0,17,247433690,521,0,0,be175004,off,,,,,,, +2303,c,insertion,nearly_sorted,2000,2463534242,3,2962,1,223775111,222778404,118277,118270,5835dc59,off,,,,,,, +2304,python,insertion,nearly_sorted,250,2463534242,2,1416,1,264680310,249557925,2277,2251,21a19c14,off,,,,,,, +2305,node,insertion,nearly_sorted,500,2463534242,20,26045,1,251472889,223134377,7962,7926,482b4785,off,,,,,,, +2306,c,bubble,random,2000,2463534242,4,40,1,240984974,240008815,1999000,1988792,41262855,off,,,,,,, +2307,python,insertion,nearly_sorted,1000,2463534242,8,76,1,249015413,233485491,30889,30860,1cfe4afd,off,,,,,,, +2308,node,quick,nearly_sorted,250,2463534242,12,25320,1,256728047,228674146,4346,1008,21a19c14,off,,,,,,, +2309,c,quick,reversed,250,2463534242,18,12781,1,242693145,241791676,31125,250,be175004,off,,,,,,, +2310,c,bubble,reversed,2000,2463534242,17,22,1,245521216,244430462,1999000,3998000,ad35ba05,off,,,,,,, +2311,node,none,nearly_sorted,1000,2463534242,22,0,10,260382557,632,0,0,1cfe4afd,off,,,,,,, +2312,node,none,sorted,1000,2463534242,5,0,9,228374281,801,0,0,8e15cb75,off,,,,,,, +2313,c,insertion,reversed,2000,2463534242,13,260,1,242583710,241704223,1999000,2000999,ad35ba05,off,,,,,,, +2314,node,none,nearly_sorted,500,2463534242,30,0,10,257760804,631,0,0,482b4785,off,,,,,,, +2315,c,insertion,sorted,250,2463534242,26,786886,1,239764286,238844683,249,0,7fd8c1f4,off,,,,,,, +2316,c,none,random,2000,2463534242,5,0,415,237360462,91,0,0,41262855,off,,,,,,, +2317,python,bubble,sorted,2000,2463534242,7,3,1,336751217,320469891,1999000,0,69842aa5,off,,,,,,, +2318,node,quick,sorted,1000,2463534242,18,296,1,275386225,245992980,499500,0,8e15cb75,off,,,,,,, +2319,node,quick,random,500,2463534242,25,16148,1,262448839,234374376,4578,4552,11ae932d,off,,,,,,, +2320,python,quick,reversed,500,2463534242,30,33,1,253260719,238099576,124750,500,d33e3559,off,,,,,,, +2321,c,quick,nearly_sorted,1000,2463534242,21,11273,1,236045577,234804191,28499,7892,1cfe4afd,off,,,,,,, +2322,c,insertion,random,2000,2463534242,14,492,1,229464319,228092503,996390,996389,41262855,off,,,,,,, +2323,node,insertion,sorted,250,2463534242,12,607595,1,251660816,223278457,249,0,7fd8c1f4,off,,,,,,, +2324,node,bubble,nearly_sorted,1000,2463534242,14,413,1,259283116,230793932,499500,59780,1cfe4afd,off,,,,,,, +2325,c,quick,nearly_sorted,250,2463534242,27,77221,1,238574798,237730351,4346,1008,21a19c14,off,,,,,,, +2326,python,bubble,random,500,2463534242,26,26,1,254431524,238367253,124750,119688,11ae932d,off,,,,,,, +2327,c,none,sorted,2000,2463534242,30,0,416,240032790,90,0,0,69842aa5,off,,,,,,, +2328,c,none,random,250,2463534242,14,0,429,239899159,201,0,0,b47adce4,off,,,,,,, +2329,node,none,nearly_sorted,1000,2463534242,15,0,10,259025073,741,0,0,1cfe4afd,off,,,,,,, +2330,node,quick,reversed,1000,2463534242,27,307,1,266451180,239797230,499500,1000,0ca43265,off,,,,,,, +2331,c,bubble,reversed,250,2463534242,13,1384,1,264023610,263132700,31125,62250,be175004,off,,,,,,, +2332,node,insertion,sorted,1000,2463534242,3,160107,1,256419519,228165561,999,0,8e15cb75,off,,,,,,, +2333,c,quick,random,250,2463534242,15,136055,1,242159987,241186882,1933,1940,b47adce4,off,,,,,,, +2334,node,insertion,random,500,2463534242,28,4059,1,270833563,242401894,60338,60339,11ae932d,off,,,,,,, +2335,c,quick,random,250,2463534242,28,136055,1,242238744,241327015,1933,1940,b47adce4,off,,,,,,, +2336,python,none,reversed,250,2463534242,3,0,17,243263405,541,0,0,be175004,off,,,,,,, +2337,node,quick,nearly_sorted,250,2463534242,16,25320,1,264995480,237052429,4346,1008,21a19c14,off,,,,,,, +2338,node,bubble,nearly_sorted,250,2463534242,1,6161,1,276742026,248232978,31125,4056,21a19c14,off,,,,,,, +2339,python,bubble,nearly_sorted,250,2463534242,29,188,1,268797337,252549325,31125,4056,21a19c14,off,,,,,,, +2340,node,none,random,1000,2463534242,24,0,10,253778019,11161,0,0,57561285,off,,,,,,, +2341,python,bubble,random,250,2463534242,2,112,1,253073238,237029053,31125,31196,b47adce4,off,,,,,,, +2342,c,quick,sorted,2000,2463534242,9,174,1,240067735,238887645,1999000,0,69842aa5,off,,,,,,, +2343,node,insertion,random,500,2463534242,4,4059,1,268857018,241353921,60338,60339,11ae932d,off,,,,,,, +2344,node,insertion,nearly_sorted,1000,2463534242,23,7758,1,257890265,229590943,30889,30860,1cfe4afd,off,,,,,,, +2345,node,quick,nearly_sorted,1000,2463534242,7,3752,1,258510479,230859209,28499,7892,1cfe4afd,off,,,,,,, +2346,node,bubble,reversed,500,2463534242,1,1534,1,257510083,231219142,124750,249500,d33e3559,off,,,,,,, +2347,c,insertion,reversed,250,2463534242,8,15645,1,240681715,239626230,31125,31374,be175004,off,,,,,,, +2348,python,insertion,reversed,1000,2463534242,26,6,1,293244303,278103091,499500,500499,0ca43265,off,,,,,,, +2349,c,bubble,nearly_sorted,1000,2463534242,8,507,1,241848512,240906864,499500,59780,1cfe4afd,off,,,,,,, +2350,c,quick,random,1000,2463534242,1,23484,1,238124915,236873310,11196,10210,57561285,off,,,,,,, +2351,python,quick,nearly_sorted,500,2463534242,24,296,1,237884278,222731461,10192,1850,482b4785,off,,,,,,, +2352,python,quick,sorted,2000,2463534242,13,2,1,301166945,285144132,1999000,0,69842aa5,off,,,,,,, +2353,node,insertion,nearly_sorted,1000,2463534242,15,7758,1,262886087,236388438,30889,30860,1cfe4afd,off,,,,,,, +2354,python,none,sorted,500,2463534242,7,0,16,225530451,531,0,0,d9a24379,off,,,,,,, +2355,python,bubble,sorted,250,2463534242,8,206,1,252246239,237715647,31125,0,7fd8c1f4,off,,,,,,, +2356,python,none,reversed,500,2463534242,12,0,16,223259705,582,0,0,d33e3559,off,,,,,,, +2357,node,none,reversed,250,2463534242,4,0,9,226303961,661,0,0,be175004,off,,,,,,, +2358,c,quick,reversed,250,2463534242,3,12781,1,241196150,240149318,31125,250,be175004,off,,,,,,, +2359,c,bubble,nearly_sorted,250,2463534242,2,7880,1,234453413,233233377,31125,4056,21a19c14,off,,,,,,, +2360,node,bubble,nearly_sorted,500,2463534242,25,1566,1,262235540,234980443,124750,14926,482b4785,off,,,,,,, +2361,python,quick,random,250,2463534242,25,1271,1,245645369,230756985,1933,1940,b47adce4,off,,,,,,, +2362,python,quick,reversed,2000,2463534242,11,2,1,265752298,250313431,1999000,2000,ad35ba05,off,,,,,,, +2363,python,bubble,reversed,1000,2463534242,26,5,1,292145873,277495670,499500,999000,0ca43265,off,,,,,,, +2364,python,none,sorted,500,2463534242,25,0,16,223402614,582,0,0,d9a24379,off,,,,,,, +2365,c,none,sorted,2000,2463534242,23,0,416,231598994,90,0,0,69842aa5,off,,,,,,, +2366,c,insertion,nearly_sorted,1000,2463534242,14,11445,1,246365808,245148358,30889,30860,1cfe4afd,off,,,,,,, +2367,c,none,random,1000,2463534242,8,0,367,204743536,80,0,0,57561285,off,,,,,,, +2368,python,quick,sorted,250,2463534242,6,133,1,260845963,245639088,31125,0,7fd8c1f4,off,,,,,,, +2369,python,insertion,random,1000,2463534242,1,10,1,250797063,234988666,249457,249457,57561285,off,,,,,,, +2370,c,insertion,nearly_sorted,2000,2463534242,25,2962,1,227813760,226428263,118277,118270,5835dc59,off,,,,,,, +2371,c,insertion,random,250,2463534242,3,26020,1,231568859,230451525,15844,15838,b47adce4,off,,,,,,, +2372,node,none,sorted,2000,2463534242,2,0,10,258882504,541,0,0,69842aa5,off,,,,,,, +2373,c,quick,random,2000,2463534242,14,10587,1,239652627,238599214,25197,24544,41262855,off,,,,,,, +2374,python,quick,nearly_sorted,500,2463534242,16,296,1,249781128,234084359,10192,1850,482b4785,off,,,,,,, +2375,node,insertion,reversed,500,2463534242,26,2178,1,262240229,233702531,124750,125249,d33e3559,off,,,,,,, +2376,python,insertion,sorted,2000,2463534242,30,1523,1,250644656,234953471,1999,0,69842aa5,off,,,,,,, +2377,node,none,random,500,2463534242,17,0,9,233247043,751,0,0,11ae932d,off,,,,,,, +2378,node,bubble,random,250,2463534242,7,5439,1,258837802,230495402,31125,31196,b47adce4,off,,,,,,, +2379,c,none,sorted,1000,2463534242,28,0,421,235807240,91,0,0,8e15cb75,off,,,,,,, +2380,node,bubble,nearly_sorted,1000,2463534242,10,413,1,266042423,237813673,499500,59780,1cfe4afd,off,,,,,,, +2381,node,quick,random,250,2463534242,4,36348,1,254840691,224588589,1933,1940,b47adce4,off,,,,,,, +2382,python,insertion,reversed,250,2463534242,30,102,1,243224221,228704762,31125,31374,be175004,off,,,,,,, +2383,node,none,nearly_sorted,1000,2463534242,1,0,10,252098782,712,0,0,1cfe4afd,off,,,,,,, +2384,node,quick,reversed,500,2463534242,18,1194,1,265526267,237630142,124750,500,d33e3559,off,,,,,,, +2385,python,insertion,nearly_sorted,1000,2463534242,6,76,1,245734784,230568895,30889,30860,1cfe4afd,off,,,,,,, +2386,c,quick,reversed,1000,2463534242,20,814,1,232406618,231288960,499500,1000,0ca43265,off,,,,,,, +2387,python,quick,sorted,2000,2463534242,10,2,1,300003820,283801545,1999000,0,69842aa5,off,,,,,,, +2388,c,none,sorted,250,2463534242,3,0,429,256206520,90,0,0,7fd8c1f4,off,,,,,,, +2389,c,none,reversed,250,2463534242,22,0,427,252547893,90,0,0,be175004,off,,,,,,, +2390,python,quick,reversed,2000,2463534242,13,2,1,284415436,265994067,1999000,2000,ad35ba05,off,,,,,,, +2391,c,insertion,random,1000,2463534242,3,1983,1,239736586,238769853,249457,249457,57561285,off,,,,,,, +2392,c,quick,reversed,250,2463534242,6,12781,1,243342591,242180635,31125,250,be175004,off,,,,,,, +2393,c,insertion,nearly_sorted,2000,2463534242,22,2962,1,225540199,224607752,118277,118270,5835dc59,off,,,,,,, +2394,node,none,random,2000,2463534242,20,0,9,247918985,912,0,0,41262855,off,,,,,,, +2395,node,none,sorted,2000,2463534242,9,0,10,295244565,1262,0,0,69842aa5,off,,,,,,, +2396,c,bubble,random,1000,2463534242,20,154,1,237833136,236927370,499500,496926,57561285,off,,,,,,, +2397,python,bubble,sorted,500,2463534242,17,46,1,255632035,239398985,124750,0,d9a24379,off,,,,,,, +2398,c,quick,nearly_sorted,1000,2463534242,25,11273,1,234693002,233575298,28499,7892,1cfe4afd,off,,,,,,, +2399,c,insertion,reversed,250,2463534242,23,15645,1,240349653,239441563,31125,31374,be175004,off,,,,,,, +2400,node,none,reversed,2000,2463534242,8,0,9,244873491,932,0,0,ad35ba05,off,,,,,,, +2401,c,none,nearly_sorted,250,2463534242,17,0,388,225418050,80,0,0,21a19c14,off,,,,,,, +2402,node,insertion,random,1000,2463534242,6,1091,1,292479459,262824415,249457,249457,57561285,off,,,,,,, +2403,python,none,random,500,2463534242,24,0,16,250967342,702,0,0,11ae932d,off,,,,,,, +2404,c,bubble,sorted,2000,2463534242,8,173,1,234429030,233359223,1999000,0,69842aa5,off,,,,,,, +2405,python,bubble,random,250,2463534242,24,112,1,247375340,231290617,31125,31196,b47adce4,off,,,,,,, +2406,node,insertion,sorted,1000,2463534242,11,160107,1,261879900,231222780,999,0,8e15cb75,off,,,,,,, +2407,node,bubble,random,1000,2463534242,27,362,1,267545361,238627862,499500,496926,57561285,off,,,,,,, +2408,node,insertion,sorted,250,2463534242,18,607595,1,263256372,234168722,249,0,7fd8c1f4,off,,,,,,, +2409,node,none,sorted,2000,2463534242,13,0,10,269717280,3767,0,0,69842aa5,off,,,,,,, +2410,c,none,nearly_sorted,250,2463534242,4,0,388,224960673,90,0,0,21a19c14,off,,,,,,, +2411,python,bubble,reversed,1000,2463534242,7,5,1,295860888,279214996,499500,999000,0ca43265,off,,,,,,, +2412,node,bubble,random,500,2463534242,30,1407,1,276297150,246162985,124750,119688,11ae932d,off,,,,,,, +2413,python,insertion,sorted,250,2463534242,12,18845,1,259452734,243725007,249,0,7fd8c1f4,off,,,,,,, +2414,python,bubble,nearly_sorted,500,2463534242,22,41,1,254205380,238354728,124750,14926,482b4785,off,,,,,,, +2415,python,quick,nearly_sorted,250,2463534242,7,882,1,265806912,250312339,4346,1008,21a19c14,off,,,,,,, +2416,c,bubble,reversed,1000,2463534242,9,87,1,245722584,244558280,499500,999000,0ca43265,off,,,,,,, +2417,python,quick,sorted,1000,2463534242,7,8,1,286685649,270236816,499500,0,8e15cb75,off,,,,,,, +2418,c,bubble,nearly_sorted,2000,2463534242,26,121,1,239910531,239007418,1999000,232556,5835dc59,off,,,,,,, +2419,python,bubble,sorted,500,2463534242,27,46,1,255893514,239851529,124750,0,d9a24379,off,,,,,,, +2420,c,bubble,sorted,1000,2463534242,9,702,1,237464937,236308241,499500,0,8e15cb75,off,,,,,,, +2421,c,bubble,nearly_sorted,500,2463534242,12,1998,1,237341396,236160593,124750,14926,482b4785,off,,,,,,, +2422,node,quick,nearly_sorted,2000,2463534242,30,1909,1,257754913,226992745,58002,11768,5835dc59,off,,,,,,, +2423,node,insertion,random,500,2463534242,8,4059,1,271511883,241589855,60338,60339,11ae932d,off,,,,,,, +2424,python,quick,sorted,2000,2463534242,22,2,1,301852694,285621838,1999000,0,69842aa5,off,,,,,,, +2425,node,bubble,reversed,250,2463534242,19,5861,1,261890524,233251965,31125,62250,be175004,off,,,,,,, +2426,c,insertion,nearly_sorted,2000,2463534242,21,2962,1,224205366,223356483,118277,118270,5835dc59,off,,,,,,, +2427,node,bubble,sorted,1000,2463534242,23,374,1,264770640,235729488,499500,0,8e15cb75,off,,,,,,, +2428,node,bubble,random,250,2463534242,6,5439,1,256095372,228371753,31125,31196,b47adce4,off,,,,,,, +2429,node,insertion,nearly_sorted,1000,2463534242,18,7758,1,270077145,241790699,30889,30860,1cfe4afd,off,,,,,,, +2430,python,quick,nearly_sorted,500,2463534242,25,296,1,239554353,223680122,10192,1850,482b4785,off,,,,,,, +2431,node,bubble,reversed,250,2463534242,6,5861,1,257602366,229066391,31125,62250,be175004,off,,,,,,, +2432,python,bubble,random,2000,2463534242,25,2,1,378915506,361917823,1999000,1988792,41262855,off,,,,,,, +2433,c,bubble,nearly_sorted,250,2463534242,1,7880,1,237027206,235909432,31125,4056,21a19c14,off,,,,,,, +2434,python,insertion,random,2000,2463534242,15,3,1,323129379,304649959,996390,996389,41262855,off,,,,,,, +2435,python,insertion,sorted,500,2463534242,8,6624,1,237930261,222246776,499,0,d9a24379,off,,,,,,, +2436,node,none,reversed,250,2463534242,6,0,9,238287907,982,0,0,be175004,off,,,,,,, +2437,python,none,sorted,2000,2463534242,16,0,15,243658865,671,0,0,69842aa5,off,,,,,,, +2438,node,quick,reversed,500,2463534242,11,1194,1,270170360,241320917,124750,500,d33e3559,off,,,,,,, +2439,node,insertion,reversed,2000,2463534242,22,144,1,264920420,234711978,1999000,2000999,ad35ba05,off,,,,,,, +2440,node,bubble,random,500,2463534242,28,1407,1,269383365,240864341,124750,119688,11ae932d,off,,,,,,, +2441,c,none,reversed,2000,2463534242,13,0,413,248240922,80,0,0,ad35ba05,off,,,,,,, +2442,c,quick,random,250,2463534242,21,136055,1,248958957,248210025,1933,1940,b47adce4,off,,,,,,, +2443,node,quick,reversed,250,2463534242,15,4755,1,269166960,240585463,31125,250,be175004,off,,,,,,, +2444,python,none,sorted,2000,2463534242,10,0,15,242654202,621,0,0,69842aa5,off,,,,,,, +2445,node,insertion,sorted,1000,2463534242,24,160107,1,256288354,227290510,999,0,8e15cb75,off,,,,,,, +2446,node,bubble,nearly_sorted,2000,2463534242,2,110,1,270841348,241236225,1999000,232556,5835dc59,off,,,,,,, +2447,node,none,nearly_sorted,1000,2463534242,27,0,10,279570717,792,0,0,1cfe4afd,off,,,,,,, +2448,node,insertion,reversed,2000,2463534242,16,144,1,265489426,236250658,1999000,2000999,ad35ba05,off,,,,,,, +2449,c,insertion,sorted,250,2463534242,2,786886,1,246394042,245517545,249,0,7fd8c1f4,off,,,,,,, +2450,node,quick,sorted,2000,2463534242,12,74,1,274212724,244851025,1999000,0,69842aa5,off,,,,,,, +2451,python,none,nearly_sorted,2000,2463534242,26,0,15,249727878,711,0,0,5835dc59,off,,,,,,, +2452,python,insertion,nearly_sorted,1000,2463534242,20,76,1,248391946,231730308,30889,30860,1cfe4afd,off,,,,,,, +2453,python,quick,nearly_sorted,1000,2463534242,20,110,1,267410117,250629579,28499,7892,1cfe4afd,off,,,,,,, +2454,python,none,random,500,2463534242,12,0,16,254128987,651,0,0,11ae932d,off,,,,,,, +2455,c,bubble,nearly_sorted,250,2463534242,24,7880,1,240409035,239281453,31125,4056,21a19c14,off,,,,,,, +2456,python,quick,nearly_sorted,500,2463534242,5,296,1,249714975,233395122,10192,1850,482b4785,off,,,,,,, +2457,node,none,nearly_sorted,500,2463534242,16,0,10,281336305,602,0,0,482b4785,off,,,,,,, +2458,c,insertion,nearly_sorted,500,2463534242,1,32350,1,245619049,244643230,7962,7926,482b4785,off,,,,,,, +2459,c,bubble,nearly_sorted,500,2463534242,26,1998,1,240586368,239626299,124750,14926,482b4785,off,,,,,,, +2460,python,bubble,sorted,2000,2463534242,18,3,1,332829796,314385948,1999000,0,69842aa5,off,,,,,,, +2461,python,insertion,reversed,1000,2463534242,14,6,1,312105839,296267330,499500,500499,0ca43265,off,,,,,,, +2462,python,quick,random,500,2463534242,6,507,1,263792137,246834546,4578,4552,11ae932d,off,,,,,,, +2463,node,insertion,random,500,2463534242,1,4059,1,273365658,244640931,60338,60339,11ae932d,off,,,,,,, +2464,python,none,nearly_sorted,2000,2463534242,17,0,15,251776288,771,0,0,5835dc59,off,,,,,,, +2465,node,none,random,2000,2463534242,11,0,9,250945211,902,0,0,41262855,off,,,,,,, +2466,node,quick,nearly_sorted,2000,2463534242,20,1909,1,259430642,229859977,58002,11768,5835dc59,off,,,,,,, +2467,c,bubble,reversed,500,2463534242,12,346,1,244340793,243492676,124750,249500,d33e3559,off,,,,,,, +2468,c,insertion,nearly_sorted,500,2463534242,15,32350,1,242676814,241577865,7962,7926,482b4785,off,,,,,,, +2469,c,bubble,reversed,250,2463534242,15,1384,1,241946656,241030920,31125,62250,be175004,off,,,,,,, +2470,node,insertion,random,2000,2463534242,9,234,1,268744658,238121703,996390,996389,41262855,off,,,,,,, +2471,c,none,sorted,500,2463534242,5,0,344,199777357,90,0,0,d9a24379,off,,,,,,, +2472,node,bubble,reversed,250,2463534242,30,5861,1,263348966,235189568,31125,62250,be175004,off,,,,,,, +2473,python,none,nearly_sorted,1000,2463534242,14,0,16,257902388,552,0,0,1cfe4afd,off,,,,,,, +2474,node,quick,sorted,250,2463534242,30,4507,1,277716561,248583147,31125,0,7fd8c1f4,off,,,,,,, +2475,c,quick,reversed,1000,2463534242,17,814,1,237967038,237048467,499500,1000,0ca43265,off,,,,,,, +2476,node,bubble,nearly_sorted,250,2463534242,16,6161,1,274453636,245988250,31125,4056,21a19c14,off,,,,,,, +2477,node,none,reversed,250,2463534242,7,0,9,235212526,621,0,0,be175004,off,,,,,,, +2478,python,quick,sorted,1000,2463534242,22,8,1,339743865,323049547,499500,0,8e15cb75,off,,,,,,, +2479,c,bubble,random,1000,2463534242,22,154,1,235815545,234688023,499500,496926,57561285,off,,,,,,, +2480,c,quick,random,1000,2463534242,4,23484,1,237166117,236203604,11196,10210,57561285,off,,,,,,, +2481,node,none,sorted,250,2463534242,25,0,9,243126898,1042,0,0,7fd8c1f4,off,,,,,,, +2482,python,none,reversed,500,2463534242,7,0,16,245905666,591,0,0,d33e3559,off,,,,,,, +2483,python,bubble,sorted,2000,2463534242,11,3,1,329275859,312680784,1999000,0,69842aa5,off,,,,,,, +2484,c,bubble,sorted,1000,2463534242,1,702,1,237274129,236181612,499500,0,8e15cb75,off,,,,,,, +2485,python,quick,sorted,2000,2463534242,8,2,1,301144127,283774495,1999000,0,69842aa5,off,,,,,,, +2486,python,quick,sorted,2000,2463534242,25,2,1,306140707,288703289,1999000,0,69842aa5,off,,,,,,, +2487,c,quick,random,1000,2463534242,22,23484,1,235386756,234446362,11196,10210,57561285,off,,,,,,, +2488,c,insertion,sorted,1000,2463534242,30,196883,1,235364611,234276993,999,0,8e15cb75,off,,,,,,, +2489,c,bubble,sorted,1000,2463534242,12,702,1,239515530,238491321,499500,0,8e15cb75,off,,,,,,, +2490,node,insertion,random,500,2463534242,3,4059,1,270939691,241805561,60338,60339,11ae932d,off,,,,,,, +2491,node,insertion,reversed,1000,2463534242,9,554,1,273677471,242822106,499500,500499,0ca43265,off,,,,,,, +2492,node,quick,random,1000,2463534242,2,6513,1,280098806,251289778,11196,10210,57561285,off,,,,,,, +2493,python,none,nearly_sorted,500,2463534242,24,0,17,265649538,932,0,0,482b4785,off,,,,,,, +2494,node,quick,random,500,2463534242,13,16148,1,279624657,249527702,4578,4552,11ae932d,off,,,,,,, +2495,node,none,sorted,2000,2463534242,11,0,10,274998448,982,0,0,69842aa5,off,,,,,,, +2496,python,insertion,reversed,1000,2463534242,15,6,1,291965425,275497699,499500,500499,0ca43265,off,,,,,,, +2497,python,none,random,2000,2463534242,24,0,16,276614458,631,0,0,41262855,off,,,,,,, +2498,python,none,random,1000,2463534242,21,0,15,242584390,651,0,0,57561285,off,,,,,,, +2499,python,insertion,sorted,250,2463534242,25,18845,1,263851577,247791988,249,0,7fd8c1f4,off,,,,,,, +2500,python,bubble,sorted,2000,2463534242,4,3,1,333329315,316438105,1999000,0,69842aa5,off,,,,,,, +2501,node,bubble,nearly_sorted,250,2463534242,29,6161,1,269763597,241487879,31125,4056,21a19c14,off,,,,,,, +2502,node,insertion,reversed,500,2463534242,8,2178,1,271155686,241731900,124750,125249,d33e3559,off,,,,,,, +2503,c,quick,reversed,250,2463534242,27,12781,1,316898792,315915134,31125,250,be175004,off,,,,,,, +2504,node,insertion,sorted,2000,2463534242,10,89220,1,305872469,273946390,1999,0,69842aa5,off,,,,,,, +2505,node,bubble,random,500,2463534242,25,1407,1,282996335,251772116,124750,119688,11ae932d,off,,,,,,, +2506,c,insertion,random,500,2463534242,25,6945,1,215967049,214937809,60338,60339,11ae932d,off,,,,,,, +2507,c,none,nearly_sorted,250,2463534242,28,0,388,219971975,90,0,0,21a19c14,off,,,,,,, +2508,c,none,sorted,500,2463534242,10,0,344,198099624,91,0,0,d9a24379,off,,,,,,, +2509,python,bubble,sorted,1000,2463534242,27,11,1,308045080,291800269,499500,0,8e15cb75,off,,,,,,, +2510,python,insertion,nearly_sorted,2000,2463534242,18,20,1,260910346,243215858,118277,118270,5835dc59,off,,,,,,, +2511,c,bubble,random,1000,2463534242,1,154,1,235078876,234099550,499500,496926,57561285,off,,,,,,, +2512,c,bubble,sorted,500,2463534242,25,2858,1,241992272,241080884,124750,0,d9a24379,off,,,,,,, +2513,node,insertion,reversed,1000,2463534242,10,554,1,264032868,233428166,499500,500499,0ca43265,off,,,,,,, +2514,node,quick,nearly_sorted,500,2463534242,10,10954,1,261710014,233335801,10192,1850,482b4785,off,,,,,,, +2515,python,bubble,nearly_sorted,1000,2463534242,17,9,1,251746292,235138653,499500,59780,1cfe4afd,off,,,,,,, +2516,c,bubble,reversed,250,2463534242,3,1384,1,243106410,242042986,31125,62250,be175004,off,,,,,,, +2517,c,insertion,sorted,250,2463534242,15,786886,1,245421719,244511122,249,0,7fd8c1f4,off,,,,,,, +2518,c,bubble,reversed,500,2463534242,6,346,1,244171386,242957843,124750,249500,d33e3559,off,,,,,,, +2519,node,bubble,random,2000,2463534242,28,98,1,272249745,243452383,1999000,1988792,41262855,off,,,,,,, +2520,c,bubble,nearly_sorted,1000,2463534242,26,507,1,240303837,239416645,499500,59780,1cfe4afd,off,,,,,,, +2521,node,insertion,random,500,2463534242,18,4059,1,250375412,220974580,60338,60339,11ae932d,off,,,,,,, +2522,python,bubble,random,2000,2463534242,16,2,1,371682811,354292422,1999000,1988792,41262855,off,,,,,,, +2523,c,none,reversed,250,2463534242,8,0,427,245580857,80,0,0,be175004,off,,,,,,, +2524,python,none,sorted,500,2463534242,26,0,16,248647784,771,0,0,d9a24379,off,,,,,,, +2525,node,insertion,random,500,2463534242,27,4059,1,273187665,244545150,60338,60339,11ae932d,off,,,,,,, +2526,node,none,random,500,2463534242,10,0,9,247584913,842,0,0,11ae932d,off,,,,,,, +2527,c,insertion,nearly_sorted,2000,2463534242,12,2962,1,223471192,222607589,118277,118270,5835dc59,off,,,,,,, +2528,python,insertion,nearly_sorted,2000,2463534242,16,20,1,258995737,241866053,118277,118270,5835dc59,off,,,,,,, +2529,node,none,reversed,500,2463534242,23,0,9,239181584,802,0,0,d33e3559,off,,,,,,, +2530,c,insertion,nearly_sorted,500,2463534242,24,32350,1,246320372,245346117,7962,7926,482b4785,off,,,,,,, +2531,c,bubble,reversed,500,2463534242,2,346,1,246033185,244905571,124750,249500,d33e3559,off,,,,,,, +2532,c,bubble,reversed,500,2463534242,28,346,1,244177838,243017332,124750,249500,d33e3559,off,,,,,,, +2533,c,quick,random,2000,2463534242,5,10587,1,241021161,239885042,25197,24544,41262855,off,,,,,,, +2534,python,insertion,nearly_sorted,1000,2463534242,21,76,1,252545960,236019007,30889,30860,1cfe4afd,off,,,,,,, +2535,c,insertion,random,250,2463534242,19,26020,1,235680775,234555024,15844,15838,b47adce4,off,,,,,,, +2536,c,quick,random,500,2463534242,5,55776,1,237738379,236589037,4578,4552,11ae932d,off,,,,,,, +2537,c,bubble,reversed,250,2463534242,23,1384,1,242706917,241701515,31125,62250,be175004,off,,,,,,, +2538,c,bubble,random,2000,2463534242,18,40,1,243596842,242541701,1999000,1988792,41262855,off,,,,,,, +2539,c,none,reversed,1000,2463534242,10,0,405,237117666,90,0,0,0ca43265,off,,,,,,, +2540,node,quick,reversed,2000,2463534242,22,80,1,278316757,248707027,1999000,2000,ad35ba05,off,,,,,,, +2541,python,bubble,sorted,2000,2463534242,24,3,1,325393902,308825511,1999000,0,69842aa5,off,,,,,,, +2542,node,insertion,sorted,250,2463534242,1,607595,1,251369005,223144777,249,0,7fd8c1f4,off,,,,,,, +2543,python,none,nearly_sorted,500,2463534242,21,0,17,268396576,501,0,0,482b4785,off,,,,,,, +2544,node,quick,sorted,500,2463534242,27,1164,1,280149051,251065932,124750,0,d9a24379,off,,,,,,, +2545,node,bubble,sorted,1000,2463534242,13,374,1,259598667,229268484,499500,0,8e15cb75,off,,,,,,, +2546,c,bubble,random,250,2463534242,23,2568,1,243226275,242329906,31125,31196,b47adce4,off,,,,,,, +2547,c,insertion,random,500,2463534242,22,6945,1,217089232,216128581,60338,60339,11ae932d,off,,,,,,, +2548,c,quick,reversed,2000,2463534242,13,209,1,244894913,243837680,1999000,2000,ad35ba05,off,,,,,,, +2549,c,bubble,nearly_sorted,1000,2463534242,25,507,1,244162737,243186410,499500,59780,1cfe4afd,off,,,,,,, +2550,python,none,random,500,2463534242,8,0,16,251526330,591,0,0,11ae932d,off,,,,,,, +2551,python,insertion,sorted,500,2463534242,30,6624,1,249050239,231990715,499,0,d9a24379,off,,,,,,, +2552,python,quick,nearly_sorted,1000,2463534242,2,110,1,259372694,243468072,28499,7892,1cfe4afd,off,,,,,,, +2553,python,none,reversed,2000,2463534242,28,0,16,259567068,711,0,0,ad35ba05,off,,,,,,, +2554,c,bubble,sorted,500,2463534242,4,2858,1,241232218,240215823,124750,0,d9a24379,off,,,,,,, +2555,python,quick,random,250,2463534242,21,1271,1,256728108,241443500,1933,1940,b47adce4,off,,,,,,, +2556,python,insertion,reversed,1000,2463534242,11,6,1,295736152,279674235,499500,500499,0ca43265,off,,,,,,, +2557,python,insertion,reversed,500,2463534242,10,22,1,249443155,233051423,124750,125249,d33e3559,off,,,,,,, +2558,python,none,nearly_sorted,1000,2463534242,2,0,16,249776179,571,0,0,1cfe4afd,off,,,,,,, +2559,c,quick,random,500,2463534242,24,55776,1,235236330,234299121,4578,4552,11ae932d,off,,,,,,, +2560,node,none,nearly_sorted,250,2463534242,27,0,10,270486502,1153,0,0,21a19c14,off,,,,,,, +2561,c,bubble,reversed,2000,2463534242,4,22,1,252511446,251602381,1999000,3998000,ad35ba05,off,,,,,,, +2562,node,quick,sorted,2000,2463534242,1,74,1,278405633,247094680,1999000,0,69842aa5,off,,,,,,, +2563,c,none,nearly_sorted,2000,2463534242,4,0,387,223269053,100,0,0,5835dc59,off,,,,,,, +2564,python,none,nearly_sorted,1000,2463534242,9,0,16,251458614,581,0,0,1cfe4afd,off,,,,,,, +2565,node,quick,sorted,1000,2463534242,4,296,1,273605717,244389623,499500,0,8e15cb75,off,,,,,,, +2566,python,insertion,sorted,1000,2463534242,26,3178,1,258761608,241850873,999,0,8e15cb75,off,,,,,,, +2567,python,none,reversed,500,2463534242,5,0,16,248644198,581,0,0,d33e3559,off,,,,,,, +2568,python,none,random,500,2463534242,2,0,16,252359442,621,0,0,11ae932d,off,,,,,,, +2569,python,insertion,reversed,250,2463534242,20,102,1,254739626,238824248,31125,31374,be175004,off,,,,,,, +2570,node,insertion,reversed,500,2463534242,22,2178,1,264205464,233888957,124750,125249,d33e3559,off,,,,,,, +2571,python,none,nearly_sorted,500,2463534242,10,0,17,265720170,501,0,0,482b4785,off,,,,,,, +2572,node,insertion,reversed,500,2463534242,21,2178,1,266604216,238517578,124750,125249,d33e3559,off,,,,,,, +2573,node,bubble,reversed,250,2463534242,12,5861,1,258498866,230495568,31125,62250,be175004,off,,,,,,, +2574,python,insertion,random,250,2463534242,4,206,1,261724863,244376148,15844,15838,b47adce4,off,,,,,,, +2575,c,none,reversed,2000,2463534242,22,0,413,242761964,90,0,0,ad35ba05,off,,,,,,, +2576,c,none,sorted,250,2463534242,1,0,429,248835346,160,0,0,7fd8c1f4,off,,,,,,, +2577,python,bubble,sorted,250,2463534242,22,206,1,256471276,240364168,31125,0,7fd8c1f4,off,,,,,,, +2578,c,quick,reversed,1000,2463534242,13,814,1,232996574,232031737,499500,1000,0ca43265,off,,,,,,, +2579,node,none,sorted,250,2463534242,15,0,9,241913204,722,0,0,7fd8c1f4,off,,,,,,, +2580,python,bubble,sorted,500,2463534242,13,46,1,279004726,262959659,124750,0,d9a24379,off,,,,,,, +2581,c,none,sorted,1000,2463534242,21,0,421,245427690,90,0,0,8e15cb75,off,,,,,,, +2582,node,insertion,reversed,1000,2463534242,30,554,1,269031575,240099861,499500,500499,0ca43265,off,,,,,,, +2583,python,insertion,reversed,250,2463534242,27,102,1,251340340,235488956,31125,31374,be175004,off,,,,,,, +2584,node,none,nearly_sorted,250,2463534242,20,0,10,261700717,712,0,0,21a19c14,off,,,,,,, +2585,python,insertion,nearly_sorted,250,2463534242,28,1416,1,265598262,249798956,2277,2251,21a19c14,off,,,,,,, +2586,python,bubble,nearly_sorted,1000,2463534242,11,9,1,255179897,239015375,499500,59780,1cfe4afd,off,,,,,,, +2587,c,quick,sorted,250,2463534242,3,10851,1,248136176,247240958,31125,0,7fd8c1f4,off,,,,,,, +2588,c,none,reversed,250,2463534242,11,0,427,243070582,80,0,0,be175004,off,,,,,,, +2589,c,bubble,reversed,1000,2463534242,21,87,1,243455525,242381092,499500,999000,0ca43265,off,,,,,,, +2590,python,insertion,reversed,500,2463534242,1,22,1,246592483,229857843,124750,125249,d33e3559,off,,,,,,, +2591,c,bubble,random,2000,2463534242,14,40,1,242752726,241780890,1999000,1988792,41262855,off,,,,,,, +2592,c,none,random,1000,2463534242,23,0,367,210420584,90,0,0,57561285,off,,,,,,, +2593,node,quick,random,500,2463534242,21,16148,1,272192529,242408589,4578,4552,11ae932d,off,,,,,,, +2594,python,insertion,random,1000,2463534242,22,10,1,261921450,245480035,249457,249457,57561285,off,,,,,,, +2595,node,insertion,sorted,2000,2463534242,16,89220,1,279337670,249502146,1999,0,69842aa5,off,,,,,,, +2596,c,quick,random,500,2463534242,2,55776,1,235027019,234083660,4578,4552,11ae932d,off,,,,,,, +2597,node,insertion,random,250,2463534242,3,15349,1,261569771,232554615,15844,15838,b47adce4,off,,,,,,, +2598,python,insertion,nearly_sorted,2000,2463534242,21,20,1,264681373,247580752,118277,118270,5835dc59,off,,,,,,, +2599,python,insertion,sorted,1000,2463534242,14,3178,1,251979669,235250517,999,0,8e15cb75,off,,,,,,, +2600,python,insertion,nearly_sorted,2000,2463534242,10,20,1,255657892,239311946,118277,118270,5835dc59,off,,,,,,, +2601,c,none,nearly_sorted,1000,2463534242,28,0,421,255466593,80,0,0,1cfe4afd,off,,,,,,, +2602,node,bubble,sorted,250,2463534242,27,8081,1,271825151,242335767,31125,0,7fd8c1f4,off,,,,,,, +2603,c,bubble,nearly_sorted,500,2463534242,1,1998,1,240509192,239383163,124750,14926,482b4785,off,,,,,,, +2604,node,insertion,nearly_sorted,2000,2463534242,20,2100,1,266904268,237831161,118277,118270,5835dc59,off,,,,,,, +2605,node,insertion,reversed,1000,2463534242,13,554,1,271530709,242454195,499500,500499,0ca43265,off,,,,,,, +2606,node,none,nearly_sorted,2000,2463534242,17,0,9,246902484,671,0,0,5835dc59,off,,,,,,, +2607,python,quick,reversed,500,2463534242,26,33,1,263355297,245575380,124750,500,d33e3559,off,,,,,,, +2608,node,insertion,random,1000,2463534242,26,1091,1,268640082,238350159,249457,249457,57561285,off,,,,,,, +2609,node,bubble,random,1000,2463534242,10,362,1,263857609,235800564,499500,496926,57561285,off,,,,,,, +2610,python,bubble,nearly_sorted,1000,2463534242,9,9,1,267672859,251583858,499500,59780,1cfe4afd,off,,,,,,, +2611,node,quick,sorted,1000,2463534242,9,296,1,272786642,243021105,499500,0,8e15cb75,off,,,,,,, +2612,c,none,nearly_sorted,250,2463534242,2,0,388,222985472,80,0,0,21a19c14,off,,,,,,, +2613,node,bubble,random,500,2463534242,12,1407,1,280451738,250612469,124750,119688,11ae932d,off,,,,,,, +2614,node,bubble,sorted,1000,2463534242,6,374,1,246007446,215763757,499500,0,8e15cb75,off,,,,,,, +2615,c,insertion,reversed,1000,2463534242,7,1033,1,247465739,246484021,499500,500499,0ca43265,off,,,,,,, +2616,node,none,nearly_sorted,1000,2463534242,21,0,10,271610669,661,0,0,1cfe4afd,off,,,,,,, +2617,node,quick,sorted,500,2463534242,5,1164,1,268876407,240280054,124750,0,d9a24379,off,,,,,,, +2618,python,insertion,reversed,2000,2463534242,9,2,1,418603002,401190062,1999000,2000999,ad35ba05,off,,,,,,, +2619,node,none,sorted,2000,2463534242,24,0,10,297984475,692,0,0,69842aa5,off,,,,,,, +2620,python,quick,reversed,500,2463534242,16,33,1,266944464,248552763,124750,500,d33e3559,off,,,,,,, +2621,python,bubble,nearly_sorted,500,2463534242,5,41,1,254854928,238880702,124750,14926,482b4785,off,,,,,,, +2622,c,quick,random,500,2463534242,13,55776,1,239833586,238549601,4578,4552,11ae932d,off,,,,,,, +2623,python,none,random,250,2463534242,24,0,17,265001623,571,0,0,b47adce4,off,,,,,,, +2624,c,quick,random,500,2463534242,20,55776,1,236224473,235143847,4578,4552,11ae932d,off,,,,,,, +2625,c,none,nearly_sorted,2000,2463534242,13,0,387,225018011,90,0,0,5835dc59,off,,,,,,, +2626,node,insertion,nearly_sorted,500,2463534242,29,26045,1,254868302,225002051,7962,7926,482b4785,off,,,,,,, +2627,node,bubble,nearly_sorted,2000,2463534242,16,110,1,263631134,234376176,1999000,232556,5835dc59,off,,,,,,, +2628,node,none,reversed,1000,2463534242,14,0,9,240865129,982,0,0,0ca43265,off,,,,,,, +2629,c,none,random,2000,2463534242,6,0,415,249516219,90,0,0,41262855,off,,,,,,, +2630,c,none,reversed,1000,2463534242,29,0,405,239669603,80,0,0,0ca43265,off,,,,,,, +2631,node,none,nearly_sorted,2000,2463534242,5,0,9,250958064,851,0,0,5835dc59,off,,,,,,, +2632,python,bubble,reversed,2000,2463534242,7,2,1,481139381,463318501,1999000,3998000,ad35ba05,off,,,,,,, +2633,python,quick,random,2000,2463534242,16,90,1,266619885,248337584,25197,24544,41262855,off,,,,,,, +2634,node,none,sorted,250,2463534242,8,0,9,237577629,922,0,0,7fd8c1f4,off,,,,,,, +2635,node,none,reversed,2000,2463534242,24,0,9,245378008,2816,0,0,ad35ba05,off,,,,,,, +2636,python,bubble,random,1000,2463534242,10,6,1,268097034,251703577,499500,496926,57561285,off,,,,,,, +2637,node,quick,sorted,250,2463534242,5,4507,1,268543380,239608421,31125,0,7fd8c1f4,off,,,,,,, +2638,node,quick,reversed,1000,2463534242,22,307,1,277474719,248191438,499500,1000,0ca43265,off,,,,,,, +2639,c,none,sorted,1000,2463534242,6,0,421,250591267,90,0,0,8e15cb75,off,,,,,,, +2640,c,insertion,reversed,2000,2463534242,4,260,1,246930776,245994432,1999000,2000999,ad35ba05,off,,,,,,, +2641,python,bubble,random,2000,2463534242,10,2,1,379494271,361178471,1999000,1988792,41262855,off,,,,,,, +2642,python,quick,reversed,1000,2463534242,29,8,1,271229875,254836882,499500,1000,0ca43265,off,,,,,,, +2643,c,none,random,250,2463534242,13,0,429,249088020,90,0,0,b47adce4,off,,,,,,, +2644,c,none,reversed,1000,2463534242,14,0,405,241823976,90,0,0,0ca43265,off,,,,,,, +2645,python,none,sorted,500,2463534242,29,0,16,250159658,601,0,0,d9a24379,off,,,,,,, +2646,python,none,reversed,2000,2463534242,13,0,16,265086232,561,0,0,ad35ba05,off,,,,,,, +2647,node,insertion,random,2000,2463534242,26,234,1,258250451,228690837,996390,996389,41262855,off,,,,,,, +2648,node,insertion,sorted,250,2463534242,16,607595,1,254499010,224287996,249,0,7fd8c1f4,off,,,,,,, +2649,python,bubble,random,2000,2463534242,21,2,1,391060547,373592371,1999000,1988792,41262855,off,,,,,,, +2650,c,bubble,sorted,500,2463534242,10,2858,1,246696838,245744914,124750,0,d9a24379,off,,,,,,, +2651,node,insertion,nearly_sorted,250,2463534242,3,79550,1,247541912,219429068,2277,2251,21a19c14,off,,,,,,, +2652,node,bubble,sorted,2000,2463534242,14,85,1,266296700,236239109,1999000,0,69842aa5,off,,,,,,, +2653,python,quick,sorted,500,2463534242,11,30,1,250111988,233989687,124750,0,d9a24379,off,,,,,,, +2654,python,none,sorted,500,2463534242,23,0,16,250772666,1283,0,0,d9a24379,off,,,,,,, +2655,python,quick,nearly_sorted,250,2463534242,5,882,1,263253307,247412678,4346,1008,21a19c14,off,,,,,,, +2656,node,bubble,nearly_sorted,250,2463534242,18,6161,1,275915747,246932523,31125,4056,21a19c14,off,,,,,,, +2657,node,none,random,1000,2463534242,28,0,10,269865908,3918,0,0,57561285,off,,,,,,, +2658,c,none,random,2000,2463534242,27,0,415,254006058,90,0,0,41262855,off,,,,,,, +2659,python,insertion,reversed,2000,2463534242,8,2,1,420712879,403463898,1999000,2000999,ad35ba05,off,,,,,,, +2660,c,insertion,reversed,1000,2463534242,17,1033,1,246512361,245579145,499500,500499,0ca43265,off,,,,,,, +2661,node,bubble,sorted,1000,2463534242,27,374,1,251332798,222791982,499500,0,8e15cb75,off,,,,,,, +2662,node,quick,nearly_sorted,1000,2463534242,24,3752,1,268696196,239574552,28499,7892,1cfe4afd,off,,,,,,, +2663,node,quick,random,1000,2463534242,21,6513,1,277800439,250286546,11196,10210,57561285,off,,,,,,, +2664,node,bubble,nearly_sorted,2000,2463534242,30,110,1,267328054,237760368,1999000,232556,5835dc59,off,,,,,,, +2665,python,insertion,reversed,2000,2463534242,22,2,1,411086998,394497249,1999000,2000999,ad35ba05,off,,,,,,, +2666,c,insertion,reversed,250,2463534242,3,15645,1,242337480,241414646,31125,31374,be175004,off,,,,,,, +2667,c,quick,reversed,1000,2463534242,9,814,1,237921021,236844054,499500,1000,0ca43265,off,,,,,,, +2668,node,none,nearly_sorted,250,2463534242,8,0,10,269965415,642,0,0,21a19c14,off,,,,,,, +2669,python,quick,reversed,1000,2463534242,22,8,1,283716437,265905910,499500,1000,0ca43265,off,,,,,,, +2670,node,bubble,reversed,500,2463534242,20,1534,1,281455458,242070677,124750,249500,d33e3559,off,,,,,,, +2671,node,bubble,sorted,2000,2463534242,7,85,1,270916467,240378779,1999000,0,69842aa5,off,,,,,,, +2672,python,quick,nearly_sorted,1000,2463534242,11,110,1,261619124,244755417,28499,7892,1cfe4afd,off,,,,,,, +2673,c,insertion,sorted,500,2463534242,23,399335,1,243779252,242806498,499,0,d9a24379,off,,,,,,, +2674,python,insertion,random,500,2463534242,8,44,1,254116442,237734408,60338,60339,11ae932d,off,,,,,,, +2675,node,bubble,nearly_sorted,1000,2463534242,5,413,1,278546046,248750624,499500,59780,1cfe4afd,off,,,,,,, +2676,node,quick,reversed,1000,2463534242,20,307,1,269169824,240291528,499500,1000,0ca43265,off,,,,,,, +2677,python,none,random,250,2463534242,10,0,17,266600380,551,0,0,b47adce4,off,,,,,,, +2678,c,insertion,reversed,250,2463534242,24,15645,1,246577183,245603369,31125,31374,be175004,off,,,,,,, +2679,c,insertion,random,2000,2463534242,24,492,1,232390768,231183686,996390,996389,41262855,off,,,,,,, +2680,c,none,nearly_sorted,2000,2463534242,9,0,387,229675279,90,0,0,5835dc59,off,,,,,,, +2681,python,insertion,random,250,2463534242,24,206,1,257225539,240606251,15844,15838,b47adce4,off,,,,,,, +2682,python,bubble,nearly_sorted,250,2463534242,1,188,1,268909578,252291148,31125,4056,21a19c14,off,,,,,,, +2683,python,quick,reversed,250,2463534242,10,151,1,268536017,253391904,31125,250,be175004,off,,,,,,, +2684,c,quick,reversed,1000,2463534242,29,814,1,234991221,233848621,499500,1000,0ca43265,off,,,,,,, +2685,python,quick,nearly_sorted,250,2463534242,21,882,1,273552608,257200623,4346,1008,21a19c14,off,,,,,,, +2686,python,quick,sorted,250,2463534242,11,133,1,257697701,241600405,31125,0,7fd8c1f4,off,,,,,,, +2687,c,bubble,sorted,250,2463534242,22,11386,1,248417784,247279983,31125,0,7fd8c1f4,off,,,,,,, +2688,node,insertion,nearly_sorted,250,2463534242,10,79550,1,251664172,220995082,2277,2251,21a19c14,off,,,,,,, +2689,node,quick,reversed,2000,2463534242,14,80,1,283965904,254026292,1999000,2000,ad35ba05,off,,,,,,, +2690,python,none,random,500,2463534242,14,0,16,247030374,641,0,0,11ae932d,off,,,,,,, +2691,python,insertion,reversed,2000,2463534242,30,2,1,410942156,394392614,1999000,2000999,ad35ba05,off,,,,,,, +2692,node,bubble,nearly_sorted,2000,2463534242,6,110,1,265798885,237399783,1999000,232556,5835dc59,off,,,,,,, +2693,python,none,random,1000,2463534242,7,0,15,244749729,641,0,0,57561285,off,,,,,,, +2694,node,none,reversed,1000,2463534242,19,0,9,243142888,1022,0,0,0ca43265,off,,,,,,, +2695,python,insertion,nearly_sorted,250,2463534242,12,1416,1,259787110,241557593,2277,2251,21a19c14,off,,,,,,, +2696,node,none,random,250,2463534242,8,0,10,273504047,1032,0,0,b47adce4,off,,,,,,, +2697,python,insertion,random,2000,2463534242,24,3,1,322785304,304700060,996390,996389,41262855,off,,,,,,, +2698,c,insertion,sorted,250,2463534242,10,786886,1,241474552,240553607,249,0,7fd8c1f4,off,,,,,,, +2699,node,quick,random,1000,2463534242,8,6513,1,273008668,244877974,11196,10210,57561285,off,,,,,,, +2700,python,none,sorted,500,2463534242,16,0,16,244493860,571,0,0,d9a24379,off,,,,,,, +2701,python,none,random,500,2463534242,19,0,16,245605944,741,0,0,11ae932d,off,,,,,,, +2702,c,none,nearly_sorted,2000,2463534242,22,0,387,225765892,91,0,0,5835dc59,off,,,,,,, +2703,node,none,sorted,2000,2463534242,18,0,10,278193386,1192,0,0,69842aa5,off,,,,,,, +2704,python,quick,nearly_sorted,250,2463534242,10,882,1,264204749,247941632,4346,1008,21a19c14,off,,,,,,, +2705,python,quick,sorted,250,2463534242,4,133,1,250315390,234629618,31125,0,7fd8c1f4,off,,,,,,, +2706,node,insertion,nearly_sorted,2000,2463534242,8,2100,1,273411253,243628050,118277,118270,5835dc59,off,,,,,,, +2707,node,insertion,random,1000,2463534242,14,1091,1,269232633,240771351,249457,249457,57561285,off,,,,,,, +2708,c,none,random,1000,2463534242,9,0,367,211715490,90,0,0,57561285,off,,,,,,, +2709,c,quick,nearly_sorted,250,2463534242,29,77221,1,246145426,245415848,4346,1008,21a19c14,off,,,,,,, +2710,c,insertion,random,250,2463534242,24,26020,1,234528373,233561293,15844,15838,b47adce4,off,,,,,,, +2711,c,quick,nearly_sorted,500,2463534242,13,31784,1,241316866,240004046,10192,1850,482b4785,off,,,,,,, +2712,python,insertion,nearly_sorted,1000,2463534242,27,76,1,252091540,235124024,30889,30860,1cfe4afd,off,,,,,,, +2713,python,bubble,nearly_sorted,1000,2463534242,4,9,1,258619180,242339960,499500,59780,1cfe4afd,off,,,,,,, +2714,python,insertion,nearly_sorted,2000,2463534242,13,20,1,268387769,251904533,118277,118270,5835dc59,off,,,,,,, +2715,python,none,sorted,250,2463534242,16,0,17,260991247,521,0,0,7fd8c1f4,off,,,,,,, +2716,node,bubble,reversed,250,2463534242,25,5861,1,263734879,235472734,31125,62250,be175004,off,,,,,,, +2717,python,insertion,random,2000,2463534242,17,3,1,323955396,306351620,996390,996389,41262855,off,,,,,,, +2718,node,bubble,random,2000,2463534242,24,98,1,275905087,243748099,1999000,1988792,41262855,off,,,,,,, +2719,node,insertion,random,250,2463534242,5,15349,1,268170231,239858164,15844,15838,b47adce4,off,,,,,,, +2720,c,insertion,sorted,500,2463534242,29,399335,1,243587060,242531532,499,0,d9a24379,off,,,,,,, +2721,python,insertion,random,500,2463534242,2,44,1,264018492,247478104,60338,60339,11ae932d,off,,,,,,, +2722,python,none,reversed,2000,2463534242,18,0,16,271846661,582,0,0,ad35ba05,off,,,,,,, +2723,node,none,nearly_sorted,1000,2463534242,14,0,10,278160782,1112,0,0,1cfe4afd,off,,,,,,, +2724,python,none,sorted,500,2463534242,8,0,16,253236675,541,0,0,d9a24379,off,,,,,,, +2725,c,quick,nearly_sorted,500,2463534242,18,31784,1,242855710,241875113,10192,1850,482b4785,off,,,,,,, +2726,python,bubble,reversed,500,2463534242,3,20,1,272041446,255626285,124750,249500,d33e3559,off,,,,,,, +2727,node,quick,nearly_sorted,2000,2463534242,24,1909,1,262039271,231710867,58002,11768,5835dc59,off,,,,,,, +2728,node,bubble,reversed,500,2463534242,14,1534,1,279899853,251802346,124750,249500,d33e3559,off,,,,,,, +2729,python,insertion,sorted,1000,2463534242,24,3178,1,261049416,243874306,999,0,8e15cb75,off,,,,,,, +2730,c,insertion,sorted,1000,2463534242,13,196883,1,237943696,236906420,999,0,8e15cb75,off,,,,,,, +2731,python,quick,sorted,250,2463534242,15,133,1,263312006,247559708,31125,0,7fd8c1f4,off,,,,,,, +2732,c,quick,random,2000,2463534242,11,10587,1,255484837,254524880,25197,24544,41262855,off,,,,,,, +2733,python,insertion,sorted,250,2463534242,20,18845,1,290857571,274816558,249,0,7fd8c1f4,off,,,,,,, +2734,node,quick,random,2000,2463534242,27,2126,1,272032507,241735865,25197,24544,41262855,off,,,,,,, +2735,node,insertion,sorted,1000,2463534242,5,160107,1,259751494,231928335,999,0,8e15cb75,off,,,,,,, +2736,c,quick,random,2000,2463534242,30,10587,1,246892496,245986224,25197,24544,41262855,off,,,,,,, +2737,node,quick,random,1000,2463534242,12,6513,1,269068683,240254435,11196,10210,57561285,off,,,,,,, +2738,python,none,sorted,250,2463534242,13,0,17,263776717,501,0,0,7fd8c1f4,off,,,,,,, +2739,c,insertion,reversed,500,2463534242,15,3967,1,238820588,237901525,124750,125249,d33e3559,off,,,,,,, +2740,node,insertion,sorted,1000,2463534242,30,160107,1,263289856,232593574,999,0,8e15cb75,off,,,,,,, +2741,node,insertion,reversed,2000,2463534242,6,144,1,273821753,242812452,1999000,2000999,ad35ba05,off,,,,,,, +2742,node,none,nearly_sorted,2000,2463534242,22,0,9,243330160,601,0,0,5835dc59,off,,,,,,, +2743,c,bubble,nearly_sorted,250,2463534242,13,7880,1,241329580,240356516,31125,4056,21a19c14,off,,,,,,, +2744,c,bubble,sorted,2000,2463534242,27,173,1,239758406,238519113,1999000,0,69842aa5,off,,,,,,, +2745,python,insertion,reversed,500,2463534242,21,22,1,250401040,234178645,124750,125249,d33e3559,off,,,,,,, +2746,node,insertion,reversed,250,2463534242,21,8539,1,270073987,241884756,31125,31374,be175004,off,,,,,,, +2747,node,bubble,sorted,500,2463534242,24,1964,1,258280858,229693703,124750,0,d9a24379,off,,,,,,, +2748,c,bubble,random,250,2463534242,1,2568,1,243685515,242737670,31125,31196,b47adce4,off,,,,,,, +2749,python,insertion,nearly_sorted,1000,2463534242,15,76,1,251447020,235192789,30889,30860,1cfe4afd,off,,,,,,, +2750,python,bubble,sorted,1000,2463534242,24,11,1,284915493,268829474,499500,0,8e15cb75,off,,,,,,, +2751,node,none,nearly_sorted,500,2463534242,20,0,10,304534691,831,0,0,482b4785,off,,,,,,, +2752,c,bubble,reversed,250,2463534242,25,1384,1,245813143,244907424,31125,62250,be175004,off,,,,,,, +2753,python,none,reversed,2000,2463534242,1,0,16,269866390,591,0,0,ad35ba05,off,,,,,,, +2754,c,insertion,nearly_sorted,250,2463534242,9,99421,1,243059872,242195535,2277,2251,21a19c14,off,,,,,,, +2755,c,quick,sorted,1000,2463534242,2,691,1,245130683,244242768,499500,0,8e15cb75,off,,,,,,, +2756,c,none,sorted,1000,2463534242,5,0,421,243545482,90,0,0,8e15cb75,off,,,,,,, +2757,c,none,sorted,2000,2463534242,22,0,416,245920905,90,0,0,69842aa5,off,,,,,,, +2758,node,quick,nearly_sorted,500,2463534242,26,10954,1,273232598,242698071,10192,1850,482b4785,off,,,,,,, +2759,c,insertion,nearly_sorted,1000,2463534242,13,11445,1,246792317,245870219,30889,30860,1cfe4afd,off,,,,,,, +2760,python,quick,nearly_sorted,1000,2463534242,27,110,1,264404524,247932831,28499,7892,1cfe4afd,off,,,,,,, +2761,c,quick,reversed,2000,2463534242,30,209,1,247828469,246847290,1999000,2000,ad35ba05,off,,,,,,, +2762,node,insertion,sorted,1000,2463534242,4,160107,1,263597773,233201441,999,0,8e15cb75,off,,,,,,, +2763,c,insertion,nearly_sorted,1000,2463534242,29,11445,1,245978454,244991794,30889,30860,1cfe4afd,off,,,,,,, +2764,node,quick,reversed,1000,2463534242,4,307,1,276439196,245713273,499500,1000,0ca43265,off,,,,,,, +2765,node,bubble,nearly_sorted,250,2463534242,6,6161,1,277626823,248062709,31125,4056,21a19c14,off,,,,,,, +2766,node,none,nearly_sorted,500,2463534242,23,0,10,273383492,712,0,0,482b4785,off,,,,,,, +2767,c,bubble,nearly_sorted,250,2463534242,20,7880,1,247326154,246253990,31125,4056,21a19c14,off,,,,,,, +2768,c,insertion,random,500,2463534242,16,6945,1,212914071,211773676,60338,60339,11ae932d,off,,,,,,, +2769,c,quick,reversed,1000,2463534242,5,814,1,240182420,239069143,499500,1000,0ca43265,off,,,,,,, +2770,c,bubble,random,250,2463534242,13,2568,1,244371371,243273574,31125,31196,b47adce4,off,,,,,,, +2771,c,quick,reversed,1000,2463534242,19,814,1,241884711,240911222,499500,1000,0ca43265,off,,,,,,, +2772,c,quick,sorted,500,2463534242,10,2762,1,240247177,239243150,124750,0,d9a24379,off,,,,,,, +2773,c,none,nearly_sorted,500,2463534242,7,0,405,235610665,90,0,0,482b4785,off,,,,,,, +2774,node,bubble,nearly_sorted,500,2463534242,5,1566,1,276846121,245877330,124750,14926,482b4785,off,,,,,,, +2775,python,bubble,random,500,2463534242,10,26,1,269733232,253571417,124750,119688,11ae932d,off,,,,,,, +2776,node,insertion,nearly_sorted,1000,2463534242,6,7758,1,275307675,244748792,30889,30860,1cfe4afd,off,,,,,,, +2777,python,bubble,sorted,2000,2463534242,30,3,1,334329990,317260267,1999000,0,69842aa5,off,,,,,,, +2778,node,none,nearly_sorted,250,2463534242,7,0,10,266715445,751,0,0,21a19c14,off,,,,,,, +2779,c,insertion,reversed,2000,2463534242,8,260,1,241382510,240419365,1999000,2000999,ad35ba05,off,,,,,,, +2780,c,quick,random,1000,2463534242,6,23484,1,236633808,235510664,11196,10210,57561285,off,,,,,,, +2781,python,insertion,sorted,1000,2463534242,21,3178,1,266517885,248876655,999,0,8e15cb75,off,,,,,,, +2782,c,insertion,nearly_sorted,1000,2463534242,2,11445,1,247250426,246297620,30889,30860,1cfe4afd,off,,,,,,, +2783,c,insertion,sorted,1000,2463534242,9,196883,1,242520994,241447552,999,0,8e15cb75,off,,,,,,, +2784,python,bubble,random,1000,2463534242,9,6,1,267345965,249719463,499500,496926,57561285,off,,,,,,, +2785,node,bubble,sorted,500,2463534242,25,1964,1,248540464,220445553,124750,0,d9a24379,off,,,,,,, +2786,python,bubble,reversed,1000,2463534242,17,5,1,297054653,280817849,499500,999000,0ca43265,off,,,,,,, +2787,python,bubble,reversed,1000,2463534242,1,5,1,289797461,272955507,499500,999000,0ca43265,off,,,,,,, +2788,python,bubble,reversed,250,2463534242,28,88,1,263393950,247216404,31125,62250,be175004,off,,,,,,, +2789,node,quick,random,2000,2463534242,19,2126,1,280395152,249899861,25197,24544,41262855,off,,,,,,, +2790,c,none,random,500,2463534242,30,0,426,256827123,80,0,0,11ae932d,off,,,,,,, +2791,python,quick,sorted,500,2463534242,23,30,1,252382965,235241444,124750,0,d9a24379,off,,,,,,, +2792,python,none,sorted,250,2463534242,9,0,17,259311198,581,0,0,7fd8c1f4,off,,,,,,, +2793,c,insertion,nearly_sorted,1000,2463534242,5,11445,1,245360704,244516633,30889,30860,1cfe4afd,off,,,,,,, +2794,node,insertion,reversed,1000,2463534242,26,554,1,273640504,244573194,499500,500499,0ca43265,off,,,,,,, +2795,python,insertion,random,500,2463534242,10,44,1,258541817,242473810,60338,60339,11ae932d,off,,,,,,, +2796,python,bubble,reversed,2000,2463534242,12,2,1,499839316,483032873,1999000,3998000,ad35ba05,off,,,,,,, +2797,node,bubble,random,1000,2463534242,28,362,1,267310670,238552107,499500,496926,57561285,off,,,,,,, +2798,python,bubble,nearly_sorted,250,2463534242,2,188,1,259328211,243635304,31125,4056,21a19c14,off,,,,,,, +2799,node,none,random,250,2463534242,6,0,10,265044263,972,0,0,b47adce4,off,,,,,,, +2800,node,quick,random,2000,2463534242,7,2126,1,262476781,230963172,25197,24544,41262855,off,,,,,,, +2801,python,insertion,sorted,500,2463534242,12,6624,1,245366285,229548945,499,0,d9a24379,off,,,,,,, +2802,c,quick,nearly_sorted,2000,2463534242,20,6145,1,249472220,248548700,58002,11768,5835dc59,off,,,,,,, +2803,python,bubble,random,2000,2463534242,8,2,1,381771117,364016051,1999000,1988792,41262855,off,,,,,,, +2804,c,bubble,nearly_sorted,500,2463534242,14,1998,1,242212194,241130247,124750,14926,482b4785,off,,,,,,, +2805,node,insertion,reversed,500,2463534242,16,2178,1,276244704,246819517,124750,125249,d33e3559,off,,,,,,, +2806,c,quick,nearly_sorted,500,2463534242,2,31784,1,244287352,243367739,10192,1850,482b4785,off,,,,,,, +2807,python,insertion,sorted,500,2463534242,18,6624,1,243963266,228153754,499,0,d9a24379,off,,,,,,, +2808,node,bubble,random,250,2463534242,13,5439,1,268705094,240351173,31125,31196,b47adce4,off,,,,,,, +2809,python,quick,sorted,250,2463534242,12,133,1,256573819,241288605,31125,0,7fd8c1f4,off,,,,,,, +2810,python,quick,nearly_sorted,250,2463534242,24,882,1,274521845,258277999,4346,1008,21a19c14,off,,,,,,, +2811,python,bubble,random,2000,2463534242,4,2,1,368443481,350561068,1999000,1988792,41262855,off,,,,,,, +2812,python,none,sorted,500,2463534242,18,0,16,254233363,571,0,0,d9a24379,off,,,,,,, +2813,node,insertion,random,250,2463534242,4,15349,1,265733815,236207469,15844,15838,b47adce4,off,,,,,,, +2814,c,none,nearly_sorted,250,2463534242,21,0,388,224117693,90,0,0,21a19c14,off,,,,,,, +2815,c,quick,reversed,2000,2463534242,24,209,1,246078620,245339846,1999000,2000,ad35ba05,off,,,,,,, +2816,python,insertion,nearly_sorted,1000,2463534242,3,76,1,251002759,234884767,30889,30860,1cfe4afd,off,,,,,,, +2817,python,bubble,sorted,2000,2463534242,9,3,1,332713579,315965451,1999000,0,69842aa5,off,,,,,,, +2818,c,none,reversed,1000,2463534242,2,0,405,232777372,90,0,0,0ca43265,off,,,,,,, +2819,c,quick,sorted,1000,2463534242,6,691,1,238410289,237646468,499500,0,8e15cb75,off,,,,,,, +2820,node,quick,nearly_sorted,2000,2463534242,8,1909,1,262680073,231582781,58002,11768,5835dc59,off,,,,,,, +2821,node,insertion,reversed,1000,2463534242,19,554,1,271520779,241793837,499500,500499,0ca43265,off,,,,,,, +2822,node,quick,sorted,2000,2463534242,3,74,1,272602858,243085320,1999000,0,69842aa5,off,,,,,,, +2823,node,insertion,sorted,2000,2463534242,3,89220,1,272210644,242652658,1999,0,69842aa5,off,,,,,,, +2824,python,quick,random,1000,2463534242,8,209,1,270117841,253419324,11196,10210,57561285,off,,,,,,, +2825,node,insertion,nearly_sorted,1000,2463534242,7,7758,1,268152077,239672830,30889,30860,1cfe4afd,off,,,,,,, +2826,node,insertion,random,1000,2463534242,2,1091,1,271095554,241441358,249457,249457,57561285,off,,,,,,, +2827,node,bubble,nearly_sorted,250,2463534242,11,6161,1,274721369,244323298,31125,4056,21a19c14,off,,,,,,, +2828,c,bubble,sorted,500,2463534242,2,2858,1,248943868,248085912,124750,0,d9a24379,off,,,,,,, +2829,c,quick,random,2000,2463534242,13,10587,1,244044838,243155803,25197,24544,41262855,off,,,,,,, +2830,c,quick,random,2000,2463534242,20,10587,1,240465600,239525560,25197,24544,41262855,off,,,,,,, +2831,python,none,reversed,500,2463534242,26,0,16,253713558,562,0,0,d33e3559,off,,,,,,, +2832,c,quick,nearly_sorted,2000,2463534242,14,6145,1,248992663,247876726,58002,11768,5835dc59,off,,,,,,, +2833,node,bubble,random,250,2463534242,11,5439,1,261708780,234560994,31125,31196,b47adce4,off,,,,,,, +2834,c,insertion,sorted,1000,2463534242,21,196883,1,236135646,235226373,999,0,8e15cb75,off,,,,,,, +2835,node,bubble,reversed,500,2463534242,6,1534,1,270478658,241633959,124750,249500,d33e3559,off,,,,,,, +2836,python,none,sorted,250,2463534242,30,0,17,262786070,501,0,0,7fd8c1f4,off,,,,,,, +2837,c,insertion,reversed,2000,2463534242,21,260,1,244743899,243864791,1999000,2000999,ad35ba05,off,,,,,,, +2838,python,quick,sorted,250,2463534242,25,133,1,257425375,241030341,31125,0,7fd8c1f4,off,,,,,,, +2839,node,quick,random,2000,2463534242,26,2126,1,278982254,248839580,25197,24544,41262855,off,,,,,,, +2840,c,bubble,random,250,2463534242,21,2568,1,250683901,249809221,31125,31196,b47adce4,off,,,,,,, +2841,node,none,reversed,1000,2463534242,8,0,9,249099752,812,0,0,0ca43265,off,,,,,,, +2842,node,quick,reversed,500,2463534242,19,1194,1,266877249,238660965,124750,500,d33e3559,off,,,,,,, +2843,c,insertion,sorted,2000,2463534242,12,101352,1,239130970,238171458,1999,0,69842aa5,off,,,,,,, +2844,c,bubble,nearly_sorted,250,2463534242,25,7880,1,238303628,237392892,31125,4056,21a19c14,off,,,,,,, +2845,node,none,sorted,250,2463534242,20,0,9,252408352,1002,0,0,7fd8c1f4,off,,,,,,, +2846,python,insertion,random,500,2463534242,26,44,1,275327746,257067267,60338,60339,11ae932d,off,,,,,,, +2847,node,none,sorted,2000,2463534242,4,0,10,289907428,1222,0,0,69842aa5,off,,,,,,, +2848,python,none,reversed,500,2463534242,29,0,16,264875326,591,0,0,d33e3559,off,,,,,,, +2849,python,insertion,reversed,500,2463534242,5,22,1,261760869,245665943,124750,125249,d33e3559,off,,,,,,, +2850,python,quick,sorted,1000,2463534242,21,8,1,291287765,274245264,499500,0,8e15cb75,off,,,,,,, +2851,c,none,random,2000,2463534242,22,0,415,245660627,80,0,0,41262855,off,,,,,,, +2852,node,quick,nearly_sorted,2000,2463534242,26,1909,1,264010636,229875252,58002,11768,5835dc59,off,,,,,,, +2853,python,bubble,nearly_sorted,250,2463534242,27,188,1,264465458,248376553,31125,4056,21a19c14,off,,,,,,, +2854,python,insertion,nearly_sorted,1000,2463534242,12,76,1,265099176,246471371,30889,30860,1cfe4afd,off,,,,,,, +2855,python,quick,sorted,2000,2463534242,9,2,1,310207873,293854370,1999000,0,69842aa5,off,,,,,,, +2856,node,insertion,reversed,2000,2463534242,27,144,1,268615486,240054867,1999000,2000999,ad35ba05,off,,,,,,, +2857,node,none,sorted,1000,2463534242,21,0,9,248262522,1022,0,0,8e15cb75,off,,,,,,, +2858,python,insertion,reversed,500,2463534242,16,22,1,255567455,239888979,124750,125249,d33e3559,off,,,,,,, +2859,c,quick,sorted,1000,2463534242,5,691,1,243257481,241929976,499500,0,8e15cb75,off,,,,,,, +2860,node,quick,random,2000,2463534242,30,2126,1,261133924,230367159,25197,24544,41262855,off,,,,,,, +2861,node,quick,sorted,2000,2463534242,21,74,1,273118405,243331255,1999000,0,69842aa5,off,,,,,,, +2862,node,none,nearly_sorted,1000,2463534242,24,0,10,269664251,882,0,0,1cfe4afd,off,,,,,,, +2863,c,bubble,nearly_sorted,1000,2463534242,7,507,1,245392003,244421035,499500,59780,1cfe4afd,off,,,,,,, +2864,c,bubble,nearly_sorted,250,2463534242,10,7880,1,238547276,237591254,31125,4056,21a19c14,off,,,,,,, +2865,c,insertion,reversed,250,2463534242,25,15645,1,255446092,254293094,31125,31374,be175004,off,,,,,,, +2866,c,bubble,reversed,500,2463534242,19,346,1,243327810,242182259,124750,249500,d33e3559,off,,,,,,, +2867,python,bubble,reversed,1000,2463534242,8,5,1,287494956,271317942,499500,999000,0ca43265,off,,,,,,, +2868,python,none,nearly_sorted,250,2463534242,4,0,17,255452147,601,0,0,21a19c14,off,,,,,,, +2869,python,insertion,random,250,2463534242,25,206,1,262510895,246991100,15844,15838,b47adce4,off,,,,,,, +2870,c,insertion,random,500,2463534242,24,6945,1,214621477,213740446,60338,60339,11ae932d,off,,,,,,, +2871,python,quick,reversed,2000,2463534242,23,2,1,279240408,262121581,1999000,2000,ad35ba05,off,,,,,,, +2872,python,insertion,reversed,1000,2463534242,9,6,1,301580204,284610385,499500,500499,0ca43265,off,,,,,,, +2873,c,insertion,nearly_sorted,1000,2463534242,23,11445,1,245228312,244217642,30889,30860,1cfe4afd,off,,,,,,, +2874,c,bubble,reversed,2000,2463534242,6,22,1,248720194,247777484,1999000,3998000,ad35ba05,off,,,,,,, +2875,python,quick,sorted,2000,2463534242,4,2,1,317680508,301285903,1999000,0,69842aa5,off,,,,,,, +2876,node,quick,sorted,500,2463534242,4,1164,1,276168500,247385538,124750,0,d9a24379,off,,,,,,, +2877,c,bubble,random,1000,2463534242,21,154,1,235653993,234737526,499500,496926,57561285,off,,,,,,, +2878,python,quick,random,500,2463534242,4,507,1,269798894,252548183,4578,4552,11ae932d,off,,,,,,, +2879,node,insertion,random,250,2463534242,24,15349,1,266872038,238591683,15844,15838,b47adce4,off,,,,,,, +2880,c,bubble,reversed,1000,2463534242,4,87,1,246235533,245280996,499500,999000,0ca43265,off,,,,,,, +2881,c,none,random,500,2463534242,27,0,426,249271544,90,0,0,11ae932d,off,,,,,,, +2882,c,none,random,250,2463534242,20,0,429,248051467,90,0,0,b47adce4,off,,,,,,, +2883,node,insertion,nearly_sorted,500,2463534242,17,26045,1,253072947,223714038,7962,7926,482b4785,off,,,,,,, +2884,python,bubble,random,500,2463534242,16,26,1,267292697,251641022,124750,119688,11ae932d,off,,,,,,, +2885,node,quick,sorted,250,2463534242,10,4507,1,271346504,242383299,31125,0,7fd8c1f4,off,,,,,,, +2886,node,bubble,reversed,2000,2463534242,14,94,1,272225501,240622877,1999000,3998000,ad35ba05,off,,,,,,, +2887,python,insertion,random,2000,2463534242,1,3,1,329992080,311713088,996390,996389,41262855,off,,,,,,, +2888,node,quick,sorted,2000,2463534242,28,74,1,272335728,243089303,1999000,0,69842aa5,off,,,,,,, +2889,c,none,sorted,500,2463534242,14,0,344,198713664,91,0,0,d9a24379,off,,,,,,, +2890,c,bubble,random,1000,2463534242,2,154,1,235136163,234439998,499500,496926,57561285,off,,,,,,, +2891,c,insertion,nearly_sorted,500,2463534242,4,32350,1,243979165,242978360,7962,7926,482b4785,off,,,,,,, +2892,node,bubble,sorted,250,2463534242,6,8081,1,261439427,231288480,31125,0,7fd8c1f4,off,,,,,,, +2893,node,none,reversed,1000,2463534242,27,0,9,249703894,932,0,0,0ca43265,off,,,,,,, +2894,c,bubble,random,1000,2463534242,4,154,1,238021482,237066110,499500,496926,57561285,off,,,,,,, +2895,python,bubble,nearly_sorted,1000,2463534242,5,9,1,253353132,237347028,499500,59780,1cfe4afd,off,,,,,,, +2896,python,insertion,random,250,2463534242,8,206,1,271511753,254729722,15844,15838,b47adce4,off,,,,,,, +2897,node,bubble,reversed,500,2463534242,5,1534,1,271502006,243391633,124750,249500,d33e3559,off,,,,,,, +2898,node,none,reversed,500,2463534242,11,0,9,237421406,861,0,0,d33e3559,off,,,,,,, +2899,node,quick,random,250,2463534242,30,36348,1,263220936,234415990,1933,1940,b47adce4,off,,,,,,, +2900,c,bubble,random,250,2463534242,3,2568,1,246037873,245176159,31125,31196,b47adce4,off,,,,,,, +2901,python,quick,sorted,500,2463534242,12,30,1,253565600,237077076,124750,0,d9a24379,off,,,,,,, +2902,c,none,random,500,2463534242,23,0,426,246123584,90,0,0,11ae932d,off,,,,,,, +2903,c,insertion,random,500,2463534242,28,6945,1,216405791,215687194,60338,60339,11ae932d,off,,,,,,, +2904,python,quick,reversed,2000,2463534242,20,2,1,286532133,268949055,1999000,2000,ad35ba05,off,,,,,,, +2905,python,insertion,random,2000,2463534242,8,3,1,316852066,299419983,996390,996389,41262855,off,,,,,,, +2906,python,bubble,nearly_sorted,500,2463534242,15,41,1,251792817,235367658,124750,14926,482b4785,off,,,,,,, +2907,node,quick,nearly_sorted,500,2463534242,8,10954,1,267025175,238101990,10192,1850,482b4785,off,,,,,,, +2908,python,quick,reversed,250,2463534242,24,151,1,261341486,244617685,31125,250,be175004,off,,,,,,, +2909,python,insertion,reversed,2000,2463534242,5,2,1,407780049,390775040,1999000,2000999,ad35ba05,off,,,,,,, +2910,python,bubble,nearly_sorted,500,2463534242,27,41,1,255362579,239456120,124750,14926,482b4785,off,,,,,,, +2911,c,insertion,sorted,500,2463534242,20,399335,1,247274713,246328819,499,0,d9a24379,off,,,,,,, +2912,c,insertion,sorted,1000,2463534242,26,196883,1,234134042,233101320,999,0,8e15cb75,off,,,,,,, +2913,python,none,nearly_sorted,250,2463534242,11,0,17,270905859,641,0,0,21a19c14,off,,,,,,, +2914,python,bubble,sorted,250,2463534242,5,206,1,271204417,253955451,31125,0,7fd8c1f4,off,,,,,,, +2915,python,quick,nearly_sorted,250,2463534242,18,882,1,282047699,265202604,4346,1008,21a19c14,off,,,,,,, +2916,python,insertion,reversed,250,2463534242,13,102,1,257109221,241260626,31125,31374,be175004,off,,,,,,, +2917,node,insertion,nearly_sorted,2000,2463534242,2,2100,1,265366858,235212750,118277,118270,5835dc59,off,,,,,,, +2918,node,none,reversed,2000,2463534242,20,0,9,247075348,912,0,0,ad35ba05,off,,,,,,, +2919,c,none,nearly_sorted,1000,2463534242,11,0,421,247927685,90,0,0,1cfe4afd,off,,,,,,, +2920,node,quick,reversed,2000,2463534242,24,80,1,278421954,245269076,1999000,2000,ad35ba05,off,,,,,,, +2921,python,quick,reversed,1000,2463534242,23,8,1,273925677,257915553,499500,1000,0ca43265,off,,,,,,, +2922,python,quick,sorted,2000,2463534242,27,2,1,298422857,281141189,1999000,0,69842aa5,off,,,,,,, +2923,c,insertion,reversed,250,2463534242,6,15645,1,246301548,245160599,31125,31374,be175004,off,,,,,,, +2924,c,none,random,500,2463534242,11,0,426,247620910,90,0,0,11ae932d,off,,,,,,, +2925,python,none,sorted,1000,2463534242,3,0,16,257267729,591,0,0,8e15cb75,off,,,,,,, +2926,node,quick,random,2000,2463534242,15,2126,1,268921159,238677741,25197,24544,41262855,off,,,,,,, +2927,c,insertion,reversed,2000,2463534242,2,260,1,240067215,239045799,1999000,2000999,ad35ba05,off,,,,,,, +2928,node,none,sorted,500,2463534242,18,0,10,271203257,832,0,0,d9a24379,off,,,,,,, +2929,python,insertion,nearly_sorted,500,2463534242,24,316,1,253866073,237487831,7962,7926,482b4785,off,,,,,,, +2930,node,bubble,nearly_sorted,2000,2463534242,28,110,1,262545460,232861515,1999000,232556,5835dc59,off,,,,,,, +2931,python,bubble,reversed,2000,2463534242,20,2,1,489432493,472938307,1999000,3998000,ad35ba05,off,,,,,,, +2932,python,insertion,sorted,500,2463534242,24,6624,1,249956287,233629090,499,0,d9a24379,off,,,,,,, +2933,python,quick,reversed,250,2463534242,9,151,1,266060137,249845638,31125,250,be175004,off,,,,,,, +2934,c,insertion,reversed,2000,2463534242,20,260,1,249723531,248802825,1999000,2000999,ad35ba05,off,,,,,,, +2935,python,bubble,reversed,250,2463534242,1,88,1,255036378,239332300,31125,62250,be175004,off,,,,,,, +2936,c,none,nearly_sorted,250,2463534242,5,0,388,223670746,80,0,0,21a19c14,off,,,,,,, +2937,python,none,nearly_sorted,1000,2463534242,13,0,16,249059757,501,0,0,1cfe4afd,off,,,,,,, +2938,node,none,nearly_sorted,250,2463534242,6,0,10,263514987,982,0,0,21a19c14,off,,,,,,, +2939,python,quick,random,250,2463534242,2,1271,1,249994730,234609596,1933,1940,b47adce4,off,,,,,,, +2940,c,insertion,reversed,2000,2463534242,27,260,1,241999804,240951142,1999000,2000999,ad35ba05,off,,,,,,, +2941,python,quick,random,250,2463534242,8,1271,1,254331677,238497025,1933,1940,b47adce4,off,,,,,,, +2942,node,none,random,2000,2463534242,1,0,9,245969586,1152,0,0,41262855,off,,,,,,, +2943,node,quick,nearly_sorted,1000,2463534242,4,3752,1,270058641,239864462,28499,7892,1cfe4afd,off,,,,,,, +2944,python,quick,sorted,500,2463534242,22,30,1,244731325,229057567,124750,0,d9a24379,off,,,,,,, +2945,python,insertion,nearly_sorted,500,2463534242,30,316,1,252105842,235719448,7962,7926,482b4785,off,,,,,,, +2946,c,quick,reversed,500,2463534242,6,3240,1,240148470,239062561,124750,500,d33e3559,off,,,,,,, +2947,c,quick,reversed,250,2463534242,4,12781,1,241403630,240344916,31125,250,be175004,off,,,,,,, +2948,node,quick,nearly_sorted,250,2463534242,23,25320,1,267755833,239471551,4346,1008,21a19c14,off,,,,,,, +2949,python,bubble,random,2000,2463534242,15,2,1,385872324,367779092,1999000,1988792,41262855,off,,,,,,, +2950,python,insertion,nearly_sorted,250,2463534242,3,1416,1,256317959,239252327,2277,2251,21a19c14,off,,,,,,, +2951,c,quick,sorted,500,2463534242,29,2762,1,245223037,244291791,124750,0,d9a24379,off,,,,,,, +2952,c,bubble,reversed,1000,2463534242,23,87,1,246784753,245860401,499500,999000,0ca43265,off,,,,,,, +2953,node,bubble,sorted,500,2463534242,21,1964,1,259702533,231251701,124750,0,d9a24379,off,,,,,,, +2954,python,bubble,random,250,2463534242,1,112,1,259055169,241148371,31125,31196,b47adce4,off,,,,,,, +2955,c,quick,nearly_sorted,1000,2463534242,5,11273,1,234806566,233820748,28499,7892,1cfe4afd,off,,,,,,, +2956,node,quick,reversed,1000,2463534242,21,307,1,275288461,243345654,499500,1000,0ca43265,off,,,,,,, +2957,node,none,sorted,250,2463534242,23,0,9,237857503,872,0,0,7fd8c1f4,off,,,,,,, +2958,node,insertion,reversed,2000,2463534242,13,144,1,279576196,248670316,1999000,2000999,ad35ba05,off,,,,,,, +2959,python,quick,nearly_sorted,1000,2463534242,26,110,1,280530436,262033597,28499,7892,1cfe4afd,off,,,,,,, +2960,python,bubble,nearly_sorted,2000,2463534242,18,3,1,373110366,351252273,1999000,232556,5835dc59,off,,,,,,, +2961,c,none,reversed,250,2463534242,10,0,427,256579449,90,0,0,be175004,off,,,,,,, +2962,node,quick,reversed,500,2463534242,6,1194,1,273400362,241882862,124750,500,d33e3559,off,,,,,,, +2963,node,bubble,reversed,1000,2463534242,28,378,1,276327068,245715957,499500,999000,0ca43265,off,,,,,,, +2964,python,insertion,sorted,1000,2463534242,30,3178,1,271433056,254450898,999,0,8e15cb75,off,,,,,,, +2965,node,bubble,reversed,250,2463534242,15,5861,1,261788531,233698054,31125,62250,be175004,off,,,,,,, +2966,node,bubble,reversed,1000,2463534242,2,378,1,272491119,244085472,499500,999000,0ca43265,off,,,,,,, +2967,node,quick,reversed,500,2463534242,25,1194,1,268085412,238565397,124750,500,d33e3559,off,,,,,,, +2968,python,quick,reversed,250,2463534242,6,151,1,267571168,252238793,31125,250,be175004,off,,,,,,, +2969,c,quick,sorted,250,2463534242,9,10851,1,245949720,244822858,31125,0,7fd8c1f4,off,,,,,,, +2970,c,none,sorted,250,2463534242,10,0,429,245189362,90,0,0,7fd8c1f4,off,,,,,,, +2971,python,insertion,reversed,250,2463534242,28,102,1,257056113,240297271,31125,31374,be175004,off,,,,,,, +2972,python,quick,reversed,500,2463534242,15,33,1,261309375,244541628,124750,500,d33e3559,off,,,,,,, +2973,node,insertion,random,500,2463534242,9,4059,1,279457042,249525154,60338,60339,11ae932d,off,,,,,,, +2974,c,none,reversed,250,2463534242,14,0,427,248788058,80,0,0,be175004,off,,,,,,, +2975,c,quick,reversed,1000,2463534242,18,814,1,239594198,238803175,499500,1000,0ca43265,off,,,,,,, +2976,node,none,random,500,2463534242,24,0,9,248874840,761,0,0,11ae932d,off,,,,,,, +2977,python,quick,random,250,2463534242,9,1271,1,266598165,250654291,1933,1940,b47adce4,off,,,,,,, +2978,python,quick,reversed,1000,2463534242,14,8,1,287310471,269801794,499500,1000,0ca43265,off,,,,,,, +2979,node,none,random,2000,2463534242,12,0,9,260202299,811,0,0,41262855,off,,,,,,, +2980,node,insertion,random,2000,2463534242,24,234,1,276893178,248264962,996390,996389,41262855,off,,,,,,, +2981,node,quick,random,500,2463534242,12,16148,1,280407324,251577174,4578,4552,11ae932d,off,,,,,,, +2982,node,quick,nearly_sorted,2000,2463534242,16,1909,1,265214191,233289731,58002,11768,5835dc59,off,,,,,,, +2983,c,none,sorted,1000,2463534242,14,0,421,241770677,140,0,0,8e15cb75,off,,,,,,, +2984,python,bubble,random,1000,2463534242,25,6,1,268927199,252358382,499500,496926,57561285,off,,,,,,, +2985,python,insertion,reversed,1000,2463534242,17,6,1,293954914,277178255,499500,500499,0ca43265,off,,,,,,, +2986,node,bubble,nearly_sorted,500,2463534242,11,1566,1,275532879,245606583,124750,14926,482b4785,off,,,,,,, +2987,c,quick,nearly_sorted,1000,2463534242,22,11273,1,240205103,239146298,28499,7892,1cfe4afd,off,,,,,,, +2988,python,none,nearly_sorted,500,2463534242,12,0,17,264866981,602,0,0,482b4785,off,,,,,,, +2989,python,quick,reversed,2000,2463534242,29,2,1,285154351,267531016,1999000,2000,ad35ba05,off,,,,,,, +2990,python,bubble,nearly_sorted,2000,2463534242,17,3,1,357155035,339452145,1999000,232556,5835dc59,off,,,,,,, +2991,python,bubble,reversed,500,2463534242,8,20,1,270851897,254415191,124750,249500,d33e3559,off,,,,,,, +2992,node,quick,sorted,250,2463534242,29,4507,1,269490766,239700788,31125,0,7fd8c1f4,off,,,,,,, +2993,c,bubble,random,2000,2463534242,5,40,1,246228681,245352355,1999000,1988792,41262855,off,,,,,,, +2994,python,bubble,reversed,2000,2463534242,15,2,1,493070372,475804780,1999000,3998000,ad35ba05,off,,,,,,, +2995,node,bubble,sorted,500,2463534242,13,1964,1,265095437,235626328,124750,0,d9a24379,off,,,,,,, +2996,node,bubble,reversed,500,2463534242,21,1534,1,275339818,246388072,124750,249500,d33e3559,off,,,,,,, +2997,c,none,reversed,250,2463534242,7,0,427,249466215,120,0,0,be175004,off,,,,,,, +2998,python,quick,sorted,1000,2463534242,19,8,1,310133457,293115664,499500,0,8e15cb75,off,,,,,,, +2999,python,bubble,random,250,2463534242,18,112,1,265189637,248781795,31125,31196,b47adce4,off,,,,,,, +3000,c,insertion,reversed,250,2463534242,4,15645,1,246556544,245605403,31125,31374,be175004,off,,,,,,, +3001,c,bubble,reversed,250,2463534242,17,1384,1,241989287,240794688,31125,62250,be175004,off,,,,,,, +3002,python,insertion,reversed,2000,2463534242,20,2,1,416835421,398954524,1999000,2000999,ad35ba05,off,,,,,,, +3003,node,bubble,nearly_sorted,1000,2463534242,30,413,1,277229519,246242302,499500,59780,1cfe4afd,off,,,,,,, +3004,python,none,sorted,2000,2463534242,26,0,15,251459594,531,0,0,69842aa5,off,,,,,,, +3005,c,insertion,reversed,250,2463534242,1,15645,1,247866601,246908561,31125,31374,be175004,off,,,,,,, +3006,python,insertion,random,500,2463534242,5,44,1,254058235,238387689,60338,60339,11ae932d,off,,,,,,, +3007,node,quick,random,2000,2463534242,20,2126,1,268243388,237916937,25197,24544,41262855,off,,,,,,, +3008,c,bubble,sorted,1000,2463534242,23,702,1,245240399,244380538,499500,0,8e15cb75,off,,,,,,, +3009,c,bubble,sorted,500,2463534242,24,2858,1,246240373,245296414,124750,0,d9a24379,off,,,,,,, +3010,c,bubble,sorted,1000,2463534242,14,702,1,244908236,243837981,499500,0,8e15cb75,off,,,,,,, +3011,python,insertion,sorted,1000,2463534242,18,3178,1,256350430,240526410,999,0,8e15cb75,off,,,,,,, +3012,python,quick,reversed,500,2463534242,9,33,1,263211208,246490978,124750,500,d33e3559,off,,,,,,, +3013,python,quick,reversed,500,2463534242,21,33,1,267242853,250443149,124750,500,d33e3559,off,,,,,,, +3014,python,bubble,random,1000,2463534242,8,6,1,283172697,266011221,499500,496926,57561285,off,,,,,,, +3015,python,none,sorted,1000,2463534242,25,0,16,272215982,651,0,0,8e15cb75,off,,,,,,, +3016,c,insertion,nearly_sorted,2000,2463534242,7,2962,1,228769462,227689569,118277,118270,5835dc59,off,,,,,,, +3017,python,bubble,random,250,2463534242,8,112,1,246605905,230366049,31125,31196,b47adce4,off,,,,,,, +3018,node,insertion,reversed,250,2463534242,1,8539,1,267588475,237501894,31125,31374,be175004,off,,,,,,, +3019,c,none,random,2000,2463534242,23,0,415,247367956,100,0,0,41262855,off,,,,,,, +3020,c,insertion,nearly_sorted,2000,2463534242,6,2962,1,224349038,223592560,118277,118270,5835dc59,off,,,,,,, +3021,node,bubble,random,2000,2463534242,14,98,1,270432731,241564778,1999000,1988792,41262855,off,,,,,,, +3022,c,none,nearly_sorted,2000,2463534242,30,0,387,227537342,90,0,0,5835dc59,off,,,,,,, +3023,c,insertion,reversed,500,2463534242,9,3967,1,240668532,239926221,124750,125249,d33e3559,off,,,,,,, +3024,c,none,reversed,2000,2463534242,7,0,413,243046436,90,0,0,ad35ba05,off,,,,,,, +3025,c,quick,sorted,250,2463534242,30,10851,1,249776281,248979035,31125,0,7fd8c1f4,off,,,,,,, +3026,python,insertion,reversed,250,2463534242,17,102,1,247442995,232056038,31125,31374,be175004,off,,,,,,, +3027,c,insertion,sorted,500,2463534242,14,399335,1,243203252,242061081,499,0,d9a24379,off,,,,,,, +3028,python,none,sorted,2000,2463534242,3,0,15,249732840,2104,0,0,69842aa5,off,,,,,,, +3029,python,bubble,reversed,500,2463534242,30,20,1,261580640,244752271,124750,249500,d33e3559,off,,,,,,, +3030,node,none,random,250,2463534242,17,0,10,280000082,752,0,0,b47adce4,off,,,,,,, +3031,node,insertion,nearly_sorted,500,2463534242,10,26045,1,261724151,231968134,7962,7926,482b4785,off,,,,,,, +3032,c,none,nearly_sorted,1000,2463534242,18,0,421,248918812,91,0,0,1cfe4afd,off,,,,,,, +3033,node,none,sorted,250,2463534242,14,0,9,247094123,701,0,0,7fd8c1f4,off,,,,,,, +3034,python,none,random,1000,2463534242,29,0,15,244940147,591,0,0,57561285,off,,,,,,, +3035,python,none,reversed,1000,2463534242,19,0,16,253144843,582,0,0,0ca43265,off,,,,,,, +3036,python,quick,nearly_sorted,500,2463534242,28,296,1,246524215,230637298,10192,1850,482b4785,off,,,,,,, +3037,python,quick,nearly_sorted,2000,2463534242,2,55,1,258671389,241923111,58002,11768,5835dc59,off,,,,,,, +3038,python,bubble,reversed,2000,2463534242,28,2,1,490438746,473750421,1999000,3998000,ad35ba05,off,,,,,,, +3039,python,insertion,reversed,2000,2463534242,10,2,1,455686695,438841516,1999000,2000999,ad35ba05,off,,,,,,, +3040,python,none,nearly_sorted,500,2463534242,15,0,17,260427593,541,0,0,482b4785,off,,,,,,, +3041,c,none,random,2000,2463534242,19,0,415,244391938,80,0,0,41262855,off,,,,,,, +3042,node,quick,random,1000,2463534242,3,6513,1,267821217,238310571,11196,10210,57561285,off,,,,,,, +3043,node,none,random,2000,2463534242,14,0,9,249378084,872,0,0,41262855,off,,,,,,, +3044,python,bubble,nearly_sorted,1000,2463534242,8,9,1,259265954,240946279,499500,59780,1cfe4afd,off,,,,,,, +3045,python,quick,sorted,2000,2463534242,7,2,1,301104683,284053875,1999000,0,69842aa5,off,,,,,,, +3046,python,insertion,nearly_sorted,500,2463534242,7,316,1,257849630,242021632,7962,7926,482b4785,off,,,,,,, +3047,node,none,random,1000,2463534242,3,0,10,263229863,921,0,0,57561285,off,,,,,,, +3048,python,none,nearly_sorted,250,2463534242,28,0,17,256914453,521,0,0,21a19c14,off,,,,,,, +3049,c,none,reversed,2000,2463534242,15,0,413,241184311,90,0,0,ad35ba05,off,,,,,,, +3050,node,bubble,reversed,500,2463534242,7,1534,1,270898775,242705239,124750,249500,d33e3559,off,,,,,,, +3051,python,quick,reversed,2000,2463534242,22,2,1,280987983,264540335,1999000,2000,ad35ba05,off,,,,,,, +3052,node,bubble,nearly_sorted,2000,2463534242,25,110,1,268811604,238641649,1999000,232556,5835dc59,off,,,,,,, +3053,node,insertion,random,1000,2463534242,21,1091,1,271060818,241199447,249457,249457,57561285,off,,,,,,, +3054,node,bubble,random,250,2463534242,29,5439,1,255912209,226638264,31125,31196,b47adce4,off,,,,,,, +3055,node,none,nearly_sorted,2000,2463534242,28,0,9,249527413,822,0,0,5835dc59,off,,,,,,, +3056,c,quick,reversed,500,2463534242,27,3240,1,242124756,241176063,124750,500,d33e3559,off,,,,,,, +3057,c,bubble,reversed,500,2463534242,5,346,1,243693666,242348571,124750,249500,d33e3559,off,,,,,,, +3058,python,insertion,reversed,500,2463534242,3,22,1,246996728,231578802,124750,125249,d33e3559,off,,,,,,, +3059,c,quick,nearly_sorted,1000,2463534242,8,11273,1,247550294,246608372,28499,7892,1cfe4afd,off,,,,,,, +3060,node,insertion,random,1000,2463534242,15,1091,1,272380705,242769922,249457,249457,57561285,off,,,,,,, +3061,c,quick,reversed,500,2463534242,12,3240,1,235847265,234997283,124750,500,d33e3559,off,,,,,,, +3062,c,bubble,sorted,250,2463534242,30,11386,1,244449167,243517352,31125,0,7fd8c1f4,off,,,,,,, +3063,c,none,nearly_sorted,500,2463534242,27,0,405,232985761,81,0,0,482b4785,off,,,,,,, +3064,node,bubble,random,250,2463534242,15,5439,1,262324106,233335491,31125,31196,b47adce4,off,,,,,,, +3065,python,bubble,random,1000,2463534242,21,6,1,272206725,256082867,499500,496926,57561285,off,,,,,,, +3066,node,none,random,250,2463534242,19,0,10,269715648,1252,0,0,b47adce4,off,,,,,,, +3067,python,bubble,sorted,500,2463534242,25,46,1,260023924,244440417,124750,0,d9a24379,off,,,,,,, +3068,python,none,reversed,1000,2463534242,23,0,16,256065427,641,0,0,0ca43265,off,,,,,,, +3069,node,insertion,reversed,250,2463534242,28,8539,1,281004132,251628588,31125,31374,be175004,off,,,,,,, +3070,python,insertion,reversed,2000,2463534242,7,2,1,424989524,407389779,1999000,2000999,ad35ba05,off,,,,,,, +3071,python,bubble,random,2000,2463534242,20,2,1,400544470,383177519,1999000,1988792,41262855,off,,,,,,, +3072,node,quick,random,1000,2463534242,11,6513,1,287773778,255949518,11196,10210,57561285,off,,,,,,, +3073,node,none,random,500,2463534242,26,0,9,251075194,1072,0,0,11ae932d,off,,,,,,, +3074,python,quick,reversed,500,2463534242,25,33,1,266611139,251065350,124750,500,d33e3559,off,,,,,,, +3075,c,none,reversed,2000,2463534242,2,0,413,244852712,91,0,0,ad35ba05,off,,,,,,, +3076,python,insertion,nearly_sorted,500,2463534242,9,316,1,248186490,232473851,7962,7926,482b4785,off,,,,,,, +3077,python,bubble,nearly_sorted,500,2463534242,30,41,1,263849931,247697386,124750,14926,482b4785,off,,,,,,, +3078,node,bubble,random,250,2463534242,1,5439,1,260556407,231948023,31125,31196,b47adce4,off,,,,,,, +3079,c,none,reversed,2000,2463534242,24,0,413,256858011,90,0,0,ad35ba05,off,,,,,,, +3080,python,none,nearly_sorted,500,2463534242,3,0,17,273201030,592,0,0,482b4785,off,,,,,,, +3081,node,bubble,nearly_sorted,1000,2463534242,1,413,1,270676208,240900572,499500,59780,1cfe4afd,off,,,,,,, +3082,python,insertion,sorted,500,2463534242,26,6624,1,238109876,222326359,499,0,d9a24379,off,,,,,,, +3083,python,quick,random,250,2463534242,29,1271,1,257002903,241640576,1933,1940,b47adce4,off,,,,,,, +3084,python,insertion,random,1000,2463534242,27,10,1,261510372,243549497,249457,249457,57561285,off,,,,,,, +3085,node,bubble,random,500,2463534242,3,1407,1,269005616,239303873,124750,119688,11ae932d,off,,,,,,, +3086,node,insertion,sorted,250,2463534242,10,607595,1,256802036,228756043,249,0,7fd8c1f4,off,,,,,,, +3087,c,none,reversed,500,2463534242,6,0,402,234526300,90,0,0,d33e3559,off,,,,,,, +3088,c,insertion,sorted,2000,2463534242,6,101352,1,243553368,242796510,1999,0,69842aa5,off,,,,,,, +3089,c,bubble,nearly_sorted,500,2463534242,10,1998,1,237033839,236102253,124750,14926,482b4785,off,,,,,,, +3090,node,insertion,nearly_sorted,2000,2463534242,1,2100,1,266964642,237558328,118277,118270,5835dc59,off,,,,,,, +3091,node,bubble,reversed,1000,2463534242,23,378,1,269329373,241161286,499500,999000,0ca43265,off,,,,,,, +3092,python,quick,nearly_sorted,2000,2463534242,13,55,1,263683663,246324321,58002,11768,5835dc59,off,,,,,,, +3093,node,insertion,nearly_sorted,250,2463534242,15,79550,1,247988219,218005345,2277,2251,21a19c14,off,,,,,,, +3094,python,none,sorted,250,2463534242,8,0,17,263422734,471,0,0,7fd8c1f4,off,,,,,,, +3095,python,quick,sorted,2000,2463534242,17,2,1,304236241,288334634,1999000,0,69842aa5,off,,,,,,, +3096,node,quick,nearly_sorted,2000,2463534242,23,1909,1,257326549,227240683,58002,11768,5835dc59,off,,,,,,, +3097,node,none,random,250,2463534242,24,0,10,265408446,751,0,0,b47adce4,off,,,,,,, +3098,node,insertion,reversed,1000,2463534242,23,554,1,260774621,232156964,499500,500499,0ca43265,off,,,,,,, +3099,python,insertion,random,1000,2463534242,5,10,1,261401737,244450828,249457,249457,57561285,off,,,,,,, +3100,node,none,nearly_sorted,1000,2463534242,26,0,10,264037236,832,0,0,1cfe4afd,off,,,,,,, +3101,node,none,random,250,2463534242,2,0,10,267269803,892,0,0,b47adce4,off,,,,,,, +3102,python,none,nearly_sorted,250,2463534242,17,0,17,261362724,501,0,0,21a19c14,off,,,,,,, +3103,python,none,sorted,500,2463534242,4,0,16,247343601,571,0,0,d9a24379,off,,,,,,, +3104,node,none,nearly_sorted,1000,2463534242,16,0,10,260244638,821,0,0,1cfe4afd,off,,,,,,, +3105,c,none,nearly_sorted,250,2463534242,20,0,388,229419830,81,0,0,21a19c14,off,,,,,,, +3106,c,insertion,reversed,1000,2463534242,15,1033,1,242194441,241444555,499500,500499,0ca43265,off,,,,,,, +3107,c,insertion,reversed,2000,2463534242,28,260,1,246042732,244871680,1999000,2000999,ad35ba05,off,,,,,,, +3108,python,bubble,random,2000,2463534242,13,2,1,371862729,354308901,1999000,1988792,41262855,off,,,,,,, +3109,node,bubble,sorted,1000,2463534242,22,374,1,261176174,231425003,499500,0,8e15cb75,off,,,,,,, +3110,c,none,reversed,500,2463534242,22,0,402,234832694,90,0,0,d33e3559,off,,,,,,, +3111,c,none,nearly_sorted,250,2463534242,14,0,388,226363412,90,0,0,21a19c14,off,,,,,,, +3112,python,insertion,reversed,500,2463534242,2,22,1,248012986,231392469,124750,125249,d33e3559,off,,,,,,, +3113,node,quick,reversed,2000,2463534242,2,80,1,277469888,247460119,1999000,2000,ad35ba05,off,,,,,,, +3114,python,insertion,sorted,250,2463534242,13,18845,1,261469734,245834894,249,0,7fd8c1f4,off,,,,,,, +3115,node,quick,random,2000,2463534242,9,2126,1,267412741,239365052,25197,24544,41262855,off,,,,,,, +3116,c,bubble,random,500,2463534242,7,635,1,234535968,233704209,124750,119688,11ae932d,off,,,,,,, +3117,node,quick,nearly_sorted,250,2463534242,6,25320,1,266916452,237657467,4346,1008,21a19c14,off,,,,,,, +3118,node,none,reversed,1000,2463534242,16,0,9,238785683,1022,0,0,0ca43265,off,,,,,,, +3119,node,bubble,random,1000,2463534242,16,362,1,273136798,243567163,499500,496926,57561285,off,,,,,,, +3120,c,bubble,random,250,2463534242,15,2568,1,242299247,241444545,31125,31196,b47adce4,off,,,,,,, +3121,node,insertion,sorted,2000,2463534242,15,89220,1,288545245,259032120,1999,0,69842aa5,off,,,,,,, +3122,node,none,random,2000,2463534242,5,0,9,251031742,1733,0,0,41262855,off,,,,,,, +3123,node,insertion,reversed,1000,2463534242,28,554,1,276594127,244764575,499500,500499,0ca43265,off,,,,,,, +3124,python,insertion,sorted,250,2463534242,23,18845,1,261880935,246120532,249,0,7fd8c1f4,off,,,,,,, +3125,node,none,random,1000,2463534242,17,0,10,267360514,862,0,0,57561285,off,,,,,,, +3126,python,bubble,sorted,250,2463534242,9,206,1,296449108,281024877,31125,0,7fd8c1f4,off,,,,,,, +3127,python,none,nearly_sorted,2000,2463534242,5,0,15,257955107,581,0,0,5835dc59,off,,,,,,, +3128,c,none,reversed,1000,2463534242,4,0,405,249715536,90,0,0,0ca43265,off,,,,,,, +3129,python,none,random,250,2463534242,12,0,17,304733954,1102,0,0,b47adce4,off,,,,,,, +3130,python,insertion,reversed,500,2463534242,18,22,1,270906731,253879341,124750,125249,d33e3559,off,,,,,,, +3131,c,insertion,nearly_sorted,2000,2463534242,24,2962,1,227258338,226292860,118277,118270,5835dc59,off,,,,,,, +3132,node,bubble,sorted,500,2463534242,14,1964,1,259079735,229405379,124750,0,d9a24379,off,,,,,,, +3133,node,none,sorted,2000,2463534242,14,0,10,285408056,1192,0,0,69842aa5,off,,,,,,, +3134,python,bubble,reversed,500,2463534242,29,20,1,261827386,245854461,124750,249500,d33e3559,off,,,,,,, +3135,c,insertion,nearly_sorted,1000,2463534242,17,11445,1,249324010,248399139,30889,30860,1cfe4afd,off,,,,,,, +3136,c,insertion,reversed,500,2463534242,20,3967,1,239305468,238113795,124750,125249,d33e3559,off,,,,,,, +3137,node,quick,reversed,250,2463534242,4,4755,1,272784218,244306276,31125,250,be175004,off,,,,,,, +3138,node,none,random,500,2463534242,3,0,9,240102252,1092,0,0,11ae932d,off,,,,,,, +3139,c,none,nearly_sorted,250,2463534242,6,0,388,221172823,90,0,0,21a19c14,off,,,,,,, +3140,node,bubble,sorted,1000,2463534242,10,374,1,262850451,234240911,499500,0,8e15cb75,off,,,,,,, +3141,python,insertion,sorted,2000,2463534242,21,1523,1,257300510,241242314,1999,0,69842aa5,off,,,,,,, +3142,c,insertion,sorted,250,2463534242,18,786886,1,245043330,243998661,249,0,7fd8c1f4,off,,,,,,, +3143,node,insertion,random,500,2463534242,20,4059,1,270379312,242743860,60338,60339,11ae932d,off,,,,,,, +3144,python,none,random,250,2463534242,25,0,17,245773517,701,0,0,b47adce4,off,,,,,,, +3145,c,insertion,random,500,2463534242,15,6945,1,212752935,211810470,60338,60339,11ae932d,off,,,,,,, +3146,node,insertion,random,250,2463534242,20,15349,1,263798949,237353869,15844,15838,b47adce4,off,,,,,,, +3147,c,bubble,random,1000,2463534242,11,154,1,235109433,234087649,499500,496926,57561285,off,,,,,,, +3148,python,bubble,reversed,2000,2463534242,29,2,1,484846048,468034761,1999000,3998000,ad35ba05,off,,,,,,, +3149,c,none,reversed,500,2463534242,27,0,402,225931823,91,0,0,d33e3559,off,,,,,,, +3150,python,insertion,nearly_sorted,1000,2463534242,13,76,1,249013661,233560287,30889,30860,1cfe4afd,off,,,,,,, +3151,c,none,sorted,2000,2463534242,16,0,416,239425051,90,0,0,69842aa5,off,,,,,,, +3152,node,insertion,reversed,1000,2463534242,22,554,1,269757596,239485629,499500,500499,0ca43265,off,,,,,,, +3153,python,insertion,random,1000,2463534242,17,10,1,256247108,240327520,249457,249457,57561285,off,,,,,,, +3154,c,bubble,sorted,250,2463534242,18,11386,1,243618819,242313876,31125,0,7fd8c1f4,off,,,,,,, +3155,c,bubble,nearly_sorted,1000,2463534242,27,507,1,242795867,241666111,499500,59780,1cfe4afd,off,,,,,,, +3156,c,insertion,reversed,2000,2463534242,29,260,1,236779192,235614760,1999000,2000999,ad35ba05,off,,,,,,, +3157,node,bubble,nearly_sorted,250,2463534242,7,6161,1,270914263,243053291,31125,4056,21a19c14,off,,,,,,, +3158,c,bubble,reversed,1000,2463534242,26,87,1,244187636,243058661,499500,999000,0ca43265,off,,,,,,, +3159,c,quick,nearly_sorted,500,2463534242,3,31784,1,239031573,237800496,10192,1850,482b4785,off,,,,,,, +3160,c,quick,random,1000,2463534242,26,23484,1,234396547,233443292,11196,10210,57561285,off,,,,,,, +3161,c,quick,nearly_sorted,2000,2463534242,7,6145,1,243686657,242510715,58002,11768,5835dc59,off,,,,,,, +3162,node,bubble,sorted,1000,2463534242,16,374,1,244857682,215918822,499500,0,8e15cb75,off,,,,,,, +3163,python,insertion,random,2000,2463534242,6,3,1,318890785,301902042,996390,996389,41262855,off,,,,,,, +3164,c,insertion,random,500,2463534242,13,6945,1,219267884,218307454,60338,60339,11ae932d,off,,,,,,, +3165,c,bubble,sorted,1000,2463534242,26,702,1,241217056,239918847,499500,0,8e15cb75,off,,,,,,, +3166,c,quick,random,2000,2463534242,22,10587,1,242278903,240931435,25197,24544,41262855,off,,,,,,, +3167,c,insertion,reversed,500,2463534242,22,3967,1,239066210,238041774,124750,125249,d33e3559,off,,,,,,, +3168,c,bubble,random,2000,2463534242,12,40,1,243601216,242550057,1999000,1988792,41262855,off,,,,,,, +3169,c,quick,sorted,1000,2463534242,19,691,1,238990707,237744531,499500,0,8e15cb75,off,,,,,,, +3170,node,bubble,sorted,1000,2463534242,30,374,1,246104649,215789668,499500,0,8e15cb75,off,,,,,,, +3171,c,bubble,nearly_sorted,1000,2463534242,1,507,1,239423117,238380654,499500,59780,1cfe4afd,off,,,,,,, +3172,node,bubble,nearly_sorted,2000,2463534242,18,110,1,264795106,236261856,1999000,232556,5835dc59,off,,,,,,, +3173,c,none,random,500,2463534242,14,0,426,237728923,90,0,0,11ae932d,off,,,,,,, +3174,c,bubble,nearly_sorted,500,2463534242,5,1998,1,236116900,235152784,124750,14926,482b4785,off,,,,,,, +3175,c,insertion,sorted,1000,2463534242,14,196883,1,237265323,236056297,999,0,8e15cb75,off,,,,,,, +3176,c,bubble,random,1000,2463534242,23,154,1,229510370,228553065,499500,496926,57561285,off,,,,,,, +3177,python,none,nearly_sorted,500,2463534242,26,0,17,236168328,532,0,0,482b4785,off,,,,,,, +3178,c,quick,sorted,2000,2463534242,21,174,1,243766316,242911674,1999000,0,69842aa5,off,,,,,,, +3179,c,quick,nearly_sorted,250,2463534242,18,77221,1,238130595,237003113,4346,1008,21a19c14,off,,,,,,, +3180,node,none,sorted,1000,2463534242,25,0,9,234666312,962,0,0,8e15cb75,off,,,,,,, +3181,python,none,nearly_sorted,500,2463534242,29,0,17,244111523,561,0,0,482b4785,off,,,,,,, +3182,node,bubble,nearly_sorted,1000,2463534242,22,413,1,263111381,235230280,499500,59780,1cfe4afd,off,,,,,,, +3183,node,none,nearly_sorted,250,2463534242,13,0,10,254454527,521,0,0,21a19c14,off,,,,,,, +3184,python,bubble,random,250,2463534242,17,112,1,244050080,228991525,31125,31196,b47adce4,off,,,,,,, +3185,python,insertion,reversed,500,2463534242,30,22,1,245363869,230099170,124750,125249,d33e3559,off,,,,,,, +3186,node,insertion,random,250,2463534242,19,15349,1,267340105,238221021,15844,15838,b47adce4,off,,,,,,, +3187,c,quick,nearly_sorted,2000,2463534242,5,6145,1,259121774,258076436,58002,11768,5835dc59,off,,,,,,, +3188,c,none,nearly_sorted,500,2463534242,16,0,405,248264897,100,0,0,482b4785,off,,,,,,, +3189,node,quick,sorted,250,2463534242,28,4507,1,276082760,244894341,31125,0,7fd8c1f4,off,,,,,,, +3190,python,bubble,reversed,500,2463534242,19,20,1,267816869,252506360,124750,249500,d33e3559,off,,,,,,, +3191,python,bubble,random,1000,2463534242,28,6,1,270263354,254579148,499500,496926,57561285,off,,,,,,, +3192,node,insertion,reversed,500,2463534242,11,2178,1,265221204,236994849,124750,125249,d33e3559,off,,,,,,, +3193,python,quick,random,2000,2463534242,25,90,1,259210900,242548443,25197,24544,41262855,off,,,,,,, +3194,c,quick,reversed,500,2463534242,29,3240,1,241507503,240623125,124750,500,d33e3559,off,,,,,,, +3195,node,none,random,2000,2463534242,19,0,9,241927601,1042,0,0,41262855,off,,,,,,, +3196,node,none,sorted,500,2463534242,25,0,10,273895610,1603,0,0,d9a24379,off,,,,,,, +3197,node,insertion,reversed,500,2463534242,5,2178,1,268458141,235129966,124750,125249,d33e3559,off,,,,,,, +3198,node,bubble,random,500,2463534242,9,1407,1,270018735,240191793,124750,119688,11ae932d,off,,,,,,, +3199,c,quick,sorted,250,2463534242,21,10851,1,244086022,243020010,31125,0,7fd8c1f4,off,,,,,,, +3200,node,none,nearly_sorted,2000,2463534242,26,0,9,258385287,1403,0,0,5835dc59,off,,,,,,, +3201,node,insertion,nearly_sorted,1000,2463534242,5,7758,1,283629182,251580958,30889,30860,1cfe4afd,off,,,,,,, +3202,c,bubble,sorted,2000,2463534242,1,173,1,237158973,236162406,1999000,0,69842aa5,off,,,,,,, +3203,python,insertion,nearly_sorted,500,2463534242,1,316,1,255857316,240131047,7962,7926,482b4785,off,,,,,,, +3204,python,bubble,nearly_sorted,250,2463534242,22,188,1,279821748,264253752,31125,4056,21a19c14,off,,,,,,, +3205,c,none,reversed,2000,2463534242,23,0,413,249819541,90,0,0,ad35ba05,off,,,,,,, +3206,c,insertion,nearly_sorted,250,2463534242,20,99421,1,247665085,246968229,2277,2251,21a19c14,off,,,,,,, +3207,python,none,reversed,500,2463534242,4,0,16,242481938,842,0,0,d33e3559,off,,,,,,, +3208,node,bubble,nearly_sorted,500,2463534242,26,1566,1,274445882,245061506,124750,14926,482b4785,off,,,,,,, +3209,c,none,sorted,2000,2463534242,3,0,416,241835567,90,0,0,69842aa5,off,,,,,,, +3210,node,bubble,reversed,500,2463534242,4,1534,1,265582312,237529373,124750,249500,d33e3559,off,,,,,,, +3211,python,insertion,sorted,500,2463534242,6,6624,1,245224339,229463214,499,0,d9a24379,off,,,,,,, +3212,c,insertion,random,250,2463534242,10,26020,1,229064043,227977567,15844,15838,b47adce4,off,,,,,,, +3213,c,insertion,reversed,500,2463534242,2,3967,1,236309652,235394968,124750,125249,d33e3559,off,,,,,,, +3214,node,bubble,sorted,1000,2463534242,5,374,1,243264045,214732974,499500,0,8e15cb75,off,,,,,,, +3215,node,quick,reversed,250,2463534242,19,4755,1,264392863,237161588,31125,250,be175004,off,,,,,,, +3216,c,insertion,reversed,2000,2463534242,23,260,1,238825497,237915171,1999000,2000999,ad35ba05,off,,,,,,, +3217,c,none,random,500,2463534242,19,0,426,238104506,91,0,0,11ae932d,off,,,,,,, +3218,c,none,reversed,2000,2463534242,20,0,413,234998284,81,0,0,ad35ba05,off,,,,,,, +3219,python,insertion,reversed,500,2463534242,20,22,1,246258326,230803682,124750,125249,d33e3559,off,,,,,,, +3220,c,bubble,nearly_sorted,500,2463534242,3,1998,1,236911479,236030340,124750,14926,482b4785,off,,,,,,, +3221,python,bubble,nearly_sorted,1000,2463534242,16,9,1,244721497,228907386,499500,59780,1cfe4afd,off,,,,,,, +3222,python,quick,sorted,2000,2463534242,2,2,1,295748154,280210740,1999000,0,69842aa5,off,,,,,,, +3223,c,insertion,reversed,1000,2463534242,24,1033,1,243504516,242616430,499500,500499,0ca43265,off,,,,,,, +3224,node,none,sorted,2000,2463534242,8,0,10,265069480,1072,0,0,69842aa5,off,,,,,,, +3225,node,quick,reversed,2000,2463534242,20,80,1,270771866,240134964,1999000,2000,ad35ba05,off,,,,,,, +3226,node,quick,sorted,250,2463534242,6,4507,1,266750441,239682153,31125,0,7fd8c1f4,off,,,,,,, +3227,node,none,random,250,2463534242,26,0,10,250102781,692,0,0,b47adce4,off,,,,,,, +3228,c,none,nearly_sorted,2000,2463534242,3,0,387,222792590,100,0,0,5835dc59,off,,,,,,, +3229,node,bubble,nearly_sorted,1000,2463534242,24,413,1,265280004,235858030,499500,59780,1cfe4afd,off,,,,,,, +3230,node,bubble,reversed,1000,2463534242,3,378,1,266801386,238901822,499500,999000,0ca43265,off,,,,,,, +3231,node,quick,sorted,250,2463534242,14,4507,1,258820910,232557884,31125,0,7fd8c1f4,off,,,,,,, +3232,node,quick,sorted,1000,2463534242,10,296,1,272695852,242733790,499500,0,8e15cb75,off,,,,,,, +3233,c,none,reversed,1000,2463534242,25,0,405,226537188,90,0,0,0ca43265,off,,,,,,, +3234,node,insertion,reversed,500,2463534242,20,2178,1,259326507,231153464,124750,125249,d33e3559,off,,,,,,, +3235,python,quick,sorted,500,2463534242,8,30,1,249967999,234818441,124750,0,d9a24379,off,,,,,,, +3236,node,none,reversed,1000,2463534242,20,0,9,231153089,832,0,0,0ca43265,off,,,,,,, +3237,python,bubble,sorted,2000,2463534242,2,3,1,330032356,313643540,1999000,0,69842aa5,off,,,,,,, +3238,node,insertion,nearly_sorted,2000,2463534242,29,2100,1,257104041,227946859,118277,118270,5835dc59,off,,,,,,, +3239,c,quick,nearly_sorted,250,2463534242,24,77221,1,243213751,242102518,4346,1008,21a19c14,off,,,,,,, +3240,python,bubble,nearly_sorted,2000,2463534242,22,3,1,395197751,379336148,1999000,232556,5835dc59,off,,,,,,, +3241,c,bubble,reversed,500,2463534242,10,346,1,240112097,239220826,124750,249500,d33e3559,off,,,,,,, +3242,node,insertion,nearly_sorted,1000,2463534242,2,7758,1,263923032,233251697,30889,30860,1cfe4afd,off,,,,,,, +3243,python,bubble,sorted,2000,2463534242,1,3,1,340132463,323515503,1999000,0,69842aa5,off,,,,,,, +3244,python,insertion,reversed,500,2463534242,11,22,1,246290338,231301219,124750,125249,d33e3559,off,,,,,,, +3245,python,insertion,nearly_sorted,1000,2463534242,25,76,1,264433248,244878254,30889,30860,1cfe4afd,off,,,,,,, +3246,node,quick,sorted,1000,2463534242,2,296,1,273675668,240918488,499500,0,8e15cb75,off,,,,,,, +3247,node,none,nearly_sorted,2000,2463534242,7,0,9,233393347,782,0,0,5835dc59,off,,,,,,, +3248,node,insertion,sorted,1000,2463534242,9,160107,1,262380010,233533408,999,0,8e15cb75,off,,,,,,, +3249,c,none,nearly_sorted,500,2463534242,20,0,405,231636784,80,0,0,482b4785,off,,,,,,, +3250,python,none,random,1000,2463534242,12,0,15,231692580,852,0,0,57561285,off,,,,,,, +3251,node,insertion,random,2000,2463534242,1,234,1,266519057,238772907,996390,996389,41262855,off,,,,,,, +3252,c,quick,random,250,2463534242,25,136055,1,232613124,231500249,1933,1940,b47adce4,off,,,,,,, +3253,python,none,random,1000,2463534242,15,0,15,222576347,781,0,0,57561285,off,,,,,,, +3254,c,bubble,reversed,2000,2463534242,1,22,1,242761702,241809536,1999000,3998000,ad35ba05,off,,,,,,, +3255,c,insertion,reversed,2000,2463534242,30,260,1,241042571,239933850,1999000,2000999,ad35ba05,off,,,,,,, +3256,c,quick,sorted,500,2463534242,16,2762,1,247611583,246449967,124750,0,d9a24379,off,,,,,,, +3257,node,bubble,sorted,500,2463534242,19,1964,1,253706946,225773130,124750,0,d9a24379,off,,,,,,, +3258,node,bubble,reversed,500,2463534242,12,1534,1,266159603,240166280,124750,249500,d33e3559,off,,,,,,, +3259,c,insertion,nearly_sorted,250,2463534242,29,99421,1,238628558,237715849,2277,2251,21a19c14,off,,,,,,, +3260,node,bubble,sorted,500,2463534242,8,1964,1,252023821,225787058,124750,0,d9a24379,off,,,,,,, +3261,python,none,random,250,2463534242,17,0,17,240762156,561,0,0,b47adce4,off,,,,,,, +3262,c,insertion,sorted,500,2463534242,8,399335,1,238713287,237565066,499,0,d9a24379,off,,,,,,, +3263,node,quick,reversed,1000,2463534242,1,307,1,266977045,238514666,499500,1000,0ca43265,off,,,,,,, +3264,c,quick,sorted,1000,2463534242,15,691,1,239777010,238839385,499500,0,8e15cb75,off,,,,,,, +3265,c,insertion,random,1000,2463534242,30,1983,1,241637978,240483746,249457,249457,57561285,off,,,,,,, +3266,python,insertion,sorted,1000,2463534242,13,3178,1,262643023,246960000,999,0,8e15cb75,off,,,,,,, +3267,python,none,sorted,250,2463534242,18,0,17,240882442,511,0,0,7fd8c1f4,off,,,,,,, +3268,python,none,nearly_sorted,250,2463534242,21,0,17,241327496,541,0,0,21a19c14,off,,,,,,, +3269,node,bubble,random,1000,2463534242,14,362,1,268321736,241236507,499500,496926,57561285,off,,,,,,, +3270,python,bubble,sorted,250,2463534242,26,206,1,255110756,239913624,31125,0,7fd8c1f4,off,,,,,,, +3271,c,insertion,random,1000,2463534242,11,1983,1,237982216,237077021,249457,249457,57561285,off,,,,,,, +3272,python,none,nearly_sorted,2000,2463534242,21,0,15,230213998,511,0,0,5835dc59,off,,,,,,, +3273,node,quick,nearly_sorted,1000,2463534242,30,3752,1,266165635,236736886,28499,7892,1cfe4afd,off,,,,,,, +3274,node,insertion,sorted,500,2463534242,30,324764,1,259545999,232484319,499,0,d9a24379,off,,,,,,, +3275,c,bubble,sorted,1000,2463534242,28,702,1,238839743,237809754,499500,0,8e15cb75,off,,,,,,, +3276,c,bubble,sorted,500,2463534242,7,2858,1,242641659,241543768,124750,0,d9a24379,off,,,,,,, +3277,c,none,sorted,1000,2463534242,10,0,421,237795447,90,0,0,8e15cb75,off,,,,,,, +3278,c,quick,sorted,2000,2463534242,17,174,1,239706909,238783305,1999000,0,69842aa5,off,,,,,,, +3279,node,bubble,random,1000,2463534242,6,362,1,266704133,236398745,499500,496926,57561285,off,,,,,,, +3280,python,bubble,random,2000,2463534242,27,2,1,367378885,350997521,1999000,1988792,41262855,off,,,,,,, +3281,c,insertion,reversed,1000,2463534242,13,1033,1,237928677,236889112,499500,500499,0ca43265,off,,,,,,, +3282,node,bubble,sorted,2000,2463534242,4,85,1,262329986,232945976,1999000,0,69842aa5,off,,,,,,, +3283,python,bubble,random,500,2463534242,25,26,1,255746017,240453823,124750,119688,11ae932d,off,,,,,,, +3284,c,insertion,sorted,500,2463534242,16,399335,1,246541999,245454863,499,0,d9a24379,off,,,,,,, +3285,python,insertion,random,500,2463534242,16,44,1,255315801,237327702,60338,60339,11ae932d,off,,,,,,, +3286,node,bubble,random,500,2463534242,18,1407,1,266960305,239816167,124750,119688,11ae932d,off,,,,,,, +3287,python,bubble,sorted,2000,2463534242,6,3,1,325751111,310178330,1999000,0,69842aa5,off,,,,,,, +3288,c,bubble,random,1000,2463534242,24,154,1,234813237,233881100,499500,496926,57561285,off,,,,,,, +3289,python,bubble,reversed,250,2463534242,5,88,1,247200522,231772922,31125,62250,be175004,off,,,,,,, +3290,python,quick,nearly_sorted,1000,2463534242,18,110,1,257828229,242376550,28499,7892,1cfe4afd,off,,,,,,, +3291,python,quick,nearly_sorted,500,2463534242,30,296,1,242927053,227805463,10192,1850,482b4785,off,,,,,,, +3292,node,bubble,reversed,250,2463534242,11,5861,1,256823747,230178376,31125,62250,be175004,off,,,,,,, +3293,python,none,sorted,500,2463534242,24,0,16,228533188,531,0,0,d9a24379,off,,,,,,, +3294,node,bubble,nearly_sorted,500,2463534242,7,1566,1,266539175,239158693,124750,14926,482b4785,off,,,,,,, +3295,c,none,reversed,500,2463534242,16,0,402,224927501,80,0,0,d33e3559,off,,,,,,, +3296,python,insertion,sorted,2000,2463534242,15,1523,1,258675298,242382902,1999,0,69842aa5,off,,,,,,, +3297,node,bubble,random,1000,2463534242,15,362,1,263937657,235021509,499500,496926,57561285,off,,,,,,, +3298,node,bubble,reversed,1000,2463534242,22,378,1,271365629,244674349,499500,999000,0ca43265,off,,,,,,, +3299,c,quick,nearly_sorted,500,2463534242,11,31784,1,236355809,235505124,10192,1850,482b4785,off,,,,,,, +3300,c,none,reversed,2000,2463534242,27,0,413,232966056,90,0,0,ad35ba05,off,,,,,,, +3301,node,bubble,reversed,500,2463534242,18,1534,1,269617293,240562082,124750,249500,d33e3559,off,,,,,,, +3302,python,none,reversed,250,2463534242,23,0,17,240696515,551,0,0,be175004,off,,,,,,, +3303,node,insertion,random,2000,2463534242,25,234,1,273933892,244994364,996390,996389,41262855,off,,,,,,, +3304,python,none,sorted,1000,2463534242,28,0,16,253024426,581,0,0,8e15cb75,off,,,,,,, +3305,python,none,reversed,500,2463534242,17,0,16,272112840,701,0,0,d33e3559,off,,,,,,, +3306,c,bubble,sorted,1000,2463534242,10,702,1,248048571,246956004,499500,0,8e15cb75,off,,,,,,, +3307,node,quick,reversed,250,2463534242,16,4755,1,281425352,250768339,31125,250,be175004,off,,,,,,, +3308,c,insertion,reversed,1000,2463534242,19,1033,1,248248997,247317482,499500,500499,0ca43265,off,,,,,,, +3309,node,insertion,random,2000,2463534242,16,234,1,272349724,243593371,996390,996389,41262855,off,,,,,,, +3310,python,bubble,random,1000,2463534242,17,6,1,276682023,258887889,499500,496926,57561285,off,,,,,,, +3311,node,quick,reversed,250,2463534242,13,4755,1,266864193,238430664,31125,250,be175004,off,,,,,,, +3312,c,bubble,reversed,1000,2463534242,24,87,1,243014076,241848369,499500,999000,0ca43265,off,,,,,,, +3313,python,insertion,sorted,2000,2463534242,25,1523,1,265080658,248031585,1999,0,69842aa5,off,,,,,,, +3314,c,none,nearly_sorted,250,2463534242,15,0,388,225199114,90,0,0,21a19c14,off,,,,,,, +3315,node,insertion,random,250,2463534242,15,15349,1,292959819,251535954,15844,15838,b47adce4,off,,,,,,, +3316,node,insertion,random,2000,2463534242,29,234,1,286695629,251233157,996390,996389,41262855,off,,,,,,, +3317,c,bubble,sorted,250,2463534242,3,11386,1,249353148,248409980,31125,0,7fd8c1f4,off,,,,,,, +3318,python,insertion,sorted,1000,2463534242,15,3178,1,271695867,254710277,999,0,8e15cb75,off,,,,,,, +3319,c,bubble,reversed,2000,2463534242,5,22,1,250628967,249761013,1999000,3998000,ad35ba05,off,,,,,,, +3320,node,none,reversed,500,2463534242,12,0,9,230976668,942,0,0,d33e3559,off,,,,,,, +3321,python,bubble,nearly_sorted,2000,2463534242,29,3,1,347848926,329766761,1999000,232556,5835dc59,off,,,,,,, +3322,c,quick,reversed,2000,2463534242,7,209,1,240902890,239766582,1999000,2000,ad35ba05,off,,,,,,, +3323,c,none,nearly_sorted,1000,2463534242,12,0,421,238152255,90,0,0,1cfe4afd,off,,,,,,, +3324,node,none,random,1000,2463534242,8,0,10,257868746,6202,0,0,57561285,off,,,,,,, +3325,node,quick,reversed,2000,2463534242,10,80,1,277286977,244355867,1999000,2000,ad35ba05,off,,,,,,, +3326,python,bubble,random,500,2463534242,3,26,1,264366161,247838564,124750,119688,11ae932d,off,,,,,,, +3327,python,quick,nearly_sorted,250,2463534242,3,882,1,251135438,236327620,4346,1008,21a19c14,off,,,,,,, +3328,python,quick,nearly_sorted,2000,2463534242,9,55,1,246583917,229720470,58002,11768,5835dc59,off,,,,,,, +3329,node,quick,reversed,1000,2463534242,24,307,1,260518301,233815548,499500,1000,0ca43265,off,,,,,,, +3330,python,none,reversed,500,2463534242,18,0,16,231585439,561,0,0,d33e3559,off,,,,,,, +3331,node,quick,sorted,250,2463534242,18,4507,1,264586396,236334100,31125,0,7fd8c1f4,off,,,,,,, +3332,node,none,random,1000,2463534242,20,0,10,253668834,832,0,0,57561285,off,,,,,,, +3333,python,bubble,sorted,1000,2463534242,13,11,1,276831384,261448946,499500,0,8e15cb75,off,,,,,,, +3334,c,insertion,reversed,250,2463534242,20,15645,1,246791677,245873887,31125,31374,be175004,off,,,,,,, +3335,python,quick,sorted,2000,2463534242,19,2,1,299926864,283182091,1999000,0,69842aa5,off,,,,,,, +3336,node,none,random,500,2463534242,12,0,9,232498089,761,0,0,11ae932d,off,,,,,,, +3337,c,quick,sorted,2000,2463534242,13,174,1,234123334,233244518,1999000,0,69842aa5,off,,,,,,, +3338,c,quick,sorted,500,2463534242,6,2762,1,241140436,240059050,124750,0,d9a24379,off,,,,,,, +3339,node,bubble,sorted,250,2463534242,28,8081,1,253267032,226842137,31125,0,7fd8c1f4,off,,,,,,, +3340,node,none,random,500,2463534242,27,0,9,233162294,842,0,0,11ae932d,off,,,,,,, +3341,c,quick,sorted,250,2463534242,10,10851,1,246817805,245781382,31125,0,7fd8c1f4,off,,,,,,, +3342,python,none,nearly_sorted,500,2463534242,14,0,17,236490160,511,0,0,482b4785,off,,,,,,, +3343,python,none,reversed,250,2463534242,12,0,17,236764014,551,0,0,be175004,off,,,,,,, +3344,python,quick,nearly_sorted,2000,2463534242,12,55,1,243251362,228070478,58002,11768,5835dc59,off,,,,,,, +3345,node,bubble,random,250,2463534242,24,5439,1,255666728,228408217,31125,31196,b47adce4,off,,,,,,, +3346,node,insertion,random,250,2463534242,12,15349,1,272214210,244294509,15844,15838,b47adce4,off,,,,,,, +3347,c,insertion,reversed,1000,2463534242,30,1033,1,240967081,239943442,499500,500499,0ca43265,off,,,,,,, +3348,c,none,nearly_sorted,1000,2463534242,9,0,421,237779697,91,0,0,1cfe4afd,off,,,,,,, +3349,python,insertion,sorted,250,2463534242,24,18845,1,264363788,249124959,249,0,7fd8c1f4,off,,,,,,, +3350,node,none,random,1000,2463534242,7,0,10,263217349,841,0,0,57561285,off,,,,,,, +3351,python,quick,nearly_sorted,1000,2463534242,9,110,1,261456438,244863912,28499,7892,1cfe4afd,off,,,,,,, +3352,c,quick,reversed,250,2463534242,24,12781,1,238510106,237373556,31125,250,be175004,off,,,,,,, +3353,node,quick,reversed,500,2463534242,7,1194,1,261287302,233225526,124750,500,d33e3559,off,,,,,,, +3354,node,none,reversed,250,2463534242,28,0,9,234930788,832,0,0,be175004,off,,,,,,, +3355,node,quick,nearly_sorted,1000,2463534242,22,3752,1,261769305,232592218,28499,7892,1cfe4afd,off,,,,,,, +3356,c,insertion,sorted,500,2463534242,5,399335,1,241079070,240019755,499,0,d9a24379,off,,,,,,, +3357,python,insertion,sorted,500,2463534242,22,6624,1,241076375,225615882,499,0,d9a24379,off,,,,,,, +3358,node,quick,reversed,250,2463534242,29,4755,1,267810357,240772578,31125,250,be175004,off,,,,,,, +3359,c,none,reversed,500,2463534242,21,0,402,225835813,90,0,0,d33e3559,off,,,,,,, +3360,python,quick,nearly_sorted,250,2463534242,15,882,1,260285915,245070534,4346,1008,21a19c14,off,,,,,,, +3361,python,bubble,random,1000,2463534242,2,6,1,272326570,256345526,499500,496926,57561285,off,,,,,,, +3362,c,insertion,nearly_sorted,500,2463534242,25,32350,1,241268217,240138098,7962,7926,482b4785,off,,,,,,, +3363,python,insertion,reversed,250,2463534242,11,102,1,262134114,246964517,31125,31374,be175004,off,,,,,,, +3364,c,quick,sorted,500,2463534242,15,2762,1,242293981,241267684,124750,0,d9a24379,off,,,,,,, +3365,python,none,reversed,1000,2463534242,12,0,16,238459651,581,0,0,0ca43265,off,,,,,,, +3366,python,insertion,sorted,1000,2463534242,20,3178,1,258435538,242999378,999,0,8e15cb75,off,,,,,,, +3367,python,quick,reversed,250,2463534242,17,151,1,254433858,239067883,31125,250,be175004,off,,,,,,, +3368,node,bubble,random,1000,2463534242,30,362,1,266119058,238045154,499500,496926,57561285,off,,,,,,, +3369,node,quick,nearly_sorted,2000,2463534242,18,1909,1,254967128,227408662,58002,11768,5835dc59,off,,,,,,, +3370,python,bubble,random,1000,2463534242,20,6,1,294411840,279049753,499500,496926,57561285,off,,,,,,, +3371,node,none,reversed,1000,2463534242,10,0,9,230136824,872,0,0,0ca43265,off,,,,,,, +3372,python,quick,sorted,1000,2463534242,20,8,1,278903326,263538630,499500,0,8e15cb75,off,,,,,,, +3373,python,quick,nearly_sorted,2000,2463534242,29,55,1,249364518,233094869,58002,11768,5835dc59,off,,,,,,, +3374,python,quick,sorted,1000,2463534242,29,8,1,281453014,266312066,499500,0,8e15cb75,off,,,,,,, +3375,python,bubble,reversed,2000,2463534242,17,2,1,471382005,455585302,1999000,3998000,ad35ba05,off,,,,,,, +3376,c,none,nearly_sorted,2000,2463534242,29,0,387,220365223,90,0,0,5835dc59,off,,,,,,, +3377,node,none,sorted,2000,2463534242,3,0,10,259467630,712,0,0,69842aa5,off,,,,,,, +3378,node,quick,nearly_sorted,500,2463534242,11,10954,1,261660381,234860174,10192,1850,482b4785,off,,,,,,, +3379,node,bubble,reversed,250,2463534242,26,5861,1,257621021,229169965,31125,62250,be175004,off,,,,,,, +3380,node,quick,nearly_sorted,250,2463534242,20,25320,1,263566633,235495735,4346,1008,21a19c14,off,,,,,,, +3381,node,insertion,nearly_sorted,250,2463534242,28,79550,1,231771558,204257187,2277,2251,21a19c14,off,,,,,,, +3382,node,bubble,sorted,250,2463534242,15,8081,1,257225931,229698360,31125,0,7fd8c1f4,off,,,,,,, +3383,node,none,nearly_sorted,500,2463534242,11,0,10,253658745,681,0,0,482b4785,off,,,,,,, +3384,c,quick,sorted,250,2463534242,24,10851,1,244062812,243201009,31125,0,7fd8c1f4,off,,,,,,, +3385,c,bubble,reversed,2000,2463534242,14,22,1,244718550,243820037,1999000,3998000,ad35ba05,off,,,,,,, +3386,python,none,nearly_sorted,250,2463534242,29,0,17,243627087,471,0,0,21a19c14,off,,,,,,, +3387,node,none,sorted,500,2463534242,29,0,10,255728975,781,0,0,d9a24379,off,,,,,,, +3388,c,bubble,nearly_sorted,250,2463534242,11,7880,1,236906209,235719256,31125,4056,21a19c14,off,,,,,,, +3389,node,none,reversed,1000,2463534242,26,0,9,229649531,752,0,0,0ca43265,off,,,,,,, +3390,node,insertion,random,1000,2463534242,25,1091,1,266799373,238433973,249457,249457,57561285,off,,,,,,, +3391,node,quick,random,500,2463534242,3,16148,1,261879672,234166459,4578,4552,11ae932d,off,,,,,,, +3392,node,none,reversed,2000,2463534242,7,0,9,231635061,1113,0,0,ad35ba05,off,,,,,,, +3393,node,bubble,nearly_sorted,250,2463534242,19,6161,1,268962826,242338187,31125,4056,21a19c14,off,,,,,,, +3394,node,bubble,reversed,250,2463534242,17,5861,1,256947780,230052763,31125,62250,be175004,off,,,,,,, +3395,python,insertion,random,2000,2463534242,19,3,1,317337173,300565053,996390,996389,41262855,off,,,,,,, +3396,node,quick,random,1000,2463534242,15,6513,1,262643604,232634286,11196,10210,57561285,off,,,,,,, +3397,c,quick,random,2000,2463534242,12,10587,1,238478769,237560997,25197,24544,41262855,off,,,,,,, +3398,c,quick,reversed,1000,2463534242,21,814,1,234422505,233436429,499500,1000,0ca43265,off,,,,,,, +3399,node,none,reversed,1000,2463534242,1,0,9,230938073,751,0,0,0ca43265,off,,,,,,, +3400,c,insertion,sorted,250,2463534242,23,786886,1,246343737,245210253,249,0,7fd8c1f4,off,,,,,,, +3401,c,bubble,nearly_sorted,1000,2463534242,16,507,1,244276783,243176964,499500,59780,1cfe4afd,off,,,,,,, +3402,python,insertion,random,500,2463534242,27,44,1,257869097,242139151,60338,60339,11ae932d,off,,,,,,, +3403,node,quick,reversed,250,2463534242,6,4755,1,265250770,237912294,31125,250,be175004,off,,,,,,, +3404,node,insertion,nearly_sorted,2000,2463534242,19,2100,1,261087107,231908451,118277,118270,5835dc59,off,,,,,,, +3405,c,quick,random,250,2463534242,8,136055,1,237179473,236206298,1933,1940,b47adce4,off,,,,,,, +3406,node,quick,reversed,250,2463534242,17,4755,1,267038881,239916077,31125,250,be175004,off,,,,,,, +3407,python,insertion,sorted,250,2463534242,10,18845,1,260286256,245588974,249,0,7fd8c1f4,off,,,,,,, +3408,c,none,sorted,500,2463534242,11,0,344,190719101,100,0,0,d9a24379,off,,,,,,, +3409,c,bubble,random,500,2463534242,18,635,1,231295146,230479045,124750,119688,11ae932d,off,,,,,,, +3410,node,insertion,reversed,1000,2463534242,15,554,1,266306018,238890216,499500,500499,0ca43265,off,,,,,,, +3411,python,insertion,random,1000,2463534242,12,10,1,250629840,234766387,249457,249457,57561285,off,,,,,,, +3412,node,insertion,nearly_sorted,250,2463534242,12,79550,1,236829926,208143833,2277,2251,21a19c14,off,,,,,,, +3413,python,bubble,reversed,1000,2463534242,13,5,1,293117394,277686867,499500,999000,0ca43265,off,,,,,,, +3414,node,none,random,1000,2463534242,12,0,10,270044154,912,0,0,57561285,off,,,,,,, +3415,python,quick,sorted,2000,2463534242,14,2,1,300304203,283768991,1999000,0,69842aa5,off,,,,,,, +3416,c,insertion,nearly_sorted,500,2463534242,14,32350,1,237295596,236315984,7962,7926,482b4785,off,,,,,,, +3417,node,quick,random,1000,2463534242,18,6513,1,267857007,240561988,11196,10210,57561285,off,,,,,,, +3418,node,quick,sorted,1000,2463534242,28,296,1,269359982,242279831,499500,0,8e15cb75,off,,,,,,, +3419,node,insertion,sorted,2000,2463534242,29,89220,1,256389282,228531969,1999,0,69842aa5,off,,,,,,, +3420,c,none,random,2000,2463534242,15,0,415,243350167,90,0,0,41262855,off,,,,,,, +3421,python,none,random,500,2463534242,26,0,16,251559511,701,0,0,11ae932d,off,,,,,,, +3422,c,none,random,2000,2463534242,16,0,415,247255005,80,0,0,41262855,off,,,,,,, +3423,python,bubble,random,250,2463534242,28,112,1,270672481,253407326,31125,31196,b47adce4,off,,,,,,, +3424,node,insertion,random,1000,2463534242,17,1091,1,268247606,238525614,249457,249457,57561285,off,,,,,,, +3425,python,bubble,reversed,1000,2463534242,6,5,1,310052503,293537509,499500,999000,0ca43265,off,,,,,,, +3426,python,bubble,reversed,250,2463534242,10,88,1,269977187,250718094,31125,62250,be175004,off,,,,,,, +3427,c,insertion,reversed,500,2463534242,28,3967,1,243339015,242237401,124750,125249,d33e3559,off,,,,,,, +3428,node,insertion,reversed,2000,2463534242,11,144,1,276685539,243023781,1999000,2000999,ad35ba05,off,,,,,,, +3429,python,insertion,reversed,2000,2463534242,15,2,1,426898249,404196399,1999000,2000999,ad35ba05,off,,,,,,, +3430,python,bubble,nearly_sorted,2000,2463534242,26,3,1,365359964,347765239,1999000,232556,5835dc59,off,,,,,,, +3431,python,bubble,sorted,2000,2463534242,20,3,1,323613435,306976724,1999000,0,69842aa5,off,,,,,,, +3432,python,quick,reversed,250,2463534242,29,151,1,263697165,248730055,31125,250,be175004,off,,,,,,, +3433,python,none,random,250,2463534242,28,0,17,248398711,491,0,0,b47adce4,off,,,,,,, +3434,node,none,sorted,2000,2463534242,27,0,10,265988222,752,0,0,69842aa5,off,,,,,,, +3435,c,none,nearly_sorted,1000,2463534242,8,0,421,240688970,90,0,0,1cfe4afd,off,,,,,,, +3436,python,quick,reversed,250,2463534242,23,151,1,261376468,245838694,31125,250,be175004,off,,,,,,, +3437,python,none,sorted,2000,2463534242,6,0,15,228680675,561,0,0,69842aa5,off,,,,,,, +3438,c,none,reversed,500,2463534242,19,0,402,242068123,90,0,0,d33e3559,off,,,,,,, +3439,python,insertion,random,2000,2463534242,30,3,1,342016745,321072182,996390,996389,41262855,off,,,,,,, +3440,node,insertion,random,1000,2463534242,29,1091,1,279505554,246270052,249457,249457,57561285,off,,,,,,, +3441,node,none,nearly_sorted,250,2463534242,28,0,10,275216927,822,0,0,21a19c14,off,,,,,,, +3442,node,none,reversed,2000,2463534242,2,0,9,266506954,881,0,0,ad35ba05,off,,,,,,, +3443,python,quick,sorted,1000,2463534242,3,8,1,295429296,277605343,499500,0,8e15cb75,off,,,,,,, +3444,node,insertion,reversed,250,2463534242,30,8539,1,277600565,245331459,31125,31374,be175004,off,,,,,,, +3445,node,none,sorted,250,2463534242,29,0,9,240318184,1002,0,0,7fd8c1f4,off,,,,,,, +3446,python,bubble,random,500,2463534242,13,26,1,261529365,245330767,124750,119688,11ae932d,off,,,,,,, +3447,node,quick,random,1000,2463534242,24,6513,1,267821387,238943486,11196,10210,57561285,off,,,,,,, +3448,c,none,random,250,2463534242,25,0,429,255953777,120,0,0,b47adce4,off,,,,,,, +3449,node,insertion,reversed,500,2463534242,30,2178,1,274549867,243419172,124750,125249,d33e3559,off,,,,,,, +3450,node,none,nearly_sorted,2000,2463534242,27,0,9,263416252,882,0,0,5835dc59,off,,,,,,, +3451,node,none,sorted,500,2463534242,4,0,10,268779624,1012,0,0,d9a24379,off,,,,,,, +3452,python,bubble,random,1000,2463534242,5,6,1,289876940,273439206,499500,496926,57561285,off,,,,,,, +3453,c,none,reversed,2000,2463534242,10,0,413,239407559,90,0,0,ad35ba05,off,,,,,,, +3454,node,bubble,sorted,500,2463534242,2,1964,1,257021175,228830111,124750,0,d9a24379,off,,,,,,, +3455,c,bubble,reversed,250,2463534242,9,1384,1,244322270,243448772,31125,62250,be175004,off,,,,,,, +3456,node,none,reversed,250,2463534242,18,0,9,245342710,1132,0,0,be175004,off,,,,,,, +3457,node,insertion,random,250,2463534242,21,15349,1,268244049,239297911,15844,15838,b47adce4,off,,,,,,, +3458,python,none,reversed,250,2463534242,5,0,17,256543592,531,0,0,be175004,off,,,,,,, +3459,python,none,reversed,2000,2463534242,9,0,16,263983695,772,0,0,ad35ba05,off,,,,,,, +3460,c,none,random,500,2463534242,7,0,426,245782385,90,0,0,11ae932d,off,,,,,,, +3461,node,quick,sorted,1000,2463534242,7,296,1,267970907,240061267,499500,0,8e15cb75,off,,,,,,, +3462,python,insertion,random,250,2463534242,26,206,1,264153745,246388062,15844,15838,b47adce4,off,,,,,,, +3463,c,none,sorted,500,2463534242,8,0,344,201562333,100,0,0,d9a24379,off,,,,,,, +3464,python,quick,random,2000,2463534242,11,90,1,264036243,245731989,25197,24544,41262855,off,,,,,,, +3465,python,bubble,sorted,250,2463534242,14,206,1,256211960,240390060,31125,0,7fd8c1f4,off,,,,,,, +3466,c,bubble,random,250,2463534242,5,2568,1,243853620,242985032,31125,31196,b47adce4,off,,,,,,, +3467,node,bubble,nearly_sorted,500,2463534242,30,1566,1,270577664,241301913,124750,14926,482b4785,off,,,,,,, +3468,node,quick,sorted,1000,2463534242,5,296,1,269530852,241198233,499500,0,8e15cb75,off,,,,,,, +3469,python,none,nearly_sorted,250,2463534242,10,0,17,264393233,511,0,0,21a19c14,off,,,,,,, +3470,node,none,random,250,2463534242,7,0,10,270812232,842,0,0,b47adce4,off,,,,,,, +3471,c,insertion,sorted,2000,2463534242,15,101352,1,256013168,255118832,1999,0,69842aa5,off,,,,,,, +3472,python,none,sorted,1000,2463534242,2,0,16,251086064,642,0,0,8e15cb75,off,,,,,,, +3473,node,insertion,random,1000,2463534242,22,1091,1,269840231,240791721,249457,249457,57561285,off,,,,,,, +3474,python,none,sorted,2000,2463534242,22,0,15,246283173,591,0,0,69842aa5,off,,,,,,, +3475,node,quick,sorted,1000,2463534242,21,296,1,268631936,240637665,499500,0,8e15cb75,off,,,,,,, +3476,node,quick,sorted,500,2463534242,3,1164,1,275478709,245185899,124750,0,d9a24379,off,,,,,,, +3477,c,none,random,250,2463534242,26,0,429,246313940,80,0,0,b47adce4,off,,,,,,, +3478,node,bubble,nearly_sorted,500,2463534242,28,1566,1,287853078,254249425,124750,14926,482b4785,off,,,,,,, +3479,c,quick,random,250,2463534242,13,136055,1,234270892,233365867,1933,1940,b47adce4,off,,,,,,, +3480,c,bubble,reversed,500,2463534242,22,346,1,240084948,239123317,124750,249500,d33e3559,off,,,,,,, +3481,c,none,sorted,250,2463534242,24,0,429,247907767,110,0,0,7fd8c1f4,off,,,,,,, +3482,c,bubble,reversed,2000,2463534242,15,22,1,251212371,250416618,1999000,3998000,ad35ba05,off,,,,,,, +3483,python,insertion,nearly_sorted,2000,2463534242,9,20,1,265891871,247775217,118277,118270,5835dc59,off,,,,,,, +3484,c,bubble,reversed,1000,2463534242,5,87,1,243576952,242653753,499500,999000,0ca43265,off,,,,,,, +3485,python,none,reversed,2000,2463534242,6,0,16,259842275,581,0,0,ad35ba05,off,,,,,,, +3486,python,none,sorted,2000,2463534242,9,0,15,252558993,561,0,0,69842aa5,off,,,,,,, +3487,python,bubble,nearly_sorted,500,2463534242,19,41,1,263193934,246518086,124750,14926,482b4785,off,,,,,,, +3488,node,quick,random,250,2463534242,12,36348,1,262032048,232823306,1933,1940,b47adce4,off,,,,,,, +3489,c,insertion,reversed,250,2463534242,29,15645,1,240682657,239777391,31125,31374,be175004,off,,,,,,, +3490,node,bubble,reversed,500,2463534242,19,1534,1,263510328,235822123,124750,249500,d33e3559,off,,,,,,, +3491,python,insertion,reversed,500,2463534242,15,22,1,253973687,238276312,124750,125249,d33e3559,off,,,,,,, +3492,c,bubble,reversed,250,2463534242,18,1384,1,242809522,241729659,31125,62250,be175004,off,,,,,,, +3493,c,bubble,random,250,2463534242,20,2568,1,242505043,241611498,31125,31196,b47adce4,off,,,,,,, +3494,node,quick,nearly_sorted,500,2463534242,16,10954,1,270838973,241522182,10192,1850,482b4785,off,,,,,,, +3495,c,insertion,random,250,2463534242,5,26020,1,234179870,233067552,15844,15838,b47adce4,off,,,,,,, +3496,node,quick,random,250,2463534242,22,36348,1,254954623,227452560,1933,1940,b47adce4,off,,,,,,, +3497,c,none,reversed,500,2463534242,15,0,402,227948804,90,0,0,d33e3559,off,,,,,,, +3498,c,none,reversed,250,2463534242,12,0,427,241263556,100,0,0,be175004,off,,,,,,, +3499,python,insertion,nearly_sorted,1000,2463534242,19,76,1,248667242,231546829,30889,30860,1cfe4afd,off,,,,,,, +3500,node,quick,reversed,500,2463534242,26,1194,1,261949772,232434376,124750,500,d33e3559,off,,,,,,, +3501,node,quick,nearly_sorted,250,2463534242,8,25320,1,268082958,239430101,4346,1008,21a19c14,off,,,,,,, +3502,node,none,sorted,500,2463534242,17,0,10,268423315,902,0,0,d9a24379,off,,,,,,, +3503,python,quick,reversed,2000,2463534242,2,2,1,280617889,262717525,1999000,2000,ad35ba05,off,,,,,,, +3504,node,bubble,random,2000,2463534242,26,98,1,274358608,243258129,1999000,1988792,41262855,off,,,,,,, +3505,node,none,nearly_sorted,500,2463534242,13,0,10,269194100,622,0,0,482b4785,off,,,,,,, +3506,c,bubble,random,2000,2463534242,24,40,1,246346852,245276907,1999000,1988792,41262855,off,,,,,,, +3507,c,bubble,random,500,2463534242,1,635,1,233167363,232248021,124750,119688,11ae932d,off,,,,,,, +3508,python,insertion,random,500,2463534242,30,44,1,250617877,234618851,60338,60339,11ae932d,off,,,,,,, +3509,python,quick,random,1000,2463534242,4,209,1,262199381,245687391,11196,10210,57561285,off,,,,,,, +3510,node,quick,sorted,1000,2463534242,27,296,1,273349608,243969869,499500,0,8e15cb75,off,,,,,,, +3511,node,none,nearly_sorted,1000,2463534242,30,0,10,271700507,891,0,0,1cfe4afd,off,,,,,,, +3512,c,bubble,reversed,250,2463534242,24,1384,1,243092706,242227845,31125,62250,be175004,off,,,,,,, +3513,c,quick,sorted,250,2463534242,13,10851,1,246270037,245385571,31125,0,7fd8c1f4,off,,,,,,, +3514,node,quick,random,1000,2463534242,6,6513,1,279333253,251115006,11196,10210,57561285,off,,,,,,, +3515,c,bubble,reversed,500,2463534242,25,346,1,242452352,241560122,124750,249500,d33e3559,off,,,,,,, +3516,c,insertion,random,250,2463534242,9,26020,1,232752017,231621216,15844,15838,b47adce4,off,,,,,,, +3517,node,insertion,random,500,2463534242,25,4059,1,254922933,226995922,60338,60339,11ae932d,off,,,,,,, +3518,python,quick,reversed,500,2463534242,28,33,1,255261631,239110948,124750,500,d33e3559,off,,,,,,, +3519,python,none,random,250,2463534242,4,0,17,264005545,551,0,0,b47adce4,off,,,,,,, +3520,c,bubble,sorted,2000,2463534242,13,173,1,235981407,234887864,1999000,0,69842aa5,off,,,,,,, +3521,c,insertion,nearly_sorted,1000,2463534242,3,11445,1,248308669,247000890,30889,30860,1cfe4afd,off,,,,,,, +3522,node,none,sorted,500,2463534242,9,0,10,272403404,1163,0,0,d9a24379,off,,,,,,, +3523,node,insertion,sorted,2000,2463534242,26,89220,1,275323146,242795537,1999,0,69842aa5,off,,,,,,, +3524,python,none,random,1000,2463534242,28,0,15,245475750,621,0,0,57561285,off,,,,,,, +3525,node,none,sorted,500,2463534242,28,0,10,268644289,862,0,0,d9a24379,off,,,,,,, +3526,python,quick,random,250,2463534242,3,1271,1,248364483,231797208,1933,1940,b47adce4,off,,,,,,, +3527,python,bubble,random,2000,2463534242,14,2,1,375712172,358657465,1999000,1988792,41262855,off,,,,,,, +3528,c,quick,random,2000,2463534242,24,10587,1,232412740,231273454,25197,24544,41262855,off,,,,,,, +3529,node,none,nearly_sorted,2000,2463534242,1,0,9,248178946,661,0,0,5835dc59,off,,,,,,, +3530,c,insertion,reversed,500,2463534242,17,3967,1,241565564,240641029,124750,125249,d33e3559,off,,,,,,, +3531,c,bubble,sorted,250,2463534242,25,11386,1,239142720,238022041,31125,0,7fd8c1f4,off,,,,,,, +3532,node,none,nearly_sorted,2000,2463534242,29,0,9,248248647,661,0,0,5835dc59,off,,,,,,, +3533,c,quick,random,1000,2463534242,11,23484,1,238520506,237375911,11196,10210,57561285,off,,,,,,, +3534,node,none,reversed,2000,2463534242,11,0,9,250816688,1222,0,0,ad35ba05,off,,,,,,, +3535,node,none,reversed,1000,2463534242,21,0,9,255609102,851,0,0,0ca43265,off,,,,,,, +3536,node,bubble,reversed,2000,2463534242,6,94,1,297694701,264613716,1999000,3998000,ad35ba05,off,,,,,,, +3537,c,insertion,nearly_sorted,500,2463534242,13,32350,1,250184916,249348980,7962,7926,482b4785,off,,,,,,, +3538,node,none,reversed,500,2463534242,24,0,9,255542165,771,0,0,d33e3559,off,,,,,,, +3539,python,quick,random,1000,2463534242,18,209,1,256978207,238426365,11196,10210,57561285,off,,,,,,, +3540,c,none,sorted,250,2463534242,14,0,429,250121797,100,0,0,7fd8c1f4,off,,,,,,, +3541,node,none,sorted,1000,2463534242,13,0,9,249552149,982,0,0,8e15cb75,off,,,,,,, +3542,c,insertion,random,2000,2463534242,18,492,1,235352518,234436081,996390,996389,41262855,off,,,,,,, +3543,node,none,random,2000,2463534242,16,0,9,251395454,832,0,0,41262855,off,,,,,,, +3544,node,insertion,random,500,2463534242,19,4059,1,299791472,265430268,60338,60339,11ae932d,off,,,,,,, +3545,python,quick,reversed,2000,2463534242,15,2,1,274796478,257754519,1999000,2000,ad35ba05,off,,,,,,, +3546,c,none,reversed,2000,2463534242,25,0,413,250550991,90,0,0,ad35ba05,off,,,,,,, +3547,node,bubble,sorted,250,2463534242,22,8081,1,261755860,230915882,31125,0,7fd8c1f4,off,,,,,,, +3548,node,quick,reversed,250,2463534242,21,4755,1,278346021,247730606,31125,250,be175004,off,,,,,,, +3549,python,bubble,reversed,250,2463534242,16,88,1,253995042,238016865,31125,62250,be175004,off,,,,,,, +3550,node,quick,sorted,2000,2463534242,10,74,1,275110652,244224792,1999000,0,69842aa5,off,,,,,,, +3551,python,quick,nearly_sorted,2000,2463534242,4,55,1,255994164,238413821,58002,11768,5835dc59,off,,,,,,, +3552,node,quick,random,2000,2463534242,21,2126,1,276643319,248158276,25197,24544,41262855,off,,,,,,, +3553,python,quick,reversed,250,2463534242,12,151,1,261524707,245219535,31125,250,be175004,off,,,,,,, +3554,python,bubble,random,500,2463534242,22,26,1,259004925,243095572,124750,119688,11ae932d,off,,,,,,, +3555,c,quick,sorted,2000,2463534242,11,174,1,247851412,246985228,1999000,0,69842aa5,off,,,,,,, +3556,python,insertion,nearly_sorted,2000,2463534242,14,20,1,272399880,253528368,118277,118270,5835dc59,off,,,,,,, +3557,python,insertion,random,2000,2463534242,3,3,1,322979607,305599644,996390,996389,41262855,off,,,,,,, +3558,node,none,nearly_sorted,500,2463534242,4,0,10,268059634,1052,0,0,482b4785,off,,,,,,, +3559,c,insertion,reversed,500,2463534242,1,3967,1,239265011,238362781,124750,125249,d33e3559,off,,,,,,, +3560,python,none,sorted,2000,2463534242,28,0,15,247746787,682,0,0,69842aa5,off,,,,,,, +3561,python,quick,reversed,500,2463534242,29,33,1,253726141,238485799,124750,500,d33e3559,off,,,,,,, +3562,c,insertion,nearly_sorted,1000,2463534242,15,11445,1,249786729,248615165,30889,30860,1cfe4afd,off,,,,,,, +3563,python,quick,sorted,500,2463534242,27,30,1,244604168,228578995,124750,0,d9a24379,off,,,,,,, +3564,python,quick,random,500,2463534242,30,507,1,266764125,249733172,4578,4552,11ae932d,off,,,,,,, +3565,node,insertion,reversed,1000,2463534242,21,554,1,270225133,241212071,499500,500499,0ca43265,off,,,,,,, +3566,c,bubble,random,1000,2463534242,9,154,1,234684218,233833382,499500,496926,57561285,off,,,,,,, +3567,c,insertion,sorted,1000,2463534242,17,196883,1,233430405,232490735,999,0,8e15cb75,off,,,,,,, +3568,node,insertion,nearly_sorted,1000,2463534242,11,7758,1,274300711,243362306,30889,30860,1cfe4afd,off,,,,,,, +3569,c,insertion,random,2000,2463534242,27,492,1,236367893,235426107,996390,996389,41262855,off,,,,,,, +3570,node,bubble,random,500,2463534242,29,1407,1,269155586,238364787,124750,119688,11ae932d,off,,,,,,, +3571,python,quick,reversed,2000,2463534242,5,2,1,278391998,260866165,1999000,2000,ad35ba05,off,,,,,,, +3572,c,bubble,nearly_sorted,1000,2463534242,28,507,1,246418166,245393102,499500,59780,1cfe4afd,off,,,,,,, +3573,node,none,sorted,500,2463534242,14,0,10,270473377,732,0,0,d9a24379,off,,,,,,, +3574,c,none,random,1000,2463534242,2,0,367,215593278,90,0,0,57561285,off,,,,,,, +3575,node,none,random,500,2463534242,8,0,9,244976234,1293,0,0,11ae932d,off,,,,,,, +3576,python,insertion,sorted,2000,2463534242,5,1523,1,263639236,246355158,1999,0,69842aa5,off,,,,,,, +3577,node,none,sorted,1000,2463534242,8,0,9,241005897,892,0,0,8e15cb75,off,,,,,,, +3578,c,quick,reversed,1000,2463534242,11,814,1,237064066,236063091,499500,1000,0ca43265,off,,,,,,, +3579,c,none,random,250,2463534242,5,0,429,249204808,141,0,0,b47adce4,off,,,,,,, +3580,node,none,sorted,1000,2463534242,4,0,9,252880487,942,0,0,8e15cb75,off,,,,,,, +3581,c,insertion,random,1000,2463534242,13,1983,1,243886842,242832747,249457,249457,57561285,off,,,,,,, +3582,c,none,random,1000,2463534242,3,0,367,211066325,90,0,0,57561285,off,,,,,,, +3583,python,quick,random,1000,2463534242,10,209,1,256636707,238812314,11196,10210,57561285,off,,,,,,, +3584,node,none,reversed,1000,2463534242,18,0,9,244110251,781,0,0,0ca43265,off,,,,,,, +3585,c,quick,nearly_sorted,2000,2463534242,19,6145,1,249764859,248629201,58002,11768,5835dc59,off,,,,,,, +3586,node,quick,reversed,1000,2463534242,29,307,1,266413096,236853975,499500,1000,0ca43265,off,,,,,,, +3587,python,quick,reversed,500,2463534242,5,33,1,254452694,238828936,124750,500,d33e3559,off,,,,,,, +3588,python,quick,reversed,500,2463534242,13,33,1,259535339,241439321,124750,500,d33e3559,off,,,,,,, +3589,python,quick,sorted,2000,2463534242,18,2,1,320235035,302984828,1999000,0,69842aa5,off,,,,,,, +3590,python,quick,random,1000,2463534242,26,209,1,258424667,242256657,11196,10210,57561285,off,,,,,,, +3591,python,none,reversed,1000,2463534242,2,0,16,258467518,541,0,0,0ca43265,off,,,,,,, +3592,c,bubble,random,500,2463534242,26,635,1,232106475,231048449,124750,119688,11ae932d,off,,,,,,, +3593,c,none,random,2000,2463534242,17,0,415,248527659,90,0,0,41262855,off,,,,,,, +3594,c,none,sorted,1000,2463534242,29,0,421,248539212,91,0,0,8e15cb75,off,,,,,,, +3595,python,bubble,nearly_sorted,500,2463534242,8,41,1,263754355,246494525,124750,14926,482b4785,off,,,,,,, +3596,python,bubble,reversed,250,2463534242,18,88,1,273256683,255948247,31125,62250,be175004,off,,,,,,, +3597,python,insertion,nearly_sorted,2000,2463534242,28,20,1,274239225,255053547,118277,118270,5835dc59,off,,,,,,, +3598,node,insertion,sorted,1000,2463534242,25,160107,1,267305390,235505025,999,0,8e15cb75,off,,,,,,, +3599,node,quick,nearly_sorted,500,2463534242,12,10954,1,277268524,248569940,10192,1850,482b4785,off,,,,,,, +3600,python,insertion,reversed,2000,2463534242,3,2,1,434662248,416514446,1999000,2000999,ad35ba05,off,,,,,,, +3601,python,insertion,reversed,1000,2463534242,18,6,1,300358614,283611139,499500,500499,0ca43265,off,,,,,,, +3602,c,insertion,random,2000,2463534242,16,492,1,241639781,240730035,996390,996389,41262855,off,,,,,,, +3603,python,bubble,sorted,1000,2463534242,25,11,1,290961624,273292799,499500,0,8e15cb75,off,,,,,,, +3604,python,none,reversed,250,2463534242,19,0,17,271924267,591,0,0,be175004,off,,,,,,, +3605,python,quick,random,2000,2463534242,1,90,1,264154112,245873954,25197,24544,41262855,off,,,,,,, +3606,python,insertion,random,2000,2463534242,21,3,1,339680950,321411145,996390,996389,41262855,off,,,,,,, +3607,c,bubble,nearly_sorted,250,2463534242,28,7880,1,246215157,245288559,31125,4056,21a19c14,off,,,,,,, +3608,python,none,random,1000,2463534242,18,0,15,258407935,652,0,0,57561285,off,,,,,,, +3609,python,none,random,250,2463534242,8,0,17,284399265,511,0,0,b47adce4,off,,,,,,, +3610,python,none,sorted,250,2463534242,4,0,17,275117511,521,0,0,7fd8c1f4,off,,,,,,, +3611,c,insertion,random,1000,2463534242,19,1983,1,244291411,243166904,249457,249457,57561285,off,,,,,,, +3612,python,insertion,sorted,2000,2463534242,10,1523,1,275806973,259332434,1999,0,69842aa5,off,,,,,,, +3613,node,none,reversed,250,2463534242,11,0,9,257008643,922,0,0,be175004,off,,,,,,, +3614,c,bubble,random,1000,2463534242,27,154,1,237525149,236618230,499500,496926,57561285,off,,,,,,, +3615,node,insertion,nearly_sorted,2000,2463534242,25,2100,1,259424171,230454038,118277,118270,5835dc59,off,,,,,,, +3616,python,bubble,nearly_sorted,2000,2463534242,27,3,1,366747813,349568548,1999000,232556,5835dc59,off,,,,,,, +3617,c,quick,nearly_sorted,2000,2463534242,27,6145,1,243884037,242811457,58002,11768,5835dc59,off,,,,,,, +3618,python,insertion,sorted,250,2463534242,28,18845,1,270943929,253774707,249,0,7fd8c1f4,off,,,,,,, +3619,python,insertion,sorted,2000,2463534242,29,1523,1,264576366,246271451,1999,0,69842aa5,off,,,,,,, +3620,c,bubble,random,2000,2463534242,8,40,1,243656941,242557853,1999000,1988792,41262855,off,,,,,,, +3621,python,insertion,sorted,250,2463534242,19,18845,1,270121679,253081448,249,0,7fd8c1f4,off,,,,,,, +3622,python,none,reversed,500,2463534242,15,0,16,254075727,541,0,0,d33e3559,off,,,,,,, +3623,python,insertion,nearly_sorted,500,2463534242,26,316,1,252908159,237002309,7962,7926,482b4785,off,,,,,,, +3624,c,none,sorted,1000,2463534242,23,0,421,251036451,140,0,0,8e15cb75,off,,,,,,, +3625,python,quick,sorted,500,2463534242,10,30,1,248952016,232241806,124750,0,d9a24379,off,,,,,,, +3626,c,bubble,nearly_sorted,500,2463534242,20,1998,1,238488946,237562732,124750,14926,482b4785,off,,,,,,, +3627,c,bubble,reversed,500,2463534242,30,346,1,247946290,246676538,124750,249500,d33e3559,off,,,,,,, +3628,c,none,random,250,2463534242,12,0,429,245297395,91,0,0,b47adce4,off,,,,,,, +3629,c,none,random,1000,2463534242,22,0,367,214812595,90,0,0,57561285,off,,,,,,, +3630,python,bubble,random,500,2463534242,23,26,1,277305513,260213440,124750,119688,11ae932d,off,,,,,,, +3631,c,none,reversed,2000,2463534242,4,0,413,243768991,90,0,0,ad35ba05,off,,,,,,, +3632,node,insertion,sorted,500,2463534242,7,324764,1,286973790,254887109,499,0,d9a24379,off,,,,,,, +3633,node,none,sorted,2000,2463534242,12,0,10,275066806,1042,0,0,69842aa5,off,,,,,,, +3634,c,quick,reversed,2000,2463534242,11,209,1,239644402,238727864,1999000,2000,ad35ba05,off,,,,,,, +3635,node,insertion,nearly_sorted,250,2463534242,13,79550,1,245120454,216419362,2277,2251,21a19c14,off,,,,,,, +3636,c,insertion,random,250,2463534242,8,26020,1,238297097,237425344,15844,15838,b47adce4,off,,,,,,, +3637,node,quick,random,1000,2463534242,1,6513,1,276138314,247880757,11196,10210,57561285,off,,,,,,, +3638,node,quick,nearly_sorted,1000,2463534242,5,3752,1,262320007,234818752,28499,7892,1cfe4afd,off,,,,,,, +3639,python,quick,reversed,500,2463534242,23,33,1,267678630,252424001,124750,500,d33e3559,off,,,,,,, +3640,python,insertion,reversed,250,2463534242,19,102,1,256818287,240925067,31125,31374,be175004,off,,,,,,, +3641,c,quick,reversed,500,2463534242,4,3240,1,240113751,238873021,124750,500,d33e3559,off,,,,,,, +3642,c,bubble,reversed,2000,2463534242,28,22,1,250197890,248994545,1999000,3998000,ad35ba05,off,,,,,,, +3643,python,quick,reversed,250,2463534242,5,151,1,263642786,247434360,31125,250,be175004,off,,,,,,, +3644,c,insertion,random,250,2463534242,14,26020,1,239967918,238808447,15844,15838,b47adce4,off,,,,,,, +3645,python,none,reversed,2000,2463534242,16,0,16,257748811,621,0,0,ad35ba05,off,,,,,,, +3646,python,bubble,random,2000,2463534242,2,2,1,394254121,377684334,1999000,1988792,41262855,off,,,,,,, +3647,node,quick,reversed,2000,2463534242,27,80,1,273520979,244004183,1999000,2000,ad35ba05,off,,,,,,, +3648,node,bubble,nearly_sorted,250,2463534242,28,6161,1,268116189,239163022,31125,4056,21a19c14,off,,,,,,, +3649,c,insertion,random,500,2463534242,11,6945,1,217702241,216765895,60338,60339,11ae932d,off,,,,,,, +3650,python,insertion,sorted,500,2463534242,23,6624,1,252392783,236231044,499,0,d9a24379,off,,,,,,, +3651,c,bubble,reversed,250,2463534242,1,1384,1,245525870,244505544,31125,62250,be175004,off,,,,,,, +3652,python,insertion,nearly_sorted,2000,2463534242,4,20,1,272799060,253257513,118277,118270,5835dc59,off,,,,,,, +3653,node,insertion,nearly_sorted,2000,2463534242,26,2100,1,284083083,254219477,118277,118270,5835dc59,off,,,,,,, +3654,c,bubble,reversed,500,2463534242,24,346,1,246165444,245231112,124750,249500,d33e3559,off,,,,,,, +3655,node,bubble,random,500,2463534242,16,1407,1,268716022,240319857,124750,119688,11ae932d,off,,,,,,, +3656,node,insertion,random,1000,2463534242,19,1091,1,269956819,242129678,249457,249457,57561285,off,,,,,,, +3657,python,quick,nearly_sorted,1000,2463534242,21,110,1,261673034,246126774,28499,7892,1cfe4afd,off,,,,,,, +3658,node,insertion,nearly_sorted,2000,2463534242,17,2100,1,265690884,235167129,118277,118270,5835dc59,off,,,,,,, +3659,c,bubble,sorted,1000,2463534242,18,702,1,240980947,240077032,499500,0,8e15cb75,off,,,,,,, +3660,node,none,nearly_sorted,250,2463534242,23,0,10,271633522,902,0,0,21a19c14,off,,,,,,, +3661,python,quick,random,2000,2463534242,27,90,1,270286357,252954668,25197,24544,41262855,off,,,,,,, +3662,node,none,random,1000,2463534242,18,0,10,278095301,892,0,0,57561285,off,,,,,,, +3663,c,none,nearly_sorted,500,2463534242,1,0,405,234554523,90,0,0,482b4785,off,,,,,,, +3664,python,insertion,random,250,2463534242,2,206,1,267890366,251144898,15844,15838,b47adce4,off,,,,,,, +3665,c,bubble,sorted,500,2463534242,30,2858,1,246198897,245251750,124750,0,d9a24379,off,,,,,,, +3666,python,insertion,random,250,2463534242,21,206,1,256296177,240306243,15844,15838,b47adce4,off,,,,,,, +3667,node,none,nearly_sorted,500,2463534242,9,0,10,279140079,681,0,0,482b4785,off,,,,,,, +3668,python,insertion,random,2000,2463534242,9,3,1,332621468,315442489,996390,996389,41262855,off,,,,,,, +3669,python,insertion,random,1000,2463534242,11,10,1,258905510,242008167,249457,249457,57561285,off,,,,,,, +3670,node,none,nearly_sorted,2000,2463534242,20,0,9,262778114,842,0,0,5835dc59,off,,,,,,, +3671,node,quick,random,250,2463534242,13,36348,1,272059700,239527612,1933,1940,b47adce4,off,,,,,,, +3672,c,insertion,random,500,2463534242,4,6945,1,215021927,214076445,60338,60339,11ae932d,off,,,,,,, +3673,c,insertion,sorted,500,2463534242,2,399335,1,244057492,243135807,499,0,d9a24379,off,,,,,,, +3674,c,none,nearly_sorted,1000,2463534242,27,0,421,251683202,90,0,0,1cfe4afd,off,,,,,,, +3675,node,quick,nearly_sorted,2000,2463534242,9,1909,1,274457885,243052557,58002,11768,5835dc59,off,,,,,,, +3676,c,quick,nearly_sorted,500,2463534242,7,31784,1,252725809,251694353,10192,1850,482b4785,off,,,,,,, +3677,node,insertion,random,250,2463534242,25,15349,1,267694809,237045701,15844,15838,b47adce4,off,,,,,,, +3678,python,none,random,2000,2463534242,8,0,16,290808246,772,0,0,41262855,off,,,,,,, +3679,python,none,sorted,500,2463534242,13,0,16,250700661,571,0,0,d9a24379,off,,,,,,, +3680,c,bubble,nearly_sorted,250,2463534242,4,7880,1,251564321,250490878,31125,4056,21a19c14,off,,,,,,, +3681,c,quick,nearly_sorted,2000,2463534242,29,6145,1,252085469,251074859,58002,11768,5835dc59,off,,,,,,, +3682,node,quick,nearly_sorted,250,2463534242,22,25320,1,276679687,246311288,4346,1008,21a19c14,off,,,,,,, +3683,python,quick,reversed,1000,2463534242,7,8,1,272208509,256144300,499500,1000,0ca43265,off,,,,,,, +3684,c,insertion,sorted,1000,2463534242,2,196883,1,242869636,241972547,999,0,8e15cb75,off,,,,,,, +3685,node,insertion,random,2000,2463534242,17,234,1,276506584,247502058,996390,996389,41262855,off,,,,,,, +3686,node,none,sorted,1000,2463534242,19,0,9,254831945,1012,0,0,8e15cb75,off,,,,,,, +3687,c,insertion,reversed,500,2463534242,4,3967,1,242293307,240926975,124750,125249,d33e3559,off,,,,,,, +3688,c,none,reversed,500,2463534242,17,0,402,233115885,80,0,0,d33e3559,off,,,,,,, +3689,node,quick,nearly_sorted,2000,2463534242,29,1909,1,263013968,233032326,58002,11768,5835dc59,off,,,,,,, +3690,python,quick,reversed,250,2463534242,30,151,1,257384628,241574564,31125,250,be175004,off,,,,,,, +3691,c,insertion,nearly_sorted,250,2463534242,6,99421,1,246802597,245835183,2277,2251,21a19c14,off,,,,,,, +3692,node,quick,random,2000,2463534242,5,2126,1,279492851,248525033,25197,24544,41262855,off,,,,,,, +3693,python,bubble,random,500,2463534242,6,26,1,260901358,245659898,124750,119688,11ae932d,off,,,,,,, +3694,node,insertion,sorted,1000,2463534242,10,160107,1,264585725,235094620,999,0,8e15cb75,off,,,,,,, +3695,python,bubble,nearly_sorted,2000,2463534242,16,3,1,376852549,357641986,1999000,232556,5835dc59,off,,,,,,, +3696,node,quick,nearly_sorted,250,2463534242,14,25320,1,295953649,266524262,4346,1008,21a19c14,off,,,,,,, +3697,c,bubble,random,500,2463534242,20,635,1,238582161,237566948,124750,119688,11ae932d,off,,,,,,, +3698,c,none,sorted,500,2463534242,25,0,344,200595832,90,0,0,d9a24379,off,,,,,,, +3699,c,none,random,1000,2463534242,27,0,367,212765599,80,0,0,57561285,off,,,,,,, +3700,c,quick,random,2000,2463534242,2,10587,1,237707100,236877302,25197,24544,41262855,off,,,,,,, +3701,python,none,random,1000,2463534242,9,0,15,245066353,751,0,0,57561285,off,,,,,,, +3702,c,quick,nearly_sorted,2000,2463534242,17,6145,1,247976327,247040474,58002,11768,5835dc59,off,,,,,,, +3703,node,quick,random,250,2463534242,18,36348,1,259963301,231668358,1933,1940,b47adce4,off,,,,,,, +3704,python,quick,random,250,2463534242,16,1271,1,267918379,252178852,1933,1940,b47adce4,off,,,,,,, +3705,c,quick,nearly_sorted,1000,2463534242,29,11273,1,244458223,243445812,28499,7892,1cfe4afd,off,,,,,,, +3706,c,bubble,random,1000,2463534242,15,154,1,236798057,235886238,499500,496926,57561285,off,,,,,,, +3707,node,none,reversed,1000,2463534242,7,0,9,239143282,882,0,0,0ca43265,off,,,,,,, +3708,node,quick,reversed,250,2463534242,9,4755,1,277117199,246750543,31125,250,be175004,off,,,,,,, +3709,node,bubble,nearly_sorted,2000,2463534242,1,110,1,269330005,238130976,1999000,232556,5835dc59,off,,,,,,, +3710,python,none,random,250,2463534242,3,0,17,264119650,572,0,0,b47adce4,off,,,,,,, +3711,node,insertion,nearly_sorted,250,2463534242,14,79550,1,252497239,224548426,2277,2251,21a19c14,off,,,,,,, +3712,c,insertion,reversed,250,2463534242,28,15645,1,244502525,243580097,31125,31374,be175004,off,,,,,,, +3713,c,quick,random,500,2463534242,7,55776,1,244084313,242984361,4578,4552,11ae932d,off,,,,,,, +3714,c,bubble,random,500,2463534242,12,635,1,232698214,231732987,124750,119688,11ae932d,off,,,,,,, +3715,c,none,reversed,250,2463534242,26,0,427,245794267,91,0,0,be175004,off,,,,,,, +3716,c,quick,sorted,1000,2463534242,10,691,1,244617819,243838392,499500,0,8e15cb75,off,,,,,,, +3717,node,none,nearly_sorted,500,2463534242,12,0,10,277308411,1032,0,0,482b4785,off,,,,,,, +3718,python,quick,random,250,2463534242,6,1271,1,258732353,241076699,1933,1940,b47adce4,off,,,,,,, +3719,c,none,sorted,500,2463534242,21,0,344,197064955,91,0,0,d9a24379,off,,,,,,, +3720,python,bubble,nearly_sorted,250,2463534242,9,188,1,258318239,242495521,31125,4056,21a19c14,off,,,,,,, +3721,python,bubble,reversed,2000,2463534242,24,2,1,508994154,492258194,1999000,3998000,ad35ba05,off,,,,,,, +3722,python,insertion,sorted,2000,2463534242,19,1523,1,258862528,242676273,1999,0,69842aa5,off,,,,,,, +3723,python,none,sorted,1000,2463534242,24,0,16,250079809,611,0,0,8e15cb75,off,,,,,,, +3724,python,insertion,random,1000,2463534242,3,10,1,274090899,257739119,249457,249457,57561285,off,,,,,,, +3725,python,insertion,reversed,1000,2463534242,10,6,1,312671880,297168828,499500,500499,0ca43265,off,,,,,,, +3726,python,insertion,sorted,2000,2463534242,9,1523,1,274326739,258105884,1999,0,69842aa5,off,,,,,,, +3727,node,none,random,2000,2463534242,3,0,9,258803880,932,0,0,41262855,off,,,,,,, +3728,node,none,nearly_sorted,2000,2463534242,15,0,9,257819614,3156,0,0,5835dc59,off,,,,,,, +3729,node,bubble,random,2000,2463534242,17,98,1,289117176,255569498,1999000,1988792,41262855,off,,,,,,, +3730,node,none,nearly_sorted,250,2463534242,16,0,10,275898374,762,0,0,21a19c14,off,,,,,,, +3731,python,quick,reversed,250,2463534242,3,151,1,257316931,241015090,31125,250,be175004,off,,,,,,, +3732,node,bubble,random,1000,2463534242,21,362,1,265766235,236403814,499500,496926,57561285,off,,,,,,, +3733,c,none,random,250,2463534242,28,0,429,252013602,91,0,0,b47adce4,off,,,,,,, +3734,python,quick,nearly_sorted,500,2463534242,21,296,1,245881961,230233698,10192,1850,482b4785,off,,,,,,, +3735,c,insertion,reversed,250,2463534242,12,15645,1,247233885,246238848,31125,31374,be175004,off,,,,,,, +3736,node,insertion,nearly_sorted,250,2463534242,6,79550,1,251709673,221230145,2277,2251,21a19c14,off,,,,,,, +3737,c,bubble,nearly_sorted,500,2463534242,19,1998,1,236813837,235929249,124750,14926,482b4785,off,,,,,,, +3738,python,quick,nearly_sorted,500,2463534242,6,296,1,243045576,227078131,10192,1850,482b4785,off,,,,,,, +3739,node,bubble,sorted,2000,2463534242,17,85,1,266998195,237725534,1999000,0,69842aa5,off,,,,,,, +3740,node,quick,sorted,500,2463534242,17,1164,1,281176566,250885410,124750,0,d9a24379,off,,,,,,, +3741,node,insertion,random,2000,2463534242,19,234,1,277727272,248169687,996390,996389,41262855,off,,,,,,, +3742,python,quick,reversed,250,2463534242,15,151,1,261355830,245303397,31125,250,be175004,off,,,,,,, +3743,c,bubble,reversed,2000,2463534242,7,22,1,246151627,245115415,1999000,3998000,ad35ba05,off,,,,,,, +3744,python,insertion,reversed,2000,2463534242,11,2,1,445751412,428605137,1999000,2000999,ad35ba05,off,,,,,,, +3745,python,none,random,2000,2463534242,27,0,16,280891262,1112,0,0,41262855,off,,,,,,, +3746,python,insertion,sorted,500,2463534242,20,6624,1,247014974,230495442,499,0,d9a24379,off,,,,,,, +3747,node,insertion,sorted,2000,2463534242,9,89220,1,287004999,257222836,1999,0,69842aa5,off,,,,,,, +3748,python,quick,random,2000,2463534242,12,90,1,259494431,241706510,25197,24544,41262855,off,,,,,,, +3749,node,insertion,random,2000,2463534242,18,234,1,279164135,247697631,996390,996389,41262855,off,,,,,,, +3750,c,insertion,nearly_sorted,500,2463534242,29,32350,1,240990915,240093263,7962,7926,482b4785,off,,,,,,, +3751,node,insertion,nearly_sorted,1000,2463534242,19,7758,1,277434783,245091438,30889,30860,1cfe4afd,off,,,,,,, +3752,node,bubble,nearly_sorted,2000,2463534242,13,110,1,267509844,237479492,1999000,232556,5835dc59,off,,,,,,, +3753,c,quick,random,500,2463534242,15,55776,1,245321952,244445466,4578,4552,11ae932d,off,,,,,,, +3754,node,quick,random,1000,2463534242,17,6513,1,272042959,242505415,11196,10210,57561285,off,,,,,,, +3755,node,bubble,nearly_sorted,1000,2463534242,16,413,1,266803130,238697303,499500,59780,1cfe4afd,off,,,,,,, +3756,c,bubble,random,500,2463534242,27,635,1,231659868,230784193,124750,119688,11ae932d,off,,,,,,, +3757,node,bubble,reversed,2000,2463534242,19,94,1,274997586,245218783,1999000,3998000,ad35ba05,off,,,,,,, +3758,python,none,nearly_sorted,1000,2463534242,23,0,16,251586352,591,0,0,1cfe4afd,off,,,,,,, +3759,python,none,random,1000,2463534242,26,0,15,243987100,652,0,0,57561285,off,,,,,,, +3760,python,quick,nearly_sorted,2000,2463534242,5,55,1,260177191,242420050,58002,11768,5835dc59,off,,,,,,, +3761,c,quick,reversed,500,2463534242,14,3240,1,241695316,240797342,124750,500,d33e3559,off,,,,,,, +3762,node,quick,random,250,2463534242,2,36348,1,261276211,232769223,1933,1940,b47adce4,off,,,,,,, +3763,python,bubble,reversed,2000,2463534242,10,2,1,532220331,514752465,1999000,3998000,ad35ba05,off,,,,,,, +3764,python,bubble,reversed,2000,2463534242,5,2,1,497849277,479757724,1999000,3998000,ad35ba05,off,,,,,,, +3765,node,insertion,nearly_sorted,250,2463534242,30,79550,1,253451477,223133213,2277,2251,21a19c14,off,,,,,,, +3766,python,quick,reversed,250,2463534242,7,151,1,290883838,274693688,31125,250,be175004,off,,,,,,, +3767,node,none,reversed,1000,2463534242,13,0,9,240900946,841,0,0,0ca43265,off,,,,,,, +3768,python,quick,nearly_sorted,250,2463534242,1,882,1,270052408,254667194,4346,1008,21a19c14,off,,,,,,, +3769,c,insertion,random,2000,2463534242,2,492,1,236422934,235519002,996390,996389,41262855,off,,,,,,, +3770,node,quick,random,1000,2463534242,29,6513,1,277879066,249123393,11196,10210,57561285,off,,,,,,, +3771,node,bubble,random,500,2463534242,10,1407,1,270158027,240990443,124750,119688,11ae932d,off,,,,,,, +3772,python,insertion,reversed,2000,2463534242,26,2,1,429515174,411877267,1999000,2000999,ad35ba05,off,,,,,,, +3773,c,insertion,reversed,500,2463534242,30,3967,1,245274835,244346063,124750,125249,d33e3559,off,,,,,,, +3774,c,insertion,reversed,250,2463534242,18,15645,1,248214011,247102380,31125,31374,be175004,off,,,,,,, +3775,node,quick,sorted,1000,2463534242,19,296,1,277201877,246384157,499500,0,8e15cb75,off,,,,,,, +3776,c,none,reversed,1000,2463534242,15,0,405,243250300,80,0,0,0ca43265,off,,,,,,, +3777,node,quick,reversed,250,2463534242,23,4755,1,271948742,242208316,31125,250,be175004,off,,,,,,, +3778,python,bubble,random,500,2463534242,1,26,1,266365058,249450885,124750,119688,11ae932d,off,,,,,,, +3779,c,quick,reversed,500,2463534242,7,3240,1,254334753,253266367,124750,500,d33e3559,off,,,,,,, +3780,node,insertion,reversed,500,2463534242,15,2178,1,269067272,237651323,124750,125249,d33e3559,off,,,,,,, +3781,node,insertion,random,1000,2463534242,4,1091,1,275325071,245068478,249457,249457,57561285,off,,,,,,, +3782,c,bubble,nearly_sorted,2000,2463534242,17,121,1,241268143,240371395,1999000,232556,5835dc59,off,,,,,,, +3783,node,bubble,sorted,250,2463534242,17,8081,1,273961074,241108621,31125,0,7fd8c1f4,off,,,,,,, +3784,c,insertion,sorted,2000,2463534242,19,101352,1,240399436,239236218,1999,0,69842aa5,off,,,,,,, +3785,python,quick,random,500,2463534242,1,507,1,270853209,254390093,4578,4552,11ae932d,off,,,,,,, +3786,python,quick,sorted,250,2463534242,26,133,1,249955165,233623318,31125,0,7fd8c1f4,off,,,,,,, +3787,python,insertion,reversed,250,2463534242,23,102,1,252889043,236060169,31125,31374,be175004,off,,,,,,, +3788,c,none,reversed,1000,2463534242,12,0,405,245956661,100,0,0,0ca43265,off,,,,,,, +3789,python,quick,random,500,2463534242,2,507,1,270032130,253334963,4578,4552,11ae932d,off,,,,,,, +3790,c,none,sorted,1000,2463534242,13,0,421,246512172,90,0,0,8e15cb75,off,,,,,,, +3791,c,bubble,sorted,500,2463534242,18,2858,1,250574826,249830091,124750,0,d9a24379,off,,,,,,, +3792,node,insertion,nearly_sorted,500,2463534242,22,26045,1,254666304,224131851,7962,7926,482b4785,off,,,,,,, +3793,c,none,reversed,2000,2463534242,19,0,413,239362323,100,0,0,ad35ba05,off,,,,,,, +3794,c,quick,reversed,1000,2463534242,25,814,1,243144492,242326139,499500,1000,0ca43265,off,,,,,,, +3795,node,insertion,nearly_sorted,250,2463534242,20,79550,1,247804001,218378763,2277,2251,21a19c14,off,,,,,,, +3796,c,bubble,sorted,250,2463534242,13,11386,1,244240355,243386205,31125,0,7fd8c1f4,off,,,,,,, +3797,node,quick,sorted,500,2463534242,21,1164,1,272425556,241667281,124750,0,d9a24379,off,,,,,,, +3798,c,insertion,nearly_sorted,250,2463534242,10,99421,1,242901486,241873310,2277,2251,21a19c14,off,,,,,,, +3799,node,none,random,2000,2463534242,21,0,9,266351403,1253,0,0,41262855,off,,,,,,, +3800,python,none,sorted,2000,2463534242,20,0,15,248710243,672,0,0,69842aa5,off,,,,,,, +3801,python,bubble,reversed,2000,2463534242,13,2,1,493187199,475880445,1999000,3998000,ad35ba05,off,,,,,,, +3802,python,insertion,reversed,250,2463534242,10,102,1,250057096,234432283,31125,31374,be175004,off,,,,,,, +3803,c,none,sorted,500,2463534242,2,0,344,203659744,90,0,0,d9a24379,off,,,,,,, +3804,node,none,reversed,500,2463534242,17,0,9,262769100,1172,0,0,d33e3559,off,,,,,,, +3805,c,none,nearly_sorted,500,2463534242,8,0,405,247070597,91,0,0,482b4785,off,,,,,,, +3806,node,none,random,500,2463534242,22,0,9,262242252,3427,0,0,11ae932d,off,,,,,,, +3807,python,none,sorted,1000,2463534242,11,0,16,253194646,611,0,0,8e15cb75,off,,,,,,, +3808,c,bubble,reversed,500,2463534242,26,346,1,247272888,246297712,124750,249500,d33e3559,off,,,,,,, +3809,node,none,sorted,1000,2463534242,17,0,9,253157647,962,0,0,8e15cb75,off,,,,,,, +3810,c,bubble,random,500,2463534242,13,635,1,240441932,239424840,124750,119688,11ae932d,off,,,,,,, +3811,python,bubble,reversed,500,2463534242,18,20,1,278482571,261877456,124750,249500,d33e3559,off,,,,,,, +3812,c,none,random,2000,2463534242,11,0,415,255142686,80,0,0,41262855,off,,,,,,, +3813,python,none,nearly_sorted,2000,2463534242,8,0,15,269684215,551,0,0,5835dc59,off,,,,,,, +3814,python,quick,nearly_sorted,1000,2463534242,22,110,1,285797770,268154327,28499,7892,1cfe4afd,off,,,,,,, +3815,c,quick,nearly_sorted,500,2463534242,25,31784,1,255836267,254813161,10192,1850,482b4785,off,,,,,,, +3816,python,none,random,500,2463534242,7,0,16,267537475,601,0,0,11ae932d,off,,,,,,, +3817,python,insertion,random,250,2463534242,20,206,1,281431994,265051184,15844,15838,b47adce4,off,,,,,,, +3818,python,quick,reversed,1000,2463534242,27,8,1,289335166,272173628,499500,1000,0ca43265,off,,,,,,, +3819,node,insertion,random,500,2463534242,26,4059,1,303919528,267753911,60338,60339,11ae932d,off,,,,,,, +3820,node,quick,sorted,1000,2463534242,26,296,1,295552207,262117806,499500,0,8e15cb75,off,,,,,,, +3821,python,quick,random,250,2463534242,27,1271,1,279696719,260761533,1933,1940,b47adce4,off,,,,,,, +3822,python,quick,random,1000,2463534242,24,209,1,274256713,254872781,11196,10210,57561285,off,,,,,,, +3823,python,none,random,2000,2463534242,2,0,16,308701198,902,0,0,41262855,off,,,,,,, +3824,node,bubble,nearly_sorted,1000,2463534242,11,413,1,294113500,262993009,499500,59780,1cfe4afd,off,,,,,,, +3825,python,none,reversed,250,2463534242,11,0,17,297373847,531,0,0,be175004,off,,,,,,, +3826,node,quick,sorted,500,2463534242,11,1164,1,288145449,258345114,124750,0,d9a24379,off,,,,,,, +3827,python,quick,reversed,1000,2463534242,20,8,1,294461008,277356259,499500,1000,0ca43265,off,,,,,,, +3828,node,insertion,reversed,2000,2463534242,17,144,1,294068741,259557985,1999000,2000999,ad35ba05,off,,,,,,, +3829,node,insertion,reversed,250,2463534242,4,8539,1,294811099,263489748,31125,31374,be175004,off,,,,,,, +3830,node,bubble,random,250,2463534242,19,5439,1,273406524,241737762,31125,31196,b47adce4,off,,,,,,, +3831,node,bubble,nearly_sorted,2000,2463534242,8,110,1,270530384,240644397,1999000,232556,5835dc59,off,,,,,,, +3832,node,none,reversed,500,2463534242,8,0,9,250013374,952,0,0,d33e3559,off,,,,,,, +3833,node,quick,nearly_sorted,1000,2463534242,28,3752,1,269419733,240004464,28499,7892,1cfe4afd,off,,,,,,, +3834,node,bubble,reversed,500,2463534242,24,1534,1,273523253,244883339,124750,249500,d33e3559,off,,,,,,, +3835,c,insertion,reversed,2000,2463534242,1,260,1,249417187,248510439,1999000,2000999,ad35ba05,off,,,,,,, +3836,python,none,reversed,2000,2463534242,4,0,16,259356263,631,0,0,ad35ba05,off,,,,,,, +3837,python,quick,sorted,250,2463534242,8,133,1,260149300,244144089,31125,0,7fd8c1f4,off,,,,,,, +3838,python,none,nearly_sorted,1000,2463534242,7,0,16,250002835,581,0,0,1cfe4afd,off,,,,,,, +3839,python,none,random,1000,2463534242,17,0,15,245573033,872,0,0,57561285,off,,,,,,, +3840,python,bubble,sorted,500,2463534242,20,46,1,260090409,243970657,124750,0,d9a24379,off,,,,,,, +3841,python,quick,reversed,250,2463534242,25,151,1,264806667,249323609,31125,250,be175004,off,,,,,,, +3842,python,insertion,sorted,2000,2463534242,1,1523,1,271713852,254930953,1999,0,69842aa5,off,,,,,,, +3843,node,none,reversed,1000,2463534242,6,0,9,247977078,882,0,0,0ca43265,off,,,,,,, +3844,node,insertion,nearly_sorted,250,2463534242,23,79550,1,243821991,215895344,2277,2251,21a19c14,off,,,,,,, +3845,c,none,sorted,2000,2463534242,11,0,416,240515561,91,0,0,69842aa5,off,,,,,,, +3846,python,none,random,1000,2463534242,19,0,15,248418549,601,0,0,57561285,off,,,,,,, +3847,node,bubble,sorted,2000,2463534242,6,85,1,270534852,240796426,1999000,0,69842aa5,off,,,,,,, +3848,c,bubble,reversed,1000,2463534242,27,87,1,244433296,243547847,499500,999000,0ca43265,off,,,,,,, +3849,c,bubble,reversed,2000,2463534242,24,22,1,249603445,248494969,1999000,3998000,ad35ba05,off,,,,,,, +3850,python,bubble,reversed,500,2463534242,7,20,1,264153242,247353160,124750,249500,d33e3559,off,,,,,,, +3851,python,quick,sorted,500,2463534242,6,30,1,252921595,234507927,124750,0,d9a24379,off,,,,,,, +3852,node,insertion,nearly_sorted,2000,2463534242,6,2100,1,271106834,239269952,118277,118270,5835dc59,off,,,,,,, +3853,node,none,sorted,1000,2463534242,3,0,9,250952264,972,0,0,8e15cb75,off,,,,,,, +3854,python,insertion,reversed,250,2463534242,2,102,1,270308829,253507869,31125,31374,be175004,off,,,,,,, +3855,c,bubble,sorted,2000,2463534242,11,173,1,239124878,238154199,1999000,0,69842aa5,off,,,,,,, +3856,python,none,random,2000,2463534242,7,0,16,276634353,641,0,0,41262855,off,,,,,,, +3857,c,none,reversed,250,2463534242,16,0,427,272438470,90,0,0,be175004,off,,,,,,, +3858,c,none,random,250,2463534242,1,0,429,252801889,81,0,0,b47adce4,off,,,,,,, +3859,python,none,sorted,2000,2463534242,7,0,15,247608437,632,0,0,69842aa5,off,,,,,,, +3860,python,insertion,random,2000,2463534242,28,3,1,355582264,337191718,996390,996389,41262855,off,,,,,,, +3861,node,none,sorted,500,2463534242,22,0,10,277766980,791,0,0,d9a24379,off,,,,,,, +3862,python,insertion,nearly_sorted,1000,2463534242,11,76,1,264027257,246139466,30889,30860,1cfe4afd,off,,,,,,, +3863,python,none,sorted,250,2463534242,1,0,17,265998401,601,0,0,7fd8c1f4,off,,,,,,, +3864,node,bubble,random,2000,2463534242,30,98,1,280281580,248118933,1999000,1988792,41262855,off,,,,,,, +3865,node,insertion,sorted,2000,2463534242,13,89220,1,273132541,243355219,1999,0,69842aa5,off,,,,,,, +3866,c,quick,reversed,500,2463534242,18,3240,1,242458526,241496934,124750,500,d33e3559,off,,,,,,, +3867,python,none,random,1000,2463534242,13,0,15,243735008,661,0,0,57561285,off,,,,,,, +3868,python,quick,random,500,2463534242,24,507,1,270521820,254487731,4578,4552,11ae932d,off,,,,,,, +3869,c,bubble,random,1000,2463534242,29,154,1,237324191,236263306,499500,496926,57561285,off,,,,,,, +3870,python,quick,random,1000,2463534242,17,209,1,270908914,254473104,11196,10210,57561285,off,,,,,,, +3871,python,none,nearly_sorted,250,2463534242,1,0,17,260607057,501,0,0,21a19c14,off,,,,,,, +3872,python,none,sorted,500,2463534242,11,0,16,243785744,601,0,0,d9a24379,off,,,,,,, +3873,python,quick,sorted,250,2463534242,10,133,1,252807750,237297856,31125,0,7fd8c1f4,off,,,,,,, +3874,node,quick,random,500,2463534242,23,16148,1,271880434,243143255,4578,4552,11ae932d,off,,,,,,, +3875,node,bubble,reversed,500,2463534242,9,1534,1,279727261,251455402,124750,249500,d33e3559,off,,,,,,, +3876,python,bubble,random,500,2463534242,7,26,1,285217839,268834347,124750,119688,11ae932d,off,,,,,,, +3877,c,none,reversed,1000,2463534242,21,0,405,246006835,111,0,0,0ca43265,off,,,,,,, +3878,c,insertion,nearly_sorted,500,2463534242,11,32350,1,246227519,245420437,7962,7926,482b4785,off,,,,,,, +3879,node,bubble,nearly_sorted,500,2463534242,12,1566,1,275314580,244493888,124750,14926,482b4785,off,,,,,,, +3880,c,bubble,random,1000,2463534242,6,154,1,237482700,236582064,499500,496926,57561285,off,,,,,,, +3881,node,none,reversed,500,2463534242,4,0,9,245769521,822,0,0,d33e3559,off,,,,,,, +3882,python,bubble,random,1000,2463534242,22,6,1,279534469,263268728,499500,496926,57561285,off,,,,,,, +3883,node,none,sorted,1000,2463534242,22,0,9,255443461,1332,0,0,8e15cb75,off,,,,,,, +3884,python,none,nearly_sorted,1000,2463534242,17,0,16,263350207,561,0,0,1cfe4afd,off,,,,,,, +3885,node,quick,reversed,1000,2463534242,5,307,1,275051648,245685600,499500,1000,0ca43265,off,,,,,,, +3886,node,quick,random,250,2463534242,20,36348,1,271730252,241042525,1933,1940,b47adce4,off,,,,,,, +3887,node,none,sorted,500,2463534242,12,0,10,292682789,1153,0,0,d9a24379,off,,,,,,, +3888,node,none,nearly_sorted,250,2463534242,14,0,10,279202217,801,0,0,21a19c14,off,,,,,,, +3889,node,insertion,random,1000,2463534242,23,1091,1,271370067,243214370,249457,249457,57561285,off,,,,,,, +3890,node,bubble,reversed,500,2463534242,3,1534,1,278489890,250514219,124750,249500,d33e3559,off,,,,,,, +3891,python,insertion,sorted,1000,2463534242,12,3178,1,264122366,248349268,999,0,8e15cb75,off,,,,,,, +3892,c,bubble,sorted,2000,2463534242,16,173,1,237171385,236244155,1999000,0,69842aa5,off,,,,,,, +3893,c,bubble,sorted,2000,2463534242,17,173,1,237957731,236917210,1999000,0,69842aa5,off,,,,,,, +3894,python,insertion,random,500,2463534242,9,44,1,260114855,242821114,60338,60339,11ae932d,off,,,,,,, +3895,python,none,nearly_sorted,1000,2463534242,4,0,16,260881451,611,0,0,1cfe4afd,off,,,,,,, +3896,python,none,random,500,2463534242,30,0,16,260239678,581,0,0,11ae932d,off,,,,,,, +3897,c,quick,reversed,1000,2463534242,16,814,1,240501669,239398841,499500,1000,0ca43265,off,,,,,,, +3898,c,insertion,random,500,2463534242,29,6945,1,214576854,213417591,60338,60339,11ae932d,off,,,,,,, +3899,node,insertion,nearly_sorted,500,2463534242,24,26045,1,257072794,227119080,7962,7926,482b4785,off,,,,,,, +3900,node,bubble,sorted,500,2463534242,6,1964,1,278195308,248467422,124750,0,d9a24379,off,,,,,,, +3901,c,bubble,random,500,2463534242,19,635,1,235379200,234544220,124750,119688,11ae932d,off,,,,,,, +3902,python,quick,random,2000,2463534242,22,90,1,264267897,246177025,25197,24544,41262855,off,,,,,,, +3903,node,none,nearly_sorted,250,2463534242,3,0,10,270117891,621,0,0,21a19c14,off,,,,,,, +3904,python,quick,random,250,2463534242,26,1271,1,257476661,241005735,1933,1940,b47adce4,off,,,,,,, +3905,c,insertion,random,1000,2463534242,7,1983,1,243407764,242287485,249457,249457,57561285,off,,,,,,, +3906,c,bubble,reversed,500,2463534242,13,346,1,244180623,243029657,124750,249500,d33e3559,off,,,,,,, +3907,node,quick,nearly_sorted,2000,2463534242,2,1909,1,260649519,231434423,58002,11768,5835dc59,off,,,,,,, +3908,python,none,nearly_sorted,250,2463534242,20,0,17,256840307,511,0,0,21a19c14,off,,,,,,, +3909,node,insertion,sorted,500,2463534242,24,324764,1,271174110,241873361,499,0,d9a24379,off,,,,,,, +3910,python,bubble,nearly_sorted,2000,2463534242,15,3,1,484952277,467083583,1999000,232556,5835dc59,off,,,,,,, +3911,python,bubble,random,1000,2463534242,19,6,1,267139040,250994562,499500,496926,57561285,off,,,,,,, +3912,node,bubble,nearly_sorted,1000,2463534242,8,413,1,268538199,240128427,499500,59780,1cfe4afd,off,,,,,,, +3913,python,quick,nearly_sorted,250,2463534242,4,882,1,265610925,250301936,4346,1008,21a19c14,off,,,,,,, +3914,node,insertion,random,2000,2463534242,2,234,1,281569392,252089091,996390,996389,41262855,off,,,,,,, +3915,node,bubble,sorted,2000,2463534242,13,85,1,267370713,237835877,1999000,0,69842aa5,off,,,,,,, +3916,c,insertion,reversed,2000,2463534242,25,260,1,246076742,245173976,1999000,2000999,ad35ba05,off,,,,,,, +3917,c,insertion,random,500,2463534242,12,6945,1,217234788,216307556,60338,60339,11ae932d,off,,,,,,, +3918,c,bubble,nearly_sorted,500,2463534242,2,1998,1,241115560,240194704,124750,14926,482b4785,off,,,,,,, +3919,python,none,nearly_sorted,2000,2463534242,23,0,15,246461235,751,0,0,5835dc59,off,,,,,,, +3920,python,none,nearly_sorted,250,2463534242,27,0,17,256157129,511,0,0,21a19c14,off,,,,,,, +3921,node,none,reversed,500,2463534242,1,0,9,244184018,802,0,0,d33e3559,off,,,,,,, +3922,node,quick,nearly_sorted,2000,2463534242,28,1909,1,266113757,236944148,58002,11768,5835dc59,off,,,,,,, +3923,c,bubble,reversed,2000,2463534242,26,22,1,248467578,247534889,1999000,3998000,ad35ba05,off,,,,,,, +3924,node,none,reversed,250,2463534242,12,0,9,254586724,631,0,0,be175004,off,,,,,,, +3925,node,none,reversed,2000,2463534242,17,0,9,251485873,1062,0,0,ad35ba05,off,,,,,,, +3926,python,quick,reversed,1000,2463534242,13,8,1,277675966,260118707,499500,1000,0ca43265,off,,,,,,, +3927,node,bubble,random,2000,2463534242,5,98,1,274359831,243871083,1999000,1988792,41262855,off,,,,,,, +3928,c,bubble,random,1000,2463534242,25,154,1,236997651,235936394,499500,496926,57561285,off,,,,,,, +3929,c,none,reversed,500,2463534242,9,0,402,235358079,100,0,0,d33e3559,off,,,,,,, +3930,python,none,sorted,250,2463534242,5,0,17,266176426,611,0,0,7fd8c1f4,off,,,,,,, +3931,c,none,sorted,500,2463534242,26,0,344,204611796,90,0,0,d9a24379,off,,,,,,, +3932,c,bubble,random,500,2463534242,8,635,1,234184466,233434384,124750,119688,11ae932d,off,,,,,,, +3933,c,insertion,reversed,500,2463534242,3,3967,1,246035593,245122770,124750,125249,d33e3559,off,,,,,,, +3934,python,none,sorted,250,2463534242,15,0,17,262816718,842,0,0,7fd8c1f4,off,,,,,,, +3935,c,insertion,sorted,250,2463534242,9,786886,1,248956504,247963612,249,0,7fd8c1f4,off,,,,,,, +3936,c,quick,random,500,2463534242,6,55776,1,244253548,243134511,4578,4552,11ae932d,off,,,,,,, +3937,c,bubble,random,250,2463534242,16,2568,1,246616159,245632675,31125,31196,b47adce4,off,,,,,,, +3938,node,insertion,sorted,500,2463534242,8,324764,1,268454272,236284326,499,0,d9a24379,off,,,,,,, +3939,c,bubble,reversed,2000,2463534242,30,22,1,250201716,249274349,1999000,3998000,ad35ba05,off,,,,,,, +3940,c,insertion,reversed,1000,2463534242,25,1033,1,247140500,246195170,499500,500499,0ca43265,off,,,,,,, +3941,python,bubble,random,500,2463534242,29,26,1,266816135,250869866,124750,119688,11ae932d,off,,,,,,, +3942,c,none,reversed,500,2463534242,30,0,402,239588566,91,0,0,d33e3559,off,,,,,,, +3943,c,insertion,random,1000,2463534242,4,1983,1,244092277,243329988,249457,249457,57561285,off,,,,,,, +3944,python,insertion,sorted,1000,2463534242,28,3178,1,266395475,249594533,999,0,8e15cb75,off,,,,,,, +3945,node,bubble,sorted,2000,2463534242,19,85,1,269745794,239212681,1999000,0,69842aa5,off,,,,,,, +3946,python,quick,reversed,250,2463534242,26,151,1,263069031,247221864,31125,250,be175004,off,,,,,,, +3947,node,quick,random,250,2463534242,16,36348,1,263935595,235423570,1933,1940,b47adce4,off,,,,,,, +3948,c,none,nearly_sorted,2000,2463534242,11,0,387,224934625,91,0,0,5835dc59,off,,,,,,, +3949,python,insertion,nearly_sorted,2000,2463534242,27,20,1,269365632,250457461,118277,118270,5835dc59,off,,,,,,, +3950,c,insertion,reversed,1000,2463534242,10,1033,1,247957310,246992234,499500,500499,0ca43265,off,,,,,,, +3951,c,none,reversed,250,2463534242,29,0,427,241923854,90,0,0,be175004,off,,,,,,, +3952,c,quick,reversed,500,2463534242,25,3240,1,240610552,239820251,124750,500,d33e3559,off,,,,,,, +3953,python,bubble,reversed,250,2463534242,21,88,1,263267033,246740138,31125,62250,be175004,off,,,,,,, +3954,c,quick,sorted,250,2463534242,18,10851,1,244240064,243312095,31125,0,7fd8c1f4,off,,,,,,, +3955,c,insertion,sorted,2000,2463534242,13,101352,1,241247928,240129227,1999,0,69842aa5,off,,,,,,, +3956,node,none,sorted,250,2463534242,4,0,9,250344724,1062,0,0,7fd8c1f4,off,,,,,,, +3957,c,insertion,random,2000,2463534242,23,492,1,231355808,230478996,996390,996389,41262855,off,,,,,,, +3958,c,insertion,random,500,2463534242,5,6945,1,220333452,219211100,60338,60339,11ae932d,off,,,,,,, +3959,python,insertion,random,500,2463534242,4,44,1,262534368,245655175,60338,60339,11ae932d,off,,,,,,, +3960,python,quick,sorted,1000,2463534242,27,8,1,289115044,273181915,499500,0,8e15cb75,off,,,,,,, +3961,c,insertion,sorted,1000,2463534242,19,196883,1,236054093,234925208,999,0,8e15cb75,off,,,,,,, +3962,c,insertion,reversed,500,2463534242,16,3967,1,251167016,249981865,124750,125249,d33e3559,off,,,,,,, +3963,c,insertion,reversed,250,2463534242,5,15645,1,243296275,242081429,31125,31374,be175004,off,,,,,,, +3964,node,insertion,reversed,2000,2463534242,5,144,1,269413180,239209836,1999000,2000999,ad35ba05,off,,,,,,, +3965,c,bubble,sorted,250,2463534242,17,11386,1,244401308,243247966,31125,0,7fd8c1f4,off,,,,,,, +3966,python,quick,sorted,250,2463534242,14,133,1,254536129,239033548,31125,0,7fd8c1f4,off,,,,,,, +3967,c,quick,reversed,2000,2463534242,25,209,1,275006815,274077172,1999000,2000,ad35ba05,off,,,,,,, +3968,node,bubble,random,2000,2463534242,18,98,1,287417831,257493804,1999000,1988792,41262855,off,,,,,,, +3969,c,insertion,sorted,2000,2463534242,10,101352,1,244204618,243275578,1999,0,69842aa5,off,,,,,,, +3970,c,bubble,reversed,1000,2463534242,18,87,1,248643456,247553234,499500,999000,0ca43265,off,,,,,,, +3971,node,quick,random,1000,2463534242,28,6513,1,282291586,252387573,11196,10210,57561285,off,,,,,,, +3972,python,none,random,2000,2463534242,1,0,16,300459243,922,0,0,41262855,off,,,,,,, +3973,c,bubble,random,2000,2463534242,27,40,1,246100491,245062916,1999000,1988792,41262855,off,,,,,,, +3974,c,none,reversed,1000,2463534242,26,0,405,250867786,90,0,0,0ca43265,off,,,,,,, +3975,python,insertion,nearly_sorted,2000,2463534242,23,20,1,267616114,250201879,118277,118270,5835dc59,off,,,,,,, +3976,python,quick,sorted,2000,2463534242,23,2,1,312262181,294922421,1999000,0,69842aa5,off,,,,,,, +3977,node,insertion,sorted,1000,2463534242,22,160107,1,271412236,240494641,999,0,8e15cb75,off,,,,,,, +3978,python,bubble,reversed,2000,2463534242,6,2,1,492399303,475171057,1999000,3998000,ad35ba05,off,,,,,,, +3979,python,insertion,nearly_sorted,2000,2463534242,6,20,1,264882687,247904683,118277,118270,5835dc59,off,,,,,,, +3980,node,bubble,reversed,500,2463534242,22,1534,1,272872157,243744627,124750,249500,d33e3559,off,,,,,,, +3981,c,bubble,random,1000,2463534242,3,154,1,238130484,237241230,499500,496926,57561285,off,,,,,,, +3982,python,insertion,random,2000,2463534242,29,3,1,322723258,305611504,996390,996389,41262855,off,,,,,,, +3983,node,none,reversed,500,2463534242,5,0,9,240301674,852,0,0,d33e3559,off,,,,,,, +3984,python,insertion,nearly_sorted,500,2463534242,27,316,1,254001157,236740396,7962,7926,482b4785,off,,,,,,, +3985,python,bubble,nearly_sorted,1000,2463534242,20,9,1,256336735,241035809,499500,59780,1cfe4afd,off,,,,,,, +3986,node,quick,nearly_sorted,2000,2463534242,12,1909,1,254724503,226862238,58002,11768,5835dc59,off,,,,,,, +3987,node,bubble,random,1000,2463534242,23,362,1,265444162,237898470,499500,496926,57561285,off,,,,,,, +3988,python,bubble,nearly_sorted,2000,2463534242,8,3,1,354630785,337841771,1999000,232556,5835dc59,off,,,,,,, +3989,python,bubble,sorted,500,2463534242,26,46,1,268658673,252465880,124750,0,d9a24379,off,,,,,,, +3990,c,insertion,sorted,2000,2463534242,29,101352,1,244480120,243382579,1999,0,69842aa5,off,,,,,,, +3991,node,none,random,500,2463534242,15,0,9,263168797,1713,0,0,11ae932d,off,,,,,,, +3992,c,bubble,nearly_sorted,500,2463534242,21,1998,1,247566729,246602732,124750,14926,482b4785,off,,,,,,, +3993,python,insertion,sorted,250,2463534242,22,18845,1,257301292,242176774,249,0,7fd8c1f4,off,,,,,,, +3994,python,bubble,reversed,250,2463534242,13,88,1,251829418,236515554,31125,62250,be175004,off,,,,,,, +3995,node,quick,nearly_sorted,500,2463534242,2,10954,1,269506606,238762873,10192,1850,482b4785,off,,,,,,, +3996,python,quick,sorted,2000,2463534242,3,2,1,308700007,291853603,1999000,0,69842aa5,off,,,,,,, +3997,python,quick,random,2000,2463534242,21,90,1,266414021,249822938,25197,24544,41262855,off,,,,,,, +3998,c,none,random,1000,2463534242,10,0,367,212168659,80,0,0,57561285,off,,,,,,, +3999,node,quick,reversed,2000,2463534242,1,80,1,276384766,248216642,1999000,2000,ad35ba05,off,,,,,,, +4000,node,none,nearly_sorted,2000,2463534242,23,0,9,248613150,3367,0,0,5835dc59,off,,,,,,, +4001,python,bubble,reversed,250,2463534242,11,88,1,263217980,245122615,31125,62250,be175004,off,,,,,,, +4002,c,bubble,nearly_sorted,1000,2463534242,9,507,1,243637956,242514110,499500,59780,1cfe4afd,off,,,,,,, +4003,node,insertion,reversed,250,2463534242,12,8539,1,269854367,239367814,31125,31374,be175004,off,,,,,,, +4004,node,bubble,sorted,1000,2463534242,21,374,1,253247475,225707423,499500,0,8e15cb75,off,,,,,,, +4005,node,insertion,reversed,250,2463534242,23,8539,1,265195196,238837745,31125,31374,be175004,off,,,,,,, +4006,c,bubble,random,250,2463534242,7,2568,1,242627913,241698211,31125,31196,b47adce4,off,,,,,,, +4007,node,quick,random,2000,2463534242,8,2126,1,259566447,229774595,25197,24544,41262855,off,,,,,,, +4008,python,insertion,nearly_sorted,1000,2463534242,17,76,1,250094216,234564729,30889,30860,1cfe4afd,off,,,,,,, +4009,c,insertion,nearly_sorted,2000,2463534242,9,2962,1,223394678,222243210,118277,118270,5835dc59,off,,,,,,, +4010,c,bubble,random,250,2463534242,24,2568,1,246822424,245604922,31125,31196,b47adce4,off,,,,,,, +4011,c,quick,sorted,500,2463534242,13,2762,1,239823728,238879059,124750,0,d9a24379,off,,,,,,, +4012,python,none,nearly_sorted,500,2463534242,13,0,17,260962483,541,0,0,482b4785,off,,,,,,, +4013,python,quick,reversed,500,2463534242,3,33,1,265341620,249155603,124750,500,d33e3559,off,,,,,,, +4014,python,quick,nearly_sorted,1000,2463534242,10,110,1,253157065,237323516,28499,7892,1cfe4afd,off,,,,,,, +4015,node,quick,random,2000,2463534242,14,2126,1,277656760,247008681,25197,24544,41262855,off,,,,,,, +4016,c,quick,reversed,1000,2463534242,6,814,1,233746730,232818610,499500,1000,0ca43265,off,,,,,,, +4017,node,bubble,random,2000,2463534242,7,98,1,269423154,242175212,1999000,1988792,41262855,off,,,,,,, +4018,node,quick,random,1000,2463534242,19,6513,1,261971308,234390854,11196,10210,57561285,off,,,,,,, +4019,c,bubble,reversed,1000,2463534242,16,87,1,239727778,238886521,499500,999000,0ca43265,off,,,,,,, +4020,c,quick,sorted,1000,2463534242,22,691,1,239829278,238880590,499500,0,8e15cb75,off,,,,,,, +4021,c,bubble,nearly_sorted,2000,2463534242,24,121,1,241440697,240308296,1999000,232556,5835dc59,off,,,,,,, +4022,node,bubble,random,1000,2463534242,20,362,1,265194946,236065532,499500,496926,57561285,off,,,,,,, +4023,python,none,nearly_sorted,1000,2463534242,12,0,16,228208369,542,0,0,1cfe4afd,off,,,,,,, +4024,c,insertion,random,2000,2463534242,28,492,1,229171955,228264615,996390,996389,41262855,off,,,,,,, +4025,node,insertion,nearly_sorted,250,2463534242,27,79550,1,233094426,205071549,2277,2251,21a19c14,off,,,,,,, +4026,node,quick,reversed,500,2463534242,22,1194,1,262654966,234284970,124750,500,d33e3559,off,,,,,,, +4027,python,none,nearly_sorted,2000,2463534242,11,0,15,230951550,2975,0,0,5835dc59,off,,,,,,, +4028,node,bubble,reversed,1000,2463534242,1,378,1,267223126,239066605,499500,999000,0ca43265,off,,,,,,, +4029,python,quick,sorted,1000,2463534242,18,8,1,288864746,272157345,499500,0,8e15cb75,off,,,,,,, +4030,node,none,sorted,2000,2463534242,22,0,10,272259073,1012,0,0,69842aa5,off,,,,,,, +4031,python,bubble,reversed,250,2463534242,29,88,1,255839082,239700599,31125,62250,be175004,off,,,,,,, +4032,c,none,random,250,2463534242,8,0,429,244220668,91,0,0,b47adce4,off,,,,,,, +4033,python,insertion,sorted,2000,2463534242,22,1523,1,261023008,244671071,1999,0,69842aa5,off,,,,,,, +4034,python,insertion,random,250,2463534242,7,206,1,251993263,237018200,15844,15838,b47adce4,off,,,,,,, +4035,c,insertion,nearly_sorted,500,2463534242,10,32350,1,257547132,256367243,7962,7926,482b4785,off,,,,,,, +4036,node,bubble,sorted,250,2463534242,8,8081,1,264034711,234995358,31125,0,7fd8c1f4,off,,,,,,, +4037,python,quick,nearly_sorted,1000,2463534242,15,110,1,256162387,240837923,28499,7892,1cfe4afd,off,,,,,,, +4038,python,bubble,nearly_sorted,2000,2463534242,3,3,1,354121040,338020722,1999000,232556,5835dc59,off,,,,,,, +4039,c,none,sorted,250,2463534242,28,0,429,238207579,90,0,0,7fd8c1f4,off,,,,,,, +4040,python,bubble,random,250,2463534242,29,112,1,246053853,230786321,31125,31196,b47adce4,off,,,,,,, +4041,c,quick,nearly_sorted,500,2463534242,22,31784,1,240532706,239453376,10192,1850,482b4785,off,,,,,,, +4042,node,insertion,nearly_sorted,1000,2463534242,29,7758,1,269664533,240482248,30889,30860,1cfe4afd,off,,,,,,, +4043,c,bubble,sorted,500,2463534242,14,2858,1,245651759,244686753,124750,0,d9a24379,off,,,,,,, +4044,node,quick,sorted,2000,2463534242,29,74,1,271516693,242074958,1999000,0,69842aa5,off,,,,,,, +4045,python,bubble,sorted,250,2463534242,1,206,1,263231135,247499809,31125,0,7fd8c1f4,off,,,,,,, +4046,node,none,reversed,2000,2463534242,28,0,9,251660794,702,0,0,ad35ba05,off,,,,,,, +4047,c,bubble,nearly_sorted,2000,2463534242,16,121,1,240680442,239694706,1999000,232556,5835dc59,off,,,,,,, +4048,node,bubble,sorted,500,2463534242,26,1964,1,254661905,225801635,124750,0,d9a24379,off,,,,,,, +4049,c,none,sorted,500,2463534242,17,0,344,200252308,100,0,0,d9a24379,off,,,,,,, +4050,python,insertion,random,2000,2463534242,11,3,1,334614735,316994197,996390,996389,41262855,off,,,,,,, +4051,python,bubble,sorted,1000,2463534242,12,11,1,303803851,288344467,499500,0,8e15cb75,off,,,,,,, +4052,node,bubble,random,2000,2463534242,21,98,1,272139268,243147415,1999000,1988792,41262855,off,,,,,,, +4053,python,insertion,reversed,1000,2463534242,1,6,1,293384334,278015583,499500,500499,0ca43265,off,,,,,,, +4054,python,bubble,reversed,2000,2463534242,22,2,1,489297680,472961269,1999000,3998000,ad35ba05,off,,,,,,, +4055,node,none,random,1000,2463534242,4,0,10,267573594,812,0,0,57561285,off,,,,,,, +4056,c,insertion,reversed,1000,2463534242,29,1033,1,244646556,243631666,499500,500499,0ca43265,off,,,,,,, +4057,c,quick,nearly_sorted,500,2463534242,8,31784,1,239424870,238289278,10192,1850,482b4785,off,,,,,,, +4058,c,quick,nearly_sorted,250,2463534242,15,77221,1,239103478,238022934,4346,1008,21a19c14,off,,,,,,, +4059,python,insertion,nearly_sorted,2000,2463534242,29,20,1,256969049,240377788,118277,118270,5835dc59,off,,,,,,, +4060,node,bubble,reversed,500,2463534242,8,1534,1,262198389,233765330,124750,249500,d33e3559,off,,,,,,, +4061,python,insertion,sorted,1000,2463534242,11,3178,1,255824695,240659249,999,0,8e15cb75,off,,,,,,, +4062,python,quick,reversed,250,2463534242,22,151,1,257545041,242449076,31125,250,be175004,off,,,,,,, +4063,node,bubble,reversed,1000,2463534242,29,378,1,271794231,242597130,499500,999000,0ca43265,off,,,,,,, +4064,node,bubble,sorted,250,2463534242,1,8081,1,245913149,219275407,31125,0,7fd8c1f4,off,,,,,,, +4065,c,quick,random,1000,2463534242,9,23484,1,237054448,236201991,11196,10210,57561285,off,,,,,,, +4066,node,quick,nearly_sorted,500,2463534242,30,10954,1,263196209,236140182,10192,1850,482b4785,off,,,,,,, +4067,python,none,sorted,250,2463534242,10,0,17,237220189,551,0,0,7fd8c1f4,off,,,,,,, +4068,c,insertion,reversed,250,2463534242,19,15645,1,245073756,244058260,31125,31374,be175004,off,,,,,,, +4069,python,bubble,random,1000,2463534242,1,6,1,269831554,253610775,499500,496926,57561285,off,,,,,,, +4070,c,quick,sorted,2000,2463534242,19,174,1,237986945,236972184,1999000,0,69842aa5,off,,,,,,, +4071,node,bubble,sorted,250,2463534242,11,8081,1,259399924,230456156,31125,0,7fd8c1f4,off,,,,,,, +4072,c,none,random,500,2463534242,8,0,426,239970253,90,0,0,11ae932d,off,,,,,,, +4073,c,insertion,reversed,1000,2463534242,3,1033,1,240923960,240162412,499500,500499,0ca43265,off,,,,,,, +4074,node,none,reversed,2000,2463534242,21,0,9,230404254,811,0,0,ad35ba05,off,,,,,,, +4075,node,quick,nearly_sorted,1000,2463534242,8,3752,1,256219063,227235700,28499,7892,1cfe4afd,off,,,,,,, +4076,node,none,reversed,250,2463534242,20,0,9,224697350,702,0,0,be175004,off,,,,,,, +4077,node,none,sorted,500,2463534242,27,0,10,257769729,762,0,0,d9a24379,off,,,,,,, +4078,c,insertion,nearly_sorted,2000,2463534242,26,2962,1,223942776,222950818,118277,118270,5835dc59,off,,,,,,, +4079,node,bubble,nearly_sorted,500,2463534242,3,1566,1,267750975,240246345,124750,14926,482b4785,off,,,,,,, +4080,python,quick,reversed,250,2463534242,8,151,1,260497734,244924137,31125,250,be175004,off,,,,,,, +4081,node,none,nearly_sorted,500,2463534242,8,0,10,277770111,721,0,0,482b4785,off,,,,,,, +4082,python,none,sorted,2000,2463534242,24,0,15,230814524,591,0,0,69842aa5,off,,,,,,, +4083,python,bubble,sorted,500,2463534242,19,46,1,276566456,261443953,124750,0,d9a24379,off,,,,,,, +4084,python,none,sorted,2000,2463534242,17,0,15,235388085,631,0,0,69842aa5,off,,,,,,, +4085,python,quick,random,1000,2463534242,6,209,1,251645183,234421418,11196,10210,57561285,off,,,,,,, +4086,c,bubble,random,500,2463534242,17,635,1,230014274,229080875,124750,119688,11ae932d,off,,,,,,, +4087,node,bubble,random,250,2463534242,3,5439,1,258778231,230665527,31125,31196,b47adce4,off,,,,,,, +4088,c,bubble,nearly_sorted,500,2463534242,4,1998,1,236557275,235619861,124750,14926,482b4785,off,,,,,,, +4089,c,bubble,random,250,2463534242,10,2568,1,242084996,241198394,31125,31196,b47adce4,off,,,,,,, +4090,c,none,nearly_sorted,2000,2463534242,20,0,387,222338478,90,0,0,5835dc59,off,,,,,,, +4091,python,quick,random,2000,2463534242,20,90,1,257255115,239931169,25197,24544,41262855,off,,,,,,, +4092,c,none,random,2000,2463534242,24,0,415,238192191,80,0,0,41262855,off,,,,,,, +4093,c,insertion,nearly_sorted,250,2463534242,14,99421,1,243129102,242395898,2277,2251,21a19c14,off,,,,,,, +4094,c,bubble,random,2000,2463534242,29,40,1,247315364,246161176,1999000,1988792,41262855,off,,,,,,, +4095,python,quick,random,500,2463534242,7,507,1,263809032,246176985,4578,4552,11ae932d,off,,,,,,, +4096,node,none,nearly_sorted,2000,2463534242,2,0,9,250203681,821,0,0,5835dc59,off,,,,,,, +4097,python,bubble,reversed,2000,2463534242,3,2,1,478330578,461455295,1999000,3998000,ad35ba05,off,,,,,,, +4098,c,quick,sorted,500,2463534242,9,2762,1,242261506,241290166,124750,0,d9a24379,off,,,,,,, +4099,python,insertion,sorted,250,2463534242,18,18845,1,260445866,244983433,249,0,7fd8c1f4,off,,,,,,, +4100,node,bubble,random,250,2463534242,25,5439,1,274774067,244206071,31125,31196,b47adce4,off,,,,,,, +4101,c,bubble,sorted,2000,2463534242,6,173,1,239267688,238376254,1999000,0,69842aa5,off,,,,,,, +4102,python,insertion,nearly_sorted,250,2463534242,20,1416,1,268694151,251971308,2277,2251,21a19c14,off,,,,,,, +4103,c,insertion,random,250,2463534242,20,26020,1,229665070,228584224,15844,15838,b47adce4,off,,,,,,, +4104,node,bubble,sorted,250,2463534242,13,8081,1,260895368,232890646,31125,0,7fd8c1f4,off,,,,,,, +4105,c,insertion,sorted,1000,2463534242,18,196883,1,240105516,239162178,999,0,8e15cb75,off,,,,,,, +4106,c,quick,nearly_sorted,1000,2463534242,23,11273,1,255389499,254211203,28499,7892,1cfe4afd,off,,,,,,, +4107,c,none,nearly_sorted,250,2463534242,16,0,388,232281233,80,0,0,21a19c14,off,,,,,,, +4108,node,bubble,random,1000,2463534242,13,362,1,277612047,243034727,499500,496926,57561285,off,,,,,,, +4109,python,insertion,random,1000,2463534242,15,10,1,266234792,246728667,249457,249457,57561285,off,,,,,,, +4110,c,none,sorted,250,2463534242,2,0,429,258858891,80,0,0,7fd8c1f4,off,,,,,,, +4111,node,bubble,random,2000,2463534242,2,98,1,275889388,243915700,1999000,1988792,41262855,off,,,,,,, +4112,python,insertion,random,1000,2463534242,24,10,1,261157078,244514081,249457,249457,57561285,off,,,,,,, +4113,node,insertion,sorted,250,2463534242,20,607595,1,264073313,234588652,249,0,7fd8c1f4,off,,,,,,, +4114,python,quick,reversed,1000,2463534242,9,8,1,285909956,268560955,499500,1000,0ca43265,off,,,,,,, +4115,python,insertion,reversed,250,2463534242,22,102,1,254814203,238214862,31125,31374,be175004,off,,,,,,, +4116,c,none,sorted,1000,2463534242,24,0,421,265225431,90,0,0,8e15cb75,off,,,,,,, +4117,c,none,nearly_sorted,500,2463534242,15,0,405,244830581,140,0,0,482b4785,off,,,,,,, +4118,python,none,nearly_sorted,2000,2463534242,1,0,15,283116271,832,0,0,5835dc59,off,,,,,,, +4119,node,quick,nearly_sorted,250,2463534242,21,25320,1,284260225,251598081,4346,1008,21a19c14,off,,,,,,, +4120,c,bubble,sorted,250,2463534242,19,11386,1,257415636,256305496,31125,0,7fd8c1f4,off,,,,,,, +4121,c,quick,reversed,250,2463534242,2,12781,1,247261157,246315303,31125,250,be175004,off,,,,,,, +4122,python,none,sorted,250,2463534242,24,0,17,271210687,501,0,0,7fd8c1f4,off,,,,,,, +4123,c,none,reversed,500,2463534242,3,0,402,224242518,90,0,0,d33e3559,off,,,,,,, +4124,node,quick,sorted,250,2463534242,22,4507,1,270165451,240450410,31125,0,7fd8c1f4,off,,,,,,, +4125,c,none,nearly_sorted,1000,2463534242,3,0,421,256617610,130,0,0,1cfe4afd,off,,,,,,, +4126,node,none,sorted,1000,2463534242,9,0,9,235851734,992,0,0,8e15cb75,off,,,,,,, +4127,c,bubble,sorted,250,2463534242,21,11386,1,248352782,247446994,31125,0,7fd8c1f4,off,,,,,,, +4128,node,insertion,random,250,2463534242,6,15349,1,290393890,261377647,15844,15838,b47adce4,off,,,,,,, +4129,python,none,sorted,1000,2463534242,9,0,16,269498460,611,0,0,8e15cb75,off,,,,,,, +4130,python,bubble,random,1000,2463534242,18,6,1,279065640,262612116,499500,496926,57561285,off,,,,,,, +4131,python,bubble,sorted,250,2463534242,23,206,1,259680290,244669144,31125,0,7fd8c1f4,off,,,,,,, +4132,node,quick,sorted,500,2463534242,10,1164,1,270137869,243252498,124750,0,d9a24379,off,,,,,,, +4133,python,none,nearly_sorted,500,2463534242,2,0,17,257527217,581,0,0,482b4785,off,,,,,,, +4134,c,quick,nearly_sorted,500,2463534242,9,31784,1,238807251,237743569,10192,1850,482b4785,off,,,,,,, +4135,node,insertion,sorted,250,2463534242,27,607595,1,255464832,224490132,249,0,7fd8c1f4,off,,,,,,, +4136,python,bubble,sorted,500,2463534242,7,46,1,257697675,242391074,124750,0,d9a24379,off,,,,,,, +4137,node,none,sorted,2000,2463534242,23,0,10,261775686,1203,0,0,69842aa5,off,,,,,,, +4138,node,quick,sorted,500,2463534242,14,1164,1,265681257,238005042,124750,0,d9a24379,off,,,,,,, +4139,c,none,reversed,250,2463534242,15,0,427,232727059,90,0,0,be175004,off,,,,,,, +4140,python,bubble,sorted,1000,2463534242,22,11,1,279506256,264023577,499500,0,8e15cb75,off,,,,,,, +4141,c,quick,nearly_sorted,1000,2463534242,24,11273,1,238944038,237990943,28499,7892,1cfe4afd,off,,,,,,, +4142,node,none,sorted,250,2463534242,26,0,9,243841473,762,0,0,7fd8c1f4,off,,,,,,, +4143,c,quick,random,250,2463534242,29,136055,1,238737085,237769374,1933,1940,b47adce4,off,,,,,,, +4144,c,insertion,reversed,2000,2463534242,6,260,1,243614161,242492100,1999000,2000999,ad35ba05,off,,,,,,, +4145,python,bubble,reversed,2000,2463534242,19,2,1,492049307,475516182,1999000,3998000,ad35ba05,off,,,,,,, +4146,c,insertion,nearly_sorted,250,2463534242,12,99421,1,241369144,240358561,2277,2251,21a19c14,off,,,,,,, +4147,python,none,random,1000,2463534242,25,0,15,259888601,822,0,0,57561285,off,,,,,,, +4148,c,quick,sorted,250,2463534242,16,10851,1,268189408,267258652,31125,0,7fd8c1f4,off,,,,,,, +4149,node,quick,sorted,500,2463534242,30,1164,1,269754790,241879611,124750,0,d9a24379,off,,,,,,, +4150,node,none,reversed,500,2463534242,10,0,9,246536939,1002,0,0,d33e3559,off,,,,,,, +4151,node,bubble,reversed,250,2463534242,4,5861,1,267782004,236999203,31125,62250,be175004,off,,,,,,, +4152,node,bubble,random,250,2463534242,22,5439,1,257165728,228222586,31125,31196,b47adce4,off,,,,,,, +4153,python,quick,random,1000,2463534242,22,209,1,264603928,248483756,11196,10210,57561285,off,,,,,,, +4154,python,quick,random,2000,2463534242,29,90,1,267680604,250398605,25197,24544,41262855,off,,,,,,, +4155,c,none,nearly_sorted,500,2463534242,5,0,405,227448305,90,0,0,482b4785,off,,,,,,, +4156,node,bubble,nearly_sorted,250,2463534242,2,6161,1,267972150,241271692,31125,4056,21a19c14,off,,,,,,, +4157,node,bubble,sorted,500,2463534242,11,1964,1,263731552,236027981,124750,0,d9a24379,off,,,,,,, +4158,node,bubble,random,250,2463534242,10,5439,1,260459501,233075641,31125,31196,b47adce4,off,,,,,,, +4159,c,quick,reversed,500,2463534242,15,3240,1,241983966,241052882,124750,500,d33e3559,off,,,,,,, +4160,python,insertion,reversed,1000,2463534242,8,6,1,289682467,273930273,499500,500499,0ca43265,off,,,,,,, +4161,c,bubble,sorted,250,2463534242,26,11386,1,235620112,234568249,31125,0,7fd8c1f4,off,,,,,,, +4162,c,bubble,sorted,250,2463534242,9,11386,1,238214730,236801634,31125,0,7fd8c1f4,off,,,,,,, +4163,python,none,reversed,1000,2463534242,15,0,16,226467307,611,0,0,0ca43265,off,,,,,,, +4164,c,quick,random,250,2463534242,22,136055,1,247990402,246947008,1933,1940,b47adce4,off,,,,,,, +4165,python,quick,nearly_sorted,1000,2463534242,25,110,1,257264363,241081214,28499,7892,1cfe4afd,off,,,,,,, +4166,node,insertion,nearly_sorted,250,2463534242,24,79550,1,244796968,213841046,2277,2251,21a19c14,off,,,,,,, +4167,c,bubble,sorted,1000,2463534242,30,702,1,238390242,237489393,499500,0,8e15cb75,off,,,,,,, +4168,node,insertion,nearly_sorted,1000,2463534242,12,7758,1,264990332,236274418,30889,30860,1cfe4afd,off,,,,,,, +4169,c,none,random,2000,2463534242,18,0,415,232515542,91,0,0,41262855,off,,,,,,, +4170,node,bubble,nearly_sorted,250,2463534242,24,6161,1,281985221,251874235,31125,4056,21a19c14,off,,,,,,, +4171,node,bubble,nearly_sorted,1000,2463534242,21,413,1,269224627,239583516,499500,59780,1cfe4afd,off,,,,,,, +4172,c,quick,nearly_sorted,2000,2463534242,8,6145,1,245976920,244893615,58002,11768,5835dc59,off,,,,,,, +4173,c,bubble,sorted,500,2463534242,27,2858,1,239968740,238398583,124750,0,d9a24379,off,,,,,,, +4174,c,quick,nearly_sorted,1000,2463534242,15,11273,1,232663578,231248487,28499,7892,1cfe4afd,off,,,,,,, +4175,node,quick,reversed,2000,2463534242,21,80,1,270389310,240679037,1999000,2000,ad35ba05,off,,,,,,, +4176,c,quick,nearly_sorted,250,2463534242,17,77221,1,242243443,241118685,4346,1008,21a19c14,off,,,,,,, +4177,c,none,random,500,2463534242,17,0,426,233511488,80,0,0,11ae932d,off,,,,,,, +4178,c,quick,nearly_sorted,2000,2463534242,11,6145,1,242664452,241922923,58002,11768,5835dc59,off,,,,,,, +4179,c,bubble,random,1000,2463534242,13,154,1,235779849,234711609,499500,496926,57561285,off,,,,,,, +4180,c,none,reversed,500,2463534242,12,0,402,216193162,81,0,0,d33e3559,off,,,,,,, +4181,node,quick,sorted,2000,2463534242,18,74,1,268682671,240305083,1999000,0,69842aa5,off,,,,,,, +4182,python,none,reversed,500,2463534242,30,0,16,215962256,561,0,0,d33e3559,off,,,,,,, +4183,python,quick,random,1000,2463534242,11,209,1,253648871,238497695,11196,10210,57561285,off,,,,,,, +4184,c,none,random,500,2463534242,9,0,426,238652873,90,0,0,11ae932d,off,,,,,,, +4185,c,insertion,reversed,500,2463534242,12,3967,1,243119244,242391010,124750,125249,d33e3559,off,,,,,,, +4186,c,quick,random,500,2463534242,18,55776,1,240981057,240046797,4578,4552,11ae932d,off,,,,,,, +4187,c,none,random,1000,2463534242,17,0,367,209573356,90,0,0,57561285,off,,,,,,, +4188,python,none,reversed,1000,2463534242,6,0,16,232306520,611,0,0,0ca43265,off,,,,,,, +4189,c,none,sorted,2000,2463534242,18,0,416,232431564,90,0,0,69842aa5,off,,,,,,, +4190,python,bubble,sorted,2000,2463534242,10,3,1,325480907,308699420,1999000,0,69842aa5,off,,,,,,, +4191,c,quick,reversed,1000,2463534242,12,814,1,234192214,233058480,499500,1000,0ca43265,off,,,,,,, +4192,python,bubble,nearly_sorted,2000,2463534242,20,3,1,355467120,337805932,1999000,232556,5835dc59,off,,,,,,, +4193,c,bubble,reversed,1000,2463534242,2,87,1,243118513,241969079,499500,999000,0ca43265,off,,,,,,, +4194,python,none,reversed,1000,2463534242,27,0,16,222889471,581,0,0,0ca43265,off,,,,,,, +4195,python,insertion,sorted,1000,2463534242,9,3178,1,258059158,243057118,999,0,8e15cb75,off,,,,,,, +4196,python,none,random,250,2463534242,6,0,17,249822019,551,0,0,b47adce4,off,,,,,,, +4197,python,bubble,random,250,2463534242,6,112,1,238449763,224000642,31125,31196,b47adce4,off,,,,,,, +4198,python,none,reversed,500,2463534242,28,0,16,241793820,541,0,0,d33e3559,off,,,,,,, +4199,c,insertion,nearly_sorted,500,2463534242,3,32350,1,242000679,240838642,7962,7926,482b4785,off,,,,,,, +4200,node,insertion,reversed,2000,2463534242,1,144,1,295130175,266253511,1999000,2000999,ad35ba05,off,,,,,,, +4201,c,quick,nearly_sorted,500,2463534242,5,31784,1,237396859,236525227,10192,1850,482b4785,off,,,,,,, +4202,node,quick,nearly_sorted,1000,2463534242,25,3752,1,257004215,228603831,28499,7892,1cfe4afd,off,,,,,,, +4203,python,bubble,reversed,500,2463534242,11,20,1,256471488,241690230,124750,249500,d33e3559,off,,,,,,, +4204,python,insertion,nearly_sorted,2000,2463534242,17,20,1,260638466,243801336,118277,118270,5835dc59,off,,,,,,, +4205,python,none,reversed,1000,2463534242,28,0,16,222014272,571,0,0,0ca43265,off,,,,,,, +4206,node,quick,nearly_sorted,500,2463534242,5,10954,1,258311315,232396933,10192,1850,482b4785,off,,,,,,, +4207,c,insertion,sorted,2000,2463534242,27,101352,1,239739200,238865958,1999,0,69842aa5,off,,,,,,, +4208,node,insertion,sorted,2000,2463534242,18,89220,1,266695447,238537699,1999,0,69842aa5,off,,,,,,, +4209,node,none,reversed,2000,2463534242,16,0,9,233611816,1162,0,0,ad35ba05,off,,,,,,, +4210,c,insertion,reversed,250,2463534242,30,15645,1,243546073,242637873,31125,31374,be175004,off,,,,,,, +4211,python,none,nearly_sorted,250,2463534242,18,0,17,225528027,501,0,0,21a19c14,off,,,,,,, +4212,node,none,reversed,500,2463534242,30,0,9,223416369,1232,0,0,d33e3559,off,,,,,,, +4213,node,bubble,nearly_sorted,250,2463534242,14,6161,1,269742288,240628078,31125,4056,21a19c14,off,,,,,,, +4214,c,none,sorted,250,2463534242,27,0,429,233431197,80,0,0,7fd8c1f4,off,,,,,,, +4215,c,bubble,sorted,2000,2463534242,4,173,1,234909308,234170703,1999000,0,69842aa5,off,,,,,,, +4216,c,insertion,random,2000,2463534242,9,492,1,228989764,227772238,996390,996389,41262855,off,,,,,,, +4217,c,bubble,random,2000,2463534242,30,40,1,237284770,235881842,1999000,1988792,41262855,off,,,,,,, +4218,python,bubble,reversed,250,2463534242,24,88,1,251946185,236720654,31125,62250,be175004,off,,,,,,, +4219,python,none,random,1000,2463534242,20,0,15,211352441,752,0,0,57561285,off,,,,,,, +4220,node,quick,sorted,1000,2463534242,23,296,1,269327920,242364374,499500,0,8e15cb75,off,,,,,,, +4221,c,bubble,sorted,2000,2463534242,3,173,1,229088028,227945248,1999000,0,69842aa5,off,,,,,,, +4222,c,quick,reversed,2000,2463534242,22,209,1,244527803,243083096,1999000,2000,ad35ba05,off,,,,,,, +4223,c,none,nearly_sorted,500,2463534242,30,0,405,226803788,90,0,0,482b4785,off,,,,,,, +4224,c,none,random,250,2463534242,19,0,429,251817165,80,0,0,b47adce4,off,,,,,,, +4225,python,quick,sorted,1000,2463534242,13,8,1,283521291,267717344,499500,0,8e15cb75,off,,,,,,, +4226,c,insertion,nearly_sorted,2000,2463534242,2,2962,1,227524276,226478779,118277,118270,5835dc59,off,,,,,,, +4227,node,bubble,nearly_sorted,1000,2463534242,27,413,1,271806025,242620471,499500,59780,1cfe4afd,off,,,,,,, +4228,python,insertion,reversed,2000,2463534242,27,2,1,416244383,400334384,1999000,2000999,ad35ba05,off,,,,,,, +4229,c,quick,nearly_sorted,1000,2463534242,2,11273,1,229294114,228116758,28499,7892,1cfe4afd,off,,,,,,, +4230,node,quick,nearly_sorted,1000,2463534242,3,3752,1,260907591,233984048,28499,7892,1cfe4afd,off,,,,,,, +4231,python,none,reversed,2000,2463534242,7,0,16,228077224,551,0,0,ad35ba05,off,,,,,,, +4232,python,bubble,random,250,2463534242,5,112,1,242227363,226990214,31125,31196,b47adce4,off,,,,,,, +4233,python,none,random,500,2463534242,15,0,16,218886850,591,0,0,11ae932d,off,,,,,,, +4234,c,none,random,1000,2463534242,24,0,367,198025514,90,0,0,57561285,off,,,,,,, +4235,c,bubble,reversed,2000,2463534242,22,22,1,244161045,243325146,1999000,3998000,ad35ba05,off,,,,,,, +4236,node,insertion,random,250,2463534242,22,15349,1,258398629,230983045,15844,15838,b47adce4,off,,,,,,, +4237,node,none,sorted,500,2463534242,10,0,10,247807299,722,0,0,d9a24379,off,,,,,,, +4238,python,none,nearly_sorted,2000,2463534242,28,0,15,212050959,582,0,0,5835dc59,off,,,,,,, +4239,c,quick,nearly_sorted,250,2463534242,26,77221,1,237210812,236074061,4346,1008,21a19c14,off,,,,,,, +4240,node,none,random,500,2463534242,21,0,9,225506216,781,0,0,11ae932d,off,,,,,,, +4241,python,bubble,sorted,1000,2463534242,9,11,1,276002369,259567207,499500,0,8e15cb75,off,,,,,,, +4242,python,insertion,random,250,2463534242,28,206,1,255272730,240289901,15844,15838,b47adce4,off,,,,,,, +4243,c,insertion,reversed,500,2463534242,26,3967,1,235612145,234710616,124750,125249,d33e3559,off,,,,,,, +4244,node,insertion,reversed,2000,2463534242,19,144,1,260608871,232128331,1999000,2000999,ad35ba05,off,,,,,,, +4245,node,bubble,random,2000,2463534242,10,98,1,267422861,238743397,1999000,1988792,41262855,off,,,,,,, +4246,c,quick,reversed,2000,2463534242,10,209,1,240134519,238957455,1999000,2000,ad35ba05,off,,,,,,, +4247,python,bubble,sorted,250,2463534242,17,206,1,254202365,237226942,31125,0,7fd8c1f4,off,,,,,,, +4248,c,bubble,nearly_sorted,500,2463534242,7,1998,1,236470494,235514734,124750,14926,482b4785,off,,,,,,, +4249,python,none,reversed,250,2463534242,29,0,17,225633274,571,0,0,be175004,off,,,,,,, +4250,c,insertion,random,2000,2463534242,10,492,1,223149288,222157179,996390,996389,41262855,off,,,,,,, +4251,node,insertion,random,2000,2463534242,13,234,1,272926796,242772222,996390,996389,41262855,off,,,,,,, +4252,node,none,reversed,1000,2463534242,3,0,9,224211118,781,0,0,0ca43265,off,,,,,,, +4253,node,insertion,sorted,250,2463534242,3,607595,1,250225152,222063854,249,0,7fd8c1f4,off,,,,,,, +4254,node,quick,reversed,250,2463534242,14,4755,1,263239570,234722118,31125,250,be175004,off,,,,,,, +4255,c,bubble,sorted,2000,2463534242,20,173,1,234293764,233273823,1999000,0,69842aa5,off,,,,,,, +4256,python,quick,random,2000,2463534242,5,90,1,255412792,236731937,25197,24544,41262855,off,,,,,,, +4257,python,insertion,sorted,2000,2463534242,17,1523,1,256382460,240475507,1999,0,69842aa5,off,,,,,,, +4258,node,quick,random,2000,2463534242,10,2126,1,267395379,239894666,25197,24544,41262855,off,,,,,,, +4259,node,bubble,sorted,500,2463534242,1,1964,1,249273087,221851545,124750,0,d9a24379,off,,,,,,, +4260,c,insertion,random,250,2463534242,6,26020,1,232392262,231231136,15844,15838,b47adce4,off,,,,,,, +4261,python,none,random,2000,2463534242,14,0,16,237898167,571,0,0,41262855,off,,,,,,, +4262,node,quick,random,1000,2463534242,27,6513,1,264597646,237144082,11196,10210,57561285,off,,,,,,, +4263,node,quick,reversed,2000,2463534242,4,80,1,263425589,236171923,1999000,2000,ad35ba05,off,,,,,,, +4264,node,bubble,sorted,2000,2463534242,1,85,1,256665691,228620961,1999000,0,69842aa5,off,,,,,,, +4265,node,none,sorted,2000,2463534242,10,0,10,251953199,1062,0,0,69842aa5,off,,,,,,, +4266,node,quick,nearly_sorted,2000,2463534242,14,1909,1,253286267,223940080,58002,11768,5835dc59,off,,,,,,, +4267,c,quick,reversed,2000,2463534242,19,209,1,239109760,238068328,1999000,2000,ad35ba05,off,,,,,,, +4268,c,bubble,sorted,500,2463534242,12,2858,1,242964003,241818707,124750,0,d9a24379,off,,,,,,, +4269,node,insertion,random,500,2463534242,15,4059,1,259054397,230504937,60338,60339,11ae932d,off,,,,,,, +4270,python,insertion,reversed,2000,2463534242,21,2,1,398057931,381910731,1999000,2000999,ad35ba05,off,,,,,,, +4271,python,none,nearly_sorted,2000,2463534242,9,0,15,217353967,561,0,0,5835dc59,off,,,,,,, +4272,python,insertion,nearly_sorted,2000,2463534242,3,20,1,256165744,240723221,118277,118270,5835dc59,off,,,,,,, +4273,node,none,nearly_sorted,2000,2463534242,25,0,9,229813588,641,0,0,5835dc59,off,,,,,,, +4274,python,none,sorted,250,2463534242,26,0,17,219279707,501,0,0,7fd8c1f4,off,,,,,,, +4275,python,bubble,sorted,2000,2463534242,15,3,1,312890931,297020104,1999000,0,69842aa5,off,,,,,,, +4276,node,bubble,reversed,1000,2463534242,7,378,1,296382924,265395446,499500,999000,0ca43265,off,,,,,,, +4277,node,bubble,nearly_sorted,1000,2463534242,25,413,1,291967369,257144642,499500,59780,1cfe4afd,off,,,,,,, +4278,c,bubble,sorted,500,2463534242,21,2858,1,241428616,240593341,124750,0,d9a24379,off,,,,,,, +4279,c,insertion,random,2000,2463534242,20,492,1,230558421,229127268,996390,996389,41262855,off,,,,,,, +4280,python,insertion,sorted,2000,2463534242,16,1523,1,301714269,284863043,1999,0,69842aa5,off,,,,,,, +4281,python,insertion,sorted,1000,2463534242,25,3178,1,256604175,241065586,999,0,8e15cb75,off,,,,,,, +4282,c,insertion,nearly_sorted,2000,2463534242,11,2962,1,219682672,218404748,118277,118270,5835dc59,off,,,,,,, +4283,node,insertion,nearly_sorted,1000,2463534242,8,7758,1,259466750,232245736,30889,30860,1cfe4afd,off,,,,,,, +4284,node,quick,sorted,1000,2463534242,25,296,1,275264107,246474290,499500,0,8e15cb75,off,,,,,,, +4285,node,bubble,nearly_sorted,250,2463534242,20,6161,1,266735171,240106622,31125,4056,21a19c14,off,,,,,,, +4286,node,insertion,sorted,250,2463534242,19,607595,1,245772275,217261875,249,0,7fd8c1f4,off,,,,,,, +4287,c,none,random,250,2463534242,23,0,429,230515454,60,0,0,b47adce4,off,,,,,,, +4288,c,insertion,random,250,2463534242,27,26020,1,233112891,232278629,15844,15838,b47adce4,off,,,,,,, +4289,c,quick,reversed,1000,2463534242,1,814,1,234384294,233260036,499500,1000,0ca43265,off,,,,,,, +4290,c,none,reversed,500,2463534242,7,0,402,215388203,90,0,0,d33e3559,off,,,,,,, +4291,node,insertion,reversed,1000,2463534242,2,554,1,265633607,238027478,499500,500499,0ca43265,off,,,,,,, +4292,c,insertion,random,250,2463534242,25,26020,1,233387822,232236629,15844,15838,b47adce4,off,,,,,,, +4293,python,bubble,sorted,1000,2463534242,10,11,1,280986183,264419632,499500,0,8e15cb75,off,,,,,,, +4294,python,quick,random,500,2463534242,12,507,1,255396072,239888984,4578,4552,11ae932d,off,,,,,,, +4295,python,none,nearly_sorted,500,2463534242,4,0,17,228504825,521,0,0,482b4785,off,,,,,,, +4296,python,bubble,sorted,1000,2463534242,17,11,1,279776914,264020976,499500,0,8e15cb75,off,,,,,,, +4297,c,bubble,nearly_sorted,1000,2463534242,15,507,1,234604197,233328793,499500,59780,1cfe4afd,off,,,,,,, +4298,python,bubble,nearly_sorted,250,2463534242,7,188,1,250530060,235362971,31125,4056,21a19c14,off,,,,,,, +4299,python,bubble,sorted,1000,2463534242,4,11,1,269700469,254855074,499500,0,8e15cb75,off,,,,,,, +4300,c,quick,sorted,500,2463534242,4,2762,1,236120187,235078114,124750,0,d9a24379,off,,,,,,, +4301,c,quick,nearly_sorted,250,2463534242,2,77221,1,237147062,235559372,4346,1008,21a19c14,off,,,,,,, +4302,c,bubble,sorted,2000,2463534242,24,173,1,230556600,228993742,1999000,0,69842aa5,off,,,,,,, +4303,c,quick,random,500,2463534242,10,55776,1,230419804,229272746,4578,4552,11ae932d,off,,,,,,, +4304,c,bubble,nearly_sorted,1000,2463534242,23,507,1,238277500,237326528,499500,59780,1cfe4afd,off,,,,,,, +4305,c,quick,sorted,1000,2463534242,21,691,1,231562796,230284109,499500,0,8e15cb75,off,,,,,,, +4306,c,none,random,1000,2463534242,5,0,367,197179259,90,0,0,57561285,off,,,,,,, +4307,python,quick,sorted,2000,2463534242,11,2,1,304679303,288113404,1999000,0,69842aa5,off,,,,,,, +4308,c,bubble,random,2000,2463534242,26,40,1,236465404,235162262,1999000,1988792,41262855,off,,,,,,, +4309,node,quick,reversed,1000,2463534242,17,307,1,262955658,233663358,499500,1000,0ca43265,off,,,,,,, +4310,node,bubble,sorted,250,2463534242,29,8081,1,259128787,230950969,31125,0,7fd8c1f4,off,,,,,,, +4311,python,none,sorted,500,2463534242,22,0,16,211631343,551,0,0,d9a24379,off,,,,,,, +4312,python,quick,reversed,2000,2463534242,12,2,1,266719993,250401141,1999000,2000,ad35ba05,off,,,,,,, +4313,c,none,sorted,1000,2463534242,1,0,421,227393071,90,0,0,8e15cb75,off,,,,,,, +4314,c,insertion,random,250,2463534242,1,26020,1,234449255,233718786,15844,15838,b47adce4,off,,,,,,, +4315,python,none,sorted,2000,2463534242,19,0,15,216317165,561,0,0,69842aa5,off,,,,,,, +4316,python,none,nearly_sorted,1000,2463534242,5,0,16,219451969,541,0,0,1cfe4afd,off,,,,,,, +4317,node,none,reversed,2000,2463534242,10,0,9,221825538,821,0,0,ad35ba05,off,,,,,,, +4318,node,none,sorted,250,2463534242,9,0,9,223557373,732,0,0,7fd8c1f4,off,,,,,,, +4319,node,insertion,random,1000,2463534242,10,1091,1,259900523,231980553,249457,249457,57561285,off,,,,,,, +4320,node,bubble,sorted,250,2463534242,18,8081,1,257221714,230016439,31125,0,7fd8c1f4,off,,,,,,, +4321,python,quick,reversed,250,2463534242,18,151,1,251663325,236203707,31125,250,be175004,off,,,,,,, +4322,c,quick,random,1000,2463534242,23,23484,1,230609811,229651075,11196,10210,57561285,off,,,,,,, +4323,python,insertion,random,500,2463534242,11,44,1,246147679,230938180,60338,60339,11ae932d,off,,,,,,, +4324,python,bubble,nearly_sorted,1000,2463534242,30,9,1,243936475,228840681,499500,59780,1cfe4afd,off,,,,,,, +4325,c,quick,random,500,2463534242,14,55776,1,235091380,234106804,4578,4552,11ae932d,off,,,,,,, +4326,python,none,reversed,250,2463534242,30,0,17,225837855,521,0,0,be175004,off,,,,,,, +4327,node,none,sorted,500,2463534242,2,0,10,240080058,772,0,0,d9a24379,off,,,,,,, +4328,c,bubble,reversed,500,2463534242,15,346,1,234187906,233020288,124750,249500,d33e3559,off,,,,,,, +4329,python,bubble,sorted,1000,2463534242,5,11,1,274862615,258895546,499500,0,8e15cb75,off,,,,,,, +4330,python,insertion,sorted,500,2463534242,5,6624,1,232577516,217067431,499,0,d9a24379,off,,,,,,, +4331,node,insertion,sorted,1000,2463534242,21,160107,1,247773907,218988571,999,0,8e15cb75,off,,,,,,, +4332,node,insertion,random,500,2463534242,11,4059,1,262907468,235652779,60338,60339,11ae932d,off,,,,,,, +4333,node,none,random,2000,2463534242,9,0,9,239662256,902,0,0,41262855,off,,,,,,, +4334,python,insertion,random,250,2463534242,19,206,1,250580649,235531376,15844,15838,b47adce4,off,,,,,,, +4335,c,none,random,500,2463534242,16,0,426,231716192,80,0,0,11ae932d,off,,,,,,, +4336,python,quick,random,250,2463534242,1,1271,1,247977709,233069435,1933,1940,b47adce4,off,,,,,,, +4337,node,none,reversed,250,2463534242,27,0,9,227956547,671,0,0,be175004,off,,,,,,, +4338,node,quick,nearly_sorted,500,2463534242,7,10954,1,255874258,226171420,10192,1850,482b4785,off,,,,,,, +4339,c,insertion,nearly_sorted,2000,2463534242,19,2962,1,222820792,221743786,118277,118270,5835dc59,off,,,,,,, +4340,python,quick,random,500,2463534242,8,507,1,250496469,235288189,4578,4552,11ae932d,off,,,,,,, +4341,node,insertion,reversed,250,2463534242,2,8539,1,265179576,239344412,31125,31374,be175004,off,,,,,,, +4342,c,bubble,random,2000,2463534242,22,40,1,234317729,233252212,1999000,1988792,41262855,off,,,,,,, +4343,c,none,nearly_sorted,250,2463534242,12,0,388,210513117,90,0,0,21a19c14,off,,,,,,, +4344,c,insertion,nearly_sorted,2000,2463534242,27,2962,1,247875399,247109301,118277,118270,5835dc59,off,,,,,,, +4345,node,bubble,nearly_sorted,2000,2463534242,14,110,1,270601586,241976839,1999000,232556,5835dc59,off,,,,,,, +4346,c,bubble,reversed,1000,2463534242,28,87,1,247561549,246512351,499500,999000,0ca43265,off,,,,,,, +4347,node,bubble,sorted,2000,2463534242,8,85,1,263213021,234198095,1999000,0,69842aa5,off,,,,,,, +4348,python,none,random,500,2463534242,25,0,16,217228273,571,0,0,11ae932d,off,,,,,,, +4349,python,insertion,reversed,2000,2463534242,1,2,1,407638305,391962398,1999000,2000999,ad35ba05,off,,,,,,, +4350,c,insertion,reversed,500,2463534242,24,3967,1,237742096,236498187,124750,125249,d33e3559,off,,,,,,, +4351,python,insertion,reversed,250,2463534242,5,102,1,245100988,229496299,31125,31374,be175004,off,,,,,,, +4352,python,none,random,1000,2463534242,23,0,15,208053348,581,0,0,57561285,off,,,,,,, +4353,c,quick,reversed,250,2463534242,9,12781,1,233191729,232143775,31125,250,be175004,off,,,,,,, +4354,python,none,reversed,500,2463534242,11,0,16,208088163,571,0,0,d33e3559,off,,,,,,, +4355,python,bubble,sorted,1000,2463534242,1,11,1,273124666,257865710,499500,0,8e15cb75,off,,,,,,, +4356,node,insertion,nearly_sorted,2000,2463534242,22,2100,1,250221374,222797674,118277,118270,5835dc59,off,,,,,,, +4357,c,none,sorted,250,2463534242,13,0,429,233708277,90,0,0,7fd8c1f4,off,,,,,,, +4358,node,bubble,random,2000,2463534242,29,98,1,264719745,236972998,1999000,1988792,41262855,off,,,,,,, +4359,python,insertion,nearly_sorted,2000,2463534242,7,20,1,250565198,234742565,118277,118270,5835dc59,off,,,,,,, +4360,c,bubble,random,500,2463534242,24,635,1,228650057,227412408,124750,119688,11ae932d,off,,,,,,, +4361,node,bubble,reversed,250,2463534242,13,5861,1,256057471,229599347,31125,62250,be175004,off,,,,,,, +4362,python,quick,nearly_sorted,250,2463534242,29,882,1,255009727,239887585,4346,1008,21a19c14,off,,,,,,, +4363,c,quick,sorted,250,2463534242,2,10851,1,236828013,235524691,31125,0,7fd8c1f4,off,,,,,,, +4364,node,bubble,random,250,2463534242,23,5439,1,257496598,228359757,31125,31196,b47adce4,off,,,,,,, +4365,node,none,random,2000,2463534242,4,0,9,257018001,881,0,0,41262855,off,,,,,,, +4366,c,quick,sorted,250,2463534242,26,10851,1,244981834,244011727,31125,0,7fd8c1f4,off,,,,,,, +4367,c,quick,sorted,250,2463534242,8,10851,1,240192890,238832922,31125,0,7fd8c1f4,off,,,,,,, +4368,c,insertion,reversed,500,2463534242,27,3967,1,235375891,234168900,124750,125249,d33e3559,off,,,,,,, +4369,python,none,random,1000,2463534242,8,0,15,210727971,741,0,0,57561285,off,,,,,,, +4370,python,bubble,random,500,2463534242,5,26,1,279748811,263795608,124750,119688,11ae932d,off,,,,,,, +4371,node,bubble,random,250,2463534242,20,5439,1,250581709,224462293,31125,31196,b47adce4,off,,,,,,, +4372,c,none,nearly_sorted,500,2463534242,12,0,405,215500515,80,0,0,482b4785,off,,,,,,, +4373,python,bubble,random,500,2463534242,14,26,1,264370850,249025454,124750,119688,11ae932d,off,,,,,,, +4374,c,none,sorted,250,2463534242,20,0,429,231271750,90,0,0,7fd8c1f4,off,,,,,,, +4375,python,none,reversed,500,2463534242,25,0,16,220754320,581,0,0,d33e3559,off,,,,,,, +4376,node,insertion,sorted,2000,2463534242,4,89220,1,251753465,223848313,1999,0,69842aa5,off,,,,,,, +4377,c,bubble,reversed,1000,2463534242,22,87,1,238422061,237436864,499500,999000,0ca43265,off,,,,,,, +4378,node,bubble,sorted,2000,2463534242,18,85,1,263955121,235055596,1999000,0,69842aa5,off,,,,,,, +4379,node,bubble,nearly_sorted,2000,2463534242,10,110,1,261229063,232693025,1999000,232556,5835dc59,off,,,,,,, +4380,c,insertion,random,2000,2463534242,3,492,1,226607549,225510385,996390,996389,41262855,off,,,,,,, +4381,python,insertion,reversed,2000,2463534242,4,2,1,412597509,396164441,1999000,2000999,ad35ba05,off,,,,,,, +4382,c,quick,random,1000,2463534242,27,23484,1,231897254,230941543,11196,10210,57561285,off,,,,,,, +4383,c,quick,random,500,2463534242,8,55776,1,239213905,237936823,4578,4552,11ae932d,off,,,,,,, +4384,python,none,reversed,1000,2463534242,30,0,16,216422652,581,0,0,0ca43265,off,,,,,,, +4385,c,bubble,reversed,500,2463534242,3,346,1,236498587,235527918,124750,249500,d33e3559,off,,,,,,, +4386,python,quick,sorted,500,2463534242,7,30,1,241436661,224195456,124750,0,d9a24379,off,,,,,,, +4387,node,none,sorted,2000,2463534242,7,0,10,250035745,781,0,0,69842aa5,off,,,,,,, +4388,c,bubble,random,500,2463534242,11,635,1,228480560,227338770,124750,119688,11ae932d,off,,,,,,, +4389,python,bubble,nearly_sorted,2000,2463534242,28,3,1,343769500,325645669,1999000,232556,5835dc59,off,,,,,,, +4390,python,bubble,nearly_sorted,500,2463534242,3,41,1,245350225,230107910,124750,14926,482b4785,off,,,,,,, +4391,python,bubble,nearly_sorted,2000,2463534242,19,3,1,352747205,336194462,1999000,232556,5835dc59,off,,,,,,, +4392,c,bubble,random,500,2463534242,2,635,1,228018644,226988414,124750,119688,11ae932d,off,,,,,,, +4393,node,insertion,sorted,1000,2463534242,7,160107,1,247414333,220226886,999,0,8e15cb75,off,,,,,,, +4394,c,quick,nearly_sorted,250,2463534242,8,77221,1,235886429,234718740,4346,1008,21a19c14,off,,,,,,, +4395,node,insertion,reversed,1000,2463534242,6,554,1,265120114,235819873,499500,500499,0ca43265,off,,,,,,, +4396,c,bubble,reversed,1000,2463534242,10,87,1,236412014,235546903,499500,999000,0ca43265,off,,,,,,, +4397,node,quick,random,500,2463534242,8,16148,1,263751259,234221125,4578,4552,11ae932d,off,,,,,,, +4398,python,none,random,250,2463534242,13,0,17,223474368,471,0,0,b47adce4,off,,,,,,, +4399,python,quick,sorted,500,2463534242,30,30,1,241033215,225374253,124750,0,d9a24379,off,,,,,,, +4400,c,none,nearly_sorted,1000,2463534242,22,0,421,228018673,90,0,0,1cfe4afd,off,,,,,,, +4401,node,none,nearly_sorted,1000,2463534242,4,0,10,253619592,621,0,0,1cfe4afd,off,,,,,,, +4402,node,none,reversed,250,2463534242,16,0,9,219566975,872,0,0,be175004,off,,,,,,, +4403,c,insertion,sorted,250,2463534242,20,786886,1,246604235,245506537,249,0,7fd8c1f4,off,,,,,,, +4404,c,none,sorted,250,2463534242,25,0,429,229636506,90,0,0,7fd8c1f4,off,,,,,,, +4405,python,quick,nearly_sorted,1000,2463534242,16,110,1,263899347,248246174,28499,7892,1cfe4afd,off,,,,,,, +4406,node,insertion,random,250,2463534242,18,15349,1,257015197,229797504,15844,15838,b47adce4,off,,,,,,, +4407,python,none,random,1000,2463534242,5,0,15,207345320,651,0,0,57561285,off,,,,,,, +4408,c,insertion,reversed,500,2463534242,8,3967,1,237750413,236869332,124750,125249,d33e3559,off,,,,,,, +4409,c,none,sorted,250,2463534242,9,0,429,228751907,101,0,0,7fd8c1f4,off,,,,,,, +4410,python,quick,random,500,2463534242,23,507,1,261696841,243696171,4578,4552,11ae932d,off,,,,,,, +4411,node,bubble,reversed,1000,2463534242,10,378,1,263155351,234460645,499500,999000,0ca43265,off,,,,,,, +4412,c,bubble,random,500,2463534242,23,635,1,223349794,222290549,124750,119688,11ae932d,off,,,,,,, +4413,c,quick,nearly_sorted,250,2463534242,11,77221,1,234084953,233194614,4346,1008,21a19c14,off,,,,,,, +4414,c,bubble,reversed,250,2463534242,6,1384,1,235719733,234804027,31125,62250,be175004,off,,,,,,, +4415,c,insertion,reversed,250,2463534242,11,15645,1,242184836,241108126,31125,31374,be175004,off,,,,,,, +4416,python,quick,random,1000,2463534242,25,209,1,248585728,232338023,11196,10210,57561285,off,,,,,,, +4417,node,insertion,sorted,2000,2463534242,20,89220,1,251943101,223960512,1999,0,69842aa5,off,,,,,,, +4418,python,bubble,nearly_sorted,2000,2463534242,25,3,1,358566109,342607941,1999000,232556,5835dc59,off,,,,,,, +4419,c,insertion,reversed,2000,2463534242,12,260,1,241726505,240683209,1999000,2000999,ad35ba05,off,,,,,,, +4420,python,insertion,reversed,1000,2463534242,7,6,1,290694182,274460520,499500,500499,0ca43265,off,,,,,,, +4421,node,bubble,random,2000,2463534242,20,98,1,265524583,236521621,1999000,1988792,41262855,off,,,,,,, +4422,python,bubble,reversed,1000,2463534242,18,5,1,291196284,275866823,499500,999000,0ca43265,off,,,,,,, +4423,python,quick,reversed,500,2463534242,22,33,1,252157262,235954801,124750,500,d33e3559,off,,,,,,, +4424,python,quick,sorted,250,2463534242,18,133,1,249052223,234181004,31125,0,7fd8c1f4,off,,,,,,, +4425,python,insertion,nearly_sorted,2000,2463534242,24,20,1,256578908,239959778,118277,118270,5835dc59,off,,,,,,, +4426,node,insertion,sorted,2000,2463534242,14,89220,1,253168557,225959916,1999,0,69842aa5,off,,,,,,, +4427,python,quick,reversed,500,2463534242,20,33,1,283994387,268829367,124750,500,d33e3559,off,,,,,,, +4428,python,none,sorted,1000,2463534242,18,0,16,215661275,561,0,0,8e15cb75,off,,,,,,, +4429,c,insertion,reversed,1000,2463534242,8,1033,1,233296155,232486197,499500,500499,0ca43265,off,,,,,,, +4430,node,bubble,random,250,2463534242,27,5439,1,248270036,219895971,31125,31196,b47adce4,off,,,,,,, +4431,node,none,nearly_sorted,1000,2463534242,8,0,10,251750970,622,0,0,1cfe4afd,off,,,,,,, +4432,node,bubble,sorted,2000,2463534242,24,85,1,261937260,234690462,1999000,0,69842aa5,off,,,,,,, +4433,c,none,sorted,250,2463534242,15,0,429,229579780,90,0,0,7fd8c1f4,off,,,,,,, +4434,node,bubble,sorted,250,2463534242,5,8081,1,251572936,224430475,31125,0,7fd8c1f4,off,,,,,,, +4435,c,none,reversed,250,2463534242,19,0,427,227822216,90,0,0,be175004,off,,,,,,, +4436,node,quick,sorted,1000,2463534242,17,296,1,269370621,240139915,499500,0,8e15cb75,off,,,,,,, +4437,c,quick,random,2000,2463534242,19,10587,1,226567925,225481279,25197,24544,41262855,off,,,,,,, +4438,node,insertion,reversed,2000,2463534242,25,144,1,291668458,262905995,1999000,2000999,ad35ba05,off,,,,,,, +4439,node,bubble,nearly_sorted,250,2463534242,17,6161,1,264247480,237257535,31125,4056,21a19c14,off,,,,,,, +4440,node,insertion,reversed,500,2463534242,27,2178,1,253299882,226348352,124750,125249,d33e3559,off,,,,,,, +4441,c,quick,random,500,2463534242,22,55776,1,230577260,229545115,4578,4552,11ae932d,off,,,,,,, +4442,python,bubble,sorted,1000,2463534242,6,11,1,277007723,260675167,499500,0,8e15cb75,off,,,,,,, +4443,python,insertion,sorted,1000,2463534242,27,3178,1,255818072,239065562,999,0,8e15cb75,off,,,,,,, +4444,python,insertion,random,500,2463534242,17,44,1,249411867,234779664,60338,60339,11ae932d,off,,,,,,, +4445,node,bubble,nearly_sorted,2000,2463534242,29,110,1,262260566,234202014,1999000,232556,5835dc59,off,,,,,,, +4446,c,insertion,sorted,2000,2463534242,24,101352,1,237351665,236149001,1999,0,69842aa5,off,,,,,,, +4447,python,quick,reversed,1000,2463534242,26,8,1,260968765,245286268,499500,1000,0ca43265,off,,,,,,, +4448,node,insertion,reversed,250,2463534242,9,8539,1,257026688,228299673,31125,31374,be175004,off,,,,,,, +4449,python,bubble,sorted,250,2463534242,13,206,1,291173051,276899069,31125,0,7fd8c1f4,off,,,,,,, +4450,c,quick,sorted,1000,2463534242,7,691,1,242635469,241409168,499500,0,8e15cb75,off,,,,,,, +4451,node,bubble,random,2000,2463534242,19,98,1,266816673,236875985,1999000,1988792,41262855,off,,,,,,, +4452,python,none,nearly_sorted,1000,2463534242,29,0,16,219433504,641,0,0,1cfe4afd,off,,,,,,, +4453,python,none,nearly_sorted,250,2463534242,7,0,17,228954389,481,0,0,21a19c14,off,,,,,,, +4454,c,none,reversed,2000,2463534242,3,0,413,229002388,80,0,0,ad35ba05,off,,,,,,, +4455,c,quick,sorted,2000,2463534242,6,174,1,234459616,233652101,1999000,0,69842aa5,off,,,,,,, +4456,python,insertion,nearly_sorted,500,2463534242,16,316,1,244824027,228289292,7962,7926,482b4785,off,,,,,,, +4457,python,none,sorted,1000,2463534242,16,0,16,219201109,561,0,0,8e15cb75,off,,,,,,, +4458,c,insertion,sorted,250,2463534242,19,786886,1,242077802,240800238,249,0,7fd8c1f4,off,,,,,,, +4459,c,insertion,reversed,500,2463534242,25,3967,1,236224413,234935256,124750,125249,d33e3559,off,,,,,,, +4460,c,quick,sorted,250,2463534242,15,10851,1,243269545,241829767,31125,0,7fd8c1f4,off,,,,,,, +4461,python,insertion,sorted,2000,2463534242,14,1523,1,280904927,263690934,1999,0,69842aa5,off,,,,,,, +4462,node,quick,nearly_sorted,1000,2463534242,29,3752,1,258399360,230991355,28499,7892,1cfe4afd,off,,,,,,, +4463,python,bubble,random,2000,2463534242,30,2,1,394782562,378366481,1999000,1988792,41262855,off,,,,,,, +4464,c,bubble,reversed,1000,2463534242,3,87,1,249037034,247892662,499500,999000,0ca43265,off,,,,,,, +4465,python,quick,nearly_sorted,2000,2463534242,8,55,1,274876098,258558649,58002,11768,5835dc59,off,,,,,,, +4466,c,insertion,random,2000,2463534242,8,492,1,226308790,225251359,996390,996389,41262855,off,,,,,,, +4467,python,quick,reversed,1000,2463534242,19,8,1,273434237,258097003,499500,1000,0ca43265,off,,,,,,, +4468,python,quick,nearly_sorted,250,2463534242,30,882,1,255402723,240446839,4346,1008,21a19c14,off,,,,,,, +4469,c,insertion,sorted,2000,2463534242,3,101352,1,235891188,234813558,1999,0,69842aa5,off,,,,,,, +4470,python,quick,nearly_sorted,2000,2463534242,14,55,1,247376463,230276941,58002,11768,5835dc59,off,,,,,,, +4471,c,quick,reversed,500,2463534242,3,3240,1,229952899,228934119,124750,500,d33e3559,off,,,,,,, +4472,python,quick,reversed,2000,2463534242,30,2,1,277697817,260987927,1999000,2000,ad35ba05,off,,,,,,, +4473,c,quick,sorted,500,2463534242,25,2762,1,242209990,240982730,124750,0,d9a24379,off,,,,,,, +4474,node,quick,random,500,2463534242,5,16148,1,276688405,247173438,4578,4552,11ae932d,off,,,,,,, +4475,python,none,nearly_sorted,2000,2463534242,13,0,15,219930957,571,0,0,5835dc59,off,,,,,,, +4476,c,none,reversed,2000,2463534242,14,0,413,231799340,91,0,0,ad35ba05,off,,,,,,, +4477,python,none,sorted,1000,2463534242,27,0,16,216481392,591,0,0,8e15cb75,off,,,,,,, +4478,c,insertion,random,250,2463534242,29,26020,1,229688404,228668352,15844,15838,b47adce4,off,,,,,,, +4479,node,quick,random,2000,2463534242,11,2126,1,266460606,238519385,25197,24544,41262855,off,,,,,,, +4480,c,insertion,sorted,1000,2463534242,4,196883,1,231628891,230479706,999,0,8e15cb75,off,,,,,,, +4481,node,insertion,reversed,1000,2463534242,25,554,1,272356486,243138437,499500,500499,0ca43265,off,,,,,,, +4482,node,bubble,sorted,1000,2463534242,11,374,1,249998806,222699674,499500,0,8e15cb75,off,,,,,,, +4483,python,none,reversed,250,2463534242,25,0,17,228577413,541,0,0,be175004,off,,,,,,, +4484,python,insertion,nearly_sorted,1000,2463534242,16,76,1,248636684,233774806,30889,30860,1cfe4afd,off,,,,,,, +4485,python,bubble,sorted,500,2463534242,18,46,1,262048374,245947735,124750,0,d9a24379,off,,,,,,, +4486,c,insertion,random,1000,2463534242,10,1983,1,240966624,239695158,249457,249457,57561285,off,,,,,,, +4487,python,quick,reversed,500,2463534242,27,33,1,256899749,240523914,124750,500,d33e3559,off,,,,,,, +4488,node,insertion,sorted,250,2463534242,25,607595,1,248674624,221283974,249,0,7fd8c1f4,off,,,,,,, +4489,python,bubble,nearly_sorted,250,2463534242,20,188,1,262581157,245483504,31125,4056,21a19c14,off,,,,,,, +4490,python,quick,random,500,2463534242,5,507,1,251780285,236423223,4578,4552,11ae932d,off,,,,,,, +4491,node,bubble,random,500,2463534242,26,1407,1,273586441,246794792,124750,119688,11ae932d,off,,,,,,, +4492,c,bubble,nearly_sorted,1000,2463534242,20,507,1,240664153,239620767,499500,59780,1cfe4afd,off,,,,,,, +4493,node,insertion,reversed,500,2463534242,14,2178,1,261068111,232194067,124750,125249,d33e3559,off,,,,,,, +4494,node,quick,nearly_sorted,1000,2463534242,6,3752,1,252332651,224313966,28499,7892,1cfe4afd,off,,,,,,, +4495,node,none,sorted,2000,2463534242,19,0,10,252186727,701,0,0,69842aa5,off,,,,,,, +4496,c,insertion,nearly_sorted,500,2463534242,27,32350,1,239669319,238506820,7962,7926,482b4785,off,,,,,,, +4497,node,bubble,sorted,250,2463534242,7,8081,1,252309597,223484282,31125,0,7fd8c1f4,off,,,,,,, +4498,c,insertion,nearly_sorted,250,2463534242,27,99421,1,231642806,230653392,2277,2251,21a19c14,off,,,,,,, +4499,c,insertion,random,250,2463534242,2,26020,1,228421198,227241620,15844,15838,b47adce4,off,,,,,,, +4500,c,none,sorted,2000,2463534242,19,0,416,225299869,90,0,0,69842aa5,off,,,,,,, +4501,node,quick,sorted,1000,2463534242,8,296,1,267388676,239938137,499500,0,8e15cb75,off,,,,,,, +4502,python,quick,reversed,1000,2463534242,25,8,1,267681626,252612191,499500,1000,0ca43265,off,,,,,,, +4503,c,bubble,nearly_sorted,2000,2463534242,28,121,1,239841410,238690213,1999000,232556,5835dc59,off,,,,,,, +4504,c,none,sorted,1000,2463534242,2,0,421,226035007,100,0,0,8e15cb75,off,,,,,,, +4505,python,bubble,reversed,250,2463534242,20,88,1,252716691,236871435,31125,62250,be175004,off,,,,,,, +4506,c,insertion,sorted,2000,2463534242,14,101352,1,238174757,236749216,1999,0,69842aa5,off,,,,,,, +4507,node,quick,nearly_sorted,2000,2463534242,15,1909,1,250410517,220897091,58002,11768,5835dc59,off,,,,,,, +4508,node,quick,reversed,1000,2463534242,25,307,1,259636107,232738845,499500,1000,0ca43265,off,,,,,,, +4509,c,none,nearly_sorted,500,2463534242,13,0,405,215062184,80,0,0,482b4785,off,,,,,,, +4510,c,none,random,500,2463534242,29,0,426,227550386,91,0,0,11ae932d,off,,,,,,, +4511,c,quick,nearly_sorted,1000,2463534242,20,11273,1,231995546,231230063,28499,7892,1cfe4afd,off,,,,,,, +4512,node,none,reversed,2000,2463534242,1,0,9,224971043,781,0,0,ad35ba05,off,,,,,,, +4513,c,insertion,sorted,250,2463534242,3,786886,1,241196711,240051835,249,0,7fd8c1f4,off,,,,,,, +4514,node,insertion,random,500,2463534242,7,4059,1,266412277,238239582,60338,60339,11ae932d,off,,,,,,, +4515,c,none,random,1000,2463534242,1,0,367,199127620,91,0,0,57561285,off,,,,,,, +4516,python,quick,reversed,2000,2463534242,28,2,1,273695297,258140243,1999000,2000,ad35ba05,off,,,,,,, +4517,c,none,random,500,2463534242,5,0,426,229870994,90,0,0,11ae932d,off,,,,,,, +4518,c,bubble,reversed,2000,2463534242,29,22,1,243886602,243160591,1999000,3998000,ad35ba05,off,,,,,,, +4519,node,bubble,reversed,1000,2463534242,17,378,1,265423734,237651062,499500,999000,0ca43265,off,,,,,,, +4520,node,quick,random,1000,2463534242,13,6513,1,255714017,228474633,11196,10210,57561285,off,,,,,,, +4521,python,none,reversed,1000,2463534242,18,0,16,216721984,571,0,0,0ca43265,off,,,,,,, +4522,python,insertion,sorted,2000,2463534242,13,1523,1,263186532,246698364,1999,0,69842aa5,off,,,,,,, +4523,c,quick,sorted,500,2463534242,27,2762,1,235726691,234473442,124750,0,d9a24379,off,,,,,,, +4524,python,none,reversed,1000,2463534242,7,0,16,221144300,541,0,0,0ca43265,off,,,,,,, +4525,c,quick,reversed,2000,2463534242,17,209,1,241145705,239864805,1999000,2000,ad35ba05,off,,,,,,, +4526,node,bubble,nearly_sorted,1000,2463534242,9,413,1,263267444,235472667,499500,59780,1cfe4afd,off,,,,,,, +4527,python,insertion,reversed,1000,2463534242,12,6,1,291331467,276139288,499500,500499,0ca43265,off,,,,,,, +4528,c,bubble,random,500,2463534242,5,635,1,224593807,223341605,124750,119688,11ae932d,off,,,,,,, +4529,python,bubble,sorted,250,2463534242,29,206,1,259083240,244907045,31125,0,7fd8c1f4,off,,,,,,, +4530,c,bubble,nearly_sorted,1000,2463534242,12,507,1,237898129,236862088,499500,59780,1cfe4afd,off,,,,,,, +4531,python,bubble,reversed,250,2463534242,7,88,1,251569681,235085674,31125,62250,be175004,off,,,,,,, +4532,python,quick,reversed,250,2463534242,20,151,1,246741975,232559451,31125,250,be175004,off,,,,,,, +4533,python,quick,random,500,2463534242,10,507,1,251924453,237433676,4578,4552,11ae932d,off,,,,,,, +4534,c,quick,sorted,250,2463534242,7,10851,1,235664393,234688894,31125,0,7fd8c1f4,off,,,,,,, +4535,python,bubble,nearly_sorted,500,2463534242,24,41,1,243819725,228175953,124750,14926,482b4785,off,,,,,,, +4536,python,bubble,sorted,500,2463534242,6,46,1,255947255,240616059,124750,0,d9a24379,off,,,,,,, +4537,python,insertion,random,500,2463534242,3,44,1,246483729,230850829,60338,60339,11ae932d,off,,,,,,, +4538,node,insertion,sorted,250,2463534242,22,607595,1,245493022,216875823,249,0,7fd8c1f4,off,,,,,,, +4539,python,none,reversed,2000,2463534242,8,0,16,226648286,611,0,0,ad35ba05,off,,,,,,, +4540,c,bubble,random,2000,2463534242,2,40,1,238232175,237068203,1999000,1988792,41262855,off,,,,,,, +4541,c,quick,sorted,2000,2463534242,18,174,1,237323432,235918299,1999000,0,69842aa5,off,,,,,,, +4542,node,quick,sorted,250,2463534242,24,4507,1,262142024,235915718,31125,0,7fd8c1f4,off,,,,,,, +4543,python,none,random,500,2463534242,3,0,16,212936940,561,0,0,11ae932d,off,,,,,,, +4544,c,quick,sorted,2000,2463534242,16,174,1,233729287,232722449,1999000,0,69842aa5,off,,,,,,, +4545,node,none,nearly_sorted,500,2463534242,25,0,10,252375701,732,0,0,482b4785,off,,,,,,, +4546,python,none,random,250,2463534242,22,0,17,230285142,510,0,0,b47adce4,off,,,,,,, +4547,c,none,random,250,2463534242,22,0,429,227279409,100,0,0,b47adce4,off,,,,,,, +4548,node,bubble,random,250,2463534242,14,5439,1,253332203,225204025,31125,31196,b47adce4,off,,,,,,, +4549,node,none,nearly_sorted,1000,2463534242,2,0,10,247555157,662,0,0,1cfe4afd,off,,,,,,, +4550,python,bubble,reversed,1000,2463534242,2,5,1,288370228,273078805,499500,999000,0ca43265,off,,,,,,, +4551,node,none,sorted,250,2463534242,12,0,9,224484711,581,0,0,7fd8c1f4,off,,,,,,, +4552,python,insertion,sorted,1000,2463534242,17,3178,1,255106448,240043371,999,0,8e15cb75,off,,,,,,, +4553,node,bubble,nearly_sorted,1000,2463534242,17,413,1,256333969,228756538,499500,59780,1cfe4afd,off,,,,,,, +4554,node,insertion,random,250,2463534242,29,15349,1,256206721,227306606,15844,15838,b47adce4,off,,,,,,, +4555,python,quick,reversed,2000,2463534242,16,2,1,274559736,258740852,1999000,2000,ad35ba05,off,,,,,,, +4556,c,quick,sorted,2000,2463534242,4,174,1,238877063,237736808,1999000,0,69842aa5,off,,,,,,, +4557,python,none,reversed,2000,2463534242,21,0,16,228361988,642,0,0,ad35ba05,off,,,,,,, +4558,node,insertion,nearly_sorted,500,2463534242,6,26045,1,240736688,212317560,7962,7926,482b4785,off,,,,,,, +4559,python,bubble,sorted,250,2463534242,18,206,1,251542300,236535460,31125,0,7fd8c1f4,off,,,,,,, +4560,node,quick,random,500,2463534242,1,16148,1,263056037,233452577,4578,4552,11ae932d,off,,,,,,, +4561,c,none,reversed,1000,2463534242,19,0,405,219449114,90,0,0,0ca43265,off,,,,,,, +4562,node,quick,reversed,1000,2463534242,14,307,1,263894979,235539015,499500,1000,0ca43265,off,,,,,,, +4563,c,none,random,250,2463534242,10,0,429,227659280,90,0,0,b47adce4,off,,,,,,, +4564,c,bubble,nearly_sorted,250,2463534242,26,7880,1,237433037,236624883,31125,4056,21a19c14,off,,,,,,, +4565,c,quick,random,250,2463534242,3,136055,1,239532753,238542648,1933,1940,b47adce4,off,,,,,,, +4566,c,bubble,random,250,2463534242,26,2568,1,239914900,238919405,31125,31196,b47adce4,off,,,,,,, +4567,node,bubble,random,1000,2463534242,26,362,1,264336365,236571319,499500,496926,57561285,off,,,,,,, +4568,python,none,sorted,1000,2463534242,14,0,16,214157818,571,0,0,8e15cb75,off,,,,,,, +4569,python,bubble,nearly_sorted,500,2463534242,25,41,1,245339174,231447433,124750,14926,482b4785,off,,,,,,, +4570,node,none,random,250,2463534242,10,0,10,250761005,2655,0,0,b47adce4,off,,,,,,, +4571,c,insertion,nearly_sorted,250,2463534242,26,99421,1,237151459,236056868,2277,2251,21a19c14,off,,,,,,, +4572,python,bubble,random,250,2463534242,14,112,1,247974023,231534551,31125,31196,b47adce4,off,,,,,,, +4573,c,insertion,nearly_sorted,250,2463534242,1,99421,1,236382599,234655903,2277,2251,21a19c14,off,,,,,,, +4574,c,quick,sorted,1000,2463534242,14,691,1,235241070,233673011,499500,0,8e15cb75,off,,,,,,, +4575,python,quick,random,1000,2463534242,20,209,1,249652718,233670214,11196,10210,57561285,off,,,,,,, +4576,c,none,sorted,2000,2463534242,24,0,416,226971151,90,0,0,69842aa5,off,,,,,,, +4577,node,bubble,random,1000,2463534242,17,362,1,255796792,228539815,499500,496926,57561285,off,,,,,,, +4578,node,quick,nearly_sorted,2000,2463534242,25,1909,1,252363860,223140025,58002,11768,5835dc59,off,,,,,,, +4579,node,bubble,nearly_sorted,500,2463534242,15,1566,1,264777833,237663817,124750,14926,482b4785,off,,,,,,, +4580,python,quick,nearly_sorted,250,2463534242,11,882,1,258447350,242726915,4346,1008,21a19c14,off,,,,,,, +4581,python,bubble,reversed,500,2463534242,21,20,1,262360363,247243995,124750,249500,d33e3559,off,,,,,,, +4582,python,bubble,reversed,2000,2463534242,21,2,1,476125655,459755786,1999000,3998000,ad35ba05,off,,,,,,, +4583,python,insertion,nearly_sorted,1000,2463534242,24,76,1,250917739,235674529,30889,30860,1cfe4afd,off,,,,,,, +4584,c,insertion,nearly_sorted,1000,2463534242,20,11445,1,247893543,246756833,30889,30860,1cfe4afd,off,,,,,,, +4585,python,bubble,random,1000,2463534242,3,6,1,273219362,256776634,499500,496926,57561285,off,,,,,,, +4586,node,none,reversed,250,2463534242,17,0,9,235825665,721,0,0,be175004,off,,,,,,, +4587,node,bubble,reversed,2000,2463534242,4,94,1,267931464,239336299,1999000,3998000,ad35ba05,off,,,,,,, +4588,python,quick,nearly_sorted,2000,2463534242,10,55,1,245279152,229357624,58002,11768,5835dc59,off,,,,,,, +4589,node,insertion,random,2000,2463534242,21,234,1,269279851,241636611,996390,996389,41262855,off,,,,,,, +4590,c,quick,nearly_sorted,250,2463534242,21,77221,1,242858626,241876865,4346,1008,21a19c14,off,,,,,,, +4591,python,insertion,reversed,250,2463534242,4,102,1,250717224,235497501,31125,31374,be175004,off,,,,,,, +4592,python,quick,nearly_sorted,250,2463534242,9,882,1,253308539,238328354,4346,1008,21a19c14,off,,,,,,, +4593,c,none,nearly_sorted,250,2463534242,25,0,388,205649112,90,0,0,21a19c14,off,,,,,,, +4594,node,quick,sorted,1000,2463534242,12,296,1,270066595,241370678,499500,0,8e15cb75,off,,,,,,, +4595,python,insertion,sorted,250,2463534242,21,18845,1,262090597,246522139,249,0,7fd8c1f4,off,,,,,,, +4596,python,bubble,sorted,1000,2463534242,14,11,1,272336199,257317734,499500,0,8e15cb75,off,,,,,,, +4597,c,quick,random,2000,2463534242,8,10587,1,236473300,234857701,25197,24544,41262855,off,,,,,,, +4598,python,insertion,random,500,2463534242,12,44,1,270046105,253520026,60338,60339,11ae932d,off,,,,,,, +4599,python,quick,sorted,2000,2463534242,6,2,1,315361111,298764010,1999000,0,69842aa5,off,,,,,,, +4600,python,none,sorted,250,2463534242,29,0,17,226805501,531,0,0,7fd8c1f4,off,,,,,,, +4601,c,insertion,sorted,500,2463534242,11,399335,1,240378276,238884839,499,0,d9a24379,off,,,,,,, +4602,c,insertion,sorted,1000,2463534242,7,196883,1,225897198,224757493,999,0,8e15cb75,off,,,,,,, +4603,node,insertion,sorted,500,2463534242,27,324764,1,259500604,230463748,499,0,d9a24379,off,,,,,,, +4604,c,insertion,sorted,500,2463534242,26,399335,1,233806099,232585493,499,0,d9a24379,off,,,,,,, +4605,python,bubble,sorted,500,2463534242,2,46,1,267032870,251609324,124750,0,d9a24379,off,,,,,,, +4606,node,bubble,sorted,250,2463534242,3,8081,1,246452180,217884216,31125,0,7fd8c1f4,off,,,,,,, +4607,node,none,random,2000,2463534242,24,0,9,239589919,591,0,0,41262855,off,,,,,,, +4608,c,quick,nearly_sorted,1000,2463534242,28,11273,1,231670298,230470830,28499,7892,1cfe4afd,off,,,,,,, +4609,node,insertion,sorted,2000,2463534242,21,89220,1,255788626,225706498,1999,0,69842aa5,off,,,,,,, +4610,node,bubble,random,1000,2463534242,19,362,1,268489670,239990305,499500,496926,57561285,off,,,,,,, +4611,node,none,random,250,2463534242,16,0,10,248725390,771,0,0,b47adce4,off,,,,,,, +4612,c,insertion,sorted,500,2463534242,27,399335,1,242437948,241412615,499,0,d9a24379,off,,,,,,, +4613,python,bubble,random,1000,2463534242,26,6,1,261850616,246187791,499500,496926,57561285,off,,,,,,, +4614,c,insertion,random,1000,2463534242,9,1983,1,240517177,239587495,249457,249457,57561285,off,,,,,,, +4615,python,insertion,random,1000,2463534242,8,10,1,258379513,241273911,249457,249457,57561285,off,,,,,,, +4616,node,none,sorted,500,2463534242,7,0,10,248821690,1132,0,0,d9a24379,off,,,,,,, +4617,node,insertion,sorted,2000,2463534242,5,89220,1,253676138,225110445,1999,0,69842aa5,off,,,,,,, +4618,c,bubble,random,500,2463534242,3,635,1,229121762,228096350,124750,119688,11ae932d,off,,,,,,, +4619,python,bubble,reversed,2000,2463534242,26,2,1,468601183,451929458,1999000,3998000,ad35ba05,off,,,,,,, +4620,c,bubble,nearly_sorted,1000,2463534242,18,507,1,238880129,237575896,499500,59780,1cfe4afd,off,,,,,,, +4621,node,insertion,nearly_sorted,250,2463534242,18,79550,1,230961008,202464966,2277,2251,21a19c14,off,,,,,,, +4622,node,none,reversed,1000,2463534242,29,0,9,222380428,761,0,0,0ca43265,off,,,,,,, +4623,c,insertion,sorted,2000,2463534242,26,101352,1,231113414,230152593,1999,0,69842aa5,off,,,,,,, +4624,node,quick,nearly_sorted,1000,2463534242,10,3752,1,250309849,223926341,28499,7892,1cfe4afd,off,,,,,,, +4625,python,quick,reversed,250,2463534242,4,151,1,259937082,244568002,31125,250,be175004,off,,,,,,, +4626,c,quick,random,2000,2463534242,1,10587,1,237715577,236578776,25197,24544,41262855,off,,,,,,, +4627,c,quick,random,250,2463534242,24,136055,1,238417953,237122666,1933,1940,b47adce4,off,,,,,,, +4628,c,bubble,sorted,500,2463534242,22,2858,1,241558929,240273942,124750,0,d9a24379,off,,,,,,, +4629,c,quick,sorted,250,2463534242,22,10851,1,243044814,241974429,31125,0,7fd8c1f4,off,,,,,,, +4630,node,insertion,nearly_sorted,250,2463534242,29,79550,1,230651178,203886626,2277,2251,21a19c14,off,,,,,,, +4631,python,quick,reversed,1000,2463534242,1,8,1,271257226,255403682,499500,1000,0ca43265,off,,,,,,, +4632,node,insertion,reversed,2000,2463534242,9,144,1,257616262,228807099,1999000,2000999,ad35ba05,off,,,,,,, +4633,python,quick,random,500,2463534242,16,507,1,252489474,237301204,4578,4552,11ae932d,off,,,,,,, +4634,node,quick,sorted,1000,2463534242,1,296,1,267882131,240308696,499500,0,8e15cb75,off,,,,,,, +4635,python,insertion,sorted,2000,2463534242,20,1523,1,254907286,239228353,1999,0,69842aa5,off,,,,,,, +4636,python,quick,nearly_sorted,500,2463534242,10,296,1,241634992,226803895,10192,1850,482b4785,off,,,,,,, +4637,c,insertion,nearly_sorted,2000,2463534242,29,2962,1,218407612,217102527,118277,118270,5835dc59,off,,,,,,, +4638,c,bubble,sorted,2000,2463534242,19,173,1,228940551,227955476,1999000,0,69842aa5,off,,,,,,, +4639,c,bubble,nearly_sorted,500,2463534242,16,1998,1,232244554,231272162,124750,14926,482b4785,off,,,,,,, +4640,c,insertion,sorted,250,2463534242,1,786886,1,241253868,240126685,249,0,7fd8c1f4,off,,,,,,, +4641,node,bubble,nearly_sorted,1000,2463534242,26,413,1,255893233,228205821,499500,59780,1cfe4afd,off,,,,,,, +4642,c,bubble,nearly_sorted,1000,2463534242,14,507,1,235035555,234004372,499500,59780,1cfe4afd,off,,,,,,, +4643,c,insertion,random,1000,2463534242,21,1983,1,239478911,238228540,249457,249457,57561285,off,,,,,,, +4644,c,bubble,random,500,2463534242,29,635,1,229004592,228044152,124750,119688,11ae932d,off,,,,,,, +4645,c,insertion,nearly_sorted,250,2463534242,11,99421,1,239708182,238782377,2277,2251,21a19c14,off,,,,,,, +4646,c,insertion,nearly_sorted,250,2463534242,17,99421,1,238074879,236899574,2277,2251,21a19c14,off,,,,,,, +4647,python,insertion,reversed,1000,2463534242,20,6,1,326356827,310255280,499500,500499,0ca43265,off,,,,,,, +4648,node,none,sorted,2000,2463534242,6,0,10,247487120,591,0,0,69842aa5,off,,,,,,, +4649,python,insertion,reversed,1000,2463534242,22,6,1,298162480,281651097,499500,500499,0ca43265,off,,,,,,, +4650,python,insertion,reversed,500,2463534242,22,22,1,242944275,227649553,124750,125249,d33e3559,off,,,,,,, +4651,python,bubble,nearly_sorted,1000,2463534242,18,9,1,251059635,234702804,499500,59780,1cfe4afd,off,,,,,,, +4652,python,insertion,random,2000,2463534242,4,3,1,312472156,296468203,996390,996389,41262855,off,,,,,,, +4653,python,insertion,nearly_sorted,250,2463534242,16,1416,1,255951172,239563907,2277,2251,21a19c14,off,,,,,,, +4654,python,none,random,250,2463534242,18,0,17,226008337,521,0,0,b47adce4,off,,,,,,, +4655,node,insertion,nearly_sorted,2000,2463534242,4,2100,1,255381635,227098133,118277,118270,5835dc59,off,,,,,,, +4656,node,bubble,nearly_sorted,2000,2463534242,22,110,1,259430011,231811885,1999000,232556,5835dc59,off,,,,,,, +4657,c,quick,reversed,2000,2463534242,29,209,1,242322030,241336673,1999000,2000,ad35ba05,off,,,,,,, +4658,python,bubble,random,500,2463534242,30,26,1,270739156,255575767,124750,119688,11ae932d,off,,,,,,, +4659,c,insertion,sorted,2000,2463534242,30,101352,1,235971238,234821414,1999,0,69842aa5,off,,,,,,, +4660,python,none,random,2000,2463534242,28,0,16,243370435,861,0,0,41262855,off,,,,,,, +4661,python,quick,nearly_sorted,500,2463534242,7,296,1,233247084,218094721,10192,1850,482b4785,off,,,,,,, +4662,c,none,sorted,2000,2463534242,5,0,416,230822056,80,0,0,69842aa5,off,,,,,,, +4663,python,none,sorted,2000,2463534242,11,0,15,215137644,621,0,0,69842aa5,off,,,,,,, +4664,c,insertion,nearly_sorted,500,2463534242,22,32350,1,236633049,235490246,7962,7926,482b4785,off,,,,,,, +4665,node,bubble,nearly_sorted,250,2463534242,23,6161,1,263107181,235842975,31125,4056,21a19c14,off,,,,,,, +4666,c,quick,sorted,250,2463534242,6,10851,1,245140070,244131020,31125,0,7fd8c1f4,off,,,,,,, +4667,node,insertion,random,2000,2463534242,7,234,1,265690344,236985567,996390,996389,41262855,off,,,,,,, +4668,node,quick,reversed,2000,2463534242,26,80,1,269700639,241586261,1999000,2000,ad35ba05,off,,,,,,, +4669,node,bubble,random,2000,2463534242,6,98,1,260156052,233343614,1999000,1988792,41262855,off,,,,,,, +4670,python,none,nearly_sorted,250,2463534242,22,0,17,226847820,441,0,0,21a19c14,off,,,,,,, +4671,python,bubble,sorted,500,2463534242,21,46,1,260445775,244010565,124750,0,d9a24379,off,,,,,,, +4672,c,insertion,sorted,2000,2463534242,4,101352,1,237711640,236637667,1999,0,69842aa5,off,,,,,,, +4673,python,quick,random,250,2463534242,17,1271,1,242397942,225883398,1933,1940,b47adce4,off,,,,,,, +4674,c,insertion,random,500,2463534242,18,6945,1,208974334,207756914,60338,60339,11ae932d,off,,,,,,, +4675,node,none,random,500,2463534242,20,0,9,224840288,732,0,0,11ae932d,off,,,,,,, +4676,node,none,reversed,2000,2463534242,3,0,9,228151774,781,0,0,ad35ba05,off,,,,,,, +4677,node,quick,sorted,2000,2463534242,9,74,1,264943805,237381248,1999000,0,69842aa5,off,,,,,,, +4678,c,quick,random,250,2463534242,7,136055,1,240305531,239168630,1933,1940,b47adce4,off,,,,,,, +4679,c,none,random,250,2463534242,30,0,429,229778142,91,0,0,b47adce4,off,,,,,,, +4680,c,quick,reversed,2000,2463534242,15,209,1,253115167,252221042,1999000,2000,ad35ba05,off,,,,,,, +4681,c,bubble,nearly_sorted,1000,2463534242,17,507,1,238797140,237675632,499500,59780,1cfe4afd,off,,,,,,, +4682,c,bubble,sorted,250,2463534242,23,11386,1,241367084,239940537,31125,0,7fd8c1f4,off,,,,,,, +4683,python,none,sorted,1000,2463534242,8,0,16,217377261,561,0,0,8e15cb75,off,,,,,,, +4684,c,bubble,nearly_sorted,1000,2463534242,13,507,1,239783432,238626654,499500,59780,1cfe4afd,off,,,,,,, +4685,python,none,reversed,2000,2463534242,11,0,16,225566769,612,0,0,ad35ba05,off,,,,,,, +4686,python,bubble,nearly_sorted,500,2463534242,11,41,1,253325862,236931383,124750,14926,482b4785,off,,,,,,, +4687,c,bubble,reversed,1000,2463534242,1,87,1,236825340,235937215,499500,999000,0ca43265,off,,,,,,, +4688,python,bubble,reversed,2000,2463534242,14,2,1,473151312,457598299,1999000,3998000,ad35ba05,off,,,,,,, +4689,python,bubble,random,1000,2463534242,4,6,1,263194685,247675558,499500,496926,57561285,off,,,,,,, +4690,node,none,random,250,2463534242,21,0,10,251645883,641,0,0,b47adce4,off,,,,,,, +4691,node,quick,random,250,2463534242,21,36348,1,247685912,222014908,1933,1940,b47adce4,off,,,,,,, +4692,c,quick,reversed,250,2463534242,21,12781,1,239956596,238808916,31125,250,be175004,off,,,,,,, +4693,node,quick,sorted,2000,2463534242,20,74,1,267434191,237272589,1999000,0,69842aa5,off,,,,,,, +4694,c,quick,nearly_sorted,250,2463534242,10,77221,1,233886311,232807671,4346,1008,21a19c14,off,,,,,,, +4695,python,quick,nearly_sorted,1000,2463534242,19,110,1,257171990,240802373,28499,7892,1cfe4afd,off,,,,,,, +4696,c,insertion,random,250,2463534242,13,26020,1,231762381,230517398,15844,15838,b47adce4,off,,,,,,, +4697,python,quick,sorted,1000,2463534242,30,8,1,279945989,264232647,499500,0,8e15cb75,off,,,,,,, +4698,node,insertion,reversed,2000,2463534242,8,144,1,285174949,259062655,1999000,2000999,ad35ba05,off,,,,,,, +4699,node,insertion,nearly_sorted,1000,2463534242,20,7758,1,257501487,227207733,30889,30860,1cfe4afd,off,,,,,,, +4700,c,bubble,random,2000,2463534242,23,40,1,238604984,237693928,1999000,1988792,41262855,off,,,,,,, +4701,node,none,nearly_sorted,2000,2463534242,30,0,9,231312697,551,0,0,5835dc59,off,,,,,,, +4702,c,insertion,random,250,2463534242,23,26020,1,234572226,233585286,15844,15838,b47adce4,off,,,,,,, +4703,node,none,reversed,1000,2463534242,23,0,9,233896771,801,0,0,0ca43265,off,,,,,,, +4704,c,quick,random,2000,2463534242,6,10587,1,248788949,247818500,25197,24544,41262855,off,,,,,,, +4705,node,bubble,sorted,2000,2463534242,23,85,1,267805999,238603260,1999000,0,69842aa5,off,,,,,,, +4706,c,bubble,reversed,2000,2463534242,25,22,1,248590968,247371153,1999000,3998000,ad35ba05,off,,,,,,, +4707,node,quick,random,2000,2463534242,18,2126,1,256954252,229052053,25197,24544,41262855,off,,,,,,, +4708,c,insertion,sorted,1000,2463534242,28,196883,1,231570490,230412701,999,0,8e15cb75,off,,,,,,, +4709,node,bubble,random,2000,2463534242,8,98,1,263623660,236092038,1999000,1988792,41262855,off,,,,,,, +4710,c,bubble,sorted,500,2463534242,8,2858,1,240691685,239600836,124750,0,d9a24379,off,,,,,,, +4711,node,quick,reversed,1000,2463534242,18,307,1,264535670,238054798,499500,1000,0ca43265,off,,,,,,, +4712,node,none,reversed,500,2463534242,14,0,9,222286532,791,0,0,d33e3559,off,,,,,,, +4713,node,insertion,random,500,2463534242,22,4059,1,244867420,218471969,60338,60339,11ae932d,off,,,,,,, +4714,c,insertion,nearly_sorted,1000,2463534242,1,11445,1,246490872,245445954,30889,30860,1cfe4afd,off,,,,,,, +4715,c,insertion,sorted,500,2463534242,18,399335,1,234245063,233143328,499,0,d9a24379,off,,,,,,, +4716,c,insertion,sorted,250,2463534242,21,786886,1,236366530,235296864,249,0,7fd8c1f4,off,,,,,,, +4717,node,insertion,reversed,500,2463534242,18,2178,1,252032085,225934433,124750,125249,d33e3559,off,,,,,,, +4718,node,bubble,random,1000,2463534242,1,362,1,262759462,234734839,499500,496926,57561285,off,,,,,,, +4719,c,quick,random,1000,2463534242,10,23484,1,234987544,233972151,11196,10210,57561285,off,,,,,,, +4720,python,none,random,1000,2463534242,2,0,15,208484836,581,0,0,57561285,off,,,,,,, +4721,node,bubble,nearly_sorted,500,2463534242,8,1566,1,262162492,235401280,124750,14926,482b4785,off,,,,,,, +4722,node,quick,sorted,1000,2463534242,24,296,1,264156840,238089433,499500,0,8e15cb75,off,,,,,,, +4723,node,bubble,reversed,2000,2463534242,15,94,1,269039390,240969267,1999000,3998000,ad35ba05,off,,,,,,, +4724,c,insertion,sorted,2000,2463534242,17,101352,1,233713787,232552220,1999,0,69842aa5,off,,,,,,, +4725,python,quick,sorted,250,2463534242,28,133,1,248201729,232756218,31125,0,7fd8c1f4,off,,,,,,, +4726,python,bubble,random,2000,2463534242,29,2,1,360440622,343963108,1999000,1988792,41262855,off,,,,,,, +4727,c,quick,nearly_sorted,250,2463534242,9,77221,1,234682052,233686927,4346,1008,21a19c14,off,,,,,,, +4728,node,none,random,2000,2463534242,10,0,9,235364671,731,0,0,41262855,off,,,,,,, +4729,node,quick,nearly_sorted,2000,2463534242,5,1909,1,255262752,226079170,58002,11768,5835dc59,off,,,,,,, +4730,python,insertion,random,1000,2463534242,9,10,1,260095929,244506623,249457,249457,57561285,off,,,,,,, +4731,c,bubble,nearly_sorted,250,2463534242,16,7880,1,239657105,238394469,31125,4056,21a19c14,off,,,,,,, +4732,python,none,nearly_sorted,1000,2463534242,28,0,16,216999563,541,0,0,1cfe4afd,off,,,,,,, +4733,node,quick,reversed,1000,2463534242,12,307,1,260061966,233070514,499500,1000,0ca43265,off,,,,,,, +4734,python,none,reversed,2000,2463534242,30,0,16,226116580,541,0,0,ad35ba05,off,,,,,,, +4735,node,quick,sorted,500,2463534242,9,1164,1,263812274,237847109,124750,0,d9a24379,off,,,,,,, +4736,c,insertion,reversed,2000,2463534242,15,260,1,240061143,239074754,1999000,2000999,ad35ba05,off,,,,,,, +4737,python,bubble,sorted,250,2463534242,2,206,1,248880712,233767477,31125,0,7fd8c1f4,off,,,,,,, +4738,node,none,random,1000,2463534242,13,0,10,246218343,791,0,0,57561285,off,,,,,,, +4739,node,insertion,reversed,1000,2463534242,5,554,1,270961120,243548132,499500,500499,0ca43265,off,,,,,,, +4740,python,none,sorted,500,2463534242,30,0,16,213086610,531,0,0,d9a24379,off,,,,,,, +4741,c,insertion,sorted,2000,2463534242,2,101352,1,233086642,231973983,1999,0,69842aa5,off,,,,,,, +4742,python,insertion,random,500,2463534242,29,44,1,246350285,230310142,60338,60339,11ae932d,off,,,,,,, +4743,node,bubble,sorted,2000,2463534242,3,85,1,260103027,233135969,1999000,0,69842aa5,off,,,,,,, +4744,node,none,sorted,1000,2463534242,16,0,9,221146686,731,0,0,8e15cb75,off,,,,,,, +4745,c,quick,reversed,2000,2463534242,14,209,1,232543404,231596619,1999000,2000,ad35ba05,off,,,,,,, +4746,python,bubble,reversed,250,2463534242,25,88,1,264324233,248097149,31125,62250,be175004,off,,,,,,, +4747,python,none,reversed,250,2463534242,9,0,17,220003103,541,0,0,be175004,off,,,,,,, +4748,python,none,sorted,1000,2463534242,10,0,16,215080898,581,0,0,8e15cb75,off,,,,,,, +4749,node,bubble,sorted,2000,2463534242,9,85,1,262716049,234315578,1999000,0,69842aa5,off,,,,,,, +4750,node,quick,nearly_sorted,1000,2463534242,23,3752,1,255239127,228304900,28499,7892,1cfe4afd,off,,,,,,, +4751,node,bubble,reversed,2000,2463534242,27,94,1,268695675,240234380,1999000,3998000,ad35ba05,off,,,,,,, +4752,node,none,reversed,1000,2463534242,25,0,9,221107031,751,0,0,0ca43265,off,,,,,,, +4753,node,none,random,500,2463534242,7,0,9,221155202,742,0,0,11ae932d,off,,,,,,, +4754,python,bubble,random,500,2463534242,27,26,1,256381739,241517488,124750,119688,11ae932d,off,,,,,,, +4755,python,none,reversed,250,2463534242,15,0,17,226969958,511,0,0,be175004,off,,,,,,, +4756,python,none,reversed,2000,2463534242,29,0,16,226108895,601,0,0,ad35ba05,off,,,,,,, +4757,node,insertion,sorted,1000,2463534242,23,160107,1,248428152,221335824,999,0,8e15cb75,off,,,,,,, +4758,node,bubble,sorted,250,2463534242,25,8081,1,251506643,224173993,31125,0,7fd8c1f4,off,,,,,,, +4759,c,insertion,random,1000,2463534242,6,1983,1,238941545,237843065,249457,249457,57561285,off,,,,,,, +4760,python,quick,random,1000,2463534242,29,209,1,257126815,240444122,11196,10210,57561285,off,,,,,,, +4761,c,bubble,nearly_sorted,500,2463534242,8,1998,1,235521856,234410134,124750,14926,482b4785,off,,,,,,, +4762,node,bubble,sorted,500,2463534242,7,1964,1,265789019,239104855,124750,0,d9a24379,off,,,,,,, +4763,node,bubble,sorted,1000,2463534242,2,374,1,253136688,226625081,499500,0,8e15cb75,off,,,,,,, +4764,python,bubble,sorted,250,2463534242,28,206,1,263582243,247921237,31125,0,7fd8c1f4,off,,,,,,, +4765,c,bubble,nearly_sorted,250,2463534242,21,7880,1,231848462,230640560,31125,4056,21a19c14,off,,,,,,, +4766,c,quick,sorted,1000,2463534242,23,691,1,242578902,241662094,499500,0,8e15cb75,off,,,,,,, +4767,c,none,reversed,2000,2463534242,29,0,413,226333314,91,0,0,ad35ba05,off,,,,,,, +4768,c,insertion,reversed,500,2463534242,14,3967,1,233236784,232417769,124750,125249,d33e3559,off,,,,,,, +4769,python,bubble,reversed,250,2463534242,17,88,1,247978731,232245614,31125,62250,be175004,off,,,,,,, +4770,node,none,nearly_sorted,2000,2463534242,19,0,9,237458305,621,0,0,5835dc59,off,,,,,,, +4771,c,insertion,nearly_sorted,250,2463534242,25,99421,1,239194789,238191279,2277,2251,21a19c14,off,,,,,,, +4772,python,bubble,reversed,1000,2463534242,10,5,1,280683953,265394775,499500,999000,0ca43265,off,,,,,,, +4773,python,quick,random,2000,2463534242,17,90,1,248178375,232481163,25197,24544,41262855,off,,,,,,, +4774,node,quick,sorted,250,2463534242,19,4507,1,258203703,232147292,31125,0,7fd8c1f4,off,,,,,,, +4775,c,bubble,random,2000,2463534242,16,40,1,236243038,235210053,1999000,1988792,41262855,off,,,,,,, +4776,node,bubble,reversed,2000,2463534242,7,94,1,261636627,232668044,1999000,3998000,ad35ba05,off,,,,,,, +4777,c,bubble,random,2000,2463534242,19,40,1,238395791,237327369,1999000,1988792,41262855,off,,,,,,, +4778,python,bubble,random,2000,2463534242,24,2,1,352880184,336670609,1999000,1988792,41262855,off,,,,,,, +4779,c,bubble,random,2000,2463534242,9,40,1,239169221,237859006,1999000,1988792,41262855,off,,,,,,, +4780,node,none,sorted,500,2463534242,21,0,10,243809587,721,0,0,d9a24379,off,,,,,,, +4781,c,quick,random,2000,2463534242,17,10587,1,240305641,239226189,25197,24544,41262855,off,,,,,,, +4782,python,none,random,2000,2463534242,23,0,16,239251175,1563,0,0,41262855,off,,,,,,, +4783,c,bubble,random,250,2463534242,9,2568,1,237626159,236570783,31125,31196,b47adce4,off,,,,,,, +4784,python,bubble,nearly_sorted,250,2463534242,11,188,1,257240430,242445553,31125,4056,21a19c14,off,,,,,,, +4785,python,insertion,nearly_sorted,2000,2463534242,20,20,1,260710981,245024027,118277,118270,5835dc59,off,,,,,,, +4786,node,quick,random,500,2463534242,2,16148,1,261446199,233071428,4578,4552,11ae932d,off,,,,,,, +4787,c,quick,nearly_sorted,2000,2463534242,9,6145,1,243844222,242662788,58002,11768,5835dc59,off,,,,,,, +4788,c,none,nearly_sorted,1000,2463534242,24,0,421,227962315,80,0,0,1cfe4afd,off,,,,,,, +4789,python,quick,sorted,1000,2463534242,8,8,1,286519342,271319884,499500,0,8e15cb75,off,,,,,,, +4790,python,quick,random,1000,2463534242,12,209,1,256618843,240474918,11196,10210,57561285,off,,,,,,, +4791,python,insertion,nearly_sorted,500,2463534242,29,316,1,250668382,235080117,7962,7926,482b4785,off,,,,,,, +4792,c,insertion,nearly_sorted,1000,2463534242,12,11445,1,241651363,240454579,30889,30860,1cfe4afd,off,,,,,,, +4793,python,bubble,random,500,2463534242,4,26,1,249682704,235023928,124750,119688,11ae932d,off,,,,,,, +4794,python,none,nearly_sorted,500,2463534242,23,0,17,226725901,551,0,0,482b4785,off,,,,,,, +4795,python,quick,sorted,1000,2463534242,10,8,1,283894170,269463379,499500,0,8e15cb75,off,,,,,,, +4796,c,none,reversed,1000,2463534242,16,0,405,220823279,80,0,0,0ca43265,off,,,,,,, +4797,node,none,sorted,1000,2463534242,10,0,9,220318103,781,0,0,8e15cb75,off,,,,,,, +4798,python,insertion,reversed,250,2463534242,21,102,1,243756548,228380844,31125,31374,be175004,off,,,,,,, +4799,c,none,sorted,250,2463534242,4,0,429,228772015,80,0,0,7fd8c1f4,off,,,,,,, +4800,node,insertion,sorted,500,2463534242,26,324764,1,254852193,227882600,499,0,d9a24379,off,,,,,,, +4801,python,insertion,random,250,2463534242,17,206,1,252566118,237044282,15844,15838,b47adce4,off,,,,,,, +4802,python,none,nearly_sorted,500,2463534242,7,0,17,225269342,531,0,0,482b4785,off,,,,,,, +4803,node,insertion,sorted,500,2463534242,15,324764,1,255328765,228729513,499,0,d9a24379,off,,,,,,, +4804,node,none,reversed,250,2463534242,3,0,9,217935016,651,0,0,be175004,off,,,,,,, +4805,node,quick,random,250,2463534242,6,36348,1,244591102,218407590,1933,1940,b47adce4,off,,,,,,, +4806,c,none,nearly_sorted,1000,2463534242,26,0,421,234747685,90,0,0,1cfe4afd,off,,,,,,, +4807,node,none,reversed,1000,2463534242,30,0,9,221299281,772,0,0,0ca43265,off,,,,,,, +4808,node,none,reversed,2000,2463534242,27,0,9,228427130,731,0,0,ad35ba05,off,,,,,,, +4809,c,none,random,250,2463534242,15,0,429,228874137,91,0,0,b47adce4,off,,,,,,, +4810,python,quick,reversed,500,2463534242,6,33,1,255819606,241108876,124750,500,d33e3559,off,,,,,,, +4811,python,bubble,reversed,1000,2463534242,30,5,1,281408942,265210441,499500,999000,0ca43265,off,,,,,,, +4812,c,bubble,random,250,2463534242,11,2568,1,233792124,232888617,31125,31196,b47adce4,off,,,,,,, +4813,node,none,sorted,500,2463534242,15,0,10,243569126,841,0,0,d9a24379,off,,,,,,, +4814,node,insertion,reversed,2000,2463534242,26,144,1,261409602,231807085,1999000,2000999,ad35ba05,off,,,,,,, +4815,python,quick,nearly_sorted,2000,2463534242,15,55,1,253625122,237235514,58002,11768,5835dc59,off,,,,,,, +4816,node,none,random,2000,2463534242,25,0,9,228819956,841,0,0,41262855,off,,,,,,, +4817,c,insertion,sorted,2000,2463534242,16,101352,1,238471243,237545388,1999,0,69842aa5,off,,,,,,, +4818,python,bubble,nearly_sorted,500,2463534242,26,41,1,254950537,238429439,124750,14926,482b4785,off,,,,,,, +4819,c,none,reversed,1000,2463534242,13,0,405,223557073,80,0,0,0ca43265,off,,,,,,, +4820,python,none,random,250,2463534242,20,0,17,227339701,541,0,0,b47adce4,off,,,,,,, +4821,node,quick,sorted,250,2463534242,13,4507,1,258529143,231755706,31125,0,7fd8c1f4,off,,,,,,, +4822,python,bubble,nearly_sorted,250,2463534242,8,188,1,253099317,237751347,31125,4056,21a19c14,off,,,,,,, +4823,c,none,sorted,2000,2463534242,21,0,416,230616353,90,0,0,69842aa5,off,,,,,,, +4824,python,quick,reversed,2000,2463534242,14,2,1,268845272,253028046,1999000,2000,ad35ba05,off,,,,,,, +4825,python,none,nearly_sorted,500,2463534242,8,0,17,251627084,571,0,0,482b4785,off,,,,,,, +4826,python,bubble,random,2000,2463534242,18,2,1,381803257,365363133,1999000,1988792,41262855,off,,,,,,, +4827,python,insertion,nearly_sorted,250,2463534242,6,1416,1,267177581,251892811,2277,2251,21a19c14,off,,,,,,, +4828,python,insertion,sorted,250,2463534242,29,18845,1,259346424,244180052,249,0,7fd8c1f4,off,,,,,,, +4829,c,quick,random,500,2463534242,9,55776,1,235529420,233941004,4578,4552,11ae932d,off,,,,,,, +4830,c,insertion,reversed,1000,2463534242,5,1033,1,234514688,233301505,499500,500499,0ca43265,off,,,,,,, +4831,node,none,sorted,2000,2463534242,28,0,10,253653756,951,0,0,69842aa5,off,,,,,,, +4832,python,insertion,sorted,1000,2463534242,2,3178,1,258466065,242650609,999,0,8e15cb75,off,,,,,,, +4833,c,bubble,reversed,250,2463534242,7,1384,1,235926856,234905602,31125,62250,be175004,off,,,,,,, +4834,python,bubble,sorted,1000,2463534242,18,11,1,278731013,263511454,499500,0,8e15cb75,off,,,,,,, +4835,c,insertion,nearly_sorted,250,2463534242,30,99421,1,236628890,235460461,2277,2251,21a19c14,off,,,,,,, +4836,c,none,random,1000,2463534242,18,0,367,198525692,100,0,0,57561285,off,,,,,,, +4837,c,bubble,nearly_sorted,250,2463534242,30,7880,1,236307348,235594724,31125,4056,21a19c14,off,,,,,,, +4838,node,quick,random,250,2463534242,28,36348,1,246699152,218284837,1933,1940,b47adce4,off,,,,,,, +4839,python,insertion,nearly_sorted,2000,2463534242,19,20,1,263943450,247372099,118277,118270,5835dc59,off,,,,,,, +4840,node,quick,nearly_sorted,2000,2463534242,6,1909,1,254068093,226018972,58002,11768,5835dc59,off,,,,,,, +4841,node,bubble,reversed,250,2463534242,20,5861,1,259166668,230332155,31125,62250,be175004,off,,,,,,, +4842,c,bubble,reversed,2000,2463534242,18,22,1,241862419,240975406,1999000,3998000,ad35ba05,off,,,,,,, +4843,python,insertion,reversed,500,2463534242,4,22,1,251595579,237055085,124750,125249,d33e3559,off,,,,,,, +4844,node,none,nearly_sorted,250,2463534242,17,0,10,248687109,602,0,0,21a19c14,off,,,,,,, +4845,c,bubble,reversed,500,2463534242,20,346,1,238919623,237965725,124750,249500,d33e3559,off,,,,,,, +4846,python,none,nearly_sorted,250,2463534242,24,0,17,233495921,521,0,0,21a19c14,off,,,,,,, +4847,python,none,nearly_sorted,1000,2463534242,22,0,16,222617239,541,0,0,1cfe4afd,off,,,,,,, +4848,python,none,sorted,1000,2463534242,22,0,16,217291742,511,0,0,8e15cb75,off,,,,,,, +4849,node,insertion,reversed,2000,2463534242,2,144,1,264637009,236227576,1999000,2000999,ad35ba05,off,,,,,,, +4850,node,insertion,random,250,2463534242,30,15349,1,248898395,222866458,15844,15838,b47adce4,off,,,,,,, +4851,c,quick,nearly_sorted,250,2463534242,14,77221,1,237774138,236605959,4346,1008,21a19c14,off,,,,,,, +4852,node,bubble,sorted,500,2463534242,10,1964,1,269071409,238089362,124750,0,d9a24379,off,,,,,,, +4853,python,none,nearly_sorted,1000,2463534242,21,0,16,215200141,561,0,0,1cfe4afd,off,,,,,,, +4854,c,insertion,reversed,500,2463534242,10,3967,1,236020761,235220611,124750,125249,d33e3559,off,,,,,,, +4855,node,insertion,sorted,1000,2463534242,13,160107,1,247402611,220443183,999,0,8e15cb75,off,,,,,,, +4856,python,none,random,500,2463534242,17,0,16,217817717,601,0,0,11ae932d,off,,,,,,, +4857,c,bubble,nearly_sorted,2000,2463534242,5,121,1,238498584,237357797,1999000,232556,5835dc59,off,,,,,,, +4858,python,bubble,sorted,250,2463534242,6,206,1,250627985,235394938,31125,0,7fd8c1f4,off,,,,,,, +4859,python,bubble,sorted,250,2463534242,27,206,1,255709309,240732288,31125,0,7fd8c1f4,off,,,,,,, +4860,c,none,sorted,250,2463534242,26,0,429,233576179,90,0,0,7fd8c1f4,off,,,,,,, +4861,python,quick,sorted,500,2463534242,15,30,1,244743047,229229716,124750,0,d9a24379,off,,,,,,, +4862,python,none,nearly_sorted,500,2463534242,30,0,17,224989337,521,0,0,482b4785,off,,,,,,, +4863,node,bubble,nearly_sorted,2000,2463534242,15,110,1,260855282,234049379,1999000,232556,5835dc59,off,,,,,,, +4864,node,quick,reversed,500,2463534242,28,1194,1,262640528,233898352,124750,500,d33e3559,off,,,,,,, +4865,c,bubble,random,250,2463534242,30,2568,1,232292304,231324300,31125,31196,b47adce4,off,,,,,,, +4866,python,none,sorted,1000,2463534242,30,0,16,220117736,591,0,0,8e15cb75,off,,,,,,, +4867,c,bubble,reversed,250,2463534242,16,1384,1,231835327,230960618,31125,62250,be175004,off,,,,,,, +4868,c,insertion,nearly_sorted,1000,2463534242,18,11445,1,237153023,236121840,30889,30860,1cfe4afd,off,,,,,,, +4869,python,insertion,sorted,1000,2463534242,23,3178,1,242441784,226423738,999,0,8e15cb75,off,,,,,,, +4870,node,none,random,2000,2463534242,13,0,9,231283262,742,0,0,41262855,off,,,,,,, +4871,python,bubble,sorted,500,2463534242,23,46,1,261228714,246558349,124750,0,d9a24379,off,,,,,,, +4872,python,bubble,reversed,500,2463534242,15,20,1,259938553,243773842,124750,249500,d33e3559,off,,,,,,, +4873,node,quick,random,250,2463534242,24,36348,1,250964115,224093640,1933,1940,b47adce4,off,,,,,,, +4874,python,insertion,sorted,1000,2463534242,7,3178,1,243166342,228220491,999,0,8e15cb75,off,,,,,,, +4875,node,bubble,reversed,250,2463534242,22,5861,1,251841309,223836276,31125,62250,be175004,off,,,,,,, +4876,python,none,reversed,250,2463534242,1,0,17,224428626,551,0,0,be175004,off,,,,,,, +4877,python,none,random,500,2463534242,28,0,16,216933700,521,0,0,11ae932d,off,,,,,,, +4878,node,quick,reversed,250,2463534242,10,4755,1,262335075,235469373,31125,250,be175004,off,,,,,,, +4879,c,quick,reversed,500,2463534242,17,3240,1,236514485,235310391,124750,500,d33e3559,off,,,,,,, +4880,node,insertion,reversed,2000,2463534242,10,144,1,290958518,261878213,1999000,2000999,ad35ba05,off,,,,,,, +4881,node,quick,reversed,250,2463534242,22,4755,1,261176384,233332038,31125,250,be175004,off,,,,,,, +4882,c,none,reversed,1000,2463534242,22,0,405,218815527,100,0,0,0ca43265,off,,,,,,, +4883,python,none,reversed,1000,2463534242,21,0,16,215712842,582,0,0,0ca43265,off,,,,,,, +4884,c,bubble,reversed,1000,2463534242,14,87,1,234892707,233803716,499500,999000,0ca43265,off,,,,,,, +4885,c,quick,reversed,250,2463534242,28,12781,1,235919330,234506363,31125,250,be175004,off,,,,,,, +4886,node,quick,nearly_sorted,1000,2463534242,12,3752,1,252583210,224622054,28499,7892,1cfe4afd,off,,,,,,, +4887,c,bubble,random,500,2463534242,15,635,1,224029337,222925368,124750,119688,11ae932d,off,,,,,,, +4888,python,insertion,sorted,250,2463534242,17,18845,1,261767512,245934281,249,0,7fd8c1f4,off,,,,,,, +4889,python,bubble,sorted,1000,2463534242,15,11,1,275285616,260547732,499500,0,8e15cb75,off,,,,,,, +4890,python,none,nearly_sorted,2000,2463534242,3,0,15,213958224,602,0,0,5835dc59,off,,,,,,, +4891,python,bubble,nearly_sorted,2000,2463534242,7,3,1,353382295,337196434,1999000,232556,5835dc59,off,,,,,,, +4892,node,none,nearly_sorted,250,2463534242,9,0,10,245850252,622,0,0,21a19c14,off,,,,,,, +4893,node,quick,sorted,500,2463534242,6,1164,1,264814673,237342445,124750,0,d9a24379,off,,,,,,, +4894,python,none,nearly_sorted,1000,2463534242,11,0,16,216867355,541,0,0,1cfe4afd,off,,,,,,, +4895,node,bubble,nearly_sorted,500,2463534242,27,1566,1,260177893,233895047,124750,14926,482b4785,off,,,,,,, +4896,c,none,random,500,2463534242,10,0,426,234958780,90,0,0,11ae932d,off,,,,,,, +4897,c,none,sorted,500,2463534242,3,0,344,185756701,80,0,0,d9a24379,off,,,,,,, +4898,c,none,nearly_sorted,2000,2463534242,18,0,387,210198538,90,0,0,5835dc59,off,,,,,,, +4899,python,insertion,random,1000,2463534242,26,10,1,257674873,242120978,249457,249457,57561285,off,,,,,,, +4900,c,quick,sorted,250,2463534242,11,10851,1,241892616,240645439,31125,0,7fd8c1f4,off,,,,,,, +4901,node,quick,sorted,1000,2463534242,20,296,1,268957565,239128843,499500,0,8e15cb75,off,,,,,,, +4902,python,quick,nearly_sorted,250,2463534242,17,882,1,257054378,242857618,4346,1008,21a19c14,off,,,,,,, +4903,c,bubble,random,2000,2463534242,17,40,1,233989445,232710759,1999000,1988792,41262855,off,,,,,,, +4904,python,insertion,random,250,2463534242,6,206,1,248991116,233658621,15844,15838,b47adce4,off,,,,,,, +4905,c,quick,random,250,2463534242,30,136055,1,235091259,234021284,1933,1940,b47adce4,off,,,,,,, +4906,python,insertion,sorted,500,2463534242,7,6624,1,228843781,213842194,499,0,d9a24379,off,,,,,,, +4907,c,bubble,random,500,2463534242,16,635,1,225722521,224798740,124750,119688,11ae932d,off,,,,,,, +4908,c,quick,nearly_sorted,1000,2463534242,16,11273,1,237525481,236436401,28499,7892,1cfe4afd,off,,,,,,, +4909,node,none,random,2000,2463534242,6,0,9,233938938,822,0,0,41262855,off,,,,,,, +4910,c,insertion,random,500,2463534242,20,6945,1,217435020,216479470,60338,60339,11ae932d,off,,,,,,, +4911,node,none,nearly_sorted,1000,2463534242,9,0,10,256614906,651,0,0,1cfe4afd,off,,,,,,, +4912,python,bubble,sorted,250,2463534242,19,206,1,252716651,237119672,31125,0,7fd8c1f4,off,,,,,,, +4913,c,none,reversed,2000,2463534242,18,0,413,224442242,80,0,0,ad35ba05,off,,,,,,, +4914,node,bubble,reversed,250,2463534242,21,5861,1,249645093,222787442,31125,62250,be175004,off,,,,,,, +4915,python,bubble,sorted,250,2463534242,10,206,1,250840014,234192879,31125,0,7fd8c1f4,off,,,,,,, +4916,python,none,random,1000,2463534242,14,0,15,210038988,631,0,0,57561285,off,,,,,,, +4917,python,bubble,reversed,1000,2463534242,27,5,1,284723563,269963794,499500,999000,0ca43265,off,,,,,,, +4918,node,bubble,sorted,2000,2463534242,22,85,1,259408883,232757019,1999000,0,69842aa5,off,,,,,,, +4919,node,none,random,1000,2463534242,27,0,10,242712011,6032,0,0,57561285,off,,,,,,, +4920,node,quick,nearly_sorted,1000,2463534242,1,3752,1,256447922,228071587,28499,7892,1cfe4afd,off,,,,,,, +4921,c,quick,random,1000,2463534242,16,23484,1,233415338,232297924,11196,10210,57561285,off,,,,,,, +4922,c,insertion,reversed,1000,2463534242,2,1033,1,237849838,236548421,499500,500499,0ca43265,off,,,,,,, +4923,node,quick,sorted,250,2463534242,25,4507,1,260936254,232295111,31125,0,7fd8c1f4,off,,,,,,, +4924,python,none,nearly_sorted,2000,2463534242,10,0,15,215259672,681,0,0,5835dc59,off,,,,,,, +4925,c,none,random,2000,2463534242,13,0,415,226277191,91,0,0,41262855,off,,,,,,, +4926,python,none,reversed,2000,2463534242,5,0,16,228625091,561,0,0,ad35ba05,off,,,,,,, +4927,python,insertion,sorted,500,2463534242,25,6624,1,235677167,221069010,499,0,d9a24379,off,,,,,,, +4928,python,quick,random,2000,2463534242,13,90,1,255560119,238833285,25197,24544,41262855,off,,,,,,, +4929,c,insertion,nearly_sorted,250,2463534242,16,99421,1,239143363,238122720,2277,2251,21a19c14,off,,,,,,, +4930,python,quick,nearly_sorted,250,2463534242,14,882,1,259993007,244290082,4346,1008,21a19c14,off,,,,,,, +4931,c,none,nearly_sorted,500,2463534242,23,0,405,221535320,100,0,0,482b4785,off,,,,,,, +4932,python,insertion,sorted,250,2463534242,7,18845,1,256057645,240597776,249,0,7fd8c1f4,off,,,,,,, +4933,python,quick,sorted,1000,2463534242,23,8,1,287192401,270997643,499500,0,8e15cb75,off,,,,,,, +4934,c,bubble,sorted,1000,2463534242,24,702,1,236350508,234977066,499500,0,8e15cb75,off,,,,,,, +4935,c,insertion,sorted,250,2463534242,8,786886,1,243216162,242248472,249,0,7fd8c1f4,off,,,,,,, +4936,c,none,reversed,250,2463534242,30,0,427,224391019,80,0,0,be175004,off,,,,,,, +4937,c,none,nearly_sorted,250,2463534242,27,0,388,210912115,80,0,0,21a19c14,off,,,,,,, +4938,python,insertion,nearly_sorted,1000,2463534242,26,76,1,245517749,229563921,30889,30860,1cfe4afd,off,,,,,,, +4939,c,bubble,reversed,500,2463534242,23,346,1,241771858,240558906,124750,249500,d33e3559,off,,,,,,, +4940,c,bubble,nearly_sorted,2000,2463534242,7,121,1,232714856,231324309,1999000,232556,5835dc59,off,,,,,,, +4941,node,none,nearly_sorted,500,2463534242,27,0,10,247700329,831,0,0,482b4785,off,,,,,,, +4942,python,quick,random,250,2463534242,20,1271,1,248614733,233490022,1933,1940,b47adce4,off,,,,,,, +4943,python,bubble,sorted,500,2463534242,12,46,1,260192259,244572888,124750,0,d9a24379,off,,,,,,, +4944,python,insertion,random,2000,2463534242,12,3,1,321820993,305284264,996390,996389,41262855,off,,,,,,, +4945,node,quick,sorted,250,2463534242,16,4507,1,266361075,238236304,31125,0,7fd8c1f4,off,,,,,,, +4946,node,bubble,reversed,500,2463534242,16,1534,1,258526628,232420698,124750,249500,d33e3559,off,,,,,,, +4947,node,none,random,1000,2463534242,23,0,10,263372550,1192,0,0,57561285,off,,,,,,, +4948,python,insertion,sorted,2000,2463534242,11,1523,1,264390904,247816312,1999,0,69842aa5,off,,,,,,, +4949,c,quick,sorted,500,2463534242,19,2762,1,246655064,245683317,124750,0,d9a24379,off,,,,,,, +4950,python,bubble,sorted,2000,2463534242,5,3,1,344362331,327564194,1999000,0,69842aa5,off,,,,,,, +4951,node,insertion,reversed,500,2463534242,3,2178,1,255382346,228145694,124750,125249,d33e3559,off,,,,,,, +4952,c,bubble,sorted,500,2463534242,17,2858,1,240286815,239089671,124750,0,d9a24379,off,,,,,,, +4953,python,insertion,random,250,2463534242,3,206,1,249213035,232054605,15844,15838,b47adce4,off,,,,,,, +4954,c,quick,nearly_sorted,2000,2463534242,16,6145,1,241672723,240615642,58002,11768,5835dc59,off,,,,,,, +4955,c,insertion,random,1000,2463534242,16,1983,1,239511562,238043934,249457,249457,57561285,off,,,,,,, +4956,node,quick,reversed,250,2463534242,5,4755,1,260009647,231160298,31125,250,be175004,off,,,,,,, +4957,node,insertion,random,1000,2463534242,7,1091,1,260184346,233145911,249457,249457,57561285,off,,,,,,, +4958,python,insertion,nearly_sorted,250,2463534242,7,1416,1,281702161,266408837,2277,2251,21a19c14,off,,,,,,, +4959,c,insertion,reversed,250,2463534242,27,15645,1,243450294,242054129,31125,31374,be175004,off,,,,,,, +4960,python,insertion,sorted,1000,2463534242,29,3178,1,243920335,228243762,999,0,8e15cb75,off,,,,,,, +4961,c,none,random,1000,2463534242,25,0,367,199802053,80,0,0,57561285,off,,,,,,, +4962,c,bubble,reversed,250,2463534242,10,1384,1,238724508,237798724,31125,62250,be175004,off,,,,,,, +4963,node,insertion,reversed,500,2463534242,13,2178,1,257957441,229298631,124750,125249,d33e3559,off,,,,,,, +4964,node,insertion,random,500,2463534242,29,4059,1,265197380,236953786,60338,60339,11ae932d,off,,,,,,, +4965,node,none,random,1000,2463534242,11,0,10,252261827,822,0,0,57561285,off,,,,,,, +4966,c,insertion,reversed,1000,2463534242,28,1033,1,240917668,239729301,499500,500499,0ca43265,off,,,,,,, +4967,node,bubble,reversed,2000,2463534242,26,94,1,267217055,238007251,1999000,3998000,ad35ba05,off,,,,,,, +4968,node,bubble,sorted,250,2463534242,4,8081,1,254066360,226909430,31125,0,7fd8c1f4,off,,,,,,, +4969,node,bubble,nearly_sorted,500,2463534242,24,1566,1,267450141,239355529,124750,14926,482b4785,off,,,,,,, +4970,node,quick,reversed,2000,2463534242,7,80,1,265017162,238096178,1999000,2000,ad35ba05,off,,,,,,, +4971,python,quick,nearly_sorted,1000,2463534242,7,110,1,260287068,243971589,28499,7892,1cfe4afd,off,,,,,,, +4972,c,bubble,reversed,500,2463534242,17,346,1,237528245,236400763,124750,249500,d33e3559,off,,,,,,, +4973,c,quick,random,1000,2463534242,15,23484,1,233332273,232322852,11196,10210,57561285,off,,,,,,, +4974,node,insertion,nearly_sorted,500,2463534242,11,26045,1,242240046,212479317,7962,7926,482b4785,off,,,,,,, +4975,python,bubble,nearly_sorted,250,2463534242,6,188,1,261440880,246628653,31125,4056,21a19c14,off,,,,,,, +4976,c,quick,random,1000,2463534242,24,23484,1,233683953,232523687,11196,10210,57561285,off,,,,,,, +4977,c,insertion,nearly_sorted,1000,2463534242,30,11445,1,247244603,245668642,30889,30860,1cfe4afd,off,,,,,,, +4978,python,quick,random,1000,2463534242,14,209,1,249089412,233087495,11196,10210,57561285,off,,,,,,, +4979,c,bubble,sorted,250,2463534242,8,11386,1,233892653,232886166,31125,0,7fd8c1f4,off,,,,,,, +4980,python,insertion,reversed,250,2463534242,18,102,1,249594579,233348144,31125,31374,be175004,off,,,,,,, +4981,c,insertion,random,2000,2463534242,19,492,1,229362653,228449052,996390,996389,41262855,off,,,,,,, +4982,python,quick,nearly_sorted,1000,2463534242,24,110,1,257988560,241975212,28499,7892,1cfe4afd,off,,,,,,, +4983,c,bubble,random,500,2463534242,9,635,1,228732021,227489583,124750,119688,11ae932d,off,,,,,,, +4984,c,none,reversed,250,2463534242,21,0,427,228726050,91,0,0,be175004,off,,,,,,, +4985,c,quick,nearly_sorted,1000,2463534242,11,11273,1,237831353,236973516,28499,7892,1cfe4afd,off,,,,,,, +4986,c,quick,reversed,500,2463534242,2,3240,1,238890920,237846852,124750,500,d33e3559,off,,,,,,, +4987,c,bubble,nearly_sorted,1000,2463534242,19,507,1,242513018,241365177,499500,59780,1cfe4afd,off,,,,,,, +4988,node,bubble,reversed,1000,2463534242,21,378,1,268721144,239946654,499500,999000,0ca43265,off,,,,,,, +4989,python,quick,reversed,1000,2463534242,18,8,1,279518328,264050221,499500,1000,0ca43265,off,,,,,,, +4990,c,insertion,nearly_sorted,2000,2463534242,1,2962,1,225651818,224735872,118277,118270,5835dc59,off,,,,,,, +4991,python,insertion,reversed,1000,2463534242,13,6,1,295718490,278754961,499500,500499,0ca43265,off,,,,,,, +4992,c,insertion,sorted,1000,2463534242,1,196883,1,226095839,224748175,999,0,8e15cb75,off,,,,,,, +4993,python,none,nearly_sorted,2000,2463534242,29,0,15,210277145,602,0,0,5835dc59,off,,,,,,, +4994,node,none,reversed,2000,2463534242,29,0,9,228096770,742,0,0,ad35ba05,off,,,,,,, +4995,node,bubble,reversed,2000,2463534242,2,94,1,271926340,243072867,1999000,3998000,ad35ba05,off,,,,,,, +4996,node,none,random,1000,2463534242,26,0,10,246164391,1062,0,0,57561285,off,,,,,,, +4997,python,insertion,reversed,250,2463534242,15,102,1,253308829,238115633,31125,31374,be175004,off,,,,,,, +4998,c,none,reversed,1000,2463534242,17,0,405,218917077,110,0,0,0ca43265,off,,,,,,, +4999,python,quick,nearly_sorted,250,2463534242,19,882,1,251668215,236205995,4346,1008,21a19c14,off,,,,,,, +5000,c,insertion,random,250,2463534242,15,26020,1,226345579,225290291,15844,15838,b47adce4,off,,,,,,, +5001,node,none,sorted,1000,2463534242,1,0,9,219072248,822,0,0,8e15cb75,off,,,,,,, +5002,c,none,random,2000,2463534242,1,0,415,230280313,90,0,0,41262855,off,,,,,,, +5003,c,bubble,nearly_sorted,1000,2463534242,29,507,1,239942449,239064196,499500,59780,1cfe4afd,off,,,,,,, +5004,node,quick,random,250,2463534242,26,36348,1,250165319,223998807,1933,1940,b47adce4,off,,,,,,, +5005,python,quick,random,1000,2463534242,7,209,1,259214026,243248319,11196,10210,57561285,off,,,,,,, +5006,node,none,random,1000,2463534242,5,0,10,246295437,792,0,0,57561285,off,,,,,,, +5007,python,insertion,nearly_sorted,1000,2463534242,10,76,1,251053564,235955480,30889,30860,1cfe4afd,off,,,,,,, +5008,c,bubble,sorted,250,2463534242,2,11386,1,239261664,237981164,31125,0,7fd8c1f4,off,,,,,,, +5009,python,quick,sorted,500,2463534242,4,30,1,249686583,234429274,124750,0,d9a24379,off,,,,,,, +5010,node,quick,random,500,2463534242,16,16148,1,266527562,238544717,4578,4552,11ae932d,off,,,,,,, +5011,c,bubble,reversed,500,2463534242,14,346,1,237703484,236607269,124750,249500,d33e3559,off,,,,,,, +5012,python,bubble,nearly_sorted,2000,2463534242,4,3,1,366833884,349869607,1999000,232556,5835dc59,off,,,,,,, +5013,python,quick,reversed,1000,2463534242,10,8,1,265022923,250361676,499500,1000,0ca43265,off,,,,,,, +5014,python,insertion,nearly_sorted,250,2463534242,9,1416,1,248772329,234639784,2277,2251,21a19c14,off,,,,,,, +5015,c,quick,random,500,2463534242,11,55776,1,237997074,236993906,4578,4552,11ae932d,off,,,,,,, +5016,node,insertion,random,1000,2463534242,3,1091,1,278127632,250913686,249457,249457,57561285,off,,,,,,, +5017,c,none,random,1000,2463534242,4,0,367,206677840,80,0,0,57561285,off,,,,,,, +5018,c,none,sorted,2000,2463534242,17,0,416,228854992,90,0,0,69842aa5,off,,,,,,, +5019,python,insertion,reversed,1000,2463534242,21,6,1,298055939,282894074,499500,500499,0ca43265,off,,,,,,, +5020,python,quick,nearly_sorted,500,2463534242,29,296,1,238881484,224137021,10192,1850,482b4785,off,,,,,,, +5021,c,bubble,reversed,2000,2463534242,21,22,1,242422246,241132422,1999000,3998000,ad35ba05,off,,,,,,, +5022,python,bubble,sorted,500,2463534242,28,46,1,255311072,240423127,124750,0,d9a24379,off,,,,,,, +5023,c,insertion,nearly_sorted,1000,2463534242,10,11445,1,240747159,239630726,30889,30860,1cfe4afd,off,,,,,,, +5024,node,none,random,500,2463534242,18,0,9,228731120,852,0,0,11ae932d,off,,,,,,, +5025,c,none,nearly_sorted,1000,2463534242,5,0,421,236495631,90,0,0,1cfe4afd,off,,,,,,, +5026,node,bubble,reversed,250,2463534242,8,5861,1,255413164,228781430,31125,62250,be175004,off,,,,,,, +5027,python,insertion,random,250,2463534242,18,206,1,259061902,244282934,15844,15838,b47adce4,off,,,,,,, +5028,c,quick,nearly_sorted,250,2463534242,13,77221,1,236827172,235585155,4346,1008,21a19c14,off,,,,,,, +5029,c,bubble,sorted,1000,2463534242,25,702,1,232773265,231838083,499500,0,8e15cb75,off,,,,,,, +5030,python,bubble,reversed,500,2463534242,5,20,1,260272300,245195010,124750,249500,d33e3559,off,,,,,,, +5031,c,quick,sorted,500,2463534242,8,2762,1,240208949,239176807,124750,0,d9a24379,off,,,,,,, +5032,c,bubble,nearly_sorted,2000,2463534242,27,121,1,237261897,236001244,1999000,232556,5835dc59,off,,,,,,, +5033,node,insertion,random,2000,2463534242,10,234,1,266401917,238403253,996390,996389,41262855,off,,,,,,, +5034,python,bubble,nearly_sorted,500,2463534242,28,41,1,251652736,237036117,124750,14926,482b4785,off,,,,,,, +5035,python,bubble,reversed,500,2463534242,28,20,1,257936653,242294251,124750,249500,d33e3559,off,,,,,,, +5036,c,none,nearly_sorted,2000,2463534242,19,0,387,212571574,90,0,0,5835dc59,off,,,,,,, +5037,c,quick,random,2000,2463534242,18,10587,1,237112577,236378120,25197,24544,41262855,off,,,,,,, +5038,python,insertion,reversed,1000,2463534242,27,6,1,300598215,283838498,499500,500499,0ca43265,off,,,,,,, +5039,python,insertion,random,2000,2463534242,20,3,1,324230060,307391615,996390,996389,41262855,off,,,,,,, +5040,node,quick,reversed,2000,2463534242,28,80,1,270604323,242686110,1999000,2000,ad35ba05,off,,,,,,, +5041,c,none,sorted,250,2463534242,8,0,429,229303421,80,0,0,7fd8c1f4,off,,,,,,, +5042,c,none,nearly_sorted,1000,2463534242,30,0,421,226947958,80,0,0,1cfe4afd,off,,,,,,, +5043,node,none,nearly_sorted,500,2463534242,10,0,10,250920644,692,0,0,482b4785,off,,,,,,, +5044,c,quick,random,250,2463534242,23,136055,1,240698497,239709725,1933,1940,b47adce4,off,,,,,,, +5045,node,none,reversed,2000,2463534242,5,0,9,226809508,771,0,0,ad35ba05,off,,,,,,, +5046,node,quick,nearly_sorted,2000,2463534242,7,1909,1,249753397,224685019,58002,11768,5835dc59,off,,,,,,, +5047,node,bubble,random,2000,2463534242,1,98,1,265580207,237697857,1999000,1988792,41262855,off,,,,,,, +5048,node,none,sorted,1000,2463534242,27,0,9,223332962,721,0,0,8e15cb75,off,,,,,,, +5049,node,none,sorted,250,2463534242,21,0,9,233365404,791,0,0,7fd8c1f4,off,,,,,,, +5050,node,insertion,random,500,2463534242,14,4059,1,266784955,239477305,60338,60339,11ae932d,off,,,,,,, +5051,node,bubble,random,2000,2463534242,3,98,1,267805187,237650957,1999000,1988792,41262855,off,,,,,,, +5052,python,bubble,random,250,2463534242,7,112,1,253185769,238441998,31125,31196,b47adce4,off,,,,,,, +5053,c,bubble,nearly_sorted,250,2463534242,7,7880,1,236779382,235717424,31125,4056,21a19c14,off,,,,,,, +5054,python,none,reversed,1000,2463534242,10,0,16,222460678,612,0,0,0ca43265,off,,,,,,, +5055,node,none,reversed,2000,2463534242,15,0,9,231147789,752,0,0,ad35ba05,off,,,,,,, +5056,c,insertion,nearly_sorted,500,2463534242,5,32350,1,234549744,233550481,7962,7926,482b4785,off,,,,,,, +5057,python,quick,sorted,250,2463534242,16,133,1,258110749,243142377,31125,0,7fd8c1f4,off,,,,,,, +5058,node,bubble,random,1000,2463534242,7,362,1,267566951,238617777,499500,496926,57561285,off,,,,,,, +5059,node,bubble,random,500,2463534242,19,1407,1,269361002,241679917,124750,119688,11ae932d,off,,,,,,, +5060,node,none,reversed,1000,2463534242,4,0,9,220021266,741,0,0,0ca43265,off,,,,,,, +5061,python,none,reversed,1000,2463534242,20,0,16,217909118,591,0,0,0ca43265,off,,,,,,, +5062,c,quick,random,250,2463534242,17,136055,1,237431706,236289661,1933,1940,b47adce4,off,,,,,,, +5063,c,bubble,nearly_sorted,500,2463534242,11,1998,1,235667319,234364226,124750,14926,482b4785,off,,,,,,, +5064,python,insertion,random,500,2463534242,1,44,1,242425023,226804680,60338,60339,11ae932d,off,,,,,,, +5065,python,none,nearly_sorted,500,2463534242,6,0,17,227309185,491,0,0,482b4785,off,,,,,,, +5066,node,insertion,random,250,2463534242,23,15349,1,254274748,225572170,15844,15838,b47adce4,off,,,,,,, +5067,python,insertion,nearly_sorted,2000,2463534242,1,20,1,258947267,243162462,118277,118270,5835dc59,off,,,,,,, +5068,c,insertion,nearly_sorted,500,2463534242,7,32350,1,246987640,245971077,7962,7926,482b4785,off,,,,,,, +5069,python,quick,reversed,1000,2463534242,4,8,1,281577972,264687461,499500,1000,0ca43265,off,,,,,,, +5070,c,bubble,nearly_sorted,1000,2463534242,22,507,1,245232684,244214033,499500,59780,1cfe4afd,off,,,,,,, +5071,node,insertion,sorted,500,2463534242,4,324764,1,263484750,234604401,499,0,d9a24379,off,,,,,,, +5072,node,insertion,sorted,2000,2463534242,30,89220,1,249215709,220513834,1999,0,69842aa5,off,,,,,,, +5073,c,bubble,random,2000,2463534242,13,40,1,237149205,236218814,1999000,1988792,41262855,off,,,,,,, +5074,node,insertion,random,500,2463534242,12,4059,1,241614604,213683742,60338,60339,11ae932d,off,,,,,,, +5075,node,bubble,reversed,250,2463534242,28,5861,1,259420303,232790988,31125,62250,be175004,off,,,,,,, +5076,node,bubble,random,2000,2463534242,13,98,1,265716413,239160212,1999000,1988792,41262855,off,,,,,,, +5077,python,insertion,reversed,2000,2463534242,12,2,1,397263291,381564298,1999000,2000999,ad35ba05,off,,,,,,, +5078,node,none,sorted,500,2463534242,20,0,10,247410395,792,0,0,d9a24379,off,,,,,,, +5079,node,quick,sorted,2000,2463534242,23,74,1,262933126,236129537,1999000,0,69842aa5,off,,,,,,, +5080,c,insertion,sorted,250,2463534242,22,786886,1,239808229,238741150,249,0,7fd8c1f4,off,,,,,,, +5081,python,bubble,reversed,250,2463534242,6,88,1,251786196,235713663,31125,62250,be175004,off,,,,,,, +5082,c,insertion,random,250,2463534242,21,26020,1,237034861,235760773,15844,15838,b47adce4,off,,,,,,, +5083,c,insertion,reversed,500,2463534242,19,3967,1,237324123,236135215,124750,125249,d33e3559,off,,,,,,, +5084,node,insertion,reversed,250,2463534242,6,8539,1,260653814,233674246,31125,31374,be175004,off,,,,,,, +5085,c,none,reversed,250,2463534242,4,0,427,227455869,91,0,0,be175004,off,,,,,,, +5086,c,quick,reversed,1000,2463534242,7,814,1,238546937,237856221,499500,1000,0ca43265,off,,,,,,, +5087,node,none,nearly_sorted,250,2463534242,12,0,10,244099068,631,0,0,21a19c14,off,,,,,,, +5088,python,quick,reversed,250,2463534242,14,151,1,260768630,244800466,31125,250,be175004,off,,,,,,, +5089,python,quick,random,250,2463534242,15,1271,1,258226577,243431458,1933,1940,b47adce4,off,,,,,,, +5090,python,quick,sorted,250,2463534242,29,133,1,251963849,236755521,31125,0,7fd8c1f4,off,,,,,,, +5091,python,none,sorted,2000,2463534242,2,0,15,213985355,601,0,0,69842aa5,off,,,,,,, +5092,node,none,nearly_sorted,250,2463534242,11,0,10,272055693,931,0,0,21a19c14,off,,,,,,, +5093,c,none,reversed,250,2463534242,23,0,427,280650159,100,0,0,be175004,off,,,,,,, +5094,c,none,nearly_sorted,2000,2463534242,23,0,387,236322015,90,0,0,5835dc59,off,,,,,,, +5095,c,bubble,reversed,250,2463534242,12,1384,1,242655615,241957828,31125,62250,be175004,off,,,,,,, +5096,python,none,reversed,250,2463534242,21,0,17,257721178,631,0,0,be175004,off,,,,,,, +5097,python,bubble,nearly_sorted,2000,2463534242,21,3,1,375212748,358669483,1999000,232556,5835dc59,off,,,,,,, +5098,python,bubble,sorted,250,2463534242,11,206,1,278578887,262359800,31125,0,7fd8c1f4,off,,,,,,, +5099,python,quick,nearly_sorted,2000,2463534242,19,55,1,254168411,237487245,58002,11768,5835dc59,off,,,,,,, +5100,python,quick,reversed,250,2463534242,1,151,1,259975483,244838497,31125,250,be175004,off,,,,,,, +5101,c,quick,sorted,1000,2463534242,18,691,1,238869218,237770159,499500,0,8e15cb75,off,,,,,,, +5102,c,insertion,reversed,250,2463534242,9,15645,1,241145035,239992932,31125,31374,be175004,off,,,,,,, +5103,c,quick,nearly_sorted,1000,2463534242,10,11273,1,232511113,231608997,28499,7892,1cfe4afd,off,,,,,,, +5104,c,none,reversed,500,2463534242,5,0,402,226815860,90,0,0,d33e3559,off,,,,,,, +5105,python,bubble,nearly_sorted,500,2463534242,21,41,1,252339533,236503377,124750,14926,482b4785,off,,,,,,, +5106,node,bubble,nearly_sorted,500,2463534242,2,1566,1,271450900,242574841,124750,14926,482b4785,off,,,,,,, +5107,c,bubble,reversed,1000,2463534242,13,87,1,244943913,243984190,499500,999000,0ca43265,off,,,,,,, +5108,node,insertion,sorted,2000,2463534242,6,89220,1,269294118,239411804,1999,0,69842aa5,off,,,,,,, +5109,c,quick,reversed,250,2463534242,20,12781,1,241299343,240462766,31125,250,be175004,off,,,,,,, +5110,node,bubble,sorted,1000,2463534242,12,374,1,263316665,233053805,499500,0,8e15cb75,off,,,,,,, +5111,node,bubble,nearly_sorted,2000,2463534242,20,110,1,264607384,235422280,1999000,232556,5835dc59,off,,,,,,, +5112,python,bubble,reversed,1000,2463534242,4,5,1,295531297,278946002,499500,999000,0ca43265,off,,,,,,, +5113,python,insertion,random,500,2463534242,13,44,1,257435565,241455781,60338,60339,11ae932d,off,,,,,,, +5114,node,none,nearly_sorted,500,2463534242,18,0,10,271365829,862,0,0,482b4785,off,,,,,,, +5115,c,insertion,reversed,2000,2463534242,22,260,1,245186328,244009122,1999000,2000999,ad35ba05,off,,,,,,, +5116,c,quick,reversed,500,2463534242,9,3240,1,239770117,238734547,124750,500,d33e3559,off,,,,,,, +5117,c,quick,sorted,500,2463534242,7,2762,1,240771214,239894160,124750,0,d9a24379,off,,,,,,, +5118,node,bubble,sorted,250,2463534242,19,8081,1,263107774,233445647,31125,0,7fd8c1f4,off,,,,,,, +5119,c,insertion,sorted,1000,2463534242,5,196883,1,231294984,230443870,999,0,8e15cb75,off,,,,,,, +5120,c,bubble,random,500,2463534242,21,635,1,230299870,229276231,124750,119688,11ae932d,off,,,,,,, +5121,c,bubble,nearly_sorted,500,2463534242,25,1998,1,234891605,233758280,124750,14926,482b4785,off,,,,,,, +5122,python,none,random,2000,2463534242,29,0,16,264023881,622,0,0,41262855,off,,,,,,, +5123,node,quick,reversed,500,2463534242,20,1194,1,258875461,231280556,124750,500,d33e3559,off,,,,,,, +5124,node,quick,nearly_sorted,1000,2463534242,21,3752,1,267796301,238282861,28499,7892,1cfe4afd,off,,,,,,, +5125,node,none,nearly_sorted,500,2463534242,24,0,10,269131001,731,0,0,482b4785,off,,,,,,, +5126,python,none,random,500,2463534242,16,0,16,250235060,771,0,0,11ae932d,off,,,,,,, +5127,c,none,random,250,2463534242,6,0,429,244178610,80,0,0,b47adce4,off,,,,,,, +5128,c,bubble,random,2000,2463534242,3,40,1,244968819,244189506,1999000,1988792,41262855,off,,,,,,, +5129,c,quick,random,250,2463534242,16,136055,1,242404234,241271592,1933,1940,b47adce4,off,,,,,,, +5130,c,quick,sorted,250,2463534242,19,10851,1,245539581,244515912,31125,0,7fd8c1f4,off,,,,,,, +5131,c,none,sorted,250,2463534242,12,0,429,245177790,91,0,0,7fd8c1f4,off,,,,,,, +5132,node,quick,nearly_sorted,2000,2463534242,1,1909,1,259248922,228188718,58002,11768,5835dc59,off,,,,,,, +5133,c,quick,sorted,2000,2463534242,23,174,1,245927357,245026468,1999000,0,69842aa5,off,,,,,,, +5134,node,bubble,sorted,1000,2463534242,15,374,1,256979940,228339737,499500,0,8e15cb75,off,,,,,,, +5135,c,bubble,random,250,2463534242,22,2568,1,243656171,242534147,31125,31196,b47adce4,off,,,,,,, +5136,c,quick,random,250,2463534242,27,136055,1,237057742,236143682,1933,1940,b47adce4,off,,,,,,, +5137,c,quick,nearly_sorted,250,2463534242,23,77221,1,249206220,248308920,4346,1008,21a19c14,off,,,,,,, +5138,node,insertion,sorted,1000,2463534242,17,160107,1,258727034,230519492,999,0,8e15cb75,off,,,,,,, +5139,node,none,sorted,1000,2463534242,23,0,9,234738758,811,0,0,8e15cb75,off,,,,,,, +5140,c,none,random,2000,2463534242,10,0,415,241577144,140,0,0,41262855,off,,,,,,, +5141,python,bubble,random,1000,2463534242,11,6,1,274689639,258511630,499500,496926,57561285,off,,,,,,, +5142,python,none,nearly_sorted,2000,2463534242,12,0,15,234992664,511,0,0,5835dc59,off,,,,,,, +5143,python,quick,sorted,250,2463534242,30,133,1,250173675,235361928,31125,0,7fd8c1f4,off,,,,,,, +5144,python,bubble,reversed,1000,2463534242,22,5,1,286560215,270881676,499500,999000,0ca43265,off,,,,,,, +5145,python,bubble,nearly_sorted,250,2463534242,16,188,1,274602215,258396039,31125,4056,21a19c14,off,,,,,,, +5146,node,insertion,sorted,2000,2463534242,1,89220,1,265381956,236632303,1999,0,69842aa5,off,,,,,,, +5147,c,quick,reversed,500,2463534242,1,3240,1,241795924,240919651,124750,500,d33e3559,off,,,,,,, +5148,node,quick,random,2000,2463534242,4,2126,1,278075034,248674780,25197,24544,41262855,off,,,,,,, +5149,c,quick,nearly_sorted,2000,2463534242,12,6145,1,248957625,248000802,58002,11768,5835dc59,off,,,,,,, +5150,node,none,reversed,250,2463534242,5,0,9,238807052,781,0,0,be175004,off,,,,,,, +5151,python,insertion,reversed,250,2463534242,12,102,1,252916295,235960794,31125,31374,be175004,off,,,,,,, +5152,python,bubble,sorted,1000,2463534242,16,11,1,286516424,270991438,499500,0,8e15cb75,off,,,,,,, +5153,c,quick,sorted,1000,2463534242,28,691,1,240322683,239105583,499500,0,8e15cb75,off,,,,,,, +5154,c,none,nearly_sorted,2000,2463534242,27,0,387,211409428,81,0,0,5835dc59,off,,,,,,, +5155,python,none,reversed,1000,2463534242,3,0,16,220798132,591,0,0,0ca43265,off,,,,,,, +5156,python,insertion,random,1000,2463534242,13,10,1,255165009,239708817,249457,249457,57561285,off,,,,,,, +5157,python,bubble,sorted,2000,2463534242,22,3,1,332698952,317218475,1999000,0,69842aa5,off,,,,,,, +5158,c,insertion,nearly_sorted,250,2463534242,15,99421,1,232676564,231262233,2277,2251,21a19c14,off,,,,,,, +5159,python,insertion,random,250,2463534242,9,206,1,251396175,236175058,15844,15838,b47adce4,off,,,,,,, +5160,node,insertion,nearly_sorted,1000,2463534242,9,7758,1,250618959,222442896,30889,30860,1cfe4afd,off,,,,,,, +5161,node,none,sorted,250,2463534242,6,0,9,219926990,752,0,0,7fd8c1f4,off,,,,,,, +5162,python,bubble,random,250,2463534242,11,112,1,243488712,229081410,31125,31196,b47adce4,off,,,,,,, +5163,python,quick,random,500,2463534242,15,507,1,259020978,243644177,4578,4552,11ae932d,off,,,,,,, +5164,c,none,random,500,2463534242,28,0,426,228321481,80,0,0,11ae932d,off,,,,,,, +5165,node,bubble,reversed,1000,2463534242,9,378,1,268748515,240932046,499500,999000,0ca43265,off,,,,,,, +5166,python,none,sorted,1000,2463534242,6,0,16,218490247,551,0,0,8e15cb75,off,,,,,,, +5167,python,none,reversed,250,2463534242,2,0,17,222157930,531,0,0,be175004,off,,,,,,, +5168,node,insertion,reversed,1000,2463534242,8,554,1,263845156,237256766,499500,500499,0ca43265,off,,,,,,, +5169,c,none,reversed,1000,2463534242,28,0,405,218152865,81,0,0,0ca43265,off,,,,,,, +5170,c,insertion,reversed,2000,2463534242,11,260,1,236073920,235070821,1999000,2000999,ad35ba05,off,,,,,,, +5171,node,bubble,reversed,1000,2463534242,15,378,1,267814715,239739548,499500,999000,0ca43265,off,,,,,,, +5172,python,none,nearly_sorted,2000,2463534242,4,0,15,215732379,521,0,0,5835dc59,off,,,,,,, +5173,python,quick,random,500,2463534242,21,507,1,260404407,245181832,4578,4552,11ae932d,off,,,,,,, +5174,python,quick,sorted,500,2463534242,20,30,1,248707416,233943521,124750,0,d9a24379,off,,,,,,, +5175,c,none,random,2000,2463534242,2,0,415,228267972,90,0,0,41262855,off,,,,,,, +5176,python,none,random,2000,2463534242,20,0,16,239838055,721,0,0,41262855,off,,,,,,, +5177,c,quick,nearly_sorted,500,2463534242,14,31784,1,235138026,233901699,10192,1850,482b4785,off,,,,,,, +5178,python,none,sorted,2000,2463534242,21,0,15,213420226,571,0,0,69842aa5,off,,,,,,, +5179,node,none,random,500,2463534242,5,0,9,222368986,771,0,0,11ae932d,off,,,,,,, +5180,c,insertion,nearly_sorted,250,2463534242,23,99421,1,233635991,232513277,2277,2251,21a19c14,off,,,,,,, +5181,python,bubble,reversed,1000,2463534242,21,5,1,285285116,268828502,499500,999000,0ca43265,off,,,,,,, +5182,node,bubble,nearly_sorted,2000,2463534242,9,110,1,261177746,233934916,1999000,232556,5835dc59,off,,,,,,, +5183,python,quick,sorted,250,2463534242,5,133,1,256110581,240713848,31125,0,7fd8c1f4,off,,,,,,, +5184,node,quick,sorted,250,2463534242,7,4507,1,263178376,235233601,31125,0,7fd8c1f4,off,,,,,,, +5185,c,bubble,sorted,250,2463534242,28,11386,1,241060755,239879492,31125,0,7fd8c1f4,off,,,,,,, +5186,python,none,random,250,2463534242,29,0,17,236784473,571,0,0,b47adce4,off,,,,,,, +5187,python,bubble,nearly_sorted,1000,2463534242,2,9,1,265453650,249653280,499500,59780,1cfe4afd,off,,,,,,, +5188,node,bubble,nearly_sorted,1000,2463534242,28,413,1,266566345,238085295,499500,59780,1cfe4afd,off,,,,,,, +5189,node,insertion,random,250,2463534242,26,15349,1,262563864,235456069,15844,15838,b47adce4,off,,,,,,, +5190,c,insertion,sorted,250,2463534242,4,786886,1,242334754,241328718,249,0,7fd8c1f4,off,,,,,,, +5191,c,quick,reversed,500,2463534242,19,3240,1,232760190,231446809,124750,500,d33e3559,off,,,,,,, +5192,c,bubble,reversed,250,2463534242,5,1384,1,233329568,232375050,31125,62250,be175004,off,,,,,,, +5193,node,quick,reversed,2000,2463534242,11,80,1,277478856,247653360,1999000,2000,ad35ba05,off,,,,,,, +5194,node,insertion,nearly_sorted,250,2463534242,9,79550,1,231276049,205032808,2277,2251,21a19c14,off,,,,,,, +5195,node,quick,nearly_sorted,1000,2463534242,18,3752,1,253268514,226203385,28499,7892,1cfe4afd,off,,,,,,, +5196,python,quick,random,1000,2463534242,19,209,1,249255063,233099323,11196,10210,57561285,off,,,,,,, +5197,c,bubble,nearly_sorted,250,2463534242,15,7880,1,236773503,235337031,31125,4056,21a19c14,off,,,,,,, +5198,python,none,sorted,2000,2463534242,25,0,15,218211203,651,0,0,69842aa5,off,,,,,,, +5199,python,bubble,reversed,500,2463534242,25,20,1,258276700,244064506,124750,249500,d33e3559,off,,,,,,, +5200,python,bubble,nearly_sorted,500,2463534242,1,41,1,325540718,309737100,124750,14926,482b4785,off,,,,,,, +5201,python,bubble,sorted,1000,2463534242,2,11,1,282024936,266644060,499500,0,8e15cb75,off,,,,,,, +5202,python,quick,random,500,2463534242,3,507,1,260466865,244694838,4578,4552,11ae932d,off,,,,,,, +5203,python,insertion,random,1000,2463534242,25,10,1,260172152,243964242,249457,249457,57561285,off,,,,,,, +5204,python,bubble,reversed,1000,2463534242,25,5,1,291396568,276360815,499500,999000,0ca43265,off,,,,,,, +5205,c,quick,reversed,500,2463534242,28,3240,1,236136308,234755181,124750,500,d33e3559,off,,,,,,, +5206,c,insertion,random,1000,2463534242,2,1983,1,244627901,243377598,249457,249457,57561285,off,,,,,,, +5207,python,insertion,sorted,1000,2463534242,22,3178,1,245810287,230300065,999,0,8e15cb75,off,,,,,,, +5208,python,quick,sorted,250,2463534242,23,133,1,256598134,241458711,31125,0,7fd8c1f4,off,,,,,,, +5209,c,bubble,reversed,250,2463534242,20,1384,1,238162344,236731393,31125,62250,be175004,off,,,,,,, +5210,node,none,sorted,1000,2463534242,26,0,9,221801844,942,0,0,8e15cb75,off,,,,,,, +5211,c,insertion,nearly_sorted,2000,2463534242,30,2962,1,220030454,219018838,118277,118270,5835dc59,off,,,,,,, +5212,node,bubble,reversed,2000,2463534242,17,94,1,268709792,237694138,1999000,3998000,ad35ba05,off,,,,,,, +5213,node,quick,reversed,250,2463534242,20,4755,1,265144180,239307968,31125,250,be175004,off,,,,,,, +5214,c,insertion,reversed,2000,2463534242,19,260,1,244171816,243276840,1999000,2000999,ad35ba05,off,,,,,,, +5215,python,none,nearly_sorted,250,2463534242,12,0,17,223729635,481,0,0,21a19c14,off,,,,,,, +5216,c,bubble,sorted,500,2463534242,19,2858,1,236557408,235454589,124750,0,d9a24379,off,,,,,,, +5217,c,insertion,sorted,1000,2463534242,3,196883,1,230284820,228881341,999,0,8e15cb75,off,,,,,,, +5218,c,insertion,nearly_sorted,2000,2463534242,5,2962,1,223712965,222088042,118277,118270,5835dc59,off,,,,,,, +5219,node,insertion,nearly_sorted,250,2463534242,22,79550,1,230222615,202165529,2277,2251,21a19c14,off,,,,,,, +5220,c,quick,random,250,2463534242,14,136055,1,236180811,234861488,1933,1940,b47adce4,off,,,,,,, +5221,python,none,nearly_sorted,250,2463534242,9,0,17,224624284,491,0,0,21a19c14,off,,,,,,, +5222,python,quick,reversed,1000,2463534242,30,8,1,272119201,257350738,499500,1000,0ca43265,off,,,,,,, +5223,node,insertion,random,1000,2463534242,24,1091,1,286396148,258672393,249457,249457,57561285,off,,,,,,, +5224,python,quick,nearly_sorted,250,2463534242,12,882,1,259573146,243863765,4346,1008,21a19c14,off,,,,,,, +5225,c,bubble,reversed,1000,2463534242,6,87,1,237495658,236536658,499500,999000,0ca43265,off,,,,,,, +5226,node,none,random,250,2463534242,4,0,10,253540073,672,0,0,b47adce4,off,,,,,,, +5227,node,quick,random,1000,2463534242,10,6513,1,258651282,231586964,11196,10210,57561285,off,,,,,,, +5228,node,none,random,2000,2463534242,28,0,9,225814754,752,0,0,41262855,off,,,,,,, +5229,node,bubble,nearly_sorted,500,2463534242,1,1566,1,267666758,239465649,124750,14926,482b4785,off,,,,,,, +5230,python,bubble,reversed,250,2463534242,30,88,1,251616999,236913051,31125,62250,be175004,off,,,,,,, +5231,c,none,random,2000,2463534242,12,0,415,229439026,90,0,0,41262855,off,,,,,,, +5232,python,quick,reversed,500,2463534242,12,33,1,248697447,232986189,124750,500,d33e3559,off,,,,,,, +5233,c,bubble,reversed,1000,2463534242,20,87,1,235298037,234305596,499500,999000,0ca43265,off,,,,,,, +5234,node,insertion,sorted,1000,2463534242,15,160107,1,246482848,218721195,999,0,8e15cb75,off,,,,,,, +5235,python,quick,reversed,500,2463534242,8,33,1,248472186,233596950,124750,500,d33e3559,off,,,,,,, +5236,python,bubble,random,500,2463534242,12,26,1,259444839,244681824,124750,119688,11ae932d,off,,,,,,, +5237,python,bubble,nearly_sorted,250,2463534242,17,188,1,265007535,250355297,31125,4056,21a19c14,off,,,,,,, +5238,python,none,random,2000,2463534242,21,0,16,236867638,671,0,0,41262855,off,,,,,,, +5239,c,quick,random,2000,2463534242,15,10587,1,236818766,235711040,25197,24544,41262855,off,,,,,,, +5240,node,insertion,random,2000,2463534242,11,234,1,266864424,237288973,996390,996389,41262855,off,,,,,,, +5241,node,none,nearly_sorted,1000,2463534242,19,0,10,247460960,561,0,0,1cfe4afd,off,,,,,,, +5242,python,none,random,2000,2463534242,6,0,16,240970868,792,0,0,41262855,off,,,,,,, +5243,python,bubble,reversed,250,2463534242,14,88,1,252926714,237848397,31125,62250,be175004,off,,,,,,, +5244,c,none,random,500,2463534242,25,0,426,227757204,81,0,0,11ae932d,off,,,,,,, +5245,c,quick,reversed,500,2463534242,24,3240,1,230176898,229389704,124750,500,d33e3559,off,,,,,,, +5246,python,quick,random,1000,2463534242,1,209,1,264513648,246996675,11196,10210,57561285,off,,,,,,, +5247,c,insertion,sorted,250,2463534242,5,786886,1,247417489,246527701,249,0,7fd8c1f4,off,,,,,,, +5248,python,bubble,random,1000,2463534242,29,6,1,262417641,246571881,499500,496926,57561285,off,,,,,,, +5249,c,none,random,500,2463534242,22,0,426,235439692,80,0,0,11ae932d,off,,,,,,, +5250,node,none,reversed,2000,2463534242,26,0,9,229444486,782,0,0,ad35ba05,off,,,,,,, +5251,node,quick,sorted,250,2463534242,15,4507,1,262665004,235382164,31125,0,7fd8c1f4,off,,,,,,, +5252,node,insertion,sorted,1000,2463534242,26,160107,1,244224725,217700572,999,0,8e15cb75,off,,,,,,, +5253,c,bubble,reversed,2000,2463534242,9,22,1,242262509,241022957,1999000,3998000,ad35ba05,off,,,,,,, +5254,python,quick,nearly_sorted,1000,2463534242,6,110,1,258405541,243172487,28499,7892,1cfe4afd,off,,,,,,, +5255,node,insertion,nearly_sorted,250,2463534242,1,79550,1,232916072,206436887,2277,2251,21a19c14,off,,,,,,, +5256,python,insertion,sorted,500,2463534242,14,6624,1,236607601,221113406,499,0,d9a24379,off,,,,,,, +5257,c,bubble,nearly_sorted,500,2463534242,24,1998,1,232257499,231040427,124750,14926,482b4785,off,,,,,,, +5258,c,quick,nearly_sorted,2000,2463534242,24,6145,1,237372904,236442161,58002,11768,5835dc59,off,,,,,,, +5259,c,none,nearly_sorted,2000,2463534242,21,0,387,211580358,91,0,0,5835dc59,off,,,,,,, +5260,c,insertion,sorted,500,2463534242,15,399335,1,238395703,237331654,499,0,d9a24379,off,,,,,,, +5261,c,bubble,reversed,500,2463534242,4,346,1,236795752,235409717,124750,249500,d33e3559,off,,,,,,, +5262,c,insertion,nearly_sorted,1000,2463534242,19,11445,1,239051060,237897789,30889,30860,1cfe4afd,off,,,,,,, +5263,python,none,random,1000,2463534242,11,0,15,209955802,641,0,0,57561285,off,,,,,,, +5264,node,quick,random,500,2463534242,19,16148,1,262427519,235874064,4578,4552,11ae932d,off,,,,,,, +5265,python,none,nearly_sorted,2000,2463534242,18,0,15,214459183,571,0,0,5835dc59,off,,,,,,, +5266,python,insertion,random,500,2463534242,23,44,1,250077977,234998712,60338,60339,11ae932d,off,,,,,,, +5267,python,insertion,sorted,250,2463534242,30,18845,1,255726679,240044841,249,0,7fd8c1f4,off,,,,,,, +5268,python,none,reversed,250,2463534242,22,0,17,225205984,541,0,0,be175004,off,,,,,,, +5269,python,none,random,1000,2463534242,1,0,15,209344306,662,0,0,57561285,off,,,,,,, +5270,c,none,random,2000,2463534242,21,0,415,228783908,90,0,0,41262855,off,,,,,,, +5271,python,none,reversed,250,2463534242,7,0,17,226705473,842,0,0,be175004,off,,,,,,, +5272,node,bubble,sorted,250,2463534242,14,8081,1,245588691,218870571,31125,0,7fd8c1f4,off,,,,,,, +5273,c,none,nearly_sorted,500,2463534242,2,0,405,218477834,80,0,0,482b4785,off,,,,,,, +5274,node,insertion,random,250,2463534242,28,15349,1,252703615,225643577,15844,15838,b47adce4,off,,,,,,, +5275,python,quick,random,2000,2463534242,18,90,1,250254365,233945128,25197,24544,41262855,off,,,,,,, +5276,python,insertion,reversed,250,2463534242,16,102,1,249849297,235101849,31125,31374,be175004,off,,,,,,, +5277,node,bubble,reversed,250,2463534242,2,5861,1,254402980,225195055,31125,62250,be175004,off,,,,,,, +5278,python,none,nearly_sorted,2000,2463534242,20,0,15,212043736,561,0,0,5835dc59,off,,,,,,, +5279,c,bubble,nearly_sorted,500,2463534242,30,1998,1,236481645,235500226,124750,14926,482b4785,off,,,,,,, +5280,python,insertion,sorted,2000,2463534242,7,1523,1,245348140,229395783,1999,0,69842aa5,off,,,,,,, +5281,python,bubble,reversed,250,2463534242,19,88,1,243884828,229094203,31125,62250,be175004,off,,,,,,, +5282,c,quick,reversed,2000,2463534242,6,209,1,232977343,231782238,1999000,2000,ad35ba05,off,,,,,,, +5283,python,insertion,random,250,2463534242,11,206,1,248830551,232504762,15844,15838,b47adce4,off,,,,,,, +5284,node,quick,sorted,2000,2463534242,7,74,1,263976401,236433453,1999000,0,69842aa5,off,,,,,,, +5285,node,none,nearly_sorted,2000,2463534242,6,0,9,229855838,601,0,0,5835dc59,off,,,,,,, +5286,c,none,random,1000,2463534242,20,0,367,198412570,80,0,0,57561285,off,,,,,,, +5287,node,none,reversed,250,2463534242,19,0,9,225170938,691,0,0,be175004,off,,,,,,, +5288,node,quick,reversed,250,2463534242,1,4755,1,256206280,229994900,31125,250,be175004,off,,,,,,, +5289,c,none,nearly_sorted,500,2463534242,3,0,405,219062639,81,0,0,482b4785,off,,,,,,, +5290,node,insertion,nearly_sorted,250,2463534242,21,79550,1,231314371,204792766,2277,2251,21a19c14,off,,,,,,, +5291,python,quick,random,1000,2463534242,5,209,1,263059984,247618826,11196,10210,57561285,off,,,,,,, +5292,python,none,random,250,2463534242,26,0,17,223114965,561,0,0,b47adce4,off,,,,,,, +5293,python,quick,random,2000,2463534242,6,90,1,249469947,233296344,25197,24544,41262855,off,,,,,,, +5294,python,insertion,nearly_sorted,1000,2463534242,2,76,1,242090163,226592374,30889,30860,1cfe4afd,off,,,,,,, +5295,node,insertion,sorted,2000,2463534242,22,89220,1,250782345,222460056,1999,0,69842aa5,off,,,,,,, +5296,c,quick,sorted,2000,2463534242,29,174,1,237454338,236258736,1999000,0,69842aa5,off,,,,,,, +5297,node,insertion,random,2000,2463534242,5,234,1,270958677,241741437,996390,996389,41262855,off,,,,,,, +5298,python,none,sorted,500,2463534242,1,0,16,213052657,531,0,0,d9a24379,off,,,,,,, +5299,c,insertion,random,250,2463534242,28,26020,1,228013976,227008963,15844,15838,b47adce4,off,,,,,,, +5300,c,bubble,nearly_sorted,250,2463534242,19,7880,1,234455507,233036618,31125,4056,21a19c14,off,,,,,,, +5301,c,quick,sorted,250,2463534242,28,10851,1,235422359,234214928,31125,0,7fd8c1f4,off,,,,,,, +5302,node,insertion,reversed,250,2463534242,10,8539,1,256840107,229731515,31125,31374,be175004,off,,,,,,, +5303,node,bubble,reversed,500,2463534242,29,1534,1,260910967,234575310,124750,249500,d33e3559,off,,,,,,, +5304,c,quick,nearly_sorted,1000,2463534242,18,11273,1,240239567,239084083,28499,7892,1cfe4afd,off,,,,,,, +5305,node,bubble,sorted,2000,2463534242,25,85,1,258090942,228501637,1999000,0,69842aa5,off,,,,,,, +5306,c,bubble,sorted,1000,2463534242,21,702,1,232563261,231403227,499500,0,8e15cb75,off,,,,,,, +5307,node,quick,nearly_sorted,250,2463534242,26,25320,1,258433634,230084443,4346,1008,21a19c14,off,,,,,,, +5308,node,bubble,reversed,500,2463534242,27,1534,1,268757722,240866755,124750,249500,d33e3559,off,,,,,,, +5309,python,insertion,nearly_sorted,250,2463534242,13,1416,1,262222234,247271940,2277,2251,21a19c14,off,,,,,,, +5310,node,none,sorted,250,2463534242,16,0,9,232822097,791,0,0,7fd8c1f4,off,,,,,,, +5311,node,quick,random,500,2463534242,29,16148,1,272541162,242639321,4578,4552,11ae932d,off,,,,,,, +5312,node,quick,sorted,2000,2463534242,27,74,1,273227630,243057373,1999000,0,69842aa5,off,,,,,,, +5313,c,insertion,nearly_sorted,1000,2463534242,9,11445,1,243032942,242129088,30889,30860,1cfe4afd,off,,,,,,, +5314,node,insertion,nearly_sorted,250,2463534242,8,79550,1,227531681,200613747,2277,2251,21a19c14,off,,,,,,, +5315,python,insertion,nearly_sorted,500,2463534242,12,316,1,248876404,233902038,7962,7926,482b4785,off,,,,,,, +5316,node,insertion,random,2000,2463534242,30,234,1,270123962,242617769,996390,996389,41262855,off,,,,,,, +5317,python,quick,random,2000,2463534242,24,90,1,260801883,243914119,25197,24544,41262855,off,,,,,,, +5318,python,quick,sorted,500,2463534242,21,30,1,239692081,224767017,124750,0,d9a24379,off,,,,,,, +5319,node,none,nearly_sorted,1000,2463534242,20,0,10,253326272,631,0,0,1cfe4afd,off,,,,,,, +5320,c,bubble,reversed,250,2463534242,29,1384,1,233811450,232913078,31125,62250,be175004,off,,,,,,, +5321,node,bubble,sorted,250,2463534242,26,8081,1,258017134,231504254,31125,0,7fd8c1f4,off,,,,,,, +5322,python,insertion,nearly_sorted,250,2463534242,1,1416,1,254080836,238959625,2277,2251,21a19c14,off,,,,,,, +5323,node,insertion,sorted,500,2463534242,1,324764,1,252224658,223864952,499,0,d9a24379,off,,,,,,, +5324,python,quick,sorted,1000,2463534242,6,8,1,277550160,261927995,499500,0,8e15cb75,off,,,,,,, +5325,node,bubble,random,1000,2463534242,12,362,1,266849185,237793536,499500,496926,57561285,off,,,,,,, +5326,node,quick,random,250,2463534242,8,36348,1,244336044,217123358,1933,1940,b47adce4,off,,,,,,, +5327,python,quick,random,250,2463534242,19,1271,1,245610303,230947966,1933,1940,b47adce4,off,,,,,,, +5328,python,none,random,500,2463534242,5,0,16,215612494,571,0,0,11ae932d,off,,,,,,, +5329,c,none,reversed,250,2463534242,13,0,427,228819634,80,0,0,be175004,off,,,,,,, +5330,node,insertion,nearly_sorted,500,2463534242,28,26045,1,243070452,214919990,7962,7926,482b4785,off,,,,,,, +5331,node,bubble,reversed,250,2463534242,24,5861,1,260273603,232009374,31125,62250,be175004,off,,,,,,, +5332,c,quick,random,500,2463534242,17,55776,1,237485826,236361659,4578,4552,11ae932d,off,,,,,,, +5333,node,insertion,sorted,250,2463534242,13,607595,1,252106326,223089421,249,0,7fd8c1f4,off,,,,,,, +5334,node,none,nearly_sorted,250,2463534242,24,0,10,247426266,801,0,0,21a19c14,off,,,,,,, +5335,c,quick,reversed,2000,2463534242,4,209,1,236074080,235038520,1999000,2000,ad35ba05,off,,,,,,, +5336,c,none,sorted,250,2463534242,21,0,429,231642566,80,0,0,7fd8c1f4,off,,,,,,, +5337,c,bubble,nearly_sorted,1000,2463534242,6,507,1,242296865,241567105,499500,59780,1cfe4afd,off,,,,,,, +5338,c,none,random,500,2463534242,21,0,426,228415256,90,0,0,11ae932d,off,,,,,,, +5339,c,none,sorted,500,2463534242,23,0,344,185111372,90,0,0,d9a24379,off,,,,,,, +5340,node,insertion,sorted,2000,2463534242,2,89220,1,255304910,225667060,1999,0,69842aa5,off,,,,,,, +5341,c,none,sorted,1000,2463534242,11,0,421,230384408,91,0,0,8e15cb75,off,,,,,,, +5342,node,quick,random,500,2463534242,27,16148,1,268027313,237664731,4578,4552,11ae932d,off,,,,,,, +5343,python,insertion,sorted,1000,2463534242,5,3178,1,243493976,227645137,999,0,8e15cb75,off,,,,,,, +5344,node,none,nearly_sorted,500,2463534242,15,0,10,257789206,701,0,0,482b4785,off,,,,,,, +5345,c,insertion,sorted,250,2463534242,14,786886,1,242576266,241403289,249,0,7fd8c1f4,off,,,,,,, +5346,c,quick,nearly_sorted,1000,2463534242,4,11273,1,242621803,240835355,28499,7892,1cfe4afd,off,,,,,,, +5347,c,bubble,random,1000,2463534242,14,154,1,232464666,231004441,499500,496926,57561285,off,,,,,,, +5348,python,none,nearly_sorted,2000,2463534242,25,0,15,220454607,601,0,0,5835dc59,off,,,,,,, +5349,c,none,nearly_sorted,1000,2463534242,10,0,421,236293161,90,0,0,1cfe4afd,off,,,,,,, +5350,node,quick,random,500,2463534242,14,16148,1,260575709,231182606,4578,4552,11ae932d,off,,,,,,, +5351,c,bubble,sorted,1000,2463534242,22,702,1,234856890,233707596,499500,0,8e15cb75,off,,,,,,, +5352,python,none,sorted,500,2463534242,9,0,16,214453833,571,0,0,d9a24379,off,,,,,,, +5353,node,quick,reversed,1000,2463534242,28,307,1,264161428,236510728,499500,1000,0ca43265,off,,,,,,, +5354,c,quick,nearly_sorted,500,2463534242,21,31784,1,239387210,238465322,10192,1850,482b4785,off,,,,,,, +5355,c,quick,reversed,500,2463534242,20,3240,1,238212579,237047807,124750,500,d33e3559,off,,,,,,, +5356,node,insertion,random,500,2463534242,16,4059,1,262799025,235061611,60338,60339,11ae932d,off,,,,,,, +5357,python,bubble,random,1000,2463534242,6,6,1,273095231,256621476,499500,496926,57561285,off,,,,,,, +5358,c,quick,reversed,250,2463534242,25,12781,1,242067603,240949847,31125,250,be175004,off,,,,,,, +5359,python,none,nearly_sorted,250,2463534242,6,0,17,224600017,521,0,0,21a19c14,off,,,,,,, +5360,c,quick,nearly_sorted,500,2463534242,16,31784,1,240485487,239189178,10192,1850,482b4785,off,,,,,,, +5361,c,none,reversed,500,2463534242,23,0,402,216323427,90,0,0,d33e3559,off,,,,,,, +5362,python,insertion,reversed,250,2463534242,7,102,1,249326838,234032476,31125,31374,be175004,off,,,,,,, +5363,python,bubble,sorted,250,2463534242,25,206,1,259075547,243380512,31125,0,7fd8c1f4,off,,,,,,, +5364,node,none,nearly_sorted,250,2463534242,18,0,10,249274549,621,0,0,21a19c14,off,,,,,,, +5365,node,bubble,reversed,2000,2463534242,1,94,1,263807685,237488235,1999000,3998000,ad35ba05,off,,,,,,, +5366,node,none,nearly_sorted,500,2463534242,5,0,10,249698715,651,0,0,482b4785,off,,,,,,, +5367,node,quick,nearly_sorted,500,2463534242,24,10954,1,260595555,232782620,10192,1850,482b4785,off,,,,,,, +5368,python,bubble,random,500,2463534242,15,26,1,259635897,243880941,124750,119688,11ae932d,off,,,,,,, +5369,node,quick,reversed,250,2463534242,11,4755,1,260283411,234187113,31125,250,be175004,off,,,,,,, +5370,node,none,nearly_sorted,500,2463534242,1,0,10,251627298,841,0,0,482b4785,off,,,,,,, +5371,c,none,reversed,250,2463534242,18,0,427,234042624,90,0,0,be175004,off,,,,,,, +5372,python,insertion,reversed,2000,2463534242,2,2,1,415974278,399859797,1999000,2000999,ad35ba05,off,,,,,,, +5373,node,none,random,2000,2463534242,2,0,9,224051969,841,0,0,41262855,off,,,,,,, +5374,node,bubble,sorted,1000,2463534242,7,374,1,239921591,211002264,499500,0,8e15cb75,off,,,,,,, +5375,c,bubble,random,250,2463534242,6,2568,1,233092563,232147051,31125,31196,b47adce4,off,,,,,,, +5376,node,quick,nearly_sorted,500,2463534242,17,10954,1,260879317,233199124,10192,1850,482b4785,off,,,,,,, +5377,c,bubble,random,250,2463534242,27,2568,1,238580598,237585304,31125,31196,b47adce4,off,,,,,,, +5378,node,bubble,random,1000,2463534242,4,362,1,262757518,233680805,499500,496926,57561285,off,,,,,,, +5379,node,quick,random,1000,2463534242,26,6513,1,263036640,236332317,11196,10210,57561285,off,,,,,,, +5380,node,quick,reversed,250,2463534242,28,4755,1,259201844,232891248,31125,250,be175004,off,,,,,,, +5381,node,quick,sorted,2000,2463534242,13,74,1,267520123,239561693,1999000,0,69842aa5,off,,,,,,, +5382,python,quick,random,1000,2463534242,16,209,1,259985753,244354034,11196,10210,57561285,off,,,,,,, +5383,python,none,nearly_sorted,500,2463534242,25,0,17,228724317,661,0,0,482b4785,off,,,,,,, +5384,node,insertion,reversed,2000,2463534242,30,144,1,263226967,233296019,1999000,2000999,ad35ba05,off,,,,,,, +5385,node,none,sorted,1000,2463534242,11,0,9,222872079,782,0,0,8e15cb75,off,,,,,,, +5386,node,bubble,nearly_sorted,250,2463534242,3,6161,1,265947225,236684173,31125,4056,21a19c14,off,,,,,,, +5387,python,quick,sorted,2000,2463534242,20,2,1,300267854,284702156,1999000,0,69842aa5,off,,,,,,, +5388,c,quick,sorted,500,2463534242,17,2762,1,241845718,240616705,124750,0,d9a24379,off,,,,,,, +5389,python,insertion,reversed,1000,2463534242,16,6,1,298330826,280900366,499500,500499,0ca43265,off,,,,,,, +5390,c,none,random,2000,2463534242,29,0,415,231196758,90,0,0,41262855,off,,,,,,, +5391,python,bubble,nearly_sorted,250,2463534242,5,188,1,258647946,242822110,31125,4056,21a19c14,off,,,,,,, +5392,node,insertion,sorted,2000,2463534242,28,89220,1,253745237,225270530,1999,0,69842aa5,off,,,,,,, +5393,python,none,reversed,500,2463534242,8,0,16,211181901,601,0,0,d33e3559,off,,,,,,, +5394,c,bubble,random,2000,2463534242,1,40,1,237356254,236110870,1999000,1988792,41262855,off,,,,,,, +5395,python,quick,sorted,2000,2463534242,24,2,1,294505477,278941811,1999000,0,69842aa5,off,,,,,,, +5396,c,bubble,nearly_sorted,250,2463534242,29,7880,1,236504848,234737155,31125,4056,21a19c14,off,,,,,,, +5397,node,quick,sorted,2000,2463534242,11,74,1,262218977,234321772,1999000,0,69842aa5,off,,,,,,, +5398,python,bubble,reversed,500,2463534242,20,20,1,258494818,243435295,124750,249500,d33e3559,off,,,,,,, +5399,python,insertion,reversed,500,2463534242,29,22,1,249188809,233639610,124750,125249,d33e3559,off,,,,,,, +5400,node,bubble,sorted,2000,2463534242,10,85,1,255510064,228505348,1999000,0,69842aa5,off,,,,,,, +5401,node,bubble,nearly_sorted,500,2463534242,20,1566,1,264076199,237709465,124750,14926,482b4785,off,,,,,,, +5402,python,bubble,reversed,500,2463534242,16,20,1,256890752,241551200,124750,249500,d33e3559,off,,,,,,, +5403,node,bubble,sorted,2000,2463534242,16,85,1,260527608,232636021,1999000,0,69842aa5,off,,,,,,, +5404,python,none,nearly_sorted,250,2463534242,5,0,17,223977721,591,0,0,21a19c14,off,,,,,,, +5405,node,insertion,reversed,1000,2463534242,1,554,1,264760110,236469390,499500,500499,0ca43265,off,,,,,,, +5406,python,insertion,nearly_sorted,250,2463534242,17,1416,1,255246302,238469126,2277,2251,21a19c14,off,,,,,,, +5407,c,insertion,random,250,2463534242,4,26020,1,229755487,228572112,15844,15838,b47adce4,off,,,,,,, +5408,node,insertion,sorted,500,2463534242,9,324764,1,257259313,230231277,499,0,d9a24379,off,,,,,,, +5409,node,none,reversed,1000,2463534242,12,0,9,219231076,762,0,0,0ca43265,off,,,,,,, +5410,node,insertion,reversed,1000,2463534242,29,554,1,275771957,247616152,499500,500499,0ca43265,off,,,,,,, +5411,python,bubble,nearly_sorted,500,2463534242,14,41,1,267547565,252703373,124750,14926,482b4785,off,,,,,,, +5412,c,quick,reversed,1000,2463534242,28,814,1,241133102,240166881,499500,1000,0ca43265,off,,,,,,, +5413,node,quick,nearly_sorted,250,2463534242,25,25320,1,257429062,228828313,4346,1008,21a19c14,off,,,,,,, +5414,node,quick,nearly_sorted,2000,2463534242,11,1909,1,245074748,218898898,58002,11768,5835dc59,off,,,,,,, +5415,python,none,nearly_sorted,500,2463534242,11,0,17,228664655,450,0,0,482b4785,off,,,,,,, +5416,node,quick,reversed,500,2463534242,1,1194,1,258818395,231365210,124750,500,d33e3559,off,,,,,,, +5417,python,none,reversed,2000,2463534242,24,0,16,225035594,501,0,0,ad35ba05,off,,,,,,, +5418,python,bubble,random,1000,2463534242,14,6,1,264859669,249825608,499500,496926,57561285,off,,,,,,, +5419,node,insertion,nearly_sorted,1000,2463534242,21,7758,1,255246289,228189930,30889,30860,1cfe4afd,off,,,,,,, +5420,node,quick,reversed,500,2463534242,12,1194,1,265249637,238036956,124750,500,d33e3559,off,,,,,,, +5421,node,insertion,nearly_sorted,1000,2463534242,22,7758,1,255828643,228882944,30889,30860,1cfe4afd,off,,,,,,, +5422,python,insertion,reversed,250,2463534242,3,102,1,253406051,238246470,31125,31374,be175004,off,,,,,,, +5423,node,quick,nearly_sorted,500,2463534242,18,10954,1,266097869,239489641,10192,1850,482b4785,off,,,,,,, +5424,python,insertion,nearly_sorted,250,2463534242,5,1416,1,269018380,253113082,2277,2251,21a19c14,off,,,,,,, +5425,c,none,random,250,2463534242,21,0,429,230007281,90,0,0,b47adce4,off,,,,,,, +5426,python,bubble,nearly_sorted,2000,2463534242,10,3,1,349830108,333589065,1999000,232556,5835dc59,off,,,,,,, +5427,node,bubble,nearly_sorted,2000,2463534242,12,110,1,259352175,231726174,1999000,232556,5835dc59,off,,,,,,, +5428,c,quick,sorted,1000,2463534242,24,691,1,238670948,237763288,499500,0,8e15cb75,off,,,,,,, +5429,c,quick,sorted,250,2463534242,1,10851,1,247639266,246737166,31125,0,7fd8c1f4,off,,,,,,, +5430,python,bubble,nearly_sorted,250,2463534242,10,188,1,285163025,269630522,31125,4056,21a19c14,off,,,,,,, +5431,c,none,nearly_sorted,1000,2463534242,25,0,421,240596126,150,0,0,1cfe4afd,off,,,,,,, +5432,node,bubble,nearly_sorted,250,2463534242,13,6161,1,279021587,247270601,31125,4056,21a19c14,off,,,,,,, +5433,c,insertion,random,250,2463534242,7,26020,1,227343950,226107452,15844,15838,b47adce4,off,,,,,,, +5434,python,insertion,reversed,500,2463534242,26,22,1,245745545,230146294,124750,125249,d33e3559,off,,,,,,, +5435,c,bubble,random,250,2463534242,25,2568,1,238314229,237043948,31125,31196,b47adce4,off,,,,,,, +5436,python,quick,sorted,1000,2463534242,2,8,1,279702093,264146050,499500,0,8e15cb75,off,,,,,,, +5437,python,quick,reversed,1000,2463534242,2,8,1,267458848,252656093,499500,1000,0ca43265,off,,,,,,, +5438,node,none,random,2000,2463534242,26,0,9,230525924,742,0,0,41262855,off,,,,,,, +5439,c,quick,nearly_sorted,250,2463534242,30,77221,1,245789249,244721958,4346,1008,21a19c14,off,,,,,,, +5440,node,quick,reversed,2000,2463534242,6,80,1,271366349,241798831,1999000,2000,ad35ba05,off,,,,,,, +5441,c,insertion,nearly_sorted,500,2463534242,16,32350,1,238564939,237396980,7962,7926,482b4785,off,,,,,,, +5442,node,quick,sorted,500,2463534242,16,1164,1,264709616,236476122,124750,0,d9a24379,off,,,,,,, +5443,c,quick,nearly_sorted,1000,2463534242,3,11273,1,241816272,240704951,28499,7892,1cfe4afd,off,,,,,,, +5444,node,bubble,reversed,1000,2463534242,8,378,1,264164954,236734711,499500,999000,0ca43265,off,,,,,,, +5445,node,insertion,reversed,250,2463534242,18,8539,1,269086409,238147940,31125,31374,be175004,off,,,,,,, +5446,node,quick,random,2000,2463534242,2,2126,1,262993068,235245469,25197,24544,41262855,off,,,,,,, +5447,c,bubble,sorted,250,2463534242,7,11386,1,239924617,238753072,31125,0,7fd8c1f4,off,,,,,,, +5448,python,quick,sorted,2000,2463534242,15,2,1,303150707,286853945,1999000,0,69842aa5,off,,,,,,, +5449,python,insertion,nearly_sorted,250,2463534242,10,1416,1,256867318,241675715,2277,2251,21a19c14,off,,,,,,, +5450,python,none,nearly_sorted,1000,2463534242,26,0,16,222755792,551,0,0,1cfe4afd,off,,,,,,, +5451,python,quick,sorted,250,2463534242,1,133,1,258005942,242228509,31125,0,7fd8c1f4,off,,,,,,, +5452,python,quick,reversed,2000,2463534242,7,2,1,276056550,260458317,1999000,2000,ad35ba05,off,,,,,,, +5453,node,none,reversed,250,2463534242,14,0,9,221068800,731,0,0,be175004,off,,,,,,, +5454,c,quick,nearly_sorted,1000,2463534242,1,11273,1,237013291,235872242,28499,7892,1cfe4afd,off,,,,,,, +5455,python,quick,sorted,500,2463534242,3,30,1,249022597,233549922,124750,0,d9a24379,off,,,,,,, +5456,c,bubble,nearly_sorted,1000,2463534242,24,507,1,239435530,238309891,499500,59780,1cfe4afd,off,,,,,,, +5457,python,insertion,reversed,2000,2463534242,13,2,1,407700742,391736116,1999000,2000999,ad35ba05,off,,,,,,, +5458,python,none,random,250,2463534242,7,0,17,233414065,601,0,0,b47adce4,off,,,,,,, +5459,node,none,reversed,500,2463534242,13,0,9,233885099,802,0,0,d33e3559,off,,,,,,, +5460,node,none,nearly_sorted,1000,2463534242,5,0,10,250256961,631,0,0,1cfe4afd,off,,,,,,, +5461,python,bubble,random,250,2463534242,20,112,1,263083168,248524132,31125,31196,b47adce4,off,,,,,,, +5462,python,bubble,random,250,2463534242,30,112,1,249636888,234568593,31125,31196,b47adce4,off,,,,,,, +5463,python,quick,reversed,250,2463534242,16,151,1,263514636,248880767,31125,250,be175004,off,,,,,,, +5464,node,quick,random,1000,2463534242,22,6513,1,259625598,232501422,11196,10210,57561285,off,,,,,,, +5465,node,insertion,nearly_sorted,250,2463534242,19,79550,1,230222875,203589492,2277,2251,21a19c14,off,,,,,,, +5466,c,insertion,random,250,2463534242,17,26020,1,226861495,225901035,15844,15838,b47adce4,off,,,,,,, +5467,python,bubble,random,2000,2463534242,7,2,1,380486511,361948459,1999000,1988792,41262855,off,,,,,,, +5468,c,insertion,sorted,2000,2463534242,1,101352,1,230722311,229452421,1999,0,69842aa5,off,,,,,,, +5469,c,bubble,random,2000,2463534242,20,40,1,238296193,237337128,1999000,1988792,41262855,off,,,,,,, +5470,c,bubble,nearly_sorted,2000,2463534242,1,121,1,232635446,231169861,1999000,232556,5835dc59,off,,,,,,, +5471,node,quick,sorted,250,2463534242,2,4507,1,257241661,229865099,31125,0,7fd8c1f4,off,,,,,,, +5472,python,insertion,nearly_sorted,500,2463534242,18,316,1,245957473,231301698,7962,7926,482b4785,off,,,,,,, +5473,c,quick,nearly_sorted,2000,2463534242,25,6145,1,245893693,244848935,58002,11768,5835dc59,off,,,,,,, +5474,python,bubble,nearly_sorted,250,2463534242,15,188,1,262462826,246717581,31125,4056,21a19c14,off,,,,,,, +5475,python,insertion,sorted,250,2463534242,27,18845,1,259417558,244525652,249,0,7fd8c1f4,off,,,,,,, +5476,c,insertion,reversed,250,2463534242,7,15645,1,242624529,241646234,31125,31374,be175004,off,,,,,,, +5477,python,none,nearly_sorted,500,2463534242,28,0,17,229036831,521,0,0,482b4785,off,,,,,,, +5478,node,none,sorted,1000,2463534242,29,0,9,221514625,721,0,0,8e15cb75,off,,,,,,, +5479,c,quick,reversed,1000,2463534242,27,814,1,235453998,234582255,499500,1000,0ca43265,off,,,,,,, +5480,c,insertion,sorted,1000,2463534242,10,196883,1,226378241,225172741,999,0,8e15cb75,off,,,,,,, +5481,python,bubble,reversed,2000,2463534242,1,2,1,505548796,489333563,1999000,3998000,ad35ba05,off,,,,,,, +5482,node,quick,sorted,500,2463534242,25,1164,1,262701833,235030330,124750,0,d9a24379,off,,,,,,, +5483,c,bubble,sorted,1000,2463534242,19,702,1,234639762,233606126,499500,0,8e15cb75,off,,,,,,, +5484,python,none,sorted,250,2463534242,20,0,17,223801641,511,0,0,7fd8c1f4,off,,,,,,, +5485,node,quick,nearly_sorted,500,2463534242,3,10954,1,261102476,233786282,10192,1850,482b4785,off,,,,,,, +5486,python,none,reversed,2000,2463534242,19,0,16,226658405,611,0,0,ad35ba05,off,,,,,,, +5487,c,none,random,250,2463534242,27,0,429,231117582,90,0,0,b47adce4,off,,,,,,, +5488,c,quick,random,250,2463534242,6,136055,1,232386099,231531178,1933,1940,b47adce4,off,,,,,,, +5489,python,bubble,random,250,2463534242,26,112,1,242650826,227827152,31125,31196,b47adce4,off,,,,,,, +5490,python,insertion,random,1000,2463534242,6,10,1,257721101,240420074,249457,249457,57561285,off,,,,,,, +5491,node,none,nearly_sorted,2000,2463534242,11,0,9,236182712,671,0,0,5835dc59,off,,,,,,, +5492,python,insertion,reversed,250,2463534242,1,102,1,249860888,234591346,31125,31374,be175004,off,,,,,,, +5493,python,insertion,nearly_sorted,2000,2463534242,30,20,1,263141477,246717680,118277,118270,5835dc59,off,,,,,,, +5494,node,quick,nearly_sorted,500,2463534242,25,10954,1,260542206,232814394,10192,1850,482b4785,off,,,,,,, +5495,node,none,sorted,2000,2463534242,21,0,10,250961271,761,0,0,69842aa5,off,,,,,,, +5496,python,quick,nearly_sorted,250,2463534242,8,882,1,252961038,236613651,4346,1008,21a19c14,off,,,,,,, +5497,c,quick,reversed,2000,2463534242,18,209,1,234503148,233413246,1999000,2000,ad35ba05,off,,,,,,, +5498,python,bubble,nearly_sorted,2000,2463534242,23,3,1,354127651,336174336,1999000,232556,5835dc59,off,,,,,,, +5499,python,bubble,sorted,2000,2463534242,16,3,1,344711315,328384002,1999000,0,69842aa5,off,,,,,,, +5500,node,insertion,random,2000,2463534242,15,234,1,264916483,237418819,996390,996389,41262855,off,,,,,,, +5501,c,quick,random,250,2463534242,9,136055,1,229562558,228576169,1933,1940,b47adce4,off,,,,,,, +5502,node,bubble,nearly_sorted,250,2463534242,22,6161,1,267101908,239020294,31125,4056,21a19c14,off,,,,,,, +5503,python,none,reversed,1000,2463534242,1,0,16,218015388,611,0,0,0ca43265,off,,,,,,, +5504,node,none,reversed,500,2463534242,6,0,9,219496022,822,0,0,d33e3559,off,,,,,,, +5505,c,bubble,reversed,250,2463534242,14,1384,1,238065653,237250355,31125,62250,be175004,off,,,,,,, +5506,node,bubble,nearly_sorted,500,2463534242,6,1566,1,267879707,239935822,124750,14926,482b4785,off,,,,,,, +5507,node,none,random,250,2463534242,11,0,10,252808212,621,0,0,b47adce4,off,,,,,,, +5508,c,quick,nearly_sorted,500,2463534242,17,31784,1,239564322,238420407,10192,1850,482b4785,off,,,,,,, +5509,c,none,random,500,2463534242,2,0,426,229281591,80,0,0,11ae932d,off,,,,,,, +5510,python,bubble,nearly_sorted,2000,2463534242,2,3,1,340058525,324266369,1999000,232556,5835dc59,off,,,,,,, +5511,c,insertion,random,1000,2463534242,1,1983,1,233064550,232172539,249457,249457,57561285,off,,,,,,, +5512,node,insertion,sorted,250,2463534242,26,607595,1,246817524,218428184,249,0,7fd8c1f4,off,,,,,,, +5513,node,none,reversed,1000,2463534242,9,0,9,229292091,792,0,0,0ca43265,off,,,,,,, +5514,c,bubble,nearly_sorted,1000,2463534242,4,507,1,241490794,240324497,499500,59780,1cfe4afd,off,,,,,,, +5515,c,bubble,sorted,250,2463534242,11,11386,1,239543993,238235773,31125,0,7fd8c1f4,off,,,,,,, +5516,node,none,random,2000,2463534242,27,0,9,230187007,1803,0,0,41262855,off,,,,,,, +5517,c,bubble,reversed,500,2463534242,18,346,1,233787946,231412825,124750,249500,d33e3559,off,,,,,,, +5518,node,quick,sorted,2000,2463534242,25,74,1,269748549,240264237,1999000,0,69842aa5,off,,,,,,, +5519,c,bubble,random,2000,2463534242,7,40,1,238170760,237184112,1999000,1988792,41262855,off,,,,,,, +5520,node,quick,nearly_sorted,1000,2463534242,17,3752,1,255587711,228801905,28499,7892,1cfe4afd,off,,,,,,, +5521,node,insertion,nearly_sorted,2000,2463534242,21,2100,1,247748349,220551092,118277,118270,5835dc59,off,,,,,,, +5522,node,insertion,reversed,1000,2463534242,16,554,1,256501854,227054216,499500,500499,0ca43265,off,,,,,,, +5523,python,quick,sorted,1000,2463534242,14,8,1,281946280,266752229,499500,0,8e15cb75,off,,,,,,, +5524,c,insertion,sorted,250,2463534242,30,786886,1,238587419,237180165,249,0,7fd8c1f4,off,,,,,,, +5525,c,insertion,nearly_sorted,250,2463534242,2,99421,1,232647710,231366528,2277,2251,21a19c14,off,,,,,,, +5526,node,quick,reversed,1000,2463534242,19,307,1,260676447,230896569,499500,1000,0ca43265,off,,,,,,, +5527,node,bubble,nearly_sorted,250,2463534242,8,6161,1,263797868,235874623,31125,4056,21a19c14,off,,,,,,, +5528,c,quick,reversed,2000,2463534242,5,209,1,240336939,239378525,1999000,2000,ad35ba05,off,,,,,,, +5529,node,bubble,sorted,500,2463534242,23,1964,1,250823422,223373575,124750,0,d9a24379,off,,,,,,, +5530,node,bubble,nearly_sorted,1000,2463534242,20,413,1,267598751,240154787,499500,59780,1cfe4afd,off,,,,,,, +5531,python,none,random,250,2463534242,21,0,17,224297992,1002,0,0,b47adce4,off,,,,,,, +5532,c,none,reversed,1000,2463534242,8,0,405,217241307,90,0,0,0ca43265,off,,,,,,, +5533,python,bubble,reversed,2000,2463534242,9,2,1,464234370,448984069,1999000,3998000,ad35ba05,off,,,,,,, +5534,node,none,nearly_sorted,500,2463534242,2,0,10,248604845,622,0,0,482b4785,off,,,,,,, +5535,python,insertion,random,2000,2463534242,5,3,1,320089652,302240540,996390,996389,41262855,off,,,,,,, +5536,node,none,nearly_sorted,1000,2463534242,7,0,10,258168898,622,0,0,1cfe4afd,off,,,,,,, +5537,python,insertion,nearly_sorted,500,2463534242,4,316,1,251179208,234677052,7962,7926,482b4785,off,,,,,,, +5538,python,bubble,sorted,2000,2463534242,25,3,1,326420607,310568992,1999000,0,69842aa5,off,,,,,,, +5539,python,bubble,sorted,500,2463534242,9,46,1,283565583,268395651,124750,0,d9a24379,off,,,,,,, +5540,c,none,nearly_sorted,1000,2463534242,7,0,421,232454647,110,0,0,1cfe4afd,off,,,,,,, +5541,c,insertion,reversed,1000,2463534242,4,1033,1,242334834,241649010,499500,500499,0ca43265,off,,,,,,, +5542,node,insertion,random,250,2463534242,7,15349,1,256082769,228061779,15844,15838,b47adce4,off,,,,,,, +5543,c,insertion,sorted,2000,2463534242,7,101352,1,237692299,236642656,1999,0,69842aa5,off,,,,,,, +5544,python,insertion,reversed,250,2463534242,26,102,1,254673433,239148872,31125,31374,be175004,off,,,,,,, +5545,c,quick,sorted,500,2463534242,26,2762,1,252155186,250835050,124750,0,d9a24379,off,,,,,,, +5546,node,bubble,sorted,1000,2463534242,1,374,1,262846163,234476916,499500,0,8e15cb75,off,,,,,,, +5547,node,none,random,500,2463534242,23,0,9,240807701,832,0,0,11ae932d,off,,,,,,, +5548,c,quick,reversed,250,2463534242,16,12781,1,249312862,248315894,31125,250,be175004,off,,,,,,, +5549,python,none,random,500,2463534242,6,0,16,217968538,531,0,0,11ae932d,off,,,,,,, +5550,c,quick,random,1000,2463534242,20,23484,1,234137102,232901796,11196,10210,57561285,off,,,,,,, +5551,node,insertion,nearly_sorted,250,2463534242,16,79550,1,232556697,205752061,2277,2251,21a19c14,off,,,,,,, +5552,node,insertion,random,2000,2463534242,3,234,1,245321401,218447024,996390,996389,41262855,off,,,,,,, +5553,c,none,nearly_sorted,1000,2463534242,17,0,421,229398411,90,0,0,1cfe4afd,off,,,,,,, +5554,python,quick,random,250,2463534242,28,1271,1,243716753,228825497,1933,1940,b47adce4,off,,,,,,, +5555,c,quick,sorted,2000,2463534242,28,174,1,238745467,237307832,1999000,0,69842aa5,off,,,,,,, +5556,c,insertion,nearly_sorted,1000,2463534242,26,11445,1,241348755,239980143,30889,30860,1cfe4afd,off,,,,,,, +5557,c,none,nearly_sorted,1000,2463534242,15,0,421,228782465,90,0,0,1cfe4afd,off,,,,,,, +5558,c,insertion,sorted,1000,2463534242,24,196883,1,227447222,226717602,999,0,8e15cb75,off,,,,,,, +5559,python,quick,random,500,2463534242,13,507,1,252994571,237926688,4578,4552,11ae932d,off,,,,,,, +5560,node,none,random,250,2463534242,13,0,10,246775706,672,0,0,b47adce4,off,,,,,,, +5561,c,quick,reversed,250,2463534242,12,12781,1,241064785,239932413,31125,250,be175004,off,,,,,,, +5562,c,none,nearly_sorted,2000,2463534242,14,0,387,215899631,90,0,0,5835dc59,off,,,,,,, +5563,c,none,random,500,2463534242,18,0,426,229249681,80,0,0,11ae932d,off,,,,,,, +5564,node,quick,reversed,2000,2463534242,9,80,1,271714824,243130625,1999000,2000,ad35ba05,off,,,,,,, +5565,node,bubble,random,1000,2463534242,9,362,1,257844139,228578913,499500,496926,57561285,off,,,,,,, +5566,node,bubble,reversed,250,2463534242,14,5861,1,260406251,233261691,31125,62250,be175004,off,,,,,,, +5567,node,quick,reversed,1000,2463534242,16,307,1,254899941,228385471,499500,1000,0ca43265,off,,,,,,, +5568,node,none,nearly_sorted,250,2463534242,21,0,10,253311956,571,0,0,21a19c14,off,,,,,,, +5569,python,bubble,nearly_sorted,500,2463534242,9,41,1,251279016,234782075,124750,14926,482b4785,off,,,,,,, +5570,node,bubble,nearly_sorted,250,2463534242,12,6161,1,271498558,244109735,31125,4056,21a19c14,off,,,,,,, +5571,c,none,nearly_sorted,500,2463534242,25,0,405,218706513,80,0,0,482b4785,off,,,,,,, +5572,node,insertion,sorted,500,2463534242,10,324764,1,258843242,231344084,499,0,d9a24379,off,,,,,,, +5573,c,quick,random,1000,2463534242,28,23484,1,234501334,233283911,11196,10210,57561285,off,,,,,,, +5574,node,quick,random,500,2463534242,11,16148,1,259429440,230942046,4578,4552,11ae932d,off,,,,,,, +5575,python,quick,sorted,250,2463534242,24,133,1,251646014,237117879,31125,0,7fd8c1f4,off,,,,,,, +5576,node,quick,sorted,1000,2463534242,15,296,1,265404228,238539318,499500,0,8e15cb75,off,,,,,,, +5577,node,bubble,sorted,2000,2463534242,21,85,1,255020127,228641594,1999000,0,69842aa5,off,,,,,,, +5578,c,quick,random,500,2463534242,23,55776,1,231971352,230875558,4578,4552,11ae932d,off,,,,,,, +5579,c,insertion,reversed,250,2463534242,22,15645,1,236675758,235576538,31125,31374,be175004,off,,,,,,, +5580,node,none,nearly_sorted,250,2463534242,29,0,10,243240200,551,0,0,21a19c14,off,,,,,,, +5581,python,bubble,reversed,500,2463534242,26,20,1,260548999,244617394,124750,249500,d33e3559,off,,,,,,, +5582,c,quick,nearly_sorted,500,2463534242,23,31784,1,232861700,231647895,10192,1850,482b4785,off,,,,,,, +5583,python,insertion,reversed,500,2463534242,17,22,1,248930043,234393353,124750,125249,d33e3559,off,,,,,,, +5584,python,quick,reversed,250,2463534242,2,151,1,251072830,235699514,31125,250,be175004,off,,,,,,, +5585,c,bubble,sorted,250,2463534242,24,11386,1,234600088,233651290,31125,0,7fd8c1f4,off,,,,,,, +5586,node,none,reversed,250,2463534242,22,0,9,218125594,672,0,0,be175004,off,,,,,,, +5587,c,quick,sorted,500,2463534242,22,2762,1,243163767,242231660,124750,0,d9a24379,off,,,,,,, +5588,node,quick,nearly_sorted,2000,2463534242,3,1909,1,244421665,217992165,58002,11768,5835dc59,off,,,,,,, +5589,c,bubble,sorted,2000,2463534242,2,173,1,228169346,227284739,1999000,0,69842aa5,off,,,,,,, +5590,c,insertion,random,500,2463534242,3,6945,1,208241621,207333079,60338,60339,11ae932d,off,,,,,,, +5591,python,none,reversed,2000,2463534242,22,0,16,225787784,571,0,0,ad35ba05,off,,,,,,, +5592,python,quick,nearly_sorted,2000,2463534242,22,55,1,239862521,224779635,58002,11768,5835dc59,off,,,,,,, +5593,c,none,reversed,2000,2463534242,11,0,413,227419932,90,0,0,ad35ba05,off,,,,,,, +5594,python,none,random,2000,2463534242,10,0,16,237526302,541,0,0,41262855,off,,,,,,, +5595,python,bubble,reversed,500,2463534242,22,20,1,278844977,264863137,124750,249500,d33e3559,off,,,,,,, +5596,python,bubble,sorted,500,2463534242,3,46,1,250449853,234723808,124750,0,d9a24379,off,,,,,,, +5597,node,insertion,reversed,2000,2463534242,15,144,1,264373564,234883857,1999000,2000999,ad35ba05,off,,,,,,, +5598,python,none,random,2000,2463534242,17,0,16,238132258,701,0,0,41262855,off,,,,,,, +5599,python,none,nearly_sorted,500,2463534242,22,0,17,233709159,501,0,0,482b4785,off,,,,,,, +5600,c,none,random,500,2463534242,3,0,426,232062202,91,0,0,11ae932d,off,,,,,,, +5601,c,quick,sorted,2000,2463534242,3,174,1,238766567,237958292,1999000,0,69842aa5,off,,,,,,, +5602,c,quick,reversed,250,2463534242,8,12781,1,245320629,243926658,31125,250,be175004,off,,,,,,, +5603,node,bubble,nearly_sorted,2000,2463534242,11,110,1,261227350,231617220,1999000,232556,5835dc59,off,,,,,,, +5604,python,none,reversed,500,2463534242,13,0,16,214028115,581,0,0,d33e3559,off,,,,,,, +5605,python,bubble,nearly_sorted,1000,2463534242,14,9,1,244609186,229905514,499500,59780,1cfe4afd,off,,,,,,, +5606,c,quick,reversed,2000,2463534242,2,209,1,232880776,231891143,1999000,2000,ad35ba05,off,,,,,,, +5607,node,none,random,250,2463534242,23,0,10,251349718,691,0,0,b47adce4,off,,,,,,, +5608,python,quick,random,1000,2463534242,28,209,1,247257719,232145345,11196,10210,57561285,off,,,,,,, +5609,node,insertion,random,250,2463534242,11,15349,1,263198675,233249596,15844,15838,b47adce4,off,,,,,,, +5610,python,insertion,sorted,500,2463534242,17,6624,1,231427092,216489819,499,0,d9a24379,off,,,,,,, +5611,c,bubble,sorted,1000,2463534242,3,702,1,232471580,231456397,499500,0,8e15cb75,off,,,,,,, +5612,c,insertion,random,500,2463534242,14,6945,1,212697000,210979612,60338,60339,11ae932d,off,,,,,,, +5613,python,insertion,nearly_sorted,250,2463534242,18,1416,1,248127269,233135528,2277,2251,21a19c14,off,,,,,,, +5614,node,insertion,sorted,1000,2463534242,18,160107,1,249820783,220058325,999,0,8e15cb75,off,,,,,,, +5615,c,bubble,reversed,2000,2463534242,16,22,1,241924331,240988271,1999000,3998000,ad35ba05,off,,,,,,, +5616,c,insertion,reversed,1000,2463534242,6,1033,1,241218275,240179185,499500,500499,0ca43265,off,,,,,,, +5617,c,bubble,sorted,250,2463534242,12,11386,1,239247358,237786339,31125,0,7fd8c1f4,off,,,,,,, +5618,node,none,reversed,2000,2463534242,12,0,9,233405670,711,0,0,ad35ba05,off,,,,,,, +5619,c,quick,reversed,1000,2463534242,8,814,1,237553223,236372711,499500,1000,0ca43265,off,,,,,,, +5620,python,bubble,nearly_sorted,250,2463534242,3,188,1,261538403,246012584,31125,4056,21a19c14,off,,,,,,, +5621,python,insertion,random,250,2463534242,22,206,1,419206283,404140929,15844,15838,b47adce4,off,,,,,,, +5622,node,none,reversed,1000,2463534242,24,0,9,221065203,761,0,0,0ca43265,off,,,,,,, +5623,python,insertion,random,2000,2463534242,16,3,1,316052395,299803552,996390,996389,41262855,off,,,,,,, +5624,c,none,sorted,500,2463534242,22,0,344,194600276,90,0,0,d9a24379,off,,,,,,, +5625,c,bubble,nearly_sorted,2000,2463534242,9,121,1,250727402,249983007,1999000,232556,5835dc59,off,,,,,,, +5626,python,quick,nearly_sorted,500,2463534242,3,296,1,245525153,229297213,10192,1850,482b4785,off,,,,,,, +5627,node,insertion,nearly_sorted,250,2463534242,11,79550,1,273079061,244178922,2277,2251,21a19c14,off,,,,,,, +5628,python,none,sorted,2000,2463534242,5,0,15,542001152,671,0,0,69842aa5,off,,,,,,, +5629,python,bubble,nearly_sorted,1000,2463534242,3,9,1,392572961,367289699,499500,59780,1cfe4afd,off,,,,,,, +5630,python,insertion,random,500,2463534242,7,44,1,407943071,377566728,60338,60339,11ae932d,off,,,,,,, +5631,c,bubble,nearly_sorted,2000,2463534242,4,121,1,345483953,344061636,1999000,232556,5835dc59,off,,,,,,, +5632,node,quick,sorted,2000,2463534242,14,74,1,321632617,278508001,1999000,0,69842aa5,off,,,,,,, +5633,c,bubble,reversed,500,2463534242,21,346,1,253320552,252361292,124750,249500,d33e3559,off,,,,,,, +5634,c,insertion,random,1000,2463534242,27,1983,1,257713494,256808839,249457,249457,57561285,off,,,,,,, +5635,c,none,reversed,250,2463534242,25,0,427,247533426,90,0,0,be175004,off,,,,,,, +5636,c,insertion,nearly_sorted,500,2463534242,18,32350,1,246053423,245302256,7962,7926,482b4785,off,,,,,,, +5637,c,quick,nearly_sorted,1000,2463534242,30,11273,1,247364632,246199336,28499,7892,1cfe4afd,off,,,,,,, +5638,node,none,random,500,2463534242,19,0,9,235386129,882,0,0,11ae932d,off,,,,,,, +5639,node,bubble,nearly_sorted,500,2463534242,29,1566,1,282495148,252322469,124750,14926,482b4785,off,,,,,,, +5640,c,none,random,2000,2463534242,7,0,415,242534077,151,0,0,41262855,off,,,,,,, +5641,c,quick,nearly_sorted,500,2463534242,28,31784,1,250355514,249429091,10192,1850,482b4785,off,,,,,,, +5642,python,insertion,sorted,500,2463534242,27,6624,1,247053677,231053844,499,0,d9a24379,off,,,,,,, +5643,node,bubble,sorted,250,2463534242,30,8081,1,259736164,233084565,31125,0,7fd8c1f4,off,,,,,,, +5644,node,quick,random,250,2463534242,25,36348,1,281019762,252122494,1933,1940,b47adce4,off,,,,,,, +5645,c,none,reversed,2000,2463534242,17,0,413,233131346,90,0,0,ad35ba05,off,,,,,,, +5646,node,none,random,500,2463534242,14,0,9,242631059,731,0,0,11ae932d,off,,,,,,, +5647,node,bubble,nearly_sorted,500,2463534242,23,1566,1,269288749,242780656,124750,14926,482b4785,off,,,,,,, +5648,c,none,reversed,250,2463534242,24,0,427,242225026,91,0,0,be175004,off,,,,,,, +5649,node,insertion,reversed,1000,2463534242,11,554,1,265090098,238254738,499500,500499,0ca43265,off,,,,,,, +5650,python,bubble,nearly_sorted,500,2463534242,20,41,1,252181417,236895738,124750,14926,482b4785,off,,,,,,, +5651,python,bubble,nearly_sorted,2000,2463534242,5,3,1,363494346,346763763,1999000,232556,5835dc59,off,,,,,,, +5652,python,none,reversed,500,2463534242,1,0,16,229958810,551,0,0,d33e3559,off,,,,,,, +5653,node,none,random,250,2463534242,20,0,10,251933562,721,0,0,b47adce4,off,,,,,,, +5654,c,insertion,reversed,1000,2463534242,1,1033,1,253318067,252238516,499500,500499,0ca43265,off,,,,,,, +5655,c,quick,random,1000,2463534242,12,23484,1,239225927,238307755,11196,10210,57561285,off,,,,,,, +5656,python,bubble,reversed,250,2463534242,12,88,1,249454022,233900905,31125,62250,be175004,off,,,,,,, +5657,node,quick,random,500,2463534242,9,16148,1,265664248,238325234,4578,4552,11ae932d,off,,,,,,, +5658,node,insertion,sorted,500,2463534242,5,324764,1,259111635,231522754,499,0,d9a24379,off,,,,,,, +5659,c,quick,reversed,250,2463534242,22,12781,1,243681464,242649674,31125,250,be175004,off,,,,,,, +5660,python,bubble,sorted,2000,2463534242,12,3,1,337143597,320854951,1999000,0,69842aa5,off,,,,,,, +5661,c,insertion,sorted,2000,2463534242,22,101352,1,238203701,237096316,1999,0,69842aa5,off,,,,,,, +5662,node,quick,nearly_sorted,250,2463534242,5,25320,1,266404002,238450175,4346,1008,21a19c14,off,,,,,,, +5663,node,bubble,reversed,1000,2463534242,20,378,1,288073563,260274447,499500,999000,0ca43265,off,,,,,,, +5664,node,bubble,nearly_sorted,2000,2463534242,27,110,1,271271151,242013037,1999000,232556,5835dc59,off,,,,,,, +5665,c,none,random,1000,2463534242,26,0,367,224183206,91,0,0,57561285,off,,,,,,, +5666,python,insertion,reversed,2000,2463534242,16,2,1,460058494,442664574,1999000,2000999,ad35ba05,off,,,,,,, +5667,python,bubble,nearly_sorted,1000,2463534242,22,9,1,272648834,256017933,499500,59780,1cfe4afd,off,,,,,,, +5668,node,none,nearly_sorted,500,2463534242,29,0,10,266719081,601,0,0,482b4785,off,,,,,,, +5669,node,bubble,random,2000,2463534242,27,98,1,280774893,250549312,1999000,1988792,41262855,off,,,,,,, +5670,c,bubble,nearly_sorted,250,2463534242,8,7880,1,253626675,252654836,31125,4056,21a19c14,off,,,,,,, +5671,c,insertion,random,1000,2463534242,17,1983,1,259801397,258792495,249457,249457,57561285,off,,,,,,, +5672,node,quick,sorted,2000,2463534242,5,74,1,293348106,256793854,1999000,0,69842aa5,off,,,,,,, +5673,node,bubble,nearly_sorted,250,2463534242,25,6161,1,283953941,253139530,31125,4056,21a19c14,off,,,,,,, +5674,python,bubble,random,250,2463534242,13,112,1,266363706,248799047,31125,31196,b47adce4,off,,,,,,, +5675,c,none,sorted,2000,2463534242,27,0,416,238701805,81,0,0,69842aa5,off,,,,,,, +5676,c,insertion,reversed,1000,2463534242,21,1033,1,243499736,242537053,499500,500499,0ca43265,off,,,,,,, +5677,c,bubble,sorted,500,2463534242,1,2858,1,244369857,243182362,124750,0,d9a24379,off,,,,,,, +5678,python,none,nearly_sorted,1000,2463534242,3,0,16,245491450,541,0,0,1cfe4afd,off,,,,,,, +5679,python,quick,nearly_sorted,250,2463534242,20,882,1,258095710,243405180,4346,1008,21a19c14,off,,,,,,, +5680,python,insertion,reversed,500,2463534242,27,22,1,253062779,238186256,124750,125249,d33e3559,off,,,,,,, +5681,python,insertion,reversed,1000,2463534242,19,6,1,293778253,278119190,499500,500499,0ca43265,off,,,,,,, +5682,c,none,sorted,500,2463534242,15,0,344,193256478,90,0,0,d9a24379,off,,,,,,, +5683,python,insertion,random,250,2463534242,12,206,1,260974847,244545842,15844,15838,b47adce4,off,,,,,,, +5684,node,bubble,sorted,500,2463534242,28,1964,1,262018583,233325168,124750,0,d9a24379,off,,,,,,, +5685,python,quick,sorted,1000,2463534242,16,8,1,286120933,270484286,499500,0,8e15cb75,off,,,,,,, +5686,node,insertion,nearly_sorted,500,2463534242,21,26045,1,246204576,218375244,7962,7926,482b4785,off,,,,,,, +5687,c,insertion,reversed,1000,2463534242,14,1033,1,257898561,257050172,499500,500499,0ca43265,off,,,,,,, +5688,python,bubble,random,2000,2463534242,17,2,1,380256520,362241024,1999000,1988792,41262855,off,,,,,,, +5689,python,quick,nearly_sorted,250,2463534242,2,882,1,264555407,249454729,4346,1008,21a19c14,off,,,,,,, +5690,python,none,random,2000,2463534242,5,0,16,255589834,1012,0,0,41262855,off,,,,,,, +5691,python,insertion,nearly_sorted,1000,2463534242,23,76,1,257051013,241704400,30889,30860,1cfe4afd,off,,,,,,, +5692,c,bubble,sorted,250,2463534242,27,11386,1,243814639,242817428,31125,0,7fd8c1f4,off,,,,,,, +5693,node,none,reversed,1000,2463534242,15,0,9,236514854,791,0,0,0ca43265,off,,,,,,, +5694,node,insertion,nearly_sorted,500,2463534242,8,26045,1,254420253,226021242,7962,7926,482b4785,off,,,,,,, +5695,c,none,reversed,2000,2463534242,5,0,413,235979553,130,0,0,ad35ba05,off,,,,,,, +5696,c,insertion,reversed,500,2463534242,21,3967,1,253350027,252599066,124750,125249,d33e3559,off,,,,,,, +5697,c,insertion,sorted,500,2463534242,28,399335,1,241173016,240159300,499,0,d9a24379,off,,,,,,, +5698,c,none,reversed,1000,2463534242,3,0,405,229137872,91,0,0,0ca43265,off,,,,,,, +5699,node,quick,sorted,2000,2463534242,2,74,1,274985073,244428781,1999000,0,69842aa5,off,,,,,,, +5700,c,bubble,nearly_sorted,1000,2463534242,3,507,1,244591442,243414489,499500,59780,1cfe4afd,off,,,,,,, +5701,c,insertion,random,500,2463534242,30,6945,1,219480503,218515144,60338,60339,11ae932d,off,,,,,,, +5702,python,quick,random,1000,2463534242,21,209,1,260672515,244072096,11196,10210,57561285,off,,,,,,, +5703,node,none,sorted,1000,2463534242,30,0,9,227813523,782,0,0,8e15cb75,off,,,,,,, +5704,node,bubble,reversed,250,2463534242,10,5861,1,270093465,243460944,31125,62250,be175004,off,,,,,,, +5705,c,none,reversed,1000,2463534242,18,0,405,236500430,101,0,0,0ca43265,off,,,,,,, +5706,python,quick,nearly_sorted,500,2463534242,19,296,1,271309065,255426280,10192,1850,482b4785,off,,,,,,, +5707,node,bubble,reversed,2000,2463534242,20,94,1,272847415,244998445,1999000,3998000,ad35ba05,off,,,,,,, +5708,node,bubble,random,250,2463534242,4,5439,1,269728001,240472081,31125,31196,b47adce4,off,,,,,,, +5709,c,bubble,nearly_sorted,500,2463534242,22,1998,1,242558633,241557276,124750,14926,482b4785,off,,,,,,, +5710,node,bubble,sorted,250,2463534242,10,8081,1,267143046,241005555,31125,0,7fd8c1f4,off,,,,,,, +5711,c,none,random,2000,2463534242,20,0,415,247901516,90,0,0,41262855,off,,,,,,, +5712,python,bubble,reversed,2000,2463534242,27,2,1,496385103,479392058,1999000,3998000,ad35ba05,off,,,,,,, +5713,python,none,random,2000,2463534242,15,0,16,259694928,902,0,0,41262855,off,,,,,,, +5714,c,quick,reversed,250,2463534242,19,12781,1,245310610,244420444,31125,250,be175004,off,,,,,,, +5715,python,bubble,sorted,250,2463534242,12,206,1,279695730,264364210,31125,0,7fd8c1f4,off,,,,,,, +5716,python,bubble,reversed,2000,2463534242,30,2,1,492525310,475668894,1999000,3998000,ad35ba05,off,,,,,,, +5717,c,bubble,sorted,250,2463534242,1,11386,1,242781721,241853742,31125,0,7fd8c1f4,off,,,,,,, +5718,python,none,reversed,1000,2463534242,26,0,16,230797843,571,0,0,0ca43265,off,,,,,,, +5719,node,none,random,500,2463534242,11,0,9,230242873,882,0,0,11ae932d,off,,,,,,, +5720,c,quick,random,250,2463534242,12,136055,1,245111506,244082821,1933,1940,b47adce4,off,,,,,,, +5721,c,quick,sorted,2000,2463534242,20,174,1,248037160,246870696,1999000,0,69842aa5,off,,,,,,, +5722,c,none,random,2000,2463534242,25,0,415,241506682,90,0,0,41262855,off,,,,,,, +5723,c,insertion,random,1000,2463534242,23,1983,1,246117504,245334797,249457,249457,57561285,off,,,,,,, +5724,python,insertion,nearly_sorted,1000,2463534242,30,76,1,269418189,253378735,30889,30860,1cfe4afd,off,,,,,,, +5725,node,insertion,sorted,500,2463534242,20,324764,1,275840846,240535655,499,0,d9a24379,off,,,,,,, +5726,c,quick,sorted,2000,2463534242,7,174,1,245565099,244605719,1999000,0,69842aa5,off,,,,,,, +5727,python,insertion,sorted,250,2463534242,26,18845,1,262480797,246778502,249,0,7fd8c1f4,off,,,,,,, +5728,c,insertion,random,1000,2463534242,26,1983,1,243907531,242923769,249457,249457,57561285,off,,,,,,, +5729,node,quick,reversed,500,2463534242,2,1194,1,272712464,243298527,124750,500,d33e3559,off,,,,,,, +5730,node,none,nearly_sorted,500,2463534242,6,0,10,269570514,1123,0,0,482b4785,off,,,,,,, +5731,python,bubble,reversed,500,2463534242,14,20,1,270759013,253412736,124750,249500,d33e3559,off,,,,,,, +5732,node,quick,nearly_sorted,250,2463534242,15,25320,1,269702074,241710372,4346,1008,21a19c14,off,,,,,,, +5733,python,quick,reversed,2000,2463534242,1,2,1,278322956,261637908,1999000,2000,ad35ba05,off,,,,,,, +5734,node,quick,random,1000,2463534242,9,6513,1,289009756,255780274,11196,10210,57561285,off,,,,,,, +5735,python,insertion,sorted,250,2463534242,9,18845,1,263699994,248473370,249,0,7fd8c1f4,off,,,,,,, +5736,python,insertion,nearly_sorted,250,2463534242,11,1416,1,258882782,243380504,2277,2251,21a19c14,off,,,,,,, +5737,c,bubble,nearly_sorted,2000,2463534242,21,121,1,248985511,247880557,1999000,232556,5835dc59,off,,,,,,, +5738,python,none,sorted,250,2463534242,7,0,17,277024705,551,0,0,7fd8c1f4,off,,,,,,, +5739,c,none,reversed,2000,2463534242,28,0,413,241809458,90,0,0,ad35ba05,off,,,,,,, +5740,python,quick,nearly_sorted,250,2463534242,28,882,1,280578392,264394144,4346,1008,21a19c14,off,,,,,,, +5741,python,none,sorted,2000,2463534242,15,0,15,240779292,521,0,0,69842aa5,off,,,,,,, +5742,node,quick,reversed,1000,2463534242,23,307,1,271937401,242131675,499500,1000,0ca43265,off,,,,,,, +5743,c,none,reversed,2000,2463534242,26,0,413,239872950,90,0,0,ad35ba05,off,,,,,,, +5744,node,quick,random,250,2463534242,7,36348,1,269518157,238575563,1933,1940,b47adce4,off,,,,,,, +5745,python,quick,nearly_sorted,500,2463534242,1,296,1,256037624,238928320,10192,1850,482b4785,off,,,,,,, +5746,c,bubble,nearly_sorted,1000,2463534242,21,507,1,265007364,263839461,499500,59780,1cfe4afd,off,,,,,,, +5747,node,bubble,nearly_sorted,250,2463534242,9,6161,1,282421780,252409870,31125,4056,21a19c14,off,,,,,,, +5748,node,bubble,random,1000,2463534242,11,362,1,272979745,244914162,499500,496926,57561285,off,,,,,,, +5749,node,insertion,nearly_sorted,1000,2463534242,1,7758,1,274993789,244076462,30889,30860,1cfe4afd,off,,,,,,, +5750,python,none,reversed,1000,2463534242,9,0,16,250539850,641,0,0,0ca43265,off,,,,,,, +5751,python,bubble,reversed,500,2463534242,10,20,1,260917139,245498613,124750,249500,d33e3559,off,,,,,,, +5752,c,quick,sorted,2000,2463534242,8,174,1,252232482,251143774,1999000,0,69842aa5,off,,,,,,, +5753,c,none,random,250,2463534242,2,0,429,246842240,80,0,0,b47adce4,off,,,,,,, +5754,node,bubble,reversed,2000,2463534242,21,94,1,281959463,253639022,1999000,3998000,ad35ba05,off,,,,,,, +5755,python,none,nearly_sorted,250,2463534242,2,0,17,250512790,481,0,0,21a19c14,off,,,,,,, +5756,c,none,nearly_sorted,1000,2463534242,23,0,421,245919422,91,0,0,1cfe4afd,off,,,,,,, +5757,c,bubble,random,2000,2463534242,25,40,1,247643964,246875724,1999000,1988792,41262855,off,,,,,,, +5758,node,bubble,random,2000,2463534242,9,98,1,278905489,248648714,1999000,1988792,41262855,off,,,,,,, +5759,c,bubble,nearly_sorted,2000,2463534242,15,121,1,254583068,253466195,1999000,232556,5835dc59,off,,,,,,, +5760,node,bubble,nearly_sorted,1000,2463534242,3,413,1,274362126,242098840,499500,59780,1cfe4afd,off,,,,,,, diff --git a/data/runs/20261002T211256Z-docker/summary.csv b/data/runs/20261002T211256Z-docker/summary.csv new file mode 100644 index 0000000..2f3084d --- /dev/null +++ b/data/runs/20261002T211256Z-docker/summary.csv @@ -0,0 +1,145 @@ +language,algorithm,order,n,runs,inner_reps,comparisons,writes,time_per_sort_ns,time_per_sort_ns_low,time_per_sort_ns_high,energy_per_sort_uj,energy_per_sort_uj_low,energy_per_sort_uj_high,gross_energy_per_sort_uj,gross_energy_per_sort_uj_low,gross_energy_per_sort_uj_high,power_w,power_w_low,power_w_high +c,bubble,random,250,30,2568,31125,31196,94171.6,93434.1,94704.1,,,,,,,,, +c,bubble,random,500,30,635,124750,119688,363224.1,360114.9,365981.2,,,,,,,,, +c,bubble,random,1000,30,154,499500,496926,1525380.0,1522021.5,1532456.4,,,,,,,,, +c,bubble,random,2000,30,40,1999000,1988792,6052615.9,5944040.8,6076117.9,,,,,,,,, +c,bubble,sorted,250,30,11386,31125,0,21218.6,21115.4,21337.1,,,,,,,,, +c,bubble,sorted,500,30,2858,124750,0,85293.1,84553.6,85740.6,,,,,,,,, +c,bubble,sorted,1000,30,702,499500,0,339997.5,337310.4,342465.2,,,,,,,,, +c,bubble,sorted,2000,30,173,1999000,0,1362116.9,1351346.9,1370210.2,,,,,,,,, +c,bubble,reversed,250,30,1384,31125,62250,174012.9,172802.7,174678.3,,,,,,,,, +c,bubble,reversed,500,30,346,124750,249500,701096.2,694232.9,704064.6,,,,,,,,, +c,bubble,reversed,1000,30,87,499500,999000,2796672.6,2769657.1,2810340.0,,,,,,,,, +c,bubble,reversed,2000,30,22,1999000,3998000,11198653.0,11143270.6,11274080.3,,,,,,,,, +c,bubble,nearly_sorted,250,30,7880,31125,4056,30252.1,30014.0,30406.1,,,,,,,,, +c,bubble,nearly_sorted,500,30,1998,124750,14926,118555.9,118174.0,119966.4,,,,,,,,, +c,bubble,nearly_sorted,1000,30,507,499500,59780,476992.5,472491.9,480280.8,,,,,,,,, +c,bubble,nearly_sorted,2000,30,121,1999000,232556,1983976.7,1977695.6,1989579.9,,,,,,,,, +c,insertion,random,250,30,26020,15844,15838,8905.9,8816.0,8962.9,,,,,,,,, +c,insertion,random,500,30,6945,60338,60339,30977.8,30792.2,31188.2,,,,,,,,, +c,insertion,random,1000,30,1983,249457,249457,122321.2,121337.8,122699.8,,,,,,,,, +c,insertion,random,2000,30,492,996390,996389,472977.6,468134.8,477067.9,,,,,,,,, +c,insertion,sorted,250,30,786886,249,0,308.7,307.0,312.1,,,,,,,,, +c,insertion,sorted,500,30,399335,499,0,604.9,601.8,608.3,,,,,,,,, +c,insertion,sorted,1000,30,196883,999,0,1192.7,1182.2,1199.3,,,,,,,,, +c,insertion,sorted,2000,30,101352,1999,0,2344.4,2330.4,2363.0,,,,,,,,, +c,insertion,reversed,250,30,15645,31125,31374,15561.0,15467.2,15667.6,,,,,,,,, +c,insertion,reversed,500,30,3967,124750,125249,60145.5,59650.7,60399.7,,,,,,,,, +c,insertion,reversed,1000,30,1033,499500,500499,235169.7,232837.3,236910.1,,,,,,,,, +c,insertion,reversed,2000,30,260,1999000,2000999,930165.2,925508.8,939941.8,,,,,,,,, +c,insertion,nearly_sorted,250,30,99421,2277,2251,2406.5,2390.3,2426.7,,,,,,,,, +c,insertion,nearly_sorted,500,30,32350,7962,7926,7491.5,7436.0,7520.6,,,,,,,,, +c,insertion,nearly_sorted,1000,30,11445,30889,30860,21377.8,21197.9,21467.4,,,,,,,,, +c,insertion,nearly_sorted,2000,30,2962,118277,118270,75877.4,75451.8,76469.9,,,,,,,,, +c,quick,random,250,30,136055,1933,1940,1763.0,1744.6,1771.7,,,,,,,,, +c,quick,random,500,30,55776,4578,4552,4274.6,4245.5,4333.5,,,,,,,,, +c,quick,random,1000,30,23484,11196,10210,9959.0,9897.4,10020.6,,,,,,,,, +c,quick,random,2000,30,10587,25197,24544,22564.3,22397.9,22732.0,,,,,,,,, +c,quick,sorted,250,30,10851,31125,0,22500.3,22357.0,22629.1,,,,,,,,, +c,quick,sorted,500,30,2762,124750,0,87848.4,87417.5,88356.5,,,,,,,,, +c,quick,sorted,1000,30,691,499500,0,350214.5,346201.9,352253.7,,,,,,,,, +c,quick,sorted,2000,30,174,1999000,0,1374326.0,1368313.7,1397689.4,,,,,,,,, +c,quick,reversed,250,30,12781,31125,250,18910.5,18831.5,19013.4,,,,,,,,, +c,quick,reversed,500,30,3240,124750,500,73939.2,73451.7,74392.9,,,,,,,,, +c,quick,reversed,1000,30,814,499500,1000,291850.0,289542.1,294335.7,,,,,,,,, +c,quick,reversed,2000,30,209,1999000,2000,1150330.4,1143847.7,1167904.3,,,,,,,,, +c,quick,nearly_sorted,250,30,77221,4346,1008,3129.5,3079.2,3152.3,,,,,,,,, +c,quick,nearly_sorted,500,30,31784,10192,1850,7570.1,7517.0,7613.4,,,,,,,,, +c,quick,nearly_sorted,1000,30,11273,28499,7892,21101.2,20882.6,21391.2,,,,,,,,, +c,quick,nearly_sorted,2000,30,6145,58002,11768,40126.1,39690.6,40455.3,,,,,,,,, +python,bubble,random,250,30,112,31125,31196,2111767.7,2074385.3,2134763.1,,,,,,,,, +python,bubble,random,500,30,26,124750,119688,9564483.6,9391892.5,9681645.1,,,,,,,,, +python,bubble,random,1000,30,6,499500,496926,42781677.1,42155922.0,43092125.0,,,,,,,,, +python,bubble,random,2000,30,2,1999000,1988792,181471428.6,178667418.4,182762577.9,,,,,,,,, +python,bubble,sorted,250,30,206,31125,0,1184972.6,1169833.0,1206765.4,,,,,,,,, +python,bubble,sorted,500,30,46,124750,0,5330880.5,5270361.3,5424245.8,,,,,,,,, +python,bubble,sorted,1000,30,11,499500,0,24049986.9,23756527.8,24277016.4,,,,,,,,, +python,bubble,sorted,2000,30,3,1999000,0,104754691.6,104242638.4,106007701.6,,,,,,,,, +python,bubble,reversed,250,30,88,31125,62250,2716619.0,2697588.5,2756050.6,,,,,,,,, +python,bubble,reversed,500,30,20,124750,249500,12303063.1,12217892.7,12539871.4,,,,,,,,, +python,bubble,reversed,1000,30,5,499500,999000,55418218.7,55101989.7,56046737.7,,,,,,,,, +python,bubble,reversed,2000,30,2,1999000,3998000,237788053.4,233732689.4,239559115.6,,,,,,,,, +python,bubble,nearly_sorted,250,30,188,31125,4056,1318031.3,1308032.9,1339604.1,,,,,,,,, +python,bubble,nearly_sorted,500,30,41,124750,14926,5832673.6,5790172.1,5896967.3,,,,,,,,, +python,bubble,nearly_sorted,1000,30,9,499500,59780,26744339.6,26276598.3,27326536.7,,,,,,,,, +python,bubble,nearly_sorted,2000,30,3,1999000,232556,113441185.7,112588049.0,116197999.7,,,,,,,,, +python,insertion,random,250,30,206,15844,15838,1199823.5,1171924.9,1208041.2,,,,,,,,, +python,insertion,random,500,30,44,60338,60339,5455794.2,5365325.5,5519101.4,,,,,,,,, +python,insertion,random,1000,30,10,249457,249457,24434592.0,24252567.3,24613932.6,,,,,,,,, +python,insertion,random,2000,30,3,996390,996389,102264153.9,101584575.2,103279458.6,,,,,,,,, +python,insertion,sorted,250,30,18845,249,0,13094.3,13021.5,13166.1,,,,,,,,, +python,insertion,sorted,500,30,6624,499,0,34423.3,33710.1,34781.4,,,,,,,,, +python,insertion,sorted,1000,30,3178,999,0,76658.8,75915.2,77649.0,,,,,,,,, +python,insertion,sorted,2000,30,1523,1999,0,162784.4,161393.5,163494.6,,,,,,,,, +python,insertion,reversed,250,30,102,31125,31374,2338277.1,2309417.5,2355962.0,,,,,,,,, +python,insertion,reversed,500,30,22,124750,125249,10660167.0,10561974.0,10737335.6,,,,,,,,, +python,insertion,reversed,1000,30,6,499500,500499,46785697.3,46477159.9,47437731.0,,,,,,,,, +python,insertion,reversed,2000,30,2,1999000,2000999,200435991.1,198159557.6,203910686.4,,,,,,,,, +python,insertion,nearly_sorted,250,30,1416,2277,2251,171568.9,170126.2,175297.4,,,,,,,,, +python,insertion,nearly_sorted,500,30,316,7962,7926,751093.4,746029.1,760453.3,,,,,,,,, +python,insertion,nearly_sorted,1000,30,76,30889,30860,3098842.5,3075888.0,3118499.3,,,,,,,,, +python,insertion,nearly_sorted,2000,30,20,118277,118270,12298432.0,12149199.1,12439725.5,,,,,,,,, +python,quick,random,250,30,1271,1933,1940,187395.7,185952.9,190738.6,,,,,,,,, +python,quick,random,500,30,507,4578,4552,485748.7,480900.4,495717.2,,,,,,,,, +python,quick,random,1000,30,209,11196,10210,1163853.7,1154214.7,1182072.2,,,,,,,,, +python,quick,random,2000,30,90,25197,24544,2753701.6,2694375.7,2780610.2,,,,,,,,, +python,quick,sorted,250,30,133,31125,0,1796652.9,1774478.6,1814790.7,,,,,,,,, +python,quick,sorted,500,30,30,124750,0,7791040.0,7659940.3,7834222.4,,,,,,,,, +python,quick,sorted,1000,30,8,499500,0,34026587.1,33772308.8,34316164.1,,,,,,,,, +python,quick,sorted,2000,30,2,1999000,0,143943463.2,142658928.7,145060731.4,,,,,,,,, +python,quick,reversed,250,30,151,31125,250,1635548.0,1627967.2,1650200.7,,,,,,,,, +python,quick,reversed,500,30,33,124750,500,7332078.8,7239448.2,7518049.2,,,,,,,,, +python,quick,reversed,1000,30,8,499500,1000,32203347.6,32051745.5,32810151.8,,,,,,,,, +python,quick,reversed,2000,30,2,1999000,2000,131171233.4,129505093.6,132346262.4,,,,,,,,, +python,quick,nearly_sorted,250,30,882,4346,1008,280999.4,276959.3,284872.7,,,,,,,,, +python,quick,nearly_sorted,500,30,296,10192,1850,777001.3,766922.3,782984.8,,,,,,,,, +python,quick,nearly_sorted,1000,30,110,28499,7892,2237545.0,2212771.2,2260325.5,,,,,,,,, +python,quick,nearly_sorted,2000,30,55,58002,11768,4257980.1,4230224.2,4319056.2,,,,,,,,, +node,bubble,random,250,30,5439,31125,31196,42938.5,42515.2,43662.7,,,,,,,,, +node,bubble,random,500,30,1407,124750,119688,174978.4,172800.2,177492.8,,,,,,,,, +node,bubble,random,1000,30,362,499500,496926,661820.9,658930.0,666088.1,,,,,,,,, +node,bubble,random,2000,30,98,1999000,1988792,2468626.5,2452015.0,2503770.9,,,,,,,,, +node,bubble,sorted,250,30,8081,31125,0,28873.5,28524.5,29065.8,,,,,,,,, +node,bubble,sorted,500,30,1964,124750,0,118182.5,116279.6,120963.0,,,,,,,,, +node,bubble,sorted,1000,30,374,499500,0,617013.7,604520.9,626413.0,,,,,,,,, +node,bubble,sorted,2000,30,85,1999000,0,2796008.2,2776750.5,2834084.7,,,,,,,,, +node,bubble,reversed,250,30,5861,31125,62250,39809.1,39522.2,40235.0,,,,,,,,, +node,bubble,reversed,500,30,1534,124750,249500,158738.2,157023.8,160590.7,,,,,,,,, +node,bubble,reversed,1000,30,378,499500,999000,644237.4,639160.3,652314.3,,,,,,,,, +node,bubble,reversed,2000,30,94,1999000,3998000,2619310.5,2598282.5,2641255.2,,,,,,,,, +node,bubble,nearly_sorted,250,30,6161,31125,4056,39984.4,39407.4,40576.1,,,,,,,,, +node,bubble,nearly_sorted,500,30,1566,124750,14926,156284.3,153922.0,158817.5,,,,,,,,, +node,bubble,nearly_sorted,1000,30,413,499500,59780,584351.5,580803.3,591249.4,,,,,,,,, +node,bubble,nearly_sorted,2000,30,110,1999000,232556,2171294.4,2146560.6,2195377.2,,,,,,,,, +node,insertion,random,250,30,15349,15844,15838,15467.3,15287.4,15724.3,,,,,,,,, +node,insertion,random,500,30,4059,60338,60339,60231.1,59243.7,60889.0,,,,,,,,, +node,insertion,random,1000,30,1091,249457,249457,222764.1,221964.7,224381.8,,,,,,,,, +node,insertion,random,2000,30,234,996390,996389,1047066.7,1030888.0,1060086.6,,,,,,,,, +node,insertion,sorted,250,30,607595,249,0,375.7,371.4,386.4,,,,,,,,, +node,insertion,sorted,500,30,324764,499,0,728.4,717.8,739.4,,,,,,,,, +node,insertion,sorted,1000,30,160107,999,0,1450.1,1437.9,1472.5,,,,,,,,, +node,insertion,sorted,2000,30,89220,1999,0,2691.2,2571.1,2772.5,,,,,,,,, +node,insertion,reversed,250,30,8539,31125,31374,28473.7,28180.9,28637.3,,,,,,,,, +node,insertion,reversed,500,30,2178,124750,125249,109269.4,107637.6,110196.0,,,,,,,,, +node,insertion,reversed,1000,30,554,499500,500499,441036.5,433309.5,443257.4,,,,,,,,, +node,insertion,reversed,2000,30,144,1999000,2000999,1678682.9,1662577.5,1729592.2,,,,,,,,, +node,insertion,nearly_sorted,250,30,79550,2277,2251,2708.7,2595.1,2762.6,,,,,,,,, +node,insertion,nearly_sorted,500,30,26045,7962,7926,8718.5,8603.0,8776.1,,,,,,,,, +node,insertion,nearly_sorted,1000,30,7758,30889,30860,30760.3,30499.4,31665.6,,,,,,,,, +node,insertion,nearly_sorted,2000,30,2100,118277,118270,112721.8,111378.4,113870.5,,,,,,,,, +node,quick,random,250,30,36348,1933,1940,6436.2,6294.9,6507.5,,,,,,,,, +node,quick,random,500,30,16148,4578,4552,15103.5,14833.9,15228.4,,,,,,,,, +node,quick,random,1000,30,6513,11196,10210,37219.5,36848.1,38233.6,,,,,,,,, +node,quick,random,2000,30,2126,25197,24544,112567.2,110757.3,114147.0,,,,,,,,, +node,quick,sorted,250,30,4507,31125,0,53384.1,52850.0,53940.8,,,,,,,,, +node,quick,sorted,500,30,1164,124750,0,209698.6,208000.7,211582.7,,,,,,,,, +node,quick,sorted,1000,30,296,499500,0,829838.1,821693.9,835780.5,,,,,,,,, +node,quick,sorted,2000,30,74,1999000,0,3309003.9,3276724.1,3330835.9,,,,,,,,, +node,quick,reversed,250,30,4755,31125,250,50778.7,50220.1,51358.1,,,,,,,,, +node,quick,reversed,500,30,1194,124750,500,200029.6,198100.6,201188.2,,,,,,,,, +node,quick,reversed,1000,30,307,499500,1000,788836.7,780961.8,801414.9,,,,,,,,, +node,quick,reversed,2000,30,80,1999000,2000,3110431.6,3095729.2,3136060.8,,,,,,,,, +node,quick,nearly_sorted,250,30,25320,4346,1008,9487.6,9383.0,9558.1,,,,,,,,, +node,quick,nearly_sorted,500,30,10954,10192,1850,21834.7,21517.8,22229.8,,,,,,,,, +node,quick,nearly_sorted,1000,30,3752,28499,7892,62802.8,61686.5,63869.9,,,,,,,,, +node,quick,nearly_sorted,2000,30,1909,58002,11768,120461.7,118986.5,121947.9,,,,,,,,, diff --git a/docs/decisions.md b/docs/decisions.md index db44c3d..baaf303 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -196,6 +196,9 @@ them is a matter of accounting, not of measurement. ## 12. Results stay pending until a quiet bare-metal run +**Status.** Superseded in part by ADR 20: times measured in Docker on an otherwise idle +machine are published, and energy stays pending until the bare-metal run. + **Context.** This version was built on a machine running many other jobs at the same time, inside Docker Desktop, where RAPL is not available. diff --git a/docs/img/time_per_sort.png b/docs/img/time_per_sort.png new file mode 100644 index 0000000000000000000000000000000000000000..b451eca2f226adb64ea257f8e2a430fdbc1d0679 GIT binary patch literal 26827 zcmdqJXH=Biwk=vLMNG&tfC>sKAQ=P+0umGf3nb^P1d%KmifBYZK#7ubhAPRDRYY>8 zK(dl4l4K|-c;j=ed-lHjop<)R?cVp}z1zm6gQJC= z?R9QmUT!|lOOKqK9y^Ng@YwvvH@NK_%z1nR7vUtRBaiRuIHFKA%*YQJT9xS+3iU_` zedm_CTkOKHtEU=%zkb$(I_wkv;ur|Aek*w6E&VKgm>dp6V%o z4)pge*nE50BO_MT$K%8B=qS{M6SpYf!?*VG4D!#%Hx6Dz zK2P{19)OQeKmYtM4-U$yI_Ar7-gaSOq^h2<(HbM979(i0Htaf@BO6Rl=$d@VmwV!) z0~(EvzF{AdoUA*8tynB@6_yrx`~$U=XZ-csH_PpnNfGbe=Wk?E=zVs1U-2oCsaIm%`I3^(^qOgw`|3l&a7F3On>X3N-bDHO zKS@+0Yl+*5@g~l)915%%`IBGX1yN?QKA=g*&?vJe{eR7<)wz1l9(H)t1( zZwzHw73<44k@eaqxk|rGZ8m%OkTBx0)V;gC%9SoFBSYvj@=+;#*g7vrg-SHOJfn_s z9dY;5Eqe5T*M>w~*DbM*DtB4Prn9T;94ou3K%1je_>fi%;KzBRpFECd7dj~8>D$A-#8+18S?rWX(Yi*)Stel)J#>B;8o@~cyx+3#-!#w>m zxhl*$r;82+6@8DEkr$Vel2V>Qg^Y@dO32EWzCaDjn{52Ah*}+BM$K_K0&l%*@OdJ(=$uT(~`lC{sE!>jENYe*N@8rk^+Iu=B!z z#^uYGw+GVG(=8iAm_?js?8-iV%rt9`KyG1ZXebw(0i#Qtj5K0G%SUjksH%pzSDPnz z$$b9&x!1b)RMlRU$EGDbk%;{j&FX4#!uR|XY}r)wmB_v%DT+L!8bcVzkVexk=RRlk z48^#kd6PX^YDCiJsPp2GyTBu5BbZSfC3_*ZeO)u7k%d!LW($IB=ux9Ghj7h2-L%io zmefM8DsaZj-@V&0(Wuswtsy^bUw<(qmBD9M#c^tvGi!p>7`-6jICYOi+LUN&X;F7@ zD1iB<7SfZcA|1k$GI*_KJesjW{+Hd(=RA`gqR;a|p$k?2HSa?v3cUtgUh7vFE@k%(c6 zk_o1_;U#q|N${fis&*DD(nqR1;LrQ;7T-T?k2foSe0kVubJ$hy=JdpbDQ2IvxsqpI zAGDDEur*qzDBLt#GcS7f8CAOOx-NGwGVR`%yDr7s=>*^8>w!Ts>@4TOntXYFBwgH# zs9n_F(!yF)vOd=r^`*9UHLte2yPMyl>rOxkEQIUi6cp1|gPAIf-5U#oxmfd0zJn`V z<6|{2Kw+6RXEuq!^kT-AeOQAhbMu3xI5sx6{7;ir`+FYgPYzH_#Y}&uU|{_aq*GWe z2SKDd%AHAg&w9Xu^aJA}8md32MzAIm<-f5BLGd3~7r^qVR)z{ArSm88wHu@Tx zoAvxoTZC>!)> zoq!)GvM7vqzls}P9IkMlj&p6ge~pzr`vtrQY)i{=F5Mz+pM6rPZTalOg`qOWQQR~+ zzPrfcbXj@1MO&;ek`N%rh&b*%4Z6~w{Grchs^vTqDf$dO@^nkysNTIx87J-~SF^w8 zN5iF?s%7Xd-JPM-URt?1;$gJBygXIC(k`(-10fvWXVl(hQ%czZQPkMB!e&rVZ1>ln zlM@q}1{EcQq3uHu8!Y#B$uVo4vd3~uQwNLriLU7W0tN2<3hnMq8)uldjdgWtcSwy9 z3lgRcwS=y;fNyWkr~LZ$cy+YMoAl}w-fMA4xHX|h5`qW>3_p$43u2f%$ihqUYAfb$1hT-k$QMVb|%Fr#gG>aE}{9^%$>kr#@RhuJu3UA=2 zUbDn^oHc6cfbrrtZTJ)2^*!H&nt=g_t0t;Gqo69$-Yc;ggg9be1y8L>5w|+sZuRra z3qt>D4UYhl6c$RUy1u?Xz3#KWnY%qU5`q;K7dM3UU$j-A^?rWdT3S{X$yx*D&N+2; z9Z^wH1I1R6+sohc$4#%Da~k#9w1fwpDuKig$re{)JDfe+Edh_kn3v5WDyru=)uK{r zJG=(dkXkRN_^6)IaJh4_Nxe(4b$^#J27;uBOQoWbkC;5fu8@j~in>*p%(^w+ac~{; z_zhTUGiB59eRegi#apTp55811H8ysJDv2jiiEl|NDk_GgcBUw0LDb@z=b3fk3fwbL zEwK&_u%6v1p@o_FM#khdF_df2=hu(V5b=86^O~HYp>dJ!<25rkpL;?JOTm=ArnhWMn6``$oBS2VH1bfS;$OQJn*a|pIutKe5L`Fu2yg|GGM5OkyfOw|$rBNeT zk6LhbLAw!$x;c7Sq#BoHL-Y*{@E;yXrKP76TB5Ggv}@#Owt5Crg*K*6Hb*ws)k&Xb zQ7k%q2TmA$?ZHzfi`lL;Ep;UeR%*6QX6?QEK%LFR&EqoOQ$?d0VAVg0Zf8I5|bp5g!V zQ@C{>$HJ)AmjN3GHmeD zZ&Ly)DAlB-BmzDs0JvSOGlIM|KZI%xGuM|8P3|c zlDDuExxYUN<+T8_?xb9a4|-x^V&=4w+5k!z(@jb^Yo zxu9f8-aSJ>PStaF+jD{H!VTpg4su-yO^!Qsfijfzo;BXvlV`?C>B15U)oKI89gnZ7*!Xis`^H-v(s_Kdu zl0))z3VR8Yq`jRbDCG0&ij2Isr6HqgN=o{QI6X3YILCb#R&4~gLB!G{DR@PY4wH_Z z<(^rp*^dYf4TVQai;cZN8r>%;>FDT8l^i*81Tz^P$&B_QuH|Ef4GrBh>fj|3{0*NK z$Hv7?PEEmyzQXOwP0}7`VPRRH?^go^0LN%?mdw6B&BE{Pxu+J^WL%-CUb03yL`jej z@!nl#3HWge0)1!7DEs4W=2@l5ViM({XNw=l6`^+Eq2-%+;I+G4&nzy-4+sok#9WMB zH9qK~XbY4R-K_^Shuo(xxdYy8}MpG3u_Q*a01kU9V;YI=HXYU&-* zYWo8cRO;oBKsar)wZl3(l#%EKPon}+?#$`aDa`T_cS8&@v&`-oOhkYAv5@!gGh=Ma z(LaCwET4+DQpXvJ0bDKiCSf3v%X@o!Pthq?Zw%x_3)#g_wM3;rAk2ZN#dq$wZh>i* zVuE-cEZ13BiA9)U6%{Tf^!0}=%DTGQx23G)3MxO3hq0-hVMs`1N$|WfYA|Vo&r)OI z(*7W6GoYUqV@u%B($ZQODAu#;!)j=2YiAwALh9E+avkK?Brd&@OjrkUa&lb^KAijv zX&?11?E3s6%8eG9D{XIWJ%Wf@zA<3sOj?f%d;2Hu#j&$GMHOKJq^wJm`6NFo2JEQM zz5&GFY!yazyJ}%RqZn?3@=V#_MVjuTg#CB#-qDIUy%OKwJ?0E+;t*xI!QLL(?=pVt z*8_;rcY|-PysC5fT`$*3b=*q&^5shrW}O`=Igs}pecfEKdb+}C)_8p--eXZ12?a2{ zir$M`e^hzQL%B|(bhQ8zOgcxPcu+k3m9oLP*bNNs8H(V@lFKs z{rYjsaWGvWhTz9gT{?TF;?=9e!#?}>TjNQAP)Bs#BvtP$3i$Anw#Jp#pdQNf+4t7E z;FNwZDtBh)5g_rq5Sw4S6CK;dGUlyQ#JBDbP;(iDK<$>&UuZu2PL>{zHK>53V~bV= z%(n*dh7h5*LRFzZfGpg!rJbFfxx#jFycCsKE}$-}!IH}EK^S=~n_7CN7S3=x!cM`5 z{jR(9egNYz6k4-!u16U>f2<^`Y40cSnfXHm5&+)8i?q(Ux&~NaHqywO`|VS^_XY%-w3;!krpn7pXp3#m0~>0uXL+97pio?AaQozhk#YETo@1- zsPZT+vF=a)`jzR$%UM86qN~-=M+; z!J5RKrJBj{@f3K^?0kHxkmDgTe}FHT<5%?c_4zStk6!rowQUY1-g?%PCV!So|D!Uz z3@sAzWch4*Ojvk$T4xGU#qBaig!j5|q4&A(qDS_f z{XNmD-4%Hv@Fr&0&h_N#IIhohR&_kN^4BCE160VFkdvfh*7}}TDp+FgdAed;@`Ku! z{S~ot50*x%EW4-JDd{%UU(00Nu9F<)IZu>8CGuZ8N2W}cr&p>-Pfw4TO^cj2II6rj zvybxalI$R3PsFTW;dh)mj>p=N*lzo#B<>$T$sNeRCbd9}QG;?q8S+a3f%eEJm5xm^ z)CY#YNwL~2D}?s?_iWhwymE`-MgM-QB|hW;zcI_DKeEf;ym9h~^p(51Xr0THCG+TG z90%E+RLvdfJHLN8B`$RSpX1#Aw+Orct%DJ^51u*P=r*QAQ#OM`TtN0#H8)|PNAJmq zXDjNy#QYN!O7b;xs<~Cy*s5-ksKU1kq3-1|0inc_mCjYK+v>xEbwSjBWM(7WW8$4t z-HvNN%=1NP>{>QsdFxZRMOBOE+|nP4Y5ILQkT_%?2!ti;yVn8j)aM+yojFG`(eiDP zYSO4*zenwZ4%`OKf?{FzLh^8y+}(EWaevgOQwW{;-P+lyyiPz^y0J;nLcs}hUh5z# z0=ed=hs#;QCB;gGd15GiHe~ES=)A?*JUe!98|IboiZsHF3eLe3*>Wpt7CXB-t45}e znCYKHXu>I4d3d$vmua|&6bUPo>V`H z+WpO;%B7Z<2EZf9lCUe`6*-d@){9t0f=@An&@uA+LKkF#dS=!f9+ZaKR;_)#u{V0g zsV-xZwV__xAR6OXPK&BAraFj1VYL%Wa}E`)m%5%u-T(IAovp|e{qGXLF2_VHsy$e0W4Gs3sU ziOQvk!?zL(8Wa@5ezuc0X|IGI_u*EywM?#(FNpeU@}QJ?C(^I7kC)Ci4V2t#65QK- zo)a@PxDht(n=v_m`hF?VTIB(@^L0Q#_wsmscuc4XuNWhv6x;{kw4(R%(b3uO-=71N zHds^{Qc{87v=a>CxlruqOe1TIy8fXTOe(sz3Lc9yC#IG-im zi!`6ZK+QISu?j0FC;&D!;ZfH_77;RWSVRPtDqWFVB@fOoDg8Dk`n_v;G<87R@z8q)y3O2npK{#?1%;+V6}RXw+VLvMDJZCZd(m}XVcS9AcC9>r z!tgW2@oy0s{^WkoS+9L4hniC}y}8i2s*{4Msk>YLEVsV$y?bxmF>N({8^D4THT>>C zsGk9dLlf+%Hf|FwAyijDnrzNFJWua*v#N4MVRdCi6@$TyJi9HZt9bbt#hHqmbb|(A z_CKFGI6AJaYi$R&+>w!KsvS=Gw9Q?<|Ku{c;%JpsVQL7AhPv2UAF|~3#4AZg$ybu$ zCp(kW)i2agvf&-E;N`gsnbjzo*?_l#0-5n#^fC9YB79i1)qs4+?p{@cgjg$|@7MaX zu_6KRUKhBJ9z78e5D+jzFIs*3swOi0sYALhA^O9)qF$|d16!58NG}zwhQ((mt%0|> zcQtf-xx_}#`R5n%;xd9ITJ`bc4@1Ml&NE9=Q-Wn>WdlXa?WeffdtJ^N^eu}Re0+S1 zPB6E9Q$OPK{qA2qzej&;Ue93B#zPn1TVzu47rP|6M@FVwWf`tp^s4I}qpYrQmEHq@93&!YMP2&_^twAN4MPcHQ)8?&V#;W<=g2oTmvTnn|YFUi)@E= zp|mFS>g}*~zs-6^kupgjSaz@E+DsG1k{cTC!t{&E81`85vg|KN>&-LBDRcA*$Kh}U zp+|}KcTAzR`H4`_)T77E zCdcj{%+q!K;K-cG$xe`dlQ}RGnfVI&VVAFIfxu#Sf=Ak{ugdrnX->g zle@bOR&b&+yO0zXrd)a1z+{uB^QoFgczBFS{Nmz#viAe7bxeTaQ74vPC|@z8XUF>F zd9KrN^@Eo|srAzy+#Eu(67`EcC0|byi@TM-oIrgi>w|=*{D;XV7uU53eZpN#<+lYR zT}5g#6gQFuKgle=bZV6-uFo!6Ra(@jyj=B$;}4X65G+dHK)Ag}_6mwB$IJS4?Kkct z>Gk6UB=OS4T~F=r5pVmpf%4^${e7Ct^x}fWxCbU4s88XHaLM~A^hpX6i}S`BdQ|3G z&Qvz4-;itm_2paiBZRZPUD+ihOIiC>Wz>fQNP1=d1<#hfbWzAErM6B{#ztY8AkK(Fk6iWZ;UnBnCO`ZQIXTg8naQ!z^ z=>L}o|1LXyv#(qh)5}TercQ`Xxny+K*@loOY3EW43xZty0a@ake_M5;ez?KO0D^MUePuSWGKm>zfbej(}qs} zQiI>;!?0MnU#1OB;u5rS(tZpeU3$;1(=+5H^SM;%ra z%FK6_36=OpW7DM*NU;fYc|5?hXF zCH|c*;9sZdkSroHU}d4F$4|_E(fYOWNfS9u(`z5e4^{G+d^ema;h+ByA!wa(;!e$& z3y(DF0-!CQ6VIo0-U^QJHTlnKO9yQ&r$9v)#h?=md$L(sS!o(dS!hMY-Yt6*=k}`2 zZ09+_safi>hj57-=|eqt%ytD0bOs~P{onTEA3KM7492fq_*E3^^~bzs0%Lh{;`qWs z-ty8t6$}&mbV0nN5gAmO~TU zQIe+@9v`3H5nbGqp=1QSQ|2dU>dgV<hI{F=U6`6i5Kdjm z;QB+eS$G3Q&T~d`_wE64SLM7nBn;e(Rulg@r=_Y*bcoB+NZRuFH^RmsMV3b}y_~!P z^zwax?nje|qgT;HVh#`Vc2p@Vz20Ir2x&;KuW15JdvnWsebytijwom|kQo+sW_=zn z?_KS^vPsn;wmFjF`YpISaJW3jd9F7kRad}@+Kn&}#ukb+GeBeFg~%O1F_3f8t}=bv z$gT52ZmyRNJj*BPvaHwc896$at>*aLHyEy(j}pxEHSo$;H#=Xye%(1}QXj>TAm*Vd zPs7s-Ct@@=H@6umLZ3XjE-(`$+G;c4I+K{2%ZXNXaw^)}8h53n+bb$}nq^1(`&+WX zn0W0h2oXG`&boVe#7v7h&Aj~<=uL~7W31cd@9M(mnUlYex+nw+lT6!1q*cOYaA8zDEktF_h%WA!Jb|RT7m;Tdi4xL)xzdE&u`kkm95un2~|}K4YR+T_bTdw zQDDcfQ>|K8Yvo-l7exZ<1FtZKzq7xEe(jzjx3#m)CMc-sLn@XQ78agzfd+C&D%@UE zYpX$@(P*Qw@B+?)=h^4PnVADL-MEGComVEl8}9@yAbs|M(xv;nRq~N^;cwq20=M>k z4VgH~&IWsN{6YdF>D!KKoP&{TqTEe0uN0NM2%2{Ur@$9Kna~=s%4Pcgr zkE}r5zB^o~(Ca%YHm5KG*wQHlD z?|=OG;gg^gMytC&aMW@CfYj00YLC8$XlAACObquPA3EWd^zFY%mml+1gPZE+HO)0v z#OC_?8h-g~z9y!b8x?095*tiI!h}}(b8uBjTU%}m!xgfFCARoAi*BcDBD%phiSLKR zZcac5kY(`5{Z#k-pr7{5YB#1#rkmLsYBDl1)w8o_yY{_CYK<}WI3#R+Y{)ZXX~egd zclYTJPtrJh^`F&{4r|P^>OFe&sA?p)!TQiDPbTor2s_G@Wzn4uG;;_JmjK=Ou%(0q zpXIDxyZ}!vQ2sRiSRlX<7CP#>IcK^o!;MR4&z=Qv$n!G?^jMM_)BjxbAG*RS?Q%q< zhUY!5hUy}pL>u2Zq)`m4-#aGf*)EM@%NG^{6lzX;yY7w^&?MkHamKAOV{A z`7LhB8e)lJJeLqOq^9l!@mgJmcHc%9tm;nQEqV z)3nYL?t{M^vNJ7NL)H5H)k+P;AQD9S2L@`~zyG!^K_cVTD(()pXnBuc60ou_z)yENk?kp9#cVC5uH$*IYP3?sNngvf! zuXQ3F<65U*_j*HzAN8Ng1`r#yBC!VR9kt{}T8KPNySbl5vbx)P7d z4F|Q;W`LB-*p?kOj)my;tjA(CWw5L>KblTD-RBVhsB*HC2l;uD-HLsN3;;?}2MP)J z-`@(o#8@VlI4w)dkgDd)Q&0-P7le7w@rB!8ZXGdGKl1M@(y&u39%mu!|HaFwdy9+j zWlzo8AhhN+q)x}ak1WMi$wM&xBLj<|y~j5Y7BejRKY=HY<>-L!Ky-#$oFi|s`8$!1 zK4(=mHMR08xw^Wj)O?7*R7vxYm!Eb6(1@^NAFmwInJxFKJ$?}#45(JqHpn|@ILqcW z%bHb2(({Wb$+DkNg?*wvUtwqQ=TKNYHvjpb{>wkW2`cOiz!hcVe}EH|oIl(bYyN)* zPTnH7%$k3R%5MFbm4(KjcX7%`cUDy55K?YyO{~*YsgH_gHVCd)JzLXVHAJ{S6x_-u z>Z4AZy|}oSmdb8)?l<&C{03_KHC%fvYFlrUp(fjkx2^&<;rg$z3914abUB58!Y00d zLlb)BJ@^~kRnoAv#Uh~P(;MW)8e3R2!gMYAkH4@oU;RDK=l^$fp8s0U{rl(YW`Y5ah7pyJS1EjDf$`(}xNo*E+QKPDZ3Xzy_4$EaQwi z?>`uF$n7Wpbjp5z$!PG<*~2Rjev=;R;s@i{)2-C(%F1r&;CqMhXyB>;`S>RzSi)cv z@CIIb+YW%?0guSRAv4?Jm$T1KXqv_VUcq2p9}8;$G)IMTdt-4=vpp&v^_n>I+b;`q zjfAG>&hN`SK;X)#W;J~E8c^kc9%Ml@qFaS2Ha1ae>g$t51PhcxP7$CEs*=wMgXdLu zwmVzp<)I+f$D(JI3`j8|^vg*kj44j$;-VU>~kUvoAe(YAp!QVkS(*X_?uvm8I8i;#nf8AN=EX%shWD2wk z!kR61=lXDch{){g>m|MC7tjtaF1;~!K^@SRYpy&Ydk1MoNQQ^Jk58eH2*BsNnN&}| z5x&|w`Yzj`{<;7_J2QG0Xw!-!ZG6a^>_rN9?%av${gHX@H=NRF`dT-A^tf;&G+264 znWvj8BjQ9|*{)ol&8X&YzE!{ML;pHof{E0M`Z&3R9%@2|&!OJU;ssL?x6jJ7Ys%$!GGXo=q;-o_Z{n zX+h0LoQknKYIB;7PA^^H20KFZpt9d!6{;qu#!NNzT(wwW8%&6IBzWNUpO=4mcEBwP2Z3l7ZW`=iPLUm6}fwZyuoo)62Zd2%Yr$Xi&Ar}EcR zkc1}Di(xVWMj#^K2u_{kj~{O+#BAJ(>ihBPM)L1Focz}bTGlf@HI)v{T=uI|iHA;H zNVLRfXBk$NW1u(cH2+;?*tySOQq?cz=d0hh9!419j$YTksG#qu7_gbj#<5eQ@_oo4 zuBS8{7e9AVe@f_BmA;#s+dH8XFKn`{J9udW2R?YmMGM$rJ368@TSK7^X^jzNXU|T1 zDs{W*<85Shgu_WOI~&m$tWpWAQx0)1*<0e=2sxmys3p~(Q;>m0BwQ)M;yuvS&x>af zWF@u{#}is`Dq2aYimxF7m1m|6L(cXwy;tx5OptF*zu(%RvQcOSY=_rKy7}-yggdy z$J(!7(I8#Z80zQnq0FRx^R4oS0>CF#{7%|GVa#a>gS>0~I(&=6mx6;z-?OBpUu7G= zt;?pRv#!1lEuh!l78SHfKbfO(T?gaTSd`dr+_+;nGD6eMNWgQ1Pcn!(+`;E)fh<8e z-=-!?Oz-vUBecs?cLglEiqw0lP??pW;p3#APX9K@?_=w4w$mI*f%;&Ob_8ie+S+b6 zJ3ALv=H!Jsj`%%mH0@tB#&F+*DFgE)FPoW@^PauEx%2&f@0@3CB&y?VkBN&0G0fpN zvyusATgU$NIz)Yh)D==_w=4ld@y$Z(rSWkd1_ta-rRr}q-Mpe^aftCSB_+PhaT;6# zCcLD1P*w($NC%kESy^l}-Q#V#Clnu-0ZJf%vjf_XoFx&9Vwgt_9kOenzn=w(aDFld zK-pMtLFfcQ(^CrV;2a6HyqVsgDD_+6S7)36e}g&XQ7zGV$^sF|7FmQ!SN!;K9}yIR zP-nlmxuN4*AYFjI_cUF3gJ~1Ka&$`fUVBT!wcr_-tdXjnnRl`b-N6!`IFOr_G4UKh z{YZ@U%4x2$QOX%8FP&?YJSa)+vN5j{cu}-%cZsygVhP>tp8YM1C zik8@EP1D(SP>uR%&8z@jm2+9@r6q{rpjG%B5$gb`wCOL9=BZvQ@bvUtY3(jF=Rwf? zYP)zicoq_GxrWurQ=rsuL)6gRcqb-(f_FhbCFrIdonep$S>_`|)CH|4LVAp?Dx4nS zu|QXJ3$##~;@cA@!yqrx4$B6Td zMO#}CX6L{+_}lw(Tp5t%^i5)4eZDdDD9nymC+*mOjglROh9jtGw8$y;`|L4mGg+4O zc*HVuQP@FyeQuf|jS{82eq-k#LIH;Hx<(h2a77jr?s;_K4Hg$LO{*n89X5`9&FbH9Fp zi)G}HxkUUG{HKRk30*&b7z|I0lw{nbxM@?$a0SFgPJIThmaWkO17(hpAcw^ARKHIe zv}4FjSKy-f0)Ncd+cHT4$zv8Wnjvd-XN_2yQeQGzfYyy>p&k0qu;vgRpmYS2L+H(m zu_J*4rW;mTSd@h_Vl%1o5DMtLWMXU#=CXDSA&UhRME(ZxRqD}JfVnbrkwuU)&b2a6 zO1vpDa=U?So>PbmofXQU)jRcKd&xcAMjd{Ui(7l|&>b?bny@x{Z54bfAhU;nh=`xx z-y}3GhpyT@;3wA*{ zikLH6V}+$8c7IjDaykueB1OGYTf7ynDgxIw7GZ7|Ey&Vcfpd2y*H56z}`;8Bd<7(p54pGu-ANT$9{P`@X0`&^beL|N6 zdFr^@N7%iPau=0rHdH9SKWq7-u`R@Y>16@ej1eRQh@C6HeswSgAJ+h<%<7MWsw0$h zb|BDijonUj9{?{|8rgwEtgH>?F=vDSBi$F2Z%C6uU}>J0zs-U?p-ZLj=xPIF@Jz3s9e$6r6Ar3XkUT0R z*pgLx_2nNn@#x|`-S)I-v)GZSB=z1PxjOO2HF}g3vte}5-kfP zYb&D&%;sVmEC-8kZ=!(QQ-{_#7yymE_vC7&;rHZ}lvM=@FgsROR;pLHEO6+0yBGTO zAT8#m*Luy0Am(rmH28~Jq%l1J@0jvD-yKk`2YT;qM|#ToTJ|(lxVFKXk}ui?50w^T z)fp;tZ0!b=lMVvpTbo0UrTPXOeY_z}8eDMN$K-!yk@m`@w1A%@So0TjC%hZp`UyO2 zIftML@K=0r!%7FxfU&Lz^m`fJBqH=0xJJF4{2l5qx;}!n56a&i#F^DNI%))FuUufc zEx~A&MpA9MAPb8WrJUw=!cDhLfN-Y&Tbay7_YuL$7OO0s!gNEgb&X<+?r+%}pe(TXj9Jso0N9Snkzz;!|nj{VxTj$|)*Gi&e<-PjdDFQOAN znYncDut8jLe>E%bmsMiszA^DX+Wcml*w&C#VBWr$ol>WFq|Hv-}A~X9H zBC~%8)Rq548=n3T+Ay+ENiqw!DYr8@P(0U@rzeC+ZH0z9%J}6SZ~Sh5`Lfpo?>Mp< z{1yZT|1*t8z33#s#`A;Is9GP?w#CHtIoXE%KmM8#5ruZn?oO3;BW7nf0Eq(m322eG ze6>wM$v#a?dqn@s-cz6C5uvKhdYh}f5$FMfkR>UIwZyM+x>F|IxH@H(& z0eFSK2?)rktn}hauLC~JW&ae{jbbgGGU2J&#Wrqeq^sgeXVN5H|Db}9DRNjul6ib~ zmJ8Y!tKj%*K?In18@YT-nn^d@?QbS- zD#phFH%2*Yra=R!^~G&?ocu|1DqSEM&PdsoXvT_ZBaMEP`tgGo6Hken-!07YiVcK* zD2koq=?Uf6pmq4(Q~mOv{<419=8CKN*_;8)jf!lKN1+Ilhx$G?-(N^8>JkfO6RxGD z#j{;0Mo5|kZ3;gP! z&cK7t!>plIuyp;gi$=|W3dkRLY-x_7J~6|~rmgUKO%pp{#rrg?qA-B?qPS;{t7)@v z73{Nk@HFw;3^-M-G&R?ijF#FZKzc}Nx%UjT#A#ry(-~6UO@>CA1=L;H2BP;496HYG zeEjSW1@^l3P*1y%)TOmeQh^<$yTv$cOk1L`OU5B=Lelfx718oj=lk!1xk?)b6aj$yt%i>TH(z^hcFxM}UM9A*SfpkKhvBDB_6OGz|=da6X z84YXZ=jTiJ+HYrB(Fs~B*S-wo_t+2urF;=$pD^&Skp-!6cS)d%91sKQw`XVBRyLNK zg~S?a^;~uw^vOJ8Rua8Wk=!lyrhEH(Q~RHyj0PU@vV10}jwRJioR<*C}At z%Q?D!1H5rsy1LQe{bxgi%v}n^YdY22F#?6LCFbq)h$#sYLmz;fmi=C=0SCC?YW=A6 z40ga!kOH@v^`KS;v(m`2-po_O%GK&E#K8hAIx4a0hWUqu09S}QICX99_w*j~yub>` zl8+>O`zwoUZ3T2A(!3gSO2x<9Mc~%~S1J4T>)m5UfwWN1;SRN!%vR=G*R236)fSQ#8s{oJZyQN1p5H6j5>lD z#6KvgO$fm;NGlXeiqpJ7r?WL$tQ0XdK_2xz)URqw05%%x_n5LOPG@^NGc0*u< zofe8O3b@SQh{3M2as5`7ONMtc7QG&dGsv}^`#GPzn55A3F?knpAR%`MZ>3%6k3pE0 zIZ}QnEEb2#rVu-P_vYw+B2t85vF`P}^lm@M5PM8>B<@=2AecG9vuT-M$rA5&o8rWU z4ydrgaY4YJ#q5th;O0 z9Z_vu-HZ>abrR4ertl)MpnEx=A($qeCCfIe}cVC1xT1qD6sVp>1De8 zZfU5qAsG|;Q2rN;%d1`A8d{Cy{G( zd{BD`GZkgdY3bnV+Lwc?&fZwS@hU;V70$G(k_t(T15gOWX*EyJr+^ca_3WxN#Kk9l zhzU%>$c`T_I@g)d6|R#8vG{Cf3Q?`6O;;AJM_MZ^pT8U|JM}~2FZsY3=F1$Q9Q==r zqO|!R83o5z_Opa>y%}mnLzou6V`FVd%{4MPQTtaY5q`2b677aZRrOXzLMqR^Ar);2 z;zW-}rz_&HnmXr>9|F{&2teZ*lssCOrGowJ!e&&hJ8A4l;ojSCU2|QTfACUuc z3N`_G@mo@l`Tz+Dzcv-aIBy`6f6kB=R_Z^5K>u$vnh?WMtP$c}*8$(1FgWz7{D6kJ}T?I318&D*Z6Y3>CFCjKSDWD5v2awzahSTBP zg-ol1e8I&w0ajW=t88pppquBa-Z~oOt+TQ{8)@eEFQ4`Fq%O@1omqb%+joEmbardm z@DHK#)<72Zb;T}Ztt!&7fAifr`mZ# z7SM-=ze$N5JXZfdL=|^x{^4Z#O`3xa3|xZ$odMH=9%SO$`+HZx!K{04b$KJAFd!g~ z{SRM%i2v-2jJ3wmIROEcl~oJsCw;qQI>;{EwY1Ep_cDD2yi!ZIOb|vMf={uDhHqHd z`%Ktlp%?avKrBoEOssw^JexP5$^L-oq^92Rv;>2C9`>3cglicU!Vzo0u)-8$2V|dp z>)9cUTZ1w_Q>AvqOhXQIX^$^~NQpSQFP3utdQE{Vo9~N0BM)h3LJ!{Nz#-@}Y1!J^ z!m=!N)$#3b5E@m+jd(ec*5kl-l>LFUvuUxCiiX6{hj;_LiOUpeX=$@B&KecCT7;j1 zisi&=UPoF+LYU2v+24Ei)1#Vkr+E!s`+7Z>tbbGzknx0)VK0NrMe$<;c z$4SLzg{MxP>Z$gkcmeji`xTteZi%`qUfXQW!B*?SbC8#aITON=?H#JU{0cZPo@jwy ziZWaZ4*FCQ z4fa%-z(1ZT-uA2n106H4ImM4oLXoRUsIaTS$1uu9)%c2t3Ii2~2FCF@2*7(vY(&Mj zd#)jV&`o(M=7)SyYruRX^eEE$n(fY5pWi3pJ{gsR7j=El?#YuU+XFcLe0}iWa`PW1d(jB8Iwgs{M6l(< z1`g60N(sG=?Ny0ZqAo3cz0o8yka-b?GjLEqoP^CRV56IRDKf!qOOcb;w#v@_ep4U7iCz$~X_*d!Rpc&o>1p@dDv;S1NGEZNo4buTF&^AvgXmo9-tY*ROhppcy4%B{ z3nc95%{~`NMq$DiH9;U`3&~VMMU%i*urQtA1)J%#S*%QCVb+WL1T0q0)P4I#XpXwx zVgs5Mq$CNti0j^R9no-$6N%pV9LDEO{h0qMy06F%2gR_q>#(!ZI~$3;Eh&whsVtDt zch&--ySsISFX|p~zFTRfxV5dV7kIdyJdaJj>0;pYzbG zdEv90)M*#k*+Z3{<4rmgZQodI`hoHmOcP)yVDadHJ5RbKK-5DMP`f(pxYJs5qr9Rb z$2FlE>?j#yZSk!_!JxzuabC>*eww8z)#U~^!fVhv`GK96rl0$=xFMa;URI24wX}G3 zbqEu9z}II7wV?P{0pC6o+YMIj2M-<~)F=6&L)k{K4~^&6D+fo1rrqLC_8&ifoFUGM zHdehSO~R~#&4=hgW!SjaRV!76+3bY5foP%vHU>*GOTY)P=5+o11i{^TXW^MT zBY#nYOYb6&>(W40FX$4{q3K6Yoap5$-RN$+IM}@FjC3oNkKB=Ec*`Wx^C*yE$dj4!I_K`jQn}w z&V=2B>f%pPg!R;s%+?7JePjBSg409Wj#C!!0LTtr@m?D~lUkRunXEP=FNzvw9N5{M zi^}lh5E7Ctw(^^UIow>TO@R;tWY0cdLOWDgKQ9i*Ff%3Xc$O}T!Hf%D;X0v#@QF;5 zbldIYqxF#^Z}?uUG2#y2KHVpyMN24c#@mzYDt@ z!A2V)sSVAC*icPU&t3*Hp<$1-;`iQL4G#jQpR739zm4$W9yHqXVwDLw(|?0+^roba ziH`*X!j~LAc1*21y|u`C_SEWwB`7|)^vdr52{-;N;6nfym$O|K2B405QMabSp~ZeB zxf)n?tBfY)aB#tEArQEWIiOYQJuESVEVk^tlKuTb_mhH8aF^%apgt4=H^Iint1>-4 z&WzUEC(2KcjZG~6Oai-p|IlJ%Mts+d3ZsDK#NlFVyl#c-f-2bV_Lh9w`*WObe5mr+ zV1+JgMtf{TnCn_cUB5Ke7HiqJNQ-u#ek-x}RgHFMLl1(Hda-S{3?j9B$LO~{6_YZr zEp`FK0vzG7VW#vJ4$Ve-PE#!w2)aY$fwr3yYD%+tohkfaj+ZXB&N2{nMK04`^DCrE z&hNlrl}99Zs!>Oy8VfU;cb{I&$04@RyMeCswyu$S6i*)#2l}VIw|q+h^>cB4%f8Disw;t zL~y|FH=mI$da7jeh1w6u8UOqu5%&J=Tf1@~?hlQKHyG;@Yj&E!DMo$aR7h#f+6p^z z!j|~;Vg!;aY;G)fo>sbP!5c_2+y1NUeSAYd1t`7$WhwptLq&`fSn3A>&}hnhdtp2% zU@>$Db_-L*Hm1IM^{Rm9IJ3jln#K9Tz-+Y)tU<5O?ss0lMMh>XFTWFYtV}oVkl}M8 zn=l;XF5(UoAE3dbQHu2jY_S{K)-lkJe`1m2Xym1|w_Gh^!ouD7d$s5kxPREhIZ2 zX!Kw%@6Dk5)SL4w>#KNqo+{%dRZf5{UK_J410R=_AnbN-uTGyibt)N@^o}&QOaToO zXX_bO7wABClIy{9MC2GvOJBR=6e~Z!I_z@-n=VB5tcAEk@wE1h4mTTklCgQOE?ha{ zb7JWTZC;^yJG`@dfb|J|`+|Y=0;;tM-ZH==r!{QX2VTq>JaoJRc3EQ2@R!$u?b=Ef z3m4M(7A5Y3q4*N)pTJP)w)>evZM`poi&lV`=V^mHEu+^K+@1Wu7!9b^6P5j z) zF!hn(Dwyj*$|Sofk~K8-^sp!H-Lpc*wZ*xe+n&I`tAhasdIz~XIt=8@!seIft#94( zgeA?klmiiA0Ozgnq@Xn>9EEJ*K0@a4ez)$}V!?RlFUe0e%+R(>Gc8Rk8+o>{i5QWbl`wpF(CmFpP#NR9YX`esuWR) zf2Xk_eLMxUMnxd486E~jg%vsK`hgW-u-ufZ)B>VmL%k@2iJ$PvJ zy69-UfS|2k3Psbg!MouC{owWyQ~&#F{rqr+QbgVb1Ul4y(Scs7OLgn( z#qi|53!~`4zzMAUqh{Q(#u_SgW$9$nXn3w@E{U=E^!~G`b)IY9g{`m4***^+A~q7- ze0r$Vx8FvG8KGQUTzK;*sOFZICg@H`&*0#o#%4keB}F$716M*v!EpN<&46`<@fgC{Cf%%Z`fQu>-;TVdWgzBS?Az zS1XCVeQka5z=BBF6shU@=`%#E)Njy1f2QAWU$b*0p>p3+M!4K~sLuR)!B6dVSl9$9 zX$XqfYG3(q8@!P;GFbTm5;GrtYVEg&gm25){Rmuuz)Rqx@>W)xX`f-n{ZA{GpKKoj z&rRy(vXQYoULAARD}3rigq$V@L|jNaBntHc{haw<-aV1+7hWb*9={Rf2#I=-K+9oU z6bOE5A1Cf&&W1(sn|eIrR9Ayknnn1r@UhU@VurwH@=otb<+u55k-nz189SpzOH96Z`faOviQU z!;p+)dT{do_+iMR{`Xz1wE9vl#)$JY8qx-=?Ohku1W%tFh445YD)9*Jf1O4=0T_;4 zWP++qVt>&5sw*_kkU4N@V4|AYLhzqBhmBOwv9@>}>@NCN<^Z^|@3T;X94OBhi z=){E0m#{vb1(spfN;YyS7ua_O+7vjbsOFevjwTG3>2i@EZ6e2aCNd>Lg`J%C7gF z-M9_HrFkP|cY*yqDQum(IM+MwIIN58V%M@t|C*bzB>{6vrYUD_HLF)*At7YGzD@nI zcZe&VuZ|8W@K?gSm%MH-aockY_=6hJ@)T6q+JK2GJwqC)`C?ZEFjga0IC?~x1AXP6 zGj__+TyGZ~j6MdmNh|^fBY3v9+iG-%6)UQUlQ2V|lp&2OuQjk9@Bsb>yW(VVyS@bo z-U6^mCE}B`L#PmLVT}#d)s!MfZrUqcq_8t&?SNQIXw#Oo7}tm+0O3P$o~)NGZKLR! zfEYyi7cShrCog*kj|VjmW{Wjmo`I>ix{UJrj@zuQKCIwQ5k@ ziZvIaHqh4VCR+9!_vi6=(6AG;#_=r(Q-*+nJba%_pZEJF_^3{!}|M&p_mG$Fx#H|QTnOd54!aC)69slWqe-6zd6v_(p z2V~v-DI{iIAV6BbT+ZF!SMi+dZf0W2ifxQw(ae+= z>4$Cn-2kRY`q_5dmB)N$X$Zn5LBI?~i?-RKa1ntBqZ4>>$IJIJ!T=_d6D!z`pHRs+ z#EY)OJ=&cm)y1g^q~xVL)5YZn5oucI}{SO?8OeD zrS${_fYHx|aJ&ekF-G8F&4hAIMahJ+&#P04+w=<_ z1rl?cT;}ZUhfXxIT-&r%BHPKW3BFYce@>V?nMVWr=mz*v*0Yz6&4B&=)=tVDesEsi z@B!L5fc0N$EHex#5;-NWmHE29y%a(P& z>BO;$;ZS-N?0B+W@~H4`pu{xLT8tyQx*H2pdv7wRz!RPuPML?j4<7V@4lkP6UUrcn zczXTyZPbH-hP$9xT0 zJP3nu*ePg$iPX^9sJ`>$_s0B;a<(*7tw87rh(4>)hd@cd>G*x-3{V5pBxFzHmzOQDs^dj`&3H84ne@`*zt zBITA?A5$Vp%p!*aYlP$RjLpm}piN7P-VY3-w4UzOxD%$`@y&4vkOZO*17{!Uf*2Uw z=r4l~jhrBqp5W{EkqRQF1R6GgQP)uMQhvmX!K=)bRDwebAx`aTJ+h%yI14D4QyQ}` zCE$wzQ@7vaCEKnL&%q%Xp>{z}w#7!Rm!^W{p|eHvE{u0fA$ne1zT%0IPt_!=^88il z?P&^?zAR*leB^cI*yg@~%o_vFC4Ta=(MM0?qJQR2RNar}Qb1^{DOFXBlg@mtb^dt^ z8th0=*~o-Hp#1CbmHQlq3G}6RLY7Pu&(F8Berat^r?}SQs8ni8%h{9tZY6<=p}U-; zy=tgbC^G*ocw;2Mb8wA3{z+^FkEvYDukwSdUR71Kbzt_Pg9CLn*tXce#=tuufZXAx z_Vv7%@?20Cpd_^^wSTVeVJ_*V#>OuJJ^^Ht&2n;%E$q5bnvFzW10doOOrBGxP8t8e zA+@h>t_v1lAf5IL_iZ+4CZ9qpe`-0~GB{ZCq1O`gzko$}ZpRDZjfgiVp+V0QNN#pn zK$P9pQp-qrl+0et6{2wVCLnu2My!=0O%9p=dDt#8keFw!=9mxg4rt6N`SUO#ZX{(! z%Nt1l*74?MW@dF4MQX7Z=z!4=cl1Ql8!B)|0`)%&_+NuXzx|M_&VFsW@#bwH{U1yr zQn!wL~zNmqH5!3xYS7pzHdDya|Ahc=1uId4Owt7(moHW+3$&P2- zIyM-Uz>+90Lj%S1B-fz2Y}fwixh(`dV8Q%faT8#|JPG?zfA5|8qD7vqXj+kxBSYrqMKz>SjN&;U@3W{&3|zf zWYgdDT68A2*j6cOJb&YVQBoZ^s)~kZ!$7^|K91YyR>s_(du(4Z)9%*vxl4|Unuumn$!Oo{qCg8L~B!UM81T6vBqPSDZ=SdAGrj>j9;D^%z$XM`yMNSN<^$wQ2RQ-5`@9Fplul5R&@(0yt{f9T~uy8iqYR# zD27pJ(e%6*4ZLzA%=zt^fUuGJY{FCyN?ca5tj%m84*iasahZ!N<@eZCdzldRJzyH; zl0v+t*skRJ>ex!(4iJz_9(7y>slaZayfjl(*h!d6=R$17$p|r_ttGjlmS@fo@Pcc8 zHHFDRq#(bwk2tzx*^L^J;7rtYt3;rFd7`;mD{u{r&5R)A|Dk4QrP<Jgq{^!l$}T|-lL=H#fpr-No_01+=50cp%Iwm$V+x8N*)w0Jx#^<05OZ(@;_7`r}F)Q%!SUt5>Bv49h7FwV3jZ$Fp&Su0QCGi4ibl)LILQ( z_38%DI3Xbcaoi$QYOh28&BbxC^(6AW5{2WD&jNVamRaYg?7Z{ny%fnW6|o;JzPBIJ zQBO}tK16WL`0^mT!#?xu82(w zQ@NVC;MmR?di5CSwGvXl7Tux7xQxSQyiE?FUI@g%Y3tPafrB#JAuVll^(u+9+((;Y zFBu}0@0F6Z8==&6oau%-T^vbY;FL*EA|EO6JbC_}{k*R+^~M`Q5|M{L*(NSP^=IZS ziyKuuQn7ZWCC}Wmp%W4h+Ry@Y^RjFlJ%jWA`v`!CZ|k=VtUZpr&JD_Los-4Z1(Tq= z-XUWIbSLtO68E8&&ikKMhp+^Oai$6 zRW#Y1v+p&WO&qj#%3h)sj2RhfX*njkxAB2vXsK$(9=gO&ZxuZ)?N1v-^-i|M906o~ z7I|TH`_fK2$T{huJ0cFgc}HO@7#O;9GgY9es!+@0mZq7RCe$>r*T>$&?N_sIAqV<2 zX!_b72uCriu>LE^ra zI(*9b$pdAuqnN3V#u9g8dBH0SYsdz)u!`vv+T0OMcdi|L=I(&7?SM=elSsN z>}-?qLa@=;7o^Q|rA2MiKF(8Z2Qnj-0nNYrh-A#2gFyRsWEi@)bmMYvel+N25;kfA z#(*twl`22rggd=|=juhvGRYE|LlP7ccXb+8~mHbOb2mj@5zlmQG8CWT0v99a${JC!QX36FYfPQ7w5*g4K zZM6p3F%T{~#nQUbURSeW Hoqqcd -> **Status: time and energy results are pending.** This file was generated from a run that checked the pipeline end to end. Operation counts are exact and shown below; time, energy, power, ratio, correlation and carbon figures are withheld until the study runs on a quiet bare-metal Linux machine ([how](bare-metal.md)). The run's raw data is not in the repository, because it holds those timings; the harness command below reproduces the counts. +> **Energy results are pending.** This run measured time and operation counts only. Energy was not measured in this run: turned off with --energy off. Energy, power, carbon estimates and the answer to whether run time is a good stand-in for energy need the study on bare-metal Linux, which reads the processor's energy counters ([how](bare-metal.md)). ## How these results were produced - Harness: greenbench 2.2.0, profile `study`, 30 measured runs per configuration after 3 warm-up runs, in a random order (order seed 1234567). - Inputs: sizes 250, 500, 1,000, 2,000; orders random, sorted, reversed, nearly sorted; xorshift32 seed 2463534242. Sort loops and startup batches are calibrated to about 240 ms, so that they last at least 200 ms. -- Machine: AMD Ryzen 7 6800H with Radeon Graphics; Linux 6.18.33.2-microsoft-standard-WSL2 x86_64; 16 CPUs online, a cgroup CPU limit of 2 CPUs, a cgroup memory limit of 2.0 GiB; scaling governor not reported; CPU pinning off. +- Machine: AMD Ryzen 7 6800H with Radeon Graphics; Linux 6.18.33.2-microsoft-standard-WSL2 x86_64; 16 CPUs online, 2 of them allowed to the harness, a cgroup memory limit of 2.0 GiB; scaling governor not reported; CPU pinning to CPUs 2, 4. - Workers: C: gcc 14.2.0; Python: 3.12.14; Node.js: v24.21.0. -- Energy: unavailable. Reason: no RAPL package zone (intel-rapl:N named package-N) under /sys/class/powercap; this CPU, kernel or virtual machine does not expose RAPL. -- Run: 5,760 measured runs, status complete, 0 worker processes retried; started 2026-09-27T19:54:53Z, finished 2026-09-27T20:30:34Z; label: pipeline check in Docker Desktop; timings withheld. -- Harness command: `build/greenbench run --profile study --energy auto --label 'pipeline check in Docker Desktop; timings withheld' --out results/study-docker` -- Report command: `PYTHONPATH=analysis python3 -m greenbench_analysis report --runs results/study-docker/runs.csv --out docs/results.md --img-dir docs/img --withhold-timings --readme README.md` +- Energy: off. Reason: turned off with --energy off. +- Run: 5,760 measured runs, status complete, 0 worker processes retried; started 2026-10-02T21:12:57Z, finished 2026-10-02T21:39:08Z; label: run-docker-timing.sh study 20261002T211256Z: a Docker container limited to CPUs 2,4 and 2g of memory without swap; energy off, because containers cannot read RAPL; Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores. +- Sort loops: every sorting run reached the 200 ms target. +- Reproduce the whole run: `./scripts/run-docker-timing.sh --cpus 2,4 --memory 2g` +- Harness command: `build/greenbench run --profile study --energy off --cpus 2,4 --label 'run-docker-timing.sh study 20261002T211256Z: a Docker container limited to CPUs 2,4 and 2g of memory without swap; energy off, because containers cannot read RAPL; Windows 11 Home laptop (ASUS TUF Gaming A15 FA507RC) on mains power in the Performance power mode, with no other containers or jobs running; Docker Desktop runs Linux in a virtual machine whose CPUs are not tied to physical cores' --out data/runs/20261002T211256Z-docker` +- Report command: `PYTHONPATH=analysis python3 -m greenbench_analysis report --runs data/runs/20261002T211256Z-docker/runs.csv --out docs/results.md --img-dir docs/img --summary-csv data/runs/20261002T211256Z-docker/summary.csv --reproduce './scripts/run-docker-timing.sh --cpus 2,4 --memory 2g' --readme README.md` +- Raw data: [data/runs/20261002T211256Z-docker/](../data/runs/20261002T211256Z-docker/), with every measured run in `runs.csv` and the settings, limits and versions in `meta.json`. ## Operation counts @@ -74,19 +77,532 @@ Comparisons between elements and writes into the array, for one sort. These are ## Time and energy per sort -**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). +Median over the measured runs of each configuration, with a 95% bootstrap confidence interval in brackets. Per-sort figures are the whole-process measurement minus the median startup cost per process for the same language, order and size, divided by the number of sorts in the process. Energy per sort is pending: this run did not measure energy, so the tables give time only (see the note at the top). + +![Time per sort, largest input, random order](img/time_per_sort.png) + +Inputs of 2,000 elements: + +| Language | Algorithm | Order | Size | Sorts per process | Time per sort | +| :--- | :--- | :--- | ---: | ---: | ---: | +| C | bubble | random | 2,000 | 40 | 6.05 ms (5.94 to 6.08) | +| C | bubble | sorted | 2,000 | 173 | 1.36 ms (1.35 to 1.37) | +| C | bubble | reversed | 2,000 | 22 | 11.2 ms (11.1 to 11.3) | +| C | bubble | nearly sorted | 2,000 | 121 | 1.98 ms (1.98 to 1.99) | +| C | insertion | random | 2,000 | 492 | 473 µs (468 to 477) | +| C | insertion | sorted | 2,000 | 101,352 | 2.34 µs (2.33 to 2.36) | +| C | insertion | reversed | 2,000 | 260 | 930 µs (926 to 940) | +| C | insertion | nearly sorted | 2,000 | 2,962 | 75.9 µs (75.5 to 76.5) | +| C | quick | random | 2,000 | 10,587 | 22.6 µs (22.4 to 22.7) | +| C | quick | sorted | 2,000 | 174 | 1.37 ms (1.37 to 1.40) | +| C | quick | reversed | 2,000 | 209 | 1.15 ms (1.14 to 1.17) | +| C | quick | nearly sorted | 2,000 | 6,145 | 40.1 µs (39.7 to 40.5) | +| Python | bubble | random | 2,000 | 2 | 181 ms (179 to 183) | +| Python | bubble | sorted | 2,000 | 3 | 105 ms (104 to 106) | +| Python | bubble | reversed | 2,000 | 2 | 238 ms (234 to 240) | +| Python | bubble | nearly sorted | 2,000 | 3 | 113 ms (113 to 116) | +| Python | insertion | random | 2,000 | 3 | 102 ms (102 to 103) | +| Python | insertion | sorted | 2,000 | 1,523 | 163 µs (161 to 163) | +| Python | insertion | reversed | 2,000 | 2 | 200 ms (198 to 204) | +| Python | insertion | nearly sorted | 2,000 | 20 | 12.3 ms (12.1 to 12.4) | +| Python | quick | random | 2,000 | 90 | 2.75 ms (2.69 to 2.78) | +| Python | quick | sorted | 2,000 | 2 | 144 ms (143 to 145) | +| Python | quick | reversed | 2,000 | 2 | 131 ms (130 to 132) | +| Python | quick | nearly sorted | 2,000 | 55 | 4.26 ms (4.23 to 4.32) | +| Node.js | bubble | random | 2,000 | 98 | 2.47 ms (2.45 to 2.50) | +| Node.js | bubble | sorted | 2,000 | 85 | 2.80 ms (2.78 to 2.83) | +| Node.js | bubble | reversed | 2,000 | 94 | 2.62 ms (2.60 to 2.64) | +| Node.js | bubble | nearly sorted | 2,000 | 110 | 2.17 ms (2.15 to 2.20) | +| Node.js | insertion | random | 2,000 | 234 | 1.05 ms (1.03 to 1.06) | +| Node.js | insertion | sorted | 2,000 | 89,220 | 2.69 µs (2.57 to 2.77) | +| Node.js | insertion | reversed | 2,000 | 144 | 1.68 ms (1.66 to 1.73) | +| Node.js | insertion | nearly sorted | 2,000 | 2,100 | 113 µs (111 to 114) | +| Node.js | quick | random | 2,000 | 2,126 | 113 µs (111 to 114) | +| Node.js | quick | sorted | 2,000 | 74 | 3.31 ms (3.28 to 3.33) | +| Node.js | quick | reversed | 2,000 | 80 | 3.11 ms (3.10 to 3.14) | +| Node.js | quick | nearly sorted | 2,000 | 1,909 | 120 µs (119 to 122) | + +
Every size + +| Language | Algorithm | Order | Size | Sorts per process | Time per sort | +| :--- | :--- | :--- | ---: | ---: | ---: | +| C | bubble | random | 250 | 2,568 | 94.2 µs (93.4 to 94.7) | +| C | bubble | random | 500 | 635 | 363 µs (360 to 366) | +| C | bubble | random | 1,000 | 154 | 1.53 ms (1.52 to 1.53) | +| C | bubble | random | 2,000 | 40 | 6.05 ms (5.94 to 6.08) | +| C | bubble | sorted | 250 | 11,386 | 21.2 µs (21.1 to 21.3) | +| C | bubble | sorted | 500 | 2,858 | 85.3 µs (84.6 to 85.7) | +| C | bubble | sorted | 1,000 | 702 | 340 µs (337 to 342) | +| C | bubble | sorted | 2,000 | 173 | 1.36 ms (1.35 to 1.37) | +| C | bubble | reversed | 250 | 1,384 | 174 µs (173 to 175) | +| C | bubble | reversed | 500 | 346 | 701 µs (694 to 704) | +| C | bubble | reversed | 1,000 | 87 | 2.80 ms (2.77 to 2.81) | +| C | bubble | reversed | 2,000 | 22 | 11.2 ms (11.1 to 11.3) | +| C | bubble | nearly sorted | 250 | 7,880 | 30.3 µs (30.0 to 30.4) | +| C | bubble | nearly sorted | 500 | 1,998 | 119 µs (118 to 120) | +| C | bubble | nearly sorted | 1,000 | 507 | 477 µs (472 to 480) | +| C | bubble | nearly sorted | 2,000 | 121 | 1.98 ms (1.98 to 1.99) | +| C | insertion | random | 250 | 26,020 | 8.91 µs (8.82 to 8.96) | +| C | insertion | random | 500 | 6,945 | 31.0 µs (30.8 to 31.2) | +| C | insertion | random | 1,000 | 1,983 | 122 µs (121 to 123) | +| C | insertion | random | 2,000 | 492 | 473 µs (468 to 477) | +| C | insertion | sorted | 250 | 786,886 | 309 ns (307 to 312) | +| C | insertion | sorted | 500 | 399,335 | 605 ns (602 to 608) | +| C | insertion | sorted | 1,000 | 196,883 | 1.19 µs (1.18 to 1.20) | +| C | insertion | sorted | 2,000 | 101,352 | 2.34 µs (2.33 to 2.36) | +| C | insertion | reversed | 250 | 15,645 | 15.6 µs (15.5 to 15.7) | +| C | insertion | reversed | 500 | 3,967 | 60.1 µs (59.7 to 60.4) | +| C | insertion | reversed | 1,000 | 1,033 | 235 µs (233 to 237) | +| C | insertion | reversed | 2,000 | 260 | 930 µs (926 to 940) | +| C | insertion | nearly sorted | 250 | 99,421 | 2.41 µs (2.39 to 2.43) | +| C | insertion | nearly sorted | 500 | 32,350 | 7.49 µs (7.44 to 7.52) | +| C | insertion | nearly sorted | 1,000 | 11,445 | 21.4 µs (21.2 to 21.5) | +| C | insertion | nearly sorted | 2,000 | 2,962 | 75.9 µs (75.5 to 76.5) | +| C | quick | random | 250 | 136,055 | 1.76 µs (1.74 to 1.77) | +| C | quick | random | 500 | 55,776 | 4.27 µs (4.25 to 4.33) | +| C | quick | random | 1,000 | 23,484 | 9.96 µs (9.90 to 10.0) | +| C | quick | random | 2,000 | 10,587 | 22.6 µs (22.4 to 22.7) | +| C | quick | sorted | 250 | 10,851 | 22.5 µs (22.4 to 22.6) | +| C | quick | sorted | 500 | 2,762 | 87.8 µs (87.4 to 88.4) | +| C | quick | sorted | 1,000 | 691 | 350 µs (346 to 352) | +| C | quick | sorted | 2,000 | 174 | 1.37 ms (1.37 to 1.40) | +| C | quick | reversed | 250 | 12,781 | 18.9 µs (18.8 to 19.0) | +| C | quick | reversed | 500 | 3,240 | 73.9 µs (73.5 to 74.4) | +| C | quick | reversed | 1,000 | 814 | 292 µs (290 to 294) | +| C | quick | reversed | 2,000 | 209 | 1.15 ms (1.14 to 1.17) | +| C | quick | nearly sorted | 250 | 77,221 | 3.13 µs (3.08 to 3.15) | +| C | quick | nearly sorted | 500 | 31,784 | 7.57 µs (7.52 to 7.61) | +| C | quick | nearly sorted | 1,000 | 11,273 | 21.1 µs (20.9 to 21.4) | +| C | quick | nearly sorted | 2,000 | 6,145 | 40.1 µs (39.7 to 40.5) | +| Python | bubble | random | 250 | 112 | 2.11 ms (2.07 to 2.13) | +| Python | bubble | random | 500 | 26 | 9.56 ms (9.39 to 9.68) | +| Python | bubble | random | 1,000 | 6 | 42.8 ms (42.2 to 43.1) | +| Python | bubble | random | 2,000 | 2 | 181 ms (179 to 183) | +| Python | bubble | sorted | 250 | 206 | 1.18 ms (1.17 to 1.21) | +| Python | bubble | sorted | 500 | 46 | 5.33 ms (5.27 to 5.42) | +| Python | bubble | sorted | 1,000 | 11 | 24.0 ms (23.8 to 24.3) | +| Python | bubble | sorted | 2,000 | 3 | 105 ms (104 to 106) | +| Python | bubble | reversed | 250 | 88 | 2.72 ms (2.70 to 2.76) | +| Python | bubble | reversed | 500 | 20 | 12.3 ms (12.2 to 12.5) | +| Python | bubble | reversed | 1,000 | 5 | 55.4 ms (55.1 to 56.0) | +| Python | bubble | reversed | 2,000 | 2 | 238 ms (234 to 240) | +| Python | bubble | nearly sorted | 250 | 188 | 1.32 ms (1.31 to 1.34) | +| Python | bubble | nearly sorted | 500 | 41 | 5.83 ms (5.79 to 5.90) | +| Python | bubble | nearly sorted | 1,000 | 9 | 26.7 ms (26.3 to 27.3) | +| Python | bubble | nearly sorted | 2,000 | 3 | 113 ms (113 to 116) | +| Python | insertion | random | 250 | 206 | 1.20 ms (1.17 to 1.21) | +| Python | insertion | random | 500 | 44 | 5.46 ms (5.37 to 5.52) | +| Python | insertion | random | 1,000 | 10 | 24.4 ms (24.3 to 24.6) | +| Python | insertion | random | 2,000 | 3 | 102 ms (102 to 103) | +| Python | insertion | sorted | 250 | 18,845 | 13.1 µs (13.0 to 13.2) | +| Python | insertion | sorted | 500 | 6,624 | 34.4 µs (33.7 to 34.8) | +| Python | insertion | sorted | 1,000 | 3,178 | 76.7 µs (75.9 to 77.6) | +| Python | insertion | sorted | 2,000 | 1,523 | 163 µs (161 to 163) | +| Python | insertion | reversed | 250 | 102 | 2.34 ms (2.31 to 2.36) | +| Python | insertion | reversed | 500 | 22 | 10.7 ms (10.6 to 10.7) | +| Python | insertion | reversed | 1,000 | 6 | 46.8 ms (46.5 to 47.4) | +| Python | insertion | reversed | 2,000 | 2 | 200 ms (198 to 204) | +| Python | insertion | nearly sorted | 250 | 1,416 | 172 µs (170 to 175) | +| Python | insertion | nearly sorted | 500 | 316 | 751 µs (746 to 760) | +| Python | insertion | nearly sorted | 1,000 | 76 | 3.10 ms (3.08 to 3.12) | +| Python | insertion | nearly sorted | 2,000 | 20 | 12.3 ms (12.1 to 12.4) | +| Python | quick | random | 250 | 1,271 | 187 µs (186 to 191) | +| Python | quick | random | 500 | 507 | 486 µs (481 to 496) | +| Python | quick | random | 1,000 | 209 | 1.16 ms (1.15 to 1.18) | +| Python | quick | random | 2,000 | 90 | 2.75 ms (2.69 to 2.78) | +| Python | quick | sorted | 250 | 133 | 1.80 ms (1.77 to 1.81) | +| Python | quick | sorted | 500 | 30 | 7.79 ms (7.66 to 7.83) | +| Python | quick | sorted | 1,000 | 8 | 34.0 ms (33.8 to 34.3) | +| Python | quick | sorted | 2,000 | 2 | 144 ms (143 to 145) | +| Python | quick | reversed | 250 | 151 | 1.64 ms (1.63 to 1.65) | +| Python | quick | reversed | 500 | 33 | 7.33 ms (7.24 to 7.52) | +| Python | quick | reversed | 1,000 | 8 | 32.2 ms (32.1 to 32.8) | +| Python | quick | reversed | 2,000 | 2 | 131 ms (130 to 132) | +| Python | quick | nearly sorted | 250 | 882 | 281 µs (277 to 285) | +| Python | quick | nearly sorted | 500 | 296 | 777 µs (767 to 783) | +| Python | quick | nearly sorted | 1,000 | 110 | 2.24 ms (2.21 to 2.26) | +| Python | quick | nearly sorted | 2,000 | 55 | 4.26 ms (4.23 to 4.32) | +| Node.js | bubble | random | 250 | 5,439 | 42.9 µs (42.5 to 43.7) | +| Node.js | bubble | random | 500 | 1,407 | 175 µs (173 to 177) | +| Node.js | bubble | random | 1,000 | 362 | 662 µs (659 to 666) | +| Node.js | bubble | random | 2,000 | 98 | 2.47 ms (2.45 to 2.50) | +| Node.js | bubble | sorted | 250 | 8,081 | 28.9 µs (28.5 to 29.1) | +| Node.js | bubble | sorted | 500 | 1,964 | 118 µs (116 to 121) | +| Node.js | bubble | sorted | 1,000 | 374 | 617 µs (605 to 626) | +| Node.js | bubble | sorted | 2,000 | 85 | 2.80 ms (2.78 to 2.83) | +| Node.js | bubble | reversed | 250 | 5,861 | 39.8 µs (39.5 to 40.2) | +| Node.js | bubble | reversed | 500 | 1,534 | 159 µs (157 to 161) | +| Node.js | bubble | reversed | 1,000 | 378 | 644 µs (639 to 652) | +| Node.js | bubble | reversed | 2,000 | 94 | 2.62 ms (2.60 to 2.64) | +| Node.js | bubble | nearly sorted | 250 | 6,161 | 40.0 µs (39.4 to 40.6) | +| Node.js | bubble | nearly sorted | 500 | 1,566 | 156 µs (154 to 159) | +| Node.js | bubble | nearly sorted | 1,000 | 413 | 584 µs (581 to 591) | +| Node.js | bubble | nearly sorted | 2,000 | 110 | 2.17 ms (2.15 to 2.20) | +| Node.js | insertion | random | 250 | 15,349 | 15.5 µs (15.3 to 15.7) | +| Node.js | insertion | random | 500 | 4,059 | 60.2 µs (59.2 to 60.9) | +| Node.js | insertion | random | 1,000 | 1,091 | 223 µs (222 to 224) | +| Node.js | insertion | random | 2,000 | 234 | 1.05 ms (1.03 to 1.06) | +| Node.js | insertion | sorted | 250 | 607,595 | 376 ns (371 to 386) | +| Node.js | insertion | sorted | 500 | 324,764 | 728 ns (718 to 739) | +| Node.js | insertion | sorted | 1,000 | 160,107 | 1.45 µs (1.44 to 1.47) | +| Node.js | insertion | sorted | 2,000 | 89,220 | 2.69 µs (2.57 to 2.77) | +| Node.js | insertion | reversed | 250 | 8,539 | 28.5 µs (28.2 to 28.6) | +| Node.js | insertion | reversed | 500 | 2,178 | 109 µs (108 to 110) | +| Node.js | insertion | reversed | 1,000 | 554 | 441 µs (433 to 443) | +| Node.js | insertion | reversed | 2,000 | 144 | 1.68 ms (1.66 to 1.73) | +| Node.js | insertion | nearly sorted | 250 | 79,550 | 2.71 µs (2.60 to 2.76) | +| Node.js | insertion | nearly sorted | 500 | 26,045 | 8.72 µs (8.60 to 8.78) | +| Node.js | insertion | nearly sorted | 1,000 | 7,758 | 30.8 µs (30.5 to 31.7) | +| Node.js | insertion | nearly sorted | 2,000 | 2,100 | 113 µs (111 to 114) | +| Node.js | quick | random | 250 | 36,348 | 6.44 µs (6.29 to 6.51) | +| Node.js | quick | random | 500 | 16,148 | 15.1 µs (14.8 to 15.2) | +| Node.js | quick | random | 1,000 | 6,513 | 37.2 µs (36.8 to 38.2) | +| Node.js | quick | random | 2,000 | 2,126 | 113 µs (111 to 114) | +| Node.js | quick | sorted | 250 | 4,507 | 53.4 µs (52.9 to 53.9) | +| Node.js | quick | sorted | 500 | 1,164 | 210 µs (208 to 212) | +| Node.js | quick | sorted | 1,000 | 296 | 830 µs (822 to 836) | +| Node.js | quick | sorted | 2,000 | 74 | 3.31 ms (3.28 to 3.33) | +| Node.js | quick | reversed | 250 | 4,755 | 50.8 µs (50.2 to 51.4) | +| Node.js | quick | reversed | 500 | 1,194 | 200 µs (198 to 201) | +| Node.js | quick | reversed | 1,000 | 307 | 789 µs (781 to 801) | +| Node.js | quick | reversed | 2,000 | 80 | 3.11 ms (3.10 to 3.14) | +| Node.js | quick | nearly sorted | 250 | 25,320 | 9.49 µs (9.38 to 9.56) | +| Node.js | quick | nearly sorted | 500 | 10,954 | 21.8 µs (21.5 to 22.2) | +| Node.js | quick | nearly sorted | 1,000 | 3,752 | 62.8 µs (61.7 to 63.9) | +| Node.js | quick | nearly sorted | 2,000 | 1,909 | 120 µs (119 to 122) | + +
+ +## Time ratios + +How many times one configuration's time per sort the other's is, with a 95% bootstrap interval in brackets. They compare time alone: whether energy follows the same ratios is the question that needs the energy study. A ratio is undefined when a time per sort is at or below zero, or is in more than 2.5% of the resamples, because the startup cost subtracted from it is as large as the sorting. + +### Between languages + +For the same algorithm and input, the first language's time per sort over the second's. Random input, largest size: + +| Algorithm | Order | Size | Languages | Time ratio | +| :--- | :--- | ---: | :--- | ---: | +| bubble | random | 2,000 | Python / C | 30 (29.5 to 30.6) | +| bubble | random | 2,000 | Node.js / C | 0.408 (0.404 to 0.419) | +| bubble | random | 2,000 | Python / Node.js | 73.5 (71.9 to 74.3) | +| insertion | random | 2,000 | Python / C | 216 (214 to 220) | +| insertion | random | 2,000 | Node.js / C | 2.21 (2.17 to 2.25) | +| insertion | random | 2,000 | Python / Node.js | 97.7 (96.4 to 99.6) | +| quick | random | 2,000 | Python / C | 122 (119 to 124) | +| quick | random | 2,000 | Node.js / C | 4.99 (4.89 to 5.07) | +| quick | random | 2,000 | Python / Node.js | 24.5 (23.8 to 25) | + +
Every order and size + +| Algorithm | Order | Size | Languages | Time ratio | +| :--- | :--- | ---: | :--- | ---: | +| bubble | random | 250 | Python / C | 22.4 (22 to 22.8) | +| bubble | random | 250 | Node.js / C | 0.456 (0.451 to 0.464) | +| bubble | random | 250 | Python / Node.js | 49.2 (48 to 50) | +| bubble | random | 500 | Python / C | 26.3 (25.8 to 26.8) | +| bubble | random | 500 | Node.js / C | 0.482 (0.474 to 0.49) | +| bubble | random | 500 | Python / Node.js | 54.7 (53.4 to 55.8) | +| bubble | random | 1,000 | Python / C | 28 (27.6 to 28.3) | +| bubble | random | 1,000 | Node.js / C | 0.434 (0.431 to 0.437) | +| bubble | random | 1,000 | Python / Node.js | 64.6 (63.6 to 65.2) | +| bubble | random | 2,000 | Python / C | 30 (29.5 to 30.6) | +| bubble | random | 2,000 | Node.js / C | 0.408 (0.404 to 0.419) | +| bubble | random | 2,000 | Python / Node.js | 73.5 (71.9 to 74.3) | +| bubble | sorted | 250 | Python / C | 55.8 (55 to 57) | +| bubble | sorted | 250 | Node.js / C | 1.36 (1.34 to 1.37) | +| bubble | sorted | 250 | Python / Node.js | 41 (40.4 to 42.1) | +| bubble | sorted | 500 | Python / C | 62.5 (61.8 to 63.9) | +| bubble | sorted | 500 | Node.js / C | 1.39 (1.36 to 1.42) | +| bubble | sorted | 500 | Python / Node.js | 45.1 (44 to 46.2) | +| bubble | sorted | 1,000 | Python / C | 70.7 (69.6 to 71.6) | +| bubble | sorted | 1,000 | Node.js / C | 1.81 (1.77 to 1.85) | +| bubble | sorted | 1,000 | Python / Node.js | 39 (38.2 to 39.9) | +| bubble | sorted | 2,000 | Python / C | 76.9 (76.2 to 78.2) | +| bubble | sorted | 2,000 | Node.js / C | 2.05 (2.03 to 2.09) | +| bubble | sorted | 2,000 | Python / Node.js | 37.5 (36.9 to 38) | +| bubble | reversed | 250 | Python / C | 15.6 (15.5 to 15.9) | +| bubble | reversed | 250 | Node.js / C | 0.229 (0.227 to 0.232) | +| bubble | reversed | 250 | Python / Node.js | 68.2 (67.4 to 69.4) | +| bubble | reversed | 500 | Python / C | 17.5 (17.4 to 17.9) | +| bubble | reversed | 500 | Node.js / C | 0.226 (0.224 to 0.23) | +| bubble | reversed | 500 | Python / Node.js | 77.5 (76.5 to 79.2) | +| bubble | reversed | 1,000 | Python / C | 19.8 (19.7 to 20.1) | +| bubble | reversed | 1,000 | Node.js / C | 0.23 (0.228 to 0.234) | +| bubble | reversed | 1,000 | Python / Node.js | 86 (84.9 to 87.2) | +| bubble | reversed | 2,000 | Python / C | 21.2 (20.8 to 21.4) | +| bubble | reversed | 2,000 | Node.js / C | 0.234 (0.231 to 0.236) | +| bubble | reversed | 2,000 | Python / Node.js | 90.8 (89.1 to 91.9) | +| bubble | nearly sorted | 250 | Python / C | 43.6 (43.2 to 44.4) | +| bubble | nearly sorted | 250 | Node.js / C | 1.32 (1.3 to 1.34) | +| bubble | nearly sorted | 250 | Python / Node.js | 33 (32.4 to 33.7) | +| bubble | nearly sorted | 500 | Python / C | 49.2 (48.5 to 49.8) | +| bubble | nearly sorted | 500 | Node.js / C | 1.32 (1.29 to 1.34) | +| bubble | nearly sorted | 500 | Python / Node.js | 37.3 (36.6 to 38.1) | +| bubble | nearly sorted | 1,000 | Python / C | 56.1 (55 to 57.5) | +| bubble | nearly sorted | 1,000 | Node.js / C | 1.23 (1.21 to 1.24) | +| bubble | nearly sorted | 1,000 | Python / Node.js | 45.8 (44.8 to 46.8) | +| bubble | nearly sorted | 2,000 | Python / C | 57.2 (56.7 to 58.7) | +| bubble | nearly sorted | 2,000 | Node.js / C | 1.09 (1.08 to 1.11) | +| bubble | nearly sorted | 2,000 | Python / Node.js | 52.2 (51.6 to 53.8) | +| insertion | random | 250 | Python / C | 135 (131 to 136) | +| insertion | random | 250 | Node.js / C | 1.74 (1.71 to 1.77) | +| insertion | random | 250 | Python / Node.js | 77.6 (75.3 to 78.6) | +| insertion | random | 500 | Python / C | 176 (173 to 179) | +| insertion | random | 500 | Node.js / C | 1.94 (1.91 to 1.97) | +| insertion | random | 500 | Python / Node.js | 90.6 (89 to 92.5) | +| insertion | random | 1,000 | Python / C | 200 (198 to 202) | +| insertion | random | 1,000 | Node.js / C | 1.82 (1.81 to 1.85) | +| insertion | random | 1,000 | Python / Node.js | 110 (109 to 111) | +| insertion | random | 2,000 | Python / C | 216 (214 to 220) | +| insertion | random | 2,000 | Node.js / C | 2.21 (2.17 to 2.25) | +| insertion | random | 2,000 | Python / Node.js | 97.7 (96.4 to 99.6) | +| insertion | sorted | 250 | Python / C | 42.4 (41.9 to 42.8) | +| insertion | sorted | 250 | Node.js / C | 1.22 (1.2 to 1.25) | +| insertion | sorted | 250 | Python / Node.js | 34.9 (33.9 to 35.3) | +| insertion | sorted | 500 | Python / C | 56.9 (55.7 to 57.6) | +| insertion | sorted | 500 | Node.js / C | 1.2 (1.18 to 1.22) | +| insertion | sorted | 500 | Python / Node.js | 47.3 (46.1 to 48.2) | +| insertion | sorted | 1,000 | Python / C | 64.3 (63.6 to 65.3) | +| insertion | sorted | 1,000 | Node.js / C | 1.22 (1.2 to 1.24) | +| insertion | sorted | 1,000 | Python / Node.js | 52.9 (51.9 to 53.7) | +| insertion | sorted | 2,000 | Python / C | 69.4 (68.7 to 70) | +| insertion | sorted | 2,000 | Node.js / C | 1.15 (1.1 to 1.18) | +| insertion | sorted | 2,000 | Python / Node.js | 60.5 (58.6 to 63.4) | +| insertion | reversed | 250 | Python / C | 150 (148 to 152) | +| insertion | reversed | 250 | Node.js / C | 1.83 (1.81 to 1.85) | +| insertion | reversed | 250 | Python / Node.js | 82.1 (80.9 to 83.2) | +| insertion | reversed | 500 | Python / C | 177 (176 to 179) | +| insertion | reversed | 500 | Node.js / C | 1.82 (1.79 to 1.84) | +| insertion | reversed | 500 | Python / Node.js | 97.6 (96.4 to 99.3) | +| insertion | reversed | 1,000 | Python / C | 199 (197 to 202) | +| insertion | reversed | 1,000 | Node.js / C | 1.88 (1.84 to 1.9) | +| insertion | reversed | 1,000 | Python / Node.js | 106 (105 to 108) | +| insertion | reversed | 2,000 | Python / C | 215 (213 to 219) | +| insertion | reversed | 2,000 | Node.js / C | 1.8 (1.78 to 1.86) | +| insertion | reversed | 2,000 | Python / Node.js | 119 (116 to 122) | +| insertion | nearly sorted | 250 | Python / C | 71.3 (70.4 to 72.8) | +| insertion | nearly sorted | 250 | Node.js / C | 1.13 (1.08 to 1.15) | +| insertion | nearly sorted | 250 | Python / Node.js | 63.3 (62 to 66.3) | +| insertion | nearly sorted | 500 | Python / C | 100 (99.5 to 102) | +| insertion | nearly sorted | 500 | Node.js / C | 1.16 (1.15 to 1.18) | +| insertion | nearly sorted | 500 | Python / Node.js | 86.1 (85.3 to 87.7) | +| insertion | nearly sorted | 1,000 | Python / C | 145 (144 to 146) | +| insertion | nearly sorted | 1,000 | Node.js / C | 1.44 (1.43 to 1.48) | +| insertion | nearly sorted | 1,000 | Python / Node.js | 101 (98 to 102) | +| insertion | nearly sorted | 2,000 | Python / C | 162 (160 to 164) | +| insertion | nearly sorted | 2,000 | Node.js / C | 1.49 (1.47 to 1.5) | +| insertion | nearly sorted | 2,000 | Python / Node.js | 109 (107 to 111) | +| quick | random | 250 | Python / C | 106 (105 to 108) | +| quick | random | 250 | Node.js / C | 3.65 (3.57 to 3.71) | +| quick | random | 250 | Python / Node.js | 29.1 (28.7 to 29.9) | +| quick | random | 500 | Python / C | 114 (112 to 116) | +| quick | random | 500 | Node.js / C | 3.53 (3.45 to 3.57) | +| quick | random | 500 | Python / Node.js | 32.2 (31.8 to 33) | +| quick | random | 1,000 | Python / C | 117 (115 to 119) | +| quick | random | 1,000 | Node.js / C | 3.74 (3.7 to 3.84) | +| quick | random | 1,000 | Python / Node.js | 31.3 (30.4 to 31.8) | +| quick | random | 2,000 | Python / C | 122 (119 to 124) | +| quick | random | 2,000 | Node.js / C | 4.99 (4.89 to 5.07) | +| quick | random | 2,000 | Python / Node.js | 24.5 (23.8 to 25) | +| quick | sorted | 250 | Python / C | 79.9 (78.7 to 80.8) | +| quick | sorted | 250 | Node.js / C | 2.37 (2.35 to 2.4) | +| quick | sorted | 250 | Python / Node.js | 33.7 (33.2 to 34.1) | +| quick | sorted | 500 | Python / C | 88.7 (87.2 to 89.5) | +| quick | sorted | 500 | Node.js / C | 2.39 (2.36 to 2.41) | +| quick | sorted | 500 | Python / Node.js | 37.2 (36.5 to 37.5) | +| quick | sorted | 1,000 | Python / C | 97.2 (96.3 to 98.8) | +| quick | sorted | 1,000 | Node.js / C | 2.37 (2.34 to 2.41) | +| quick | sorted | 1,000 | Python / Node.js | 41 (40.6 to 41.6) | +| quick | sorted | 2,000 | Python / C | 105 (102 to 106) | +| quick | sorted | 2,000 | Node.js / C | 2.41 (2.37 to 2.43) | +| quick | sorted | 2,000 | Python / Node.js | 43.5 (43 to 44) | +| quick | reversed | 250 | Python / C | 86.5 (85.8 to 87.4) | +| quick | reversed | 250 | Node.js / C | 2.69 (2.65 to 2.72) | +| quick | reversed | 250 | Python / Node.js | 32.2 (31.8 to 32.7) | +| quick | reversed | 500 | Python / C | 99.2 (97.8 to 102) | +| quick | reversed | 500 | Node.js / C | 2.71 (2.68 to 2.73) | +| quick | reversed | 500 | Python / Node.js | 36.7 (36.1 to 37.7) | +| quick | reversed | 1,000 | Python / C | 110 (109 to 113) | +| quick | reversed | 1,000 | Node.js / C | 2.7 (2.67 to 2.75) | +| quick | reversed | 1,000 | Python / Node.js | 40.8 (40.1 to 41.7) | +| quick | reversed | 2,000 | Python / C | 114 (112 to 115) | +| quick | reversed | 2,000 | Node.js / C | 2.7 (2.66 to 2.73) | +| quick | reversed | 2,000 | Python / Node.js | 42.2 (41.5 to 42.6) | +| quick | nearly sorted | 250 | Python / C | 89.8 (88.3 to 91.8) | +| quick | nearly sorted | 250 | Node.js / C | 3.03 (2.99 to 3.09) | +| quick | nearly sorted | 250 | Python / Node.js | 29.6 (29.1 to 30.2) | +| quick | nearly sorted | 500 | Python / C | 103 (101 to 104) | +| quick | nearly sorted | 500 | Node.js / C | 2.88 (2.84 to 2.94) | +| quick | nearly sorted | 500 | Python / Node.js | 35.6 (34.8 to 36.2) | +| quick | nearly sorted | 1,000 | Python / C | 106 (104 to 108) | +| quick | nearly sorted | 1,000 | Node.js / C | 2.98 (2.91 to 3.04) | +| quick | nearly sorted | 1,000 | Python / Node.js | 35.6 (35 to 36.4) | +| quick | nearly sorted | 2,000 | Python / C | 106 (105 to 108) | +| quick | nearly sorted | 2,000 | Node.js / C | 3 (2.95 to 3.05) | +| quick | nearly sorted | 2,000 | Python / Node.js | 35.3 (34.8 to 36.1) | + +
+ +### Between input orders + +For the same language and algorithm, each input order's time per sort over random input's. Sorted against random, largest size: + +| Language | Algorithm | Size | Orders | Time ratio | +| :--- | :--- | ---: | :--- | ---: | +| C | bubble | 2,000 | sorted / random | 0.225 (0.223 to 0.23) | +| C | insertion | 2,000 | sorted / random | 0.00496 (0.0049 to 0.00502) | +| C | quick | 2,000 | sorted / random | 60.9 (60.3 to 62.1) | +| Python | bubble | 2,000 | sorted / random | 0.577 (0.572 to 0.589) | +| Python | insertion | 2,000 | sorted / random | 0.00159 (0.00157 to 0.00161) | +| Python | quick | 2,000 | sorted / random | 52.3 (51.5 to 53.5) | +| Node.js | bubble | 2,000 | sorted / random | 1.13 (1.11 to 1.15) | +| Node.js | insertion | 2,000 | sorted / random | 0.00257 (0.00246 to 0.00266) | +| Node.js | quick | 2,000 | sorted / random | 29.4 (28.9 to 30) | + +
Every order and size + +| Language | Algorithm | Size | Orders | Time ratio | +| :--- | :--- | ---: | :--- | ---: | +| C | bubble | 250 | sorted / random | 0.225 (0.223 to 0.228) | +| C | bubble | 500 | sorted / random | 0.235 (0.232 to 0.237) | +| C | bubble | 1,000 | sorted / random | 0.223 (0.221 to 0.225) | +| C | bubble | 2,000 | sorted / random | 0.225 (0.223 to 0.23) | +| C | bubble | 250 | reversed / random | 1.85 (1.83 to 1.86) | +| C | bubble | 500 | reversed / random | 1.93 (1.91 to 1.95) | +| C | bubble | 1,000 | reversed / random | 1.83 (1.82 to 1.84) | +| C | bubble | 2,000 | reversed / random | 1.85 (1.84 to 1.89) | +| C | bubble | 250 | nearly sorted / random | 0.321 (0.318 to 0.325) | +| C | bubble | 500 | nearly sorted / random | 0.326 (0.324 to 0.331) | +| C | bubble | 1,000 | nearly sorted / random | 0.313 (0.31 to 0.315) | +| C | bubble | 2,000 | nearly sorted / random | 0.328 (0.326 to 0.334) | +| C | insertion | 250 | sorted / random | 0.0347 (0.0344 to 0.0351) | +| C | insertion | 500 | sorted / random | 0.0195 (0.0194 to 0.0197) | +| C | insertion | 1,000 | sorted / random | 0.00975 (0.00966 to 0.00985) | +| C | insertion | 2,000 | sorted / random | 0.00496 (0.0049 to 0.00502) | +| C | insertion | 250 | reversed / random | 1.75 (1.73 to 1.77) | +| C | insertion | 500 | reversed / random | 1.94 (1.92 to 1.96) | +| C | insertion | 1,000 | reversed / random | 1.92 (1.9 to 1.95) | +| C | insertion | 2,000 | reversed / random | 1.97 (1.95 to 2) | +| C | insertion | 250 | nearly sorted / random | 0.27 (0.268 to 0.273) | +| C | insertion | 500 | nearly sorted / random | 0.242 (0.239 to 0.244) | +| C | insertion | 1,000 | nearly sorted / random | 0.175 (0.173 to 0.177) | +| C | insertion | 2,000 | nearly sorted / random | 0.16 (0.158 to 0.162) | +| C | quick | 250 | sorted / random | 12.8 (12.7 to 12.9) | +| C | quick | 500 | sorted / random | 20.6 (20.2 to 20.7) | +| C | quick | 1,000 | sorted / random | 35.2 (34.7 to 35.5) | +| C | quick | 2,000 | sorted / random | 60.9 (60.3 to 62.1) | +| C | quick | 250 | reversed / random | 10.7 (10.7 to 10.9) | +| C | quick | 500 | reversed / random | 17.3 (17 to 17.5) | +| C | quick | 1,000 | reversed / random | 29.3 (29 to 29.6) | +| C | quick | 2,000 | reversed / random | 51 (50.4 to 51.8) | +| C | quick | 250 | nearly sorted / random | 1.78 (1.75 to 1.8) | +| C | quick | 500 | nearly sorted / random | 1.77 (1.74 to 1.79) | +| C | quick | 1,000 | nearly sorted / random | 2.12 (2.09 to 2.15) | +| C | quick | 2,000 | nearly sorted / random | 1.78 (1.75 to 1.8) | +| Python | bubble | 250 | sorted / random | 0.561 (0.55 to 0.579) | +| Python | bubble | 500 | sorted / random | 0.557 (0.548 to 0.571) | +| Python | bubble | 1,000 | sorted / random | 0.562 (0.554 to 0.572) | +| Python | bubble | 2,000 | sorted / random | 0.577 (0.572 to 0.589) | +| Python | bubble | 250 | reversed / random | 1.29 (1.27 to 1.32) | +| Python | bubble | 500 | reversed / random | 1.29 (1.27 to 1.32) | +| Python | bubble | 1,000 | reversed / random | 1.3 (1.28 to 1.32) | +| Python | bubble | 2,000 | reversed / random | 1.31 (1.29 to 1.33) | +| Python | bubble | 250 | nearly sorted / random | 0.624 (0.615 to 0.641) | +| Python | bubble | 500 | nearly sorted / random | 0.61 (0.6 to 0.623) | +| Python | bubble | 1,000 | nearly sorted / random | 0.625 (0.613 to 0.643) | +| Python | bubble | 2,000 | nearly sorted / random | 0.625 (0.618 to 0.644) | +| Python | insertion | 250 | sorted / random | 0.0109 (0.0108 to 0.0112) | +| Python | insertion | 500 | sorted / random | 0.00631 (0.00616 to 0.00642) | +| Python | insertion | 1,000 | sorted / random | 0.00314 (0.0031 to 0.00319) | +| Python | insertion | 2,000 | sorted / random | 0.00159 (0.00157 to 0.00161) | +| Python | insertion | 250 | reversed / random | 1.95 (1.92 to 2) | +| Python | insertion | 500 | reversed / random | 1.95 (1.93 to 1.99) | +| Python | insertion | 1,000 | reversed / random | 1.91 (1.89 to 1.95) | +| Python | insertion | 2,000 | reversed / random | 1.96 (1.93 to 1.99) | +| Python | insertion | 250 | nearly sorted / random | 0.143 (0.141 to 0.147) | +| Python | insertion | 500 | nearly sorted / random | 0.138 (0.136 to 0.14) | +| Python | insertion | 1,000 | nearly sorted / random | 0.127 (0.126 to 0.128) | +| Python | insertion | 2,000 | nearly sorted / random | 0.12 (0.118 to 0.122) | +| Python | quick | 250 | sorted / random | 9.59 (9.4 to 9.71) | +| Python | quick | 500 | sorted / random | 16 (15.6 to 16.2) | +| Python | quick | 1,000 | sorted / random | 29.2 (28.8 to 29.6) | +| Python | quick | 2,000 | sorted / random | 52.3 (51.5 to 53.5) | +| Python | quick | 250 | reversed / random | 8.73 (8.58 to 8.84) | +| Python | quick | 500 | reversed / random | 15.1 (14.7 to 15.5) | +| Python | quick | 1,000 | reversed / random | 27.7 (27.2 to 28.2) | +| Python | quick | 2,000 | reversed / random | 47.6 (46.9 to 48.8) | +| Python | quick | 250 | nearly sorted / random | 1.5 (1.47 to 1.53) | +| Python | quick | 500 | nearly sorted / random | 1.6 (1.56 to 1.62) | +| Python | quick | 1,000 | nearly sorted / random | 1.92 (1.89 to 1.95) | +| Python | quick | 2,000 | nearly sorted / random | 1.55 (1.53 to 1.59) | +| Node.js | bubble | 250 | sorted / random | 0.672 (0.659 to 0.681) | +| Node.js | bubble | 500 | sorted / random | 0.675 (0.659 to 0.696) | +| Node.js | bubble | 1,000 | sorted / random | 0.932 (0.911 to 0.947) | +| Node.js | bubble | 2,000 | sorted / random | 1.13 (1.11 to 1.15) | +| Node.js | bubble | 250 | reversed / random | 0.927 (0.911 to 0.94) | +| Node.js | bubble | 500 | reversed / random | 0.907 (0.891 to 0.923) | +| Node.js | bubble | 1,000 | reversed / random | 0.973 (0.963 to 0.986) | +| Node.js | bubble | 2,000 | reversed / random | 1.06 (1.04 to 1.07) | +| Node.js | bubble | 250 | nearly sorted / random | 0.931 (0.911 to 0.949) | +| Node.js | bubble | 500 | nearly sorted / random | 0.893 (0.874 to 0.916) | +| Node.js | bubble | 1,000 | nearly sorted / random | 0.883 (0.875 to 0.894) | +| Node.js | bubble | 2,000 | nearly sorted / random | 0.88 (0.861 to 0.891) | +| Node.js | insertion | 250 | sorted / random | 0.0243 (0.0238 to 0.025) | +| Node.js | insertion | 500 | sorted / random | 0.0121 (0.0119 to 0.0124) | +| Node.js | insertion | 1,000 | sorted / random | 0.00651 (0.00643 to 0.00661) | +| Node.js | insertion | 2,000 | sorted / random | 0.00257 (0.00246 to 0.00266) | +| Node.js | insertion | 250 | reversed / random | 1.84 (1.8 to 1.87) | +| Node.js | insertion | 500 | reversed / random | 1.81 (1.78 to 1.85) | +| Node.js | insertion | 1,000 | reversed / random | 1.98 (1.94 to 1.99) | +| Node.js | insertion | 2,000 | reversed / random | 1.6 (1.58 to 1.66) | +| Node.js | insertion | 250 | nearly sorted / random | 0.175 (0.167 to 0.179) | +| Node.js | insertion | 500 | nearly sorted / random | 0.145 (0.142 to 0.147) | +| Node.js | insertion | 1,000 | nearly sorted / random | 0.138 (0.137 to 0.142) | +| Node.js | insertion | 2,000 | nearly sorted / random | 0.108 (0.106 to 0.11) | +| Node.js | quick | 250 | sorted / random | 8.29 (8.17 to 8.49) | +| Node.js | quick | 500 | sorted / random | 13.9 (13.7 to 14.2) | +| Node.js | quick | 1,000 | sorted / random | 22.3 (21.6 to 22.6) | +| Node.js | quick | 2,000 | sorted / random | 29.4 (28.9 to 30) | +| Node.js | quick | 250 | reversed / random | 7.89 (7.76 to 8.1) | +| Node.js | quick | 500 | reversed / random | 13.2 (13.1 to 13.5) | +| Node.js | quick | 1,000 | reversed / random | 21.2 (20.6 to 21.6) | +| Node.js | quick | 2,000 | reversed / random | 27.6 (27.2 to 28.2) | +| Node.js | quick | 250 | nearly sorted / random | 1.47 (1.45 to 1.51) | +| Node.js | quick | 500 | nearly sorted / random | 1.45 (1.42 to 1.48) | +| Node.js | quick | 1,000 | nearly sorted / random | 1.69 (1.63 to 1.72) | +| Node.js | quick | 2,000 | nearly sorted / random | 1.07 (1.05 to 1.1) | + +
## Is run time a good stand-in for energy? -**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). +**Pending.** The answer compares the power drawn while sorting across configurations, so it needs energy as well as time. This run did not measure energy (turned off with --energy off); the study on bare-metal Linux will fill this section in ([how](bare-metal.md)). ## Startup cost per language -**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). +A startup run starts the worker, which generates the input and exits without sorting, several times back to back, so that each measured batch lasts about as long as a sorting run. Figures are per process; their median is subtracted from the sorting runs above. Random order shown. + +Energy per process is pending: this run did not measure energy. + +| Language | Size | Processes per batch | Time per process | +| :--- | ---: | ---: | ---: | +| C | 250 | 429 | 569 µs (561 to 577) | +| C | 500 | 426 | 560 µs (550 to 572) | +| C | 1,000 | 367 | 573 µs (569 to 580) | +| C | 2,000 | 415 | 584 µs (575 to 592) | +| Python | 250 | 17 | 14.4 ms (14.2 to 14.8) | +| Python | 500 | 16 | 15.4 ms (14.6 to 15.6) | +| Python | 1,000 | 15 | 15.6 ms (15.0 to 16.3) | +| Python | 2,000 | 16 | 16.3 ms (15.8 to 16.9) | +| Node.js | 250 | 10 | 26.1 ms (25.5 to 26.8) | +| Node.js | 500 | 9 | 26.1 ms (25.7 to 26.9) | +| Node.js | 1,000 | 10 | 26.4 ms (26.0 to 26.9) | +| Node.js | 2,000 | 9 | 27.0 ms (26.4 to 27.6) | ## Carbon estimates -**Pending.** The numbers for this section are withheld: this run was made to check that the pipeline works, on a machine busy with other jobs, so its timings would mislead. A run on a quiet bare-metal Linux machine will fill it in; see [bare-metal.md](bare-metal.md). +**Pending.** Carbon estimates are converted from energy; the grid intensities they will use are below. This run did not measure energy (turned off with --energy off); the study on bare-metal Linux will fill this section in ([how](bare-metal.md)). ### Grid intensity diff --git a/docs/study.md b/docs/study.md index 39f8546..13930fa 100644 --- a/docs/study.md +++ b/docs/study.md @@ -11,7 +11,9 @@ identically in three languages, sorting identical inputs. > The results, and whether they include energy yet, are in [results.md](results.md), which > the report script generates from a recorded run. Energy can only be measured on Linux > running directly on hardware ([how](bare-metal.md)); until the study has run there, -> results.md shows exact operation counts and marks every time, energy and carbon figure as +> results.md comes from a run in Docker with energy off +> ([ADR 20](decisions.md#20-publish-time-from-a-quiet-docker-run-and-keep-energy-pending)). +> It gives time and exact operation counts and marks every energy, power and carbon figure as > pending. ## Questions @@ -244,7 +246,6 @@ lists, JavaScript arrays). None uses a built-in sort. the languages in general. The study script uses the system Python (3.12 on Ubuntu 24.04) and the Node.js 24 LTS that [bare-metal.md](bare-metal.md) installs from nodejs.org, and warns if an older Node.js is on the path. - ### Scope - One machine at a time, single-threaded sorts (Node.js adds its runtime's helper threads), @@ -262,6 +263,7 @@ lists, JavaScript arrays). None uses a built-in sort. make && make test # build and unit tests, on any Linux make setup # a virtual environment with the analysis packages make smoke # a short run of all three languages with energy off +./scripts/run-docker-timing.sh # with Docker: time for the study profile, energy off ./scripts/run-study.sh --check # on bare-metal Linux: can this machine measure energy? ./scripts/run-study.sh # the study itself; writes docs/results.md ``` From bbe7bab1a33c0e2715b1cb085823855374be12cd Mon Sep 17 00:00:00 2001 From: Farah Sharif <129452512+fasharif@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:46:09 +0400 Subject: [PATCH 51/51] docs: explain why the C bubble sort is slower than Node.js In the Docker time run, C bubble sort took 6.05 ms per sort on random input of 2,000 elements against 2.47 ms for Node.js, with identical counts. GCC 14.2 at -O2 vectorises the swap into one 8-byte load and store of both elements; the next comparison's load half overlaps that store and cannot be forwarded, so every swap stalls. A harness check in the same container setup measured C bubble sort at 2,000 elements with the study's build and with -fno-tree-slp-vectorize added: 6.12 against 1.34 ms per sort on random input, and 11.3 against 1.40 ms on reversed input. The threats to validity give the commands. The study keeps the plain -O2 build, and its C bubble sort results describe that build. Co-Authored-By: Claude Opus 5.5 --- docs/study.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/study.md b/docs/study.md index 13930fa..3cfe1d5 100644 --- a/docs/study.md +++ b/docs/study.md @@ -246,6 +246,30 @@ lists, JavaScript arrays). None uses a built-in sort. the languages in general. The study script uses the system Python (3.12 on Ubuntu 24.04) and the Node.js 24 LTS that [bare-metal.md](bare-metal.md) installs from nodejs.org, and warns if an older Node.js is on the path. +- **Compiler code generation.** GCC 14.2 at `-O2` compiles the C bubble sort's swap into one + 8-byte load and one 8-byte store that cover both neighbouring elements (SLP + vectorisation). The next comparison loads 8 bytes that half overlap that store, which the + processor cannot forward from its store buffer, so each swap waits for the store to reach + the cache. The C bubble sort is therefore slow wherever it swaps often, and in the time run + in Docker it was slower than Node.js on random and reversed input. A check on 2026-10-03, + in the same setup as that run (Docker Desktop on a Windows 11 laptop with an AMD Ryzen 7 + 6800H, a container on CPUs 2 and 4 with 2 GiB of memory), measured C bubble sort at 2,000 + elements with the harness, built as in the study and with `-fno-tree-slp-vectorize` added: + 6.12 ms per sort on random input and 11.3 ms on reversed input, against 1.34 ms and + 1.40 ms (medians of 30 runs, each 95% interval within 1.5% of its median). The commands ran + inside `docker run --rm --network none --cpuset-cpus 2,4 --memory 2g --memory-swap 2g greenbench`: + + ```bash + make all && make BUILD=build/noslp CFLAGS="-O2 -fno-tree-slp-vectorize" all + build/greenbench run --profile study --languages c --algorithms bubble --sizes 2000 \ + --energy off --cpus 2,4 --out results/o2 + build/noslp/greenbench run --profile study --languages c --algorithms bubble --sizes 2000 \ + --energy off --cpus 2,4 --out results/noslp + ``` + + The study keeps the plain `-O2` build: it measures what a default optimised build of this + code does, and C bubble sort results describe that build and compiler. + ### Scope - One machine at a time, single-threaded sorts (Node.js adds its runtime's helper threads),