From 020838a11b65cbcee2840eb3d3e2711e480170ab Mon Sep 17 00:00:00 2001 From: u64max Date: Mon, 9 Nov 2026 10:33:56 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F=20Read=20.evolve=20ru?= =?UTF-8?q?n=20files=20directly=20instead=20of=20Evolve=20exports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evolve Charts now reads the datakit Parquet tables in the .evolve file Evolve saves for every run, and calculates the scores the way Evolve does, with multipliers generated from score.rs by tools/sync_scores.py. The Export zip, CSV and deep analysis JSON inputs are gone. sync_categories.py no longer swallows the arm after a single-line vec![...], includes Evolve's deprecated pass names, and writes LF. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 +- README.md | 45 ++-- evolve-charts.py | 3 +- evolve-charts.py.lock | 5 +- evolve_charts/categories.py | 5 + evolve_charts/loader.py | 478 +++++++++++++++------------------ evolve_charts/scope_catalog.py | 93 ++++++- evolve_charts/score_catalog.py | 198 ++++++++++++++ evolve_charts/scores.py | 123 +++++++++ evolve_charts/stats.py | 8 +- evolve_charts/ui.py | 50 ++-- exports/README.txt | 2 - results/README.txt | 3 + tests/conftest.py | 173 ++++++++---- tests/test_loader.py | 116 ++++---- tests/test_scores.py | 60 +++++ tests/test_stats.py | 27 +- tests/test_ui.py | 15 +- tools/package.py | 4 +- tools/sync_categories.py | 21 +- tools/sync_scores.py | 131 +++++++++ 21 files changed, 1088 insertions(+), 476 deletions(-) create mode 100644 evolve_charts/score_catalog.py create mode 100644 evolve_charts/scores.py delete mode 100644 exports/README.txt create mode 100644 results/README.txt create mode 100644 tests/test_scores.py create mode 100644 tools/sync_scores.py diff --git a/.gitignore b/.gitignore index e8d6f5c..d44216a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ __pycache__/ .venv/ .pytest_cache/ -exports/* -!exports/README.txt +results/* +!results/README.txt dist/ VERSION requirements.txt diff --git a/README.md b/README.md index 08c2e8a..9e8cf45 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # πŸ“Š Evolve Charts -**Turn Evolve deep analysis exports into publication-ready charts** +**Turn Evolve benchmark runs into publication-ready charts** [![Banner](./docs/images/banner.png)](https://traverseresearch.nl) @@ -29,29 +29,28 @@ your web browser, and **everything stays on your computer**. Nothing is uploaded the libraries the tool uses (about 150 MB). This needs an internet connection, takes a few minutes and only happens once. Keep the window that opens running while you use the tool; close it when you're done. -3. **Export your results from Evolve.** After a benchmark run, click **Export** on the score screen. Do - this on every GPU or setting you want to compare, using the same benchmark on each. Scores and - per-frame data are always included; deep analysis (per-pass timings) needs an Evolve Professional - licence. -4. **Add your results.** Your browser opens Evolve Charts. Drag the export files onto *Add Evolve - exports*, or put them in the `exports` folder so they load automatically every time. +3. **Run your benchmarks in Evolve.** Evolve saves every run as a `.evolve` file, in + `Documents\Evolve\results` on Windows. Run on every GPU or setting you want to compare, using the + same benchmark on each. +4. **Add your results.** Your browser opens Evolve Charts. Drag the `.evolve` files onto *Add Evolve + runs*, or put them in the `results` folder so they load automatically every time. 5. **Name your results** on their cards (for example "RTX 4070 Ti" or "Gen 1"). These names appear in every chart. -To update, download the zip again and replace the old folder (keep your `exports` folder if you use it). +To update, download the zip again and replace the old folder (keep your `results` folder if you use it). ## πŸ—ΊοΈ What's in the tool -The tool reads every Evolve export: the **Export** zip from the score screen, and the files Evolve's -command line writes (`evolve_results.csv`, `scores.csv`, the per-frame CSV and `deep_analysis.json`). -It has a page per question, in the bar at the top: +The tool reads the `.evolve` file Evolve saves for every run: the measurements of every frame and pass, +the GPU sensor readings and the system information. It calculates the scores from those the way Evolve +does. It has a page per question, in the bar at the top: | Page | What it shows | Needs | |---|---|---| -| **Results** | Load and name exports; GPU, driver, OS, Evolve version and what each file contains | | +| **Results** | Load and name runs; GPU, driver, OS, Evolve version, benchmark and what each file contains | | | **Scores** | Evolve's scores side by side, with the workload category each one scores, and the differences | scores | -| **GPU time** | Where the GPU spends its time each frame, per workload category; then one workload category over the run and the passes it's made of | per-frame data; the passes need deep analysis | -| **Passes** | Every render pass in one searchable table; tick passes to plot them over the run | deep analysis | +| **GPU time** | Where the GPU spends its time each frame, per workload category; then one workload category over the run and the passes it's made of | per-frame data; the passes need pass timings | +| **Passes** | Every render pass in one searchable table; tick passes to plot them over the run | pass timings | | **GPU sensors** | Clocks, power, temperature and more during the run, to spot throttling | per-frame data | | **Export** | CSV files for Excel, Google Sheets, Datawrapper and Flourish | | @@ -78,13 +77,13 @@ drivers and Evolve version. **Help** explains how the numbers are made. because a GPU can idle between passes or run passes at the same time. - **Whiskers** on bar charts show the range covering 90% of frames (5th to 95th percentile). - Only compare results from the **same Evolve version and benchmark settings**. The tool warns you - when the Evolve versions or graphics APIs differ. + when the benchmarks, Evolve versions or graphics APIs differ. ## ❓ Troubleshooting -- **A result is missing from *Passes***: its export has no deep analysis, which - needs an Evolve Professional licence. Its card shows βœ— Deep analysis; scores and workload - categories still work. +- **A result is missing from *Passes***: its run has no pass timings. Its card shows βœ— Passes; scores + and workload categories still work. +- **"was saved by a newer Evolve"**: download the latest Evolve Charts. - **The browser doesn't open**: open yourself while the black window is open. - **Anything else**: send a screenshot of the black window to Traverse Research. @@ -93,16 +92,18 @@ drivers and Evolve version. **Help** explains how the numbers are made. - Run without the launcher: `uv run evolve-charts.py`. Dependencies are declared inline ([PEP 723](https://peps.python.org/pep-0723/)), so no virtual environment is needed. - Code lives in `evolve_charts/`: - - `loader.py` reads export zips into tidy tables. + - `loader.py` reads `.evolve` files (a zip of Parquet tables and a JSON document) into tidy tables. + - `scores.py` calculates Evolve's scores from those tables. - `categories.py` describes Evolve's workload categories and which pass belongs to which. - `stats.py` computes the summaries. - `charts.py` builds the Plotly figures. - `ui.py` is the Streamlit page. - The look follows Evolve's `crates/evolve/src/ui/styling.rs`: theme colours and the Spartan font are set in `.streamlit/config.toml`, and the fonts, logo and icon live in `static/`. Chart colours are in `charts.py`. -- Which pass belongs to which workload category comes from Evolve's Rust source. After pass names - change in Evolve, regenerate the list with `python tools/sync_categories.py` (it reads - `../evolve/crates/evolve-frame-bucketing/src/lib.rs` by default). +- Which pass belongs to which workload category, and the score multipliers, come from Evolve's Rust + source. After pass names or score calibrations change in Evolve, regenerate them with + `python tools/sync_categories.py` and `python tools/sync_scores.py` (both read `../evolve` by default), + then `uvx ruff format evolve_charts`. - Dependencies are locked in `evolve-charts.py.lock`; refresh it with `uv lock --script evolve-charts.py --upgrade`. - Tests: `uv export --script evolve-charts.py --no-hashes > requirements.txt`, then `uv venv && uv pip install -r requirements.txt pytest && uv run --no-project pytest`. CI runs the same. diff --git a/evolve-charts.py b/evolve-charts.py index 8c79bee..b77f5b1 100644 --- a/evolve-charts.py +++ b/evolve-charts.py @@ -4,9 +4,10 @@ # "streamlit>=1.65", # "plotly>=6", # "pandas>=2.2", +# "pyarrow>=18", # ] # /// -"""Evolve Charts: turn Evolve deep analysis exports into charts. +"""Evolve Charts: turn Evolve benchmark runs (.evolve files) into charts. Start it by double-clicking `Start Evolve Charts.bat` (Windows) or `start-evolve-charts.sh` (macOS/Linux), or run `uv run evolve-charts.py`. A page opens in your browser; nothing leaves diff --git a/evolve-charts.py.lock b/evolve-charts.py.lock index d0e408f..cd2ffb0 100644 --- a/evolve-charts.py.lock +++ b/evolve-charts.py.lock @@ -6,10 +6,10 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version < '3.12' and sys_platform == 'win32'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version < '3.12' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] @@ -17,6 +17,7 @@ resolution-markers = [ requirements = [ { name = "pandas", specifier = ">=2.2" }, { name = "plotly", specifier = ">=6" }, + { name = "pyarrow", specifier = ">=18" }, { name = "streamlit", specifier = ">=1.65" }, ] diff --git a/evolve_charts/categories.py b/evolve_charts/categories.py index a5a624b..3b23ba2 100644 --- a/evolve_charts/categories.py +++ b/evolve_charts/categories.py @@ -23,6 +23,11 @@ WORKLOAD_ORDER = list(WORKLOAD_DESCRIPTIONS) +def workload_variants(scope: str) -> list[str]: + """The `WorkloadCategory` variants the pass counts towards, as named in Evolve's source.""" + return WORKLOADS.get(scope, []) + + def workloads(scope: str) -> list[str]: """The workload categories the pass is part of, in Evolve's order; empty if Evolve leaves it uncategorized. diff --git a/evolve_charts/loader.py b/evolve_charts/loader.py index ade7bc6..d6c5537 100644 --- a/evolve_charts/loader.py +++ b/evolve_charts/loader.py @@ -1,14 +1,18 @@ -"""Reads every Evolve export into tidy pandas tables. +"""Reads Evolve's `.evolve` run files into tidy pandas tables. -Evolve exports three kinds of data, alone or bundled in the score screen's Export zip: +Evolve saves every benchmark run as a `.evolve` file (in `Documents/Evolve/results` on Windows). It's a +zip, called a datakit recording, of: -- **scores** (`scores.csv`): one score per workload category per loop, higher is better. -- **per frame** (`frametimes.csv`, also inside `evolve_results.csv`): GPU time per workload - category, plus GPU sensor readings, for every frame. -- **deep analysis** (`deep_analysis.json`, Evolve Professional): the GPU time of every render pass - in every frame. +- `datakit.json`: the format and its version. +- `tables/.parquet`: the measurements, one Parquet table each: + - `frames`: per frame, the GPU time of every workload category and the driver time. + - `gpu_scopes`: the GPU start and duration of every pass in every frame. + - `cpu_scopes`: the duration of every driver call. + - `metrics`: GPU sensor readings, sampled a few times per second. + - `command_buffers`: when each command buffer was submitted. +- `documents/client_data`: JSON with the system, driver and benchmark settings. -`evolve_results.csv` combines system information, scores and per-frame data in one file. +The scores aren't in the file; they're calculated from the tables the way Evolve does (`scores.py`). """ import hashlib @@ -16,66 +20,47 @@ import json import zipfile from dataclasses import dataclass, field -from pathlib import PurePath +import numpy as np import pandas as pd +import pyarrow.parquet as pq -from .categories import WORKLOAD_ORDER, workloads +from . import scores +from .categories import workloads -DEEP_ANALYSIS = "deep_analysis.json" +DATAKIT_VERSION = 1 +TABLES = ["frames", "gpu_scopes", "cpu_scopes", "metrics"] -METADATA_COLUMNS = [ - "Evolve Version", - "System Name", - "CPU", - "GPU", - "OS Name", - "OS Version", - "GPU Driver Version", - "Rendering Backend", -] - -# Per-frame CSV column -> workload category name (as Evolve shows it). +# `frames` column -> workload category name (as Evolve shows it). WORKLOAD_COLUMNS = { - "Ray Tracing (ns)": "Raytracing", - "Acceleration Structure Build (ns)": "Acceleration Structure Builds", - "Rasterization (ns)": "Rasterization", - "Compute (ns)": "Compute", - "Workgraphs (ns)": "Workgraphs", - "NRC (ns)": "NRC", - "Upscaling Performance (ns)": "Upscaling", + "gpu_ray_tracing_ns": "Raytracing", + "gpu_acceleration_structure_builds_ns": "Acceleration Structure Builds", + "gpu_rasterization_ns": "Rasterization", + "gpu_compute_ns": "Compute", + "gpu_workgraphs_ns": "Workgraphs", + "gpu_nrc_ns": "NRC", + "gpu_upscaler_performance_ns": "Upscaling", } -# Sensor columns in `Result.frames`, filled from deep analysis (GpuMetrics fields) or the per-frame CSV. -METRIC_FIELDS = { - "usage_percentage": "gpu_usage_pct", - "clock_speed_in_mhz": "gpu_clock_mhz", - "vram_clock_speed_in_mhz": "vram_clock_mhz", - "power_usage_in_w": "gpu_power_w", - "board_power_usage_in_w": "board_power_w", - "soc_power_usage_in_w": "soc_power_w", - "voltage_in_mv": "voltage_mv", - "vram_usage_in_mb": "vram_used_mb", - "edge_temperature_in_c": "edge_temp_c", - "hotspot_temperature_in_c": "hotspot_temp_c", -} -FAN_FIELDS = {"Rpm": "fan_rpm", "Percent": "fan_pct"} -METRIC_CSV_COLUMNS = { - # Evolve writes board power into the column it calls "Energy". - "Energy (W)": "board_power_w", - "Gpu Usage (%)": "gpu_usage_pct", - "Clock Speed (Mhz)": "gpu_clock_mhz", - "Vram Clock Speed (Mhz)": "vram_clock_mhz", - "Gpu Voltage (mV)": "voltage_mv", - "Fan Speed (rpm)": "fan_rpm", - "Fan Speed (%)": "fan_pct", - "Edge Temperature (C)": "edge_temp_c", - "Hot Spot Temperature (C)": "hotspot_temp_c", +# `metrics` column -> sensor column in `Result.frames`. +METRIC_COLUMNS = { + "gpu_usage_percent": "gpu_usage_pct", + "gpu_clock_mhz": "gpu_clock_mhz", + "vram_clock_mhz": "vram_clock_mhz", + "gpu_power_w": "gpu_power_w", + "board_power_w": "board_power_w", + "soc_power_w": "soc_power_w", + "voltage_mv": "voltage_mv", + "vram_usage_mib": "vram_used_mb", + "fan_rpm": "fan_rpm", + "fan_percent": "fan_pct", + "edge_temperature_c": "edge_temp_c", + "hotspot_temperature_c": "hotspot_temp_c", } -SENSOR_COLUMNS = list(dict.fromkeys([*METRIC_FIELDS.values(), *FAN_FIELDS.values(), *METRIC_CSV_COLUMNS.values()])) +SENSOR_COLUMNS = list(METRIC_COLUMNS.values()) -class ExportError(Exception): +class LoadError(Exception): """A problem with an input file, worded for the person using the tool.""" @@ -84,16 +69,16 @@ class Result: id: str filename: str metadata: dict[str, str] = field(default_factory=dict) - # Score name -> score, averaged over loops. Scores Evolve didn't produce for this run are absent. + # Score name -> score. Scores the benchmark doesn't have are absent. scores: dict[str, float] = field(default_factory=dict) # One row per (loop, frame): GPU milliseconds per workload category, one column each. workloads: pd.DataFrame | None = None - # One row per (loop, frame): sensor readings, and from deep analysis the GPU time sum and span. + # One row per (loop, frame): sensor readings, and the GPU time sum and span of the passes. frames: pd.DataFrame | None = None - # Deep analysis only. One row per (loop, frame, pass): ms is the pass' own GPU time in that frame; - # top_ms only counts instances not nested inside another scope, so summing it never double-counts. + # One row per (loop, frame, pass): ms is the pass' own GPU time in that frame; top_ms only counts + # instances not nested inside another scope, so summing it never double-counts. passes: pd.DataFrame | None = None - # Deep analysis only: total GPU milliseconds per pass that Evolve doesn't assign to a workload category. + # Total GPU milliseconds per pass that Evolve doesn't assign to a workload category. left_out: dict[str, float] = field(default_factory=dict) warnings: list[str] = field(default_factory=list) @@ -109,129 +94,51 @@ def contents(self) -> list[str]: for name, present in [ ("Scores", bool(self.scores)), ("Per frame", self.workloads is not None), - ("Deep analysis", self.passes is not None), + ("Passes", self.passes is not None), ] if present ] -def load_export(data: bytes, filename: str) -> Result: +def load_recording(data: bytes, filename: str) -> Result: result = Result(id=hashlib.sha256(data).hexdigest()[:12], filename=filename) - suffix = PurePath(filename).suffix.lower() - if suffix == ".zip": - _read_zip(result, data) - elif suffix == ".csv": - _read_csv(result, data, filename) - elif suffix == ".json": - _read_deep_analysis(result, data, filename) - else: - raise ExportError(f"{filename}: use an Evolve export (.zip, .csv or deep analysis .json).") - - if not result.contents: - raise ExportError(f"{filename} contains no Evolve scores, per-frame data or deep analysis.") - if result.workloads is None and result.passes is not None: - result.workloads = _workloads_from_passes(result.passes, result.frames) - if not result.metadata: - result.warnings.append(f"{filename} has no system information (GPU, driver); it's labelled by file name.") - return result - - -def _read_zip(result: Result, data: bytes) -> None: - try: - archive = zipfile.ZipFile(io.BytesIO(data)) - except zipfile.BadZipFile: - raise ExportError(f"{result.filename} is not a zip file.") from None - members = {PurePath(name).name: name for name in archive.namelist()} - - # The combined CSV holds everything the separate CSVs do, plus system information. - for name in ["evolve_results.csv", "frametimes.csv", "scores.csv"]: - if name in members: - _read_csv(result, archive.read(members[name]), f"{result.filename}/{name}") - if DEEP_ANALYSIS in members: - _read_deep_analysis(result, archive.read(members[DEEP_ANALYSIS]), result.filename) - - -def _read_csv(result: Result, data: bytes, name: str) -> None: - try: - table = pd.read_csv(io.BytesIO(data), encoding="utf-8-sig", dtype=dict.fromkeys(METADATA_COLUMNS, str)) - except (pd.errors.ParserError, pd.errors.EmptyDataError, UnicodeDecodeError): - raise ExportError(f"{name} can't be read as a CSV file.") from None - columns = list(table.columns) - - if "Evolve Version" in columns and not result.metadata: - first = table.iloc[0] - result.metadata = {key: str(first[key]) for key in METADATA_COLUMNS if key in table and pd.notna(first[key])} - if columns[:1] == ["Loop"] or "Evolve Version" in columns: - loop = "Loop" if "Loop" in columns else "Loop Index" - score_columns = columns[1:] if loop == "Loop" else columns[len(_metadata_prefix(columns)) : columns.index(loop)] - if not result.scores: - result.scores = _scores(table.drop_duplicates(loop), score_columns) - if {"Loop Index", "Frame", "Benchmark Time (ns)"} <= set(columns) and result.workloads is None: - result.workloads, sensors = _per_frame(table) - if result.frames is None: - result.frames = sensors - if not columns or not (result.scores or result.workloads is not None or "Evolve Version" in columns): - raise ExportError(f"{name} isn't an Evolve scores or per-frame export.") - - -def _metadata_prefix(columns: list[str]) -> list[str]: - return [column for column in columns if column in METADATA_COLUMNS] - - -def _scores(per_loop: pd.DataFrame, columns: list[str]) -> dict[str, float]: - scores = {} - for column in columns: - values = pd.to_numeric(per_loop[column], errors="coerce") - # Evolve writes 0 for scores it didn't calculate for this benchmark (e.g. Work Graphs). - values = values[values > 0] - if len(values): - scores[column] = float(values.mean()) - return scores - - -def _per_frame(table: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: - base = pd.DataFrame( - { - "loop": table["Loop Index"].astype(int), - "frame": table["Frame"].astype(int), - "seq_s": table["Benchmark Time (ns)"] / 1e9, - } + tables, client_data = _read_container(data, filename) + + run_info = client_data.get("run_info") or {} + variant = run_info.get("variant") + if not variant: + raise LoadError(f"{filename} doesn't say which benchmark it ran.") + platform = client_data.get("platform") or "Desktop" + version = client_data.get("evolve_version") + version = (version["major"], version["minor"]) if version else None + result.metadata = _metadata(client_data, variant) + + frames = tables["frames"] + gpu_scopes = tables["gpu_scopes"] + if not len(frames): + raise LoadError(f"{filename} has no frames; the run didn't finish.") + result.workloads = _workloads(frames) + metrics = tables["metrics"] + sensors = _sensors(frames, metrics) + result.scores = scores.calculate( + variant, + platform, + version, + len(frames), + gpu_scopes, + tables["cpu_scopes"]["len_ns"], + frames["flip"], + frames["ssim"], + metrics, ) - workloads = base.copy() - for column, name in WORKLOAD_COLUMNS.items(): - if column in table and table[column].notna().any(): - # Evolve leaves the cell empty when a workload took no time in that frame. - workloads[name] = table[column].fillna(0) / 1e6 - sensors = base.copy() - for column, name in METRIC_CSV_COLUMNS.items(): - sensors[name] = pd.to_numeric(table[column], errors="coerce") if column in table else float("nan") - for name in SENSOR_COLUMNS: - if name not in sensors: - sensors[name] = float("nan") - return workloads, sensors - - -def _read_deep_analysis(result: Result, data: bytes, filename: str) -> None: - text = data.decode("utf-8") - try: - loops = json.loads(text) - except json.JSONDecodeError: - # Evolve only closes the array after the last loop, so an interrupted run misses the final "]". - try: - loops = json.loads(text.rstrip().rstrip(",") + "]") - except json.JSONDecodeError: - raise ExportError(f"{filename}: {DEEP_ANALYSIS} is damaged and can't be read.") from None - result.warnings.append(f"{filename}: the run was interrupted; using the {len(loops)} completed loop(s).") - if not isinstance(loops, list) or (loops and "per_frame_results" not in loops[0]): - raise ExportError(f"{filename} isn't an Evolve deep analysis file.") - - passes, frames, left_out = _tabulate(loops) + + passes, totals, left_out = _passes(gpu_scopes, frames) + result.frames = sensors.join(totals, on="frame") + result.frames[["gpu_sum_ms", "left_out_ms"]] = result.frames[["gpu_sum_ms", "left_out_ms"]].fillna(0.0) if passes.empty: - result.warnings.append( - f"{filename}: its deep analysis has no GPU work that Evolve assigns to a workload category." - ) - return - result.passes, result.frames, result.left_out = passes, frames, left_out + result.warnings.append(f"{filename} has no GPU work that Evolve assigns to a workload category.") + return result + result.passes, result.left_out = passes, left_out nested = passes["ms"].sum() - passes["top_ms"].sum() if nested > 0: @@ -239,96 +146,133 @@ def _read_deep_analysis(result: Result, data: bytes, filename: str) -> None: f"{filename}: {nested / passes['ms'].sum():.1%} of measured GPU time is nested inside other passes. " "Totals only count it once." ) + return result -def _tabulate(loops: list[dict]) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, float]]: - # Pass name -> whether Evolve assigns it to a workload category. - categorized: dict[str, bool] = {} - left_out: dict[str, float] = {} - pass_rows = [] - frame_rows = [] - for loop in loops: - loop_index = loop["loop_index"] - for frame_index, frame in enumerate(loop["per_frame_results"]): - seq_s = frame["sequence_time_ns"] / 1e9 - own, top = {}, {} - left_out_ms = 0.0 - first_start, last_end = None, None - for command_buffer in frame["command_buffer_timings"].values(): - scopes = [] - for name, timings in command_buffer["scope_timings"].items(): - if name not in categorized: - categorized[name] = bool(workloads(name)) - for timing in timings: - start, end = timing["start"], timing["end"] - first_start = start if first_start is None else min(first_start, start) - last_end = end if last_end is None else max(last_end, end) - if not categorized[name]: - left_out[name] = left_out.get(name, 0.0) + (end - start) / 1e6 - left_out_ms += (end - start) / 1e6 - else: - scopes.append((start, end, name)) - - # Sorted by start, longest first: a scope ending before the current outer one is nested in it. - scopes.sort(key=lambda scope: (scope[0], -scope[1])) - outer_end = None - for start, end, name in scopes: - duration = (end - start) / 1e6 - own[name] = own.get(name, 0.0) + duration - if outer_end is None or end > outer_end: - top[name] = top.get(name, 0.0) + duration - outer_end = end - - for name, ms in own.items(): - pass_rows.append((loop_index, frame_index, seq_s, name, ms, top.get(name, 0.0))) - - row = { - "loop": loop_index, - "frame": frame_index, - "seq_s": seq_s, - "gpu_sum_ms": sum(top.values()), - "gpu_span_ms": (last_end - first_start) / 1e6 if first_start is not None else float("nan"), - "left_out_ms": left_out_ms, - } - row.update(_metrics(frame.get("metrics"))) - frame_rows.append(row) - - passes = pd.DataFrame(pass_rows, columns=["loop", "frame", "seq_s", "pass", "ms", "top_ms"]) - - frames = pd.DataFrame(frame_rows) - for column in SENSOR_COLUMNS: - if column not in frames: - frames[column] = float("nan") - return passes, frames, left_out - - -def _workloads_from_passes(passes: pd.DataFrame, frames: pd.DataFrame) -> pd.DataFrame: - """Per-frame GPU time per workload category, the way Evolve sums it for its per-frame export.""" - membership = pd.DataFrame( - [(name, workload) for name in passes["pass"].unique() for workload in workloads(name)], - columns=["pass", "workload"], - ) - if membership.empty: - return frames[["loop", "frame", "seq_s"]].copy() - per_workload = passes.merge(membership, on="pass").pivot_table( - index=["loop", "frame"], columns="workload", values="ms", aggfunc="sum" +def _read_container(data: bytes, filename: str) -> tuple[dict[str, pd.DataFrame], dict]: + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + manifest = json.loads(archive.read("datakit.json")) + except (zipfile.BadZipFile, KeyError, json.JSONDecodeError): + raise LoadError( + f"{filename} isn't an Evolve run file. Use the .evolve files Evolve saves for every run." + ) from None + if manifest.get("format") != "datakit": + raise LoadError(f"{filename} isn't an Evolve run file.") + if manifest.get("version") != DATAKIT_VERSION: + raise LoadError(f"{filename} was saved by a newer Evolve. Download the latest Evolve Charts to read it.") + + names = set(archive.namelist()) + tables = {} + for name in TABLES: + path = f"tables/{name}.parquet" + if path not in names: + raise LoadError(f"{filename} has no {name} table; it was saved by an Evolve version this tool can't read.") + tables[name] = pq.read_table(io.BytesIO(archive.read(path))).to_pandas() + try: + client_data = json.loads(archive.read("documents/client_data")) + except (KeyError, json.JSONDecodeError): + raise LoadError(f"{filename} has no readable system information.") from None + return tables, client_data + + +def _metadata(client_data: dict, variant: str) -> dict[str, str]: + system = client_data.get("system_info") or {} + os_info = system.get("os_info") or {} + driver = client_data.get("driver_info") or {} + version = client_data.get("evolve_version") + driver_version = driver.get("version") + metadata = { + "Evolve Version": f"{version['major']}.{version['minor']}" if version else "", + "Benchmark": variant, + "System Name": system.get("host_name") or "", + "CPU": (system.get("cpu_name") or "").strip(), + "GPU": driver.get("device_name") or "", + "OS Name": os_info.get("os_type") or "", + "OS Version": _os_version(os_info.get("version")), + "GPU Driver Version": ( + f"{driver_version['major']}.{driver_version['minor']}.{driver_version['patch']}" if driver_version else "" + ), + "Rendering Backend": client_data.get("rendering_backend") or "", + } + return {key: value for key, value in metadata.items() if value} + + +def _os_version(version: dict | str | None) -> str: + """The `os_info` crate's version: {"Semantic": [10, 0, 26200]}, {"Rolling": ...}, {"Custom": ...}.""" + if isinstance(version, dict): + for kind, value in version.items(): + if kind == "Semantic": + return ".".join(str(part) for part in value) + return str(value) if value else kind + return "" if version in (None, "Unknown") else str(version) + + +def _base(frames: pd.DataFrame) -> pd.DataFrame: + return pd.DataFrame({"loop": 1, "frame": frames["frame"].astype(int), "seq_s": frames["simulation_time_ns"] / 1e9}) + + +def _workloads(frames: pd.DataFrame) -> pd.DataFrame: + table = _base(frames) + for column, name in WORKLOAD_COLUMNS.items(): + if column in frames and frames[column].notna().any(): + # Empty when a workload took no time in that frame. + table[name] = frames[column].fillna(0) / 1e6 + return table + + +def _sensors(frames: pd.DataFrame, metrics: pd.DataFrame) -> pd.DataFrame: + """Each frame gets the latest sensor sample taken at or before it, like Evolve's own exports.""" + table = _base(frames) + sample_times = metrics["simulation_time_ns"].to_numpy(dtype=float) if len(metrics) else np.array([]) + rows = np.searchsorted(sample_times, frames["simulation_time_ns"].to_numpy(dtype=float), side="right") - 1 + valid = rows >= 0 + if len(metrics): + valid &= ~np.isnan(sample_times[np.clip(rows, 0, None)]) + for column, name in METRIC_COLUMNS.items(): + values = np.full(len(frames), np.nan) + if column in metrics and len(metrics): + readings = pd.to_numeric(metrics[column], errors="coerce").to_numpy(dtype=float) + values[valid] = readings[rows[valid]] + table[name] = values + return table + + +def _passes(gpu_scopes: pd.DataFrame, frames: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, float]]: + """Per (frame, pass) GPU time, and per frame the summed and spanned GPU time and what was left out.""" + scopes = pd.DataFrame( + { + "frame": gpu_scopes["frame"].astype(int), + "command_buffer": gpu_scopes["command_buffer"].astype(int), + "pass": gpu_scopes["name"].astype(str), + "start": gpu_scopes["start_ns"].astype("int64"), + "len": gpu_scopes["len_ns"].astype("int64"), + } ) - table = frames[["loop", "frame", "seq_s"]].join(per_workload, on=["loop", "frame"]) - present = [name for name in WORKLOAD_ORDER if name in table] - return table[["loop", "frame", "seq_s", *present]].fillna({name: 0.0 for name in present}) - - -def _metrics(metrics: dict | None) -> dict[str, float]: - if not metrics: - return {} - values = {} - for key, column in METRIC_FIELDS.items(): - value = metrics.get(key) - if isinstance(value, (int, float)): - values[column] = float(value) - fan = metrics.get("fan_speed") - if isinstance(fan, dict): - for unit, value in fan.items(): - if unit in FAN_FIELDS and isinstance(value, (int, float)): - values[FAN_FIELDS[unit]] = float(value) - return values + scopes["end"] = scopes["start"] + scopes["len"] + categorized = {name: bool(workloads(name)) for name in scopes["pass"].unique()} + scopes["categorized"] = scopes["pass"].map(categorized).astype(bool) + + totals = scopes.groupby("frame").agg(first_start=("start", "min"), last_end=("end", "max")) + totals["gpu_span_ms"] = (totals["last_end"] - totals["first_start"]) / 1e6 + totals = totals[["gpu_span_ms"]] + + left = scopes[~scopes["categorized"]] + totals["left_out_ms"] = left.groupby("frame")["len"].sum() / 1e6 + left_out = (left.groupby("pass")["len"].sum() / 1e6).to_dict() + + # Sorted by start, longest first: a scope ending no later than every earlier one in its command buffer is + # nested inside one of them. + kept = scopes[scopes["categorized"]] + kept = kept.iloc[np.lexsort((-kept["end"], kept["start"], kept["command_buffer"], kept["frame"]))] + earlier_end = kept.groupby(["frame", "command_buffer"])["end"].cummax() + earlier_end = earlier_end.groupby([kept["frame"], kept["command_buffer"]]).shift(1) + kept = kept.assign(ms=kept["len"] / 1e6, top_ms=np.where(earlier_end.isna() | (kept["end"] > earlier_end), 1, 0)) + kept["top_ms"] *= kept["ms"] + + passes = kept.groupby(["frame", "pass"], as_index=False, sort=True)[["ms", "top_ms"]].sum() + totals["gpu_sum_ms"] = passes.groupby("frame")["top_ms"].sum() + seq_s = frames.set_index("frame")["simulation_time_ns"] / 1e9 + passes.insert(0, "loop", 1) + passes.insert(2, "seq_s", passes["frame"].map(seq_s)) + return passes[["loop", "frame", "seq_s", "pass", "ms", "top_ms"]], totals, left_out diff --git a/evolve_charts/scope_catalog.py b/evolve_charts/scope_catalog.py index b5a71ce..ba19564 100644 --- a/evolve_charts/scope_catalog.py +++ b/evolve_charts/scope_catalog.py @@ -163,7 +163,8 @@ "update_bones": ["Compute"], "apply_vertex_bones": ["Compute"], "reduce_aabbs": ["Compute"], - "channel_animation": ["Compute"], + "channel_animation_batched": ["Compute"], + "terrain-tiles": ["Compute"], "visualize-hierarchy-transforms": ["Compute"], "override_transforms": ["Compute"], "calculate_world_transforms": ["Compute"], @@ -254,7 +255,6 @@ "fused-mlp-forward-training-64": ["Compute"], "fused-mlp-forward-training-16": ["Compute"], "fused-mlp-backward-64": ["Compute"], - "fused-mlp-backward-32": ["Compute"], "fused-mlp-backward-16": ["Compute"], "sky-cubemap": ["Compute"], "clear-counters": ["Compute"], @@ -264,17 +264,11 @@ "Clear texture pipeline": ["Compute"], "generate_paths": ["Workgraphs"], "process_bounce": ["Workgraphs"], - "fused-mlp-forward-inference-coopvec-48-32-4": ["Nrc"], - "fused-mlp-forward-inference-coopvec-32-32-8": ["Nrc"], - "fused-mlp-forward-inference-coopvec-32-64-8": ["Nrc"], - "fused-mlp-forward-training-coopvec-48-32-4": ["Nrc"], - "fused-mlp-forward-training-coopvec-32-32-8": ["Nrc"], - "fused-mlp-forward-training-coopvec-32-64-8": ["Nrc"], - "fused-mlp-backward-coopvec-48-32-4": ["Nrc"], - "fused-mlp-backward-coopvec-32-32-8": ["Nrc"], - "fused-mlp-backward-coopvec-32-64-8": ["Nrc"], - "fused-mlp-forward-inference-coopmat-32-32-8": ["Nrc"], + "nrc-mlp-high-forward-inference-coopmat-nv-amd": ["Nrc"], + "nrc-mlp-high-forward-inference-coopmat-arm": ["Nrc"], + "nrc-mlp-high-forward-inference-coopmat-qcom": ["Nrc"], "fused-mlp-forward-training-32": ["Nrc"], + "fused-mlp-backward-32": ["Nrc"], "Fsr execute": ["UpscalingPerformance"], "Xess execute": ["UpscalingPerformance"], "DLSS evaluate": ["UpscalingPerformance"], @@ -317,6 +311,80 @@ "box_downsample": ["Uncategorized"], "crop_center": ["Uncategorized"], "fallback-upscaler": ["Uncategorized"], + "init_path_tracing_vbuffer_refactored": ["Compute"], + "shading_refactored": ["Compute"], + "neubtf_shading_refactored": ["Compute"], + "resolve_refactored": ["Compute"], + "fill_indirect_args_refactored": ["Compute"], + "fill_shading_neubtf_indirect_args_refactored": ["Compute"], + "water-compositing": ["Compute"], + "resolve_aabbs": ["Compute"], + "reset_aabbs": ["Compute"], + "channel_animation": ["Compute"], + "draw_mesh_selection": ["Uncategorized"], + "breda-text pipeline": ["Uncategorized"], + "begin render pass": ["Uncategorized"], + "render pass": ["Uncategorized"], + "upscaling-raster-pipeline-sample": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_0": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_1": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_2": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_3": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_4": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_5": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_6": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_7": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_8": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_9": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_10": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_11": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_12": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_13": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_14": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_15": ["Uncategorized"], + "draw_mesh_selection_0": ["Uncategorized"], + "breda-manipulators-action-triangles_0": ["Uncategorized"], + "breda-manipulators-draw-triangles_2": ["Uncategorized"], + "breda-manipulators-action-lines_0": ["Uncategorized"], + "breda-manipulators-draw-lines_2": ["Uncategorized"], + "vbuffer-mesh-parts-prefix-sum-hierarchy": ["Rasterization"], + "vbuffer-mesh-parts-prefix-sum": ["Rasterization"], + "vbuffer-cull-instances": ["Rasterization"], + "vbuffer-raster-buckets-prefix-sum-hierarchy": ["Rasterization"], + "vbuffer-raster-buckets-prefix-sum": ["Rasterization"], + "vbuffer-build-instanced-draw-calls": ["Rasterization"], + "vbuffer-group-instances": ["Rasterization"], + "vbuffer": ["Rasterization"], + "vbuffer-indirect_0": ["Rasterization"], + "vbuffer-indirect_1": ["Rasterization"], + "vbuffer-indirect_2": ["Rasterization"], + "vbuffer-with-alpha-test": ["Rasterization"], + "vbuffer-indirect-with-alpha-test": ["Rasterization"], + "vbuffer-indirect-with-alpha-test_0": ["Rasterization"], + "vbuffer-indirect-with-alpha-test_1": ["Rasterization"], + "vbuffer-indirect-with-alpha-test_2": ["Rasterization"], + "particle-alpha-blend": ["Rasterization"], + "particle-alpha-blend-indirect": ["Rasterization"], + "particle-alpha-blend-indirect_0": ["Rasterization"], + "particle-alpha-blend-indirect_1": ["Rasterization"], + "particle-alpha-blend-indirect_2": ["Rasterization"], + "alpha-blend": ["Rasterization"], + "alpha-blend-indirect_0": ["Rasterization"], + "alpha-blend-indirect_1": ["Rasterization"], + "alpha-blend-indirect_2": ["Rasterization"], + "decal": ["Rasterization"], + "decal-indirect_0": ["Rasterization"], + "decal-indirect_1": ["Rasterization"], + "decal-indirect_2": ["Rasterization"], + "build-gbuffers_0": ["Rasterization"], + "trace-validation-rays": ["RayTracing"], + "trace-cache-diffuse-rays": ["RayTracing"], + "water-shading-inline": ["RayTracing"], + "water-shading-pipeline": ["RayTracing"], + "trace_rays_refactored": ["RayTracing"], + "trace_shadow_rays_refactored": ["RayTracing"], + "transmission-ray-tracing-pipeline": ["RayTracing"], + "transmission-ray-tracing-inline": ["RayTracing"], } # variant -> the name Evolve shows @@ -369,7 +437,6 @@ "nrc-pt-apply-nrc", "nrc-wfpt-apply-nrc", "fused-mlp-backward-64", - "fused-mlp-backward-32", "fused-mlp-backward-16", "fused-mlp-forward-inference-64", "fused-mlp-forward-inference-32", diff --git a/evolve_charts/score_catalog.py b/evolve_charts/score_catalog.py new file mode 100644 index 0000000..b63e105 --- /dev/null +++ b/evolve_charts/score_catalog.py @@ -0,0 +1,198 @@ +# Generated by tools/sync_scores.py from evolve/crates/evolve-highscore-api/src/score.rs +# and evolve/crates/evolve-datakit-scores/src/lib.rs. Do not edit by hand. + +# Score variants, in the order Evolve lists them +SCORE_ORDER = [ + "RayTracing", + "AccelerationStructureBuilds", + "Rasterization", + "Compute", + "Workgraphs", + "Nrc", + "Driver", + "Energy", + "UpscalerImageQuality1", + "UpscalerImageQuality2", + "UpscalerPerformance", +] + +# Score variant -> the name Evolve shows +SCORE_NAMES = { + "RayTracing": "Ray Tracing", + "AccelerationStructureBuilds": "Acceleration Structure", + "Rasterization": "Rasterization", + "Compute": "Compute", + "Workgraphs": "Work Graphs", + "Nrc": "NRC", + "Driver": "Driver", + "Energy": "Energy Consumption", + "UpscalerImageQuality1": "Image Quality 1", + "UpscalerImageQuality2": "Image Quality 2", + "UpscalerPerformance": "Upscaler Performance", +} + +# WorkloadCategory variant -> the Score of its GPU time +GPU_SCORES = { + "RayTracing": "RayTracing", + "AccelerationStructureBuilds": "AccelerationStructureBuilds", + "Rasterization": "Rasterization", + "Compute": "Compute", + "Workgraphs": "Workgraphs", + "Nrc": "Nrc", + "UpscalingPerformance": "UpscalerPerformance", +} + +# Platform -> BenchmarkVariant -> Score -> multiplier; a score a benchmark doesn't have is absent +MULTIPLIERS = { + "Desktop": { + "UpscalingQualcommToggleMode": {}, + "UpscalingQualcommBackToBack": {}, + "UpscalingQualcommPowerEfficiency": {}, + "Upscaling": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0, "UpscalerPerformance": 1.0049}, + "UpscalingQualcommImageQuality": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0}, + "Nrc": {"Nrc": 30.0}, + "RayTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "RayTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "PathTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 127.0, + "Compute": 190.0, + "Driver": 12.12, + "Energy": 0.003, + }, + "PathTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 615.0, + "Compute": 28.24, + "Driver": 14.59, + "Energy": 0.003, + }, + "PathTracingWorkGraph": { + "AccelerationStructureBuilds": 5.43, + "Driver": 54.63, + "Energy": 0.003, + "Workgraphs": 400.0, + }, + }, + "Phone": { + "UpscalingQualcommToggleMode": {}, + "UpscalingQualcommBackToBack": {}, + "UpscalingQualcommPowerEfficiency": {}, + "Upscaling": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0, "UpscalerPerformance": 10.0}, + "UpscalingQualcommImageQuality": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0}, + "Nrc": {}, + "RayTracingInline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 290.83, + "Rasterization": 42.76, + "Compute": 237.28, + "Driver": 40.0, + "Energy": 19350.0, + }, + "RayTracingPipeline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 290.83, + "Rasterization": 42.76, + "Compute": 237.28, + "Driver": 40.0, + "Energy": 19350.0, + }, + "PathTracingInline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 993.0, + "Compute": 487.0, + "Driver": 41.0, + "Energy": 19350.0, + }, + "PathTracingPipeline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 1948.0, + "Compute": 276.24, + "Driver": 38.5, + "Energy": 19350.0, + }, + "PathTracingWorkGraph": {"AccelerationStructureBuilds": 130.32, "Workgraphs": 400.0}, + }, + "Handheld": { + "UpscalingQualcommToggleMode": {}, + "UpscalingQualcommBackToBack": {}, + "UpscalingQualcommPowerEfficiency": {}, + "Upscaling": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0, "UpscalerPerformance": 1.0049}, + "UpscalingQualcommImageQuality": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0}, + "Nrc": {"Nrc": 30.0}, + "RayTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "RayTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "PathTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 127.0, + "Compute": 190.0, + "Driver": 12.12, + "Energy": 0.003, + }, + "PathTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 615.0, + "Compute": 28.24, + "Driver": 14.59, + "Energy": 0.003, + }, + "PathTracingWorkGraph": { + "AccelerationStructureBuilds": 5.43, + "Driver": 54.63, + "Energy": 0.003, + "Workgraphs": 400.0, + }, + }, +} + +# Phone Driver scores of runs before this Evolve version use PHONE_DRIVER_WITH_PRESENT +DRIVER_EXCLUDES_PRESENT = (17, 0) + +# BenchmarkVariant -> phone Driver multiplier from before DRIVER_EXCLUDES_PRESENT +PHONE_DRIVER_WITH_PRESENT = { + "RayTracingInline": 553.17, + "RayTracingPipeline": 553.17, + "PathTracingInline": 1224.0, + "PathTracingPipeline": 2040.0, +} + +# Energy score = ENERGY_SCALE * exp(-multiplier * average power in W) +ENERGY_SCALE = 10600.0 + +# FLIP and SSIM pooling and normalization constants +IMAGE_QUALITY = { + "WORST_CASE_FRACTION": 0.1, + "WORST_CASE_POOL_WEIGHT": 0.5, + "BEST_FLIP": 0.3156, + "WORST_FLIP": 0.4255, + "BEST_SSIM": 0.777, + "WORST_SSIM": 0.6102, +} diff --git a/evolve_charts/scores.py b/evolve_charts/scores.py new file mode 100644 index 0000000..e7c5f1b --- /dev/null +++ b/evolve_charts/scores.py @@ -0,0 +1,123 @@ +"""Evolve's scores, calculated from a run's tables the way Evolve does (`evolve-datakit-scores`). + +A `.evolve` file holds the measurements, not the scores. The calibrated multipliers come from +`score_catalog.py`, generated from Evolve's Rust source by `tools/sync_scores.py`. +""" + +import math + +import numpy as np +import pandas as pd + +from .categories import workload_variants +from .score_catalog import ( + DRIVER_EXCLUDES_PRESENT, + ENERGY_SCALE, + GPU_SCORES, + IMAGE_QUALITY, + MULTIPLIERS, + PHONE_DRIVER_WITH_PRESENT, + SCORE_NAMES, + SCORE_ORDER, +) + + +def multiplier(score: str, variant: str, platform: str, version: tuple[int, int] | None = None) -> float | None: + if score == "Driver" and platform == "Phone" and (version is None or version < DRIVER_EXCLUDES_PRESENT): + return PHONE_DRIVER_WITH_PRESENT.get(variant) + return MULTIPLIERS.get(platform, {}).get(variant, {}).get(score) + + +def calculate( + variant: str, + platform: str, + version: tuple[int, int] | None, + num_frames: int, + gpu_scopes: pd.DataFrame, + cpu_scope_ns: pd.Series, + flip: pd.Series, + ssim: pd.Series, + metrics: pd.DataFrame, +) -> dict[str, float]: + """Score name -> score, for every score the benchmark has on this platform, in Evolve's order. + + `gpu_scopes` has a `name` and `len_ns` column; `cpu_scope_ns` holds the duration of every driver call. + """ + has = MULTIPLIERS.get(platform, {}).get(variant, {}) + scores: dict[str, int] = {} + + def per_frame(score: str, total_ns: int) -> int | None: + factor = multiplier(score, variant, platform, version) + if factor is None or total_ns == 0 or num_frames == 0: + return None + return _u32(factor / (round(total_ns / num_frames) / 1e9)) + + if any(score in has for score in GPU_SCORES.values()): + per_name = gpu_scopes.groupby("name", observed=True)["len_ns"].sum() + totals = dict.fromkeys(GPU_SCORES, 0) + for name, total in per_name.items(): + for workload in workload_variants(name): + if workload in totals: + totals[workload] += int(total) + for workload, score in GPU_SCORES.items(): + if (value := per_frame(score, totals[workload])) is not None: + scores[score] = value + + if "Driver" in has and (value := per_frame("Driver", int(cpu_scope_ns.sum()))) is not None: + scores["Driver"] = value + + for score, errors in [("UpscalerImageQuality1", flip), ("UpscalerImageQuality2", ssim)]: + if score in has and (value := _image_quality(variant, score, errors.dropna().to_numpy())) is not None: + scores[score] = value + + if "Energy" in has and (power := _average_power_w(metrics)) is not None: + energy = np.float32(ENERGY_SCALE) * np.exp(np.float32(-has["Energy"]) * power) + scores["Energy"] = _u32(float(energy)) + + return {SCORE_NAMES[score]: float(scores[score]) for score in SCORE_ORDER if score in scores} + + +def _u32(value: float) -> int: + """Rust's saturating `as u32`.""" + return 0 if math.isnan(value) else int(min(max(value, 0), 2**32 - 1)) + + +def _f32_sum(values: np.ndarray) -> np.float32: + """Summed one value at a time in f32, like Rust's `iter().sum::()`.""" + return np.cumsum(values.astype(np.float32), dtype=np.float32)[-1] + + +def _average_power_w(metrics: pd.DataFrame) -> np.float32 | None: + """With an energy counter in the first and last samples, the energy used over the time between + them; otherwise the mean sampled board power, or GPU power for a sample without it.""" + if metrics.empty: + return None + energy = metrics["gpu_energy_j"] + if pd.notna(energy.iloc[0]) and pd.notna(energy.iloc[-1]): + joules = np.float32(energy.iloc[-1] - energy.iloc[0]) + elapsed_ns = max(int(metrics["sample_time_ns"].iloc[-1]) - int(metrics["sample_time_ns"].iloc[0]), 0) + return joules / np.float32(elapsed_ns / 1e9) + power = metrics["board_power_w"].fillna(metrics["gpu_power_w"]).dropna().to_numpy() + return _f32_sum(power) / np.float32(len(power)) if len(power) else None + + +def _image_quality(variant: str, score: str, errors: np.ndarray) -> int | None: + if not len(errors): + return None + if variant == "UpscalingQualcommImageQuality": + return 0 + errors = errors.astype(np.float32) + worse_is_higher = score == "UpscalerImageQuality1" + mean = _f32_sum(errors) / np.float32(len(errors)) + ordered = np.sort(errors) + count = max(math.ceil(np.float32(len(errors)) * np.float32(IMAGE_QUALITY["WORST_CASE_FRACTION"])), 1) + worst = ordered[len(ordered) - count :] if worse_is_higher else ordered[:count] + weight = np.float32(IMAGE_QUALITY["WORST_CASE_POOL_WEIGHT"]) + pooled = (np.float32(1) - weight) * mean + weight * (_f32_sum(worst) / np.float32(len(worst))) + if worse_is_higher: + best, worst_value = np.float32(IMAGE_QUALITY["BEST_FLIP"]), np.float32(IMAGE_QUALITY["WORST_FLIP"]) + normalized = (worst_value - pooled) / (worst_value - best) + else: + best, worst_value = np.float32(IMAGE_QUALITY["BEST_SSIM"]), np.float32(IMAGE_QUALITY["WORST_SSIM"]) + normalized = (pooled - worst_value) / (best - worst_value) + return _u32(float(max(normalized, np.float32(0)) * np.float32(10_000))) diff --git a/evolve_charts/stats.py b/evolve_charts/stats.py index 2c96087..368424f 100644 --- a/evolve_charts/stats.py +++ b/evolve_charts/stats.py @@ -61,7 +61,7 @@ def per_frame(result: Result, skip_s: float, level: Level, within: Within = None def summary(results: dict[str, Result], skip_s: float, level: Level, within: Within = None) -> pd.DataFrame: """One row per (result, item) with mean/median/p5/p95 milliseconds per frame. - Results without data for the level (e.g. no deep analysis) are left out. + Results without data for the level (e.g. no pass timings) are left out. """ rows = [] for label, result in results.items(): @@ -106,7 +106,7 @@ def ordered(items, level: Level) -> list[str]: def frame_totals(results: dict[str, Result], skip_s: float) -> pd.DataFrame: - """Deep analysis only: GPU time per frame of all categorized passes, what was left out, and the span.""" + """Pass timings only: GPU time per frame of all categorized passes, what was left out, and the span.""" rows = [] for label, result in results.items(): if result.passes is None: @@ -155,7 +155,7 @@ def telemetry_summary(results: dict[str, Result], skip_s: float) -> pd.DataFrame def scores_table(results: dict[str, Result]) -> pd.DataFrame: - """Scores as rows (in the order Evolve exports them), results as columns.""" + """Scores as rows (in Evolve's order), results as columns.""" names = list(dict.fromkeys(name for result in results.values() for name in result.scores)) return pd.DataFrame( {label: [result.scores.get(name) for name in names] for label, result in results.items() if result.scores}, @@ -201,7 +201,7 @@ def wide_for_spreadsheets(item_summary: pd.DataFrame, value: str = "mean_ms") -> def _smooth(by_time: pd.DataFrame, smooth_s: float) -> pd.DataFrame: - """Centred rolling mean over `smooth_s` seconds of benchmark time; exports differ a lot in frames per second.""" + """Centred rolling mean over `smooth_s` seconds of benchmark time; runs differ a lot in frames per second.""" if smooth_s <= 0 or len(by_time) < 2: return by_time step = pd.Series(by_time.index).diff().median() diff --git a/evolve_charts/ui.py b/evolve_charts/ui.py index ba1b9fe..ec4bcb7 100644 --- a/evolve_charts/ui.py +++ b/evolve_charts/ui.py @@ -2,7 +2,7 @@ Started by `evolve-charts.py`. Pages: -- Results: load and name exports, see what each contains and how Evolve's measurements relate. +- Results: load and name `.evolve` run files, see what each contains and how Evolve's measurements relate. - Scores, GPU time, Passes, GPU sensors: the analysis, with the loaded results and the "compare against" choice in a bar at the top. - Export: CSV files for spreadsheets and newsroom chart tools. @@ -18,7 +18,7 @@ from . import charts, stats from .categories import WORKLOAD_DESCRIPTIONS, WORKLOAD_ORDER, is_utility, workloads -from .loader import ExportError, Result, load_export +from .loader import LoadError, Result, load_recording APP_DIR = Path(__file__).resolve().parent.parent STATIC_DIR = APP_DIR / "static" @@ -145,11 +145,11 @@ def main() -> None: @st.cache_data(show_spinner=False, max_entries=32) def _load_file(path: str, modified: float) -> Result: - return load_export(Path(path).read_bytes(), Path(path).name) + return load_recording(Path(path).read_bytes(), Path(path).name) -def _exports_dir() -> Path: - return Path(os.environ.get("EVOLVE_CHARTS_EXPORTS", Path(__file__).resolve().parent.parent / "exports")) +def _results_dir() -> Path: + return Path(os.environ.get("EVOLVE_CHARTS_RESULTS", Path(__file__).resolve().parent.parent / "results")) def _store() -> dict: @@ -164,8 +164,8 @@ def _on_upload() -> None: for upload in st.session_state.get(f"uploader-{store['upload_key']}") or []: try: with st.spinner(f"Reading {upload.name}…"): - result = load_export(upload.getvalue(), upload.name) - except ExportError as error: + result = load_recording(upload.getvalue(), upload.name) + except LoadError as error: store["errors"][upload.name] = str(error) continue store["errors"].pop(upload.name, None) @@ -178,16 +178,16 @@ def _on_upload() -> None: def _loaded_results() -> dict[str, Result]: store = _store() loaded: list[Result] = [] - folder = _exports_dir() + folder = _results_dir() paths = sorted(folder.iterdir()) if folder.is_dir() else [] for path in paths: - if path.suffix.lower() not in {".zip", ".csv", ".json"}: + if path.suffix.lower() != ".evolve": continue try: with st.spinner(f"Reading {path.name}…"): loaded.append(_load_file(str(path), path.stat().st_mtime)) store["errors"].pop(path.name, None) - except ExportError as error: + except LoadError as error: store["errors"][path.name] = str(error) loaded += store["uploaded"].values() @@ -294,7 +294,7 @@ def _missing(present: set[str], what: str) -> bool: st.info(f"None of the loaded results contain {what}. See the Results page for what each file contains.") return True if missing: - st.caption(f"Not included, because their exports contain no {what}: {', '.join(missing)}.") + st.caption(f"Not included, because their runs have no {what}: {', '.join(missing)}.") return False @@ -329,19 +329,19 @@ def _results_page() -> None: results = st.session_state["results"] _title( "Results", - "Load Evolve exports to compare them. Everything stays on this computer.", + "Load Evolve runs to compare them. Everything stays on this computer.", ) columns = st.columns(CARDS_PER_ROW) with columns[0].container(border=True, height="stretch"): st.file_uploader( - "**Add Evolve exports**", - type=["zip", "csv", "json"], + "**Add Evolve runs**", + type=["evolve"], accept_multiple_files=True, key=f"uploader-{store['upload_key']}", on_change=_on_upload, - help="The .zip from Export on Evolve's score screen, or the CSV and JSON files Evolve's command line " - "exports. Files in the `exports` folder next to this tool load automatically.", + help="The .evolve file Evolve saves for every run, in Documents\\Evolve\\results on Windows. " + "Files in the `results` folder next to this tool load automatically.", ) for error in store["errors"].values(): st.error(error) @@ -351,9 +351,9 @@ def _results_page() -> None: """ **How to start** -1. In Evolve, finish a benchmark and click **Export** on the score screen. +1. Run a benchmark in Evolve. Evolve saves every run as a **.evolve** file in *Documents\\Evolve\\results*. 2. Repeat on every GPU or setting you want to compare, with the same benchmark. -3. Drop the files in the box on the left. +3. Drop the .evolve files in the box on the left. """ ) colors = charts.result_colors(list(results)) @@ -366,7 +366,11 @@ def _results_page() -> None: if results: problems = [] - for key, what in [("Evolve Version", "Evolve versions"), ("Rendering Backend", "graphics APIs")]: + for key, what in [ + ("Benchmark", "benchmarks"), + ("Evolve Version", "Evolve versions"), + ("Rendering Backend", "graphics APIs"), + ]: values = {result.metadata.get(key, "unknown") for result in results.values()} if len(values) > 1: problems.append(f"different {what} ({', '.join(sorted(values))})") @@ -411,7 +415,7 @@ def _result_card(result: Result, label: str, color: str) -> None: duration = frames["seq_s"].max() - frames["seq_s"].min() if len(frames) else 0 lines.append(f"{len(frames):,} frames Β· {duration // 60:.0f}:{duration % 60:02.0f} min") contents = "
".join( - f"{'βœ“' if name in result.contents else 'βœ—'} {name}" for name in ["Scores", "Per frame", "Deep analysis"] + f"{'βœ“' if name in result.contents else 'βœ—'} {name}" for name in ["Scores", "Per frame", "Passes"] ) body = "
".join(line for line in lines if line) st.markdown(f'
{body}
{contents}
', unsafe_allow_html=True) @@ -542,7 +546,7 @@ def _gpu_time_page() -> None: inside = stats.summary(context.results, WARM_UP_S, "pass", within=("workload", item)) inside = inside[inside["mean_ms"] > 0] if inside.empty: - st.caption("Load exports with deep analysis to see the passes this workload category is made of.") + st.caption("These runs have no pass timings to show the passes this workload category is made of.") return ranked = inside.groupby("item")["mean_ms"].max().sort_values(ascending=False) items = ranked.index[:TOP_PASSES].tolist() @@ -572,12 +576,12 @@ def _passes_page() -> None: context = _context() _title("Passes", "Single blocks of GPU work, named as in Evolve. Tick passes to follow them over the run.") summary = stats.summary(context.results, WARM_UP_S, "pass") - if _missing(set(summary["result"]), "deep analysis"): + if _missing(set(summary["result"]), "pass timings"): return labels = [label for label in context.labels if label in set(summary["result"])] baseline = context.baseline if context.baseline in labels else labels[0] if baseline != context.baseline: - st.caption(f"Compared against {baseline}: {context.baseline} has no deep analysis.") + st.caption(f"Compared against {baseline}: {context.baseline} has no pass timings.") table = stats.comparison_table(summary, labels, baseline) # Copies, clears and other support work still count towards their workload category, but aren't listed. utility = table["item"].map(is_utility) diff --git a/exports/README.txt b/exports/README.txt deleted file mode 100644 index 350b906..0000000 --- a/exports/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -Put Evolve export .zip files in this folder and Evolve Charts loads them automatically. -You can also drag them into the upload box in the tool instead. diff --git a/results/README.txt b/results/README.txt new file mode 100644 index 0000000..bbb40bc --- /dev/null +++ b/results/README.txt @@ -0,0 +1,3 @@ +Put .evolve files in this folder and Evolve Charts loads them automatically. +Evolve saves one for every benchmark run, in Documents\Evolve\results on Windows. +You can also drag them into the upload box in the tool instead. diff --git a/tests/conftest.py b/tests/conftest.py index 4b34ecd..41e2807 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,31 +4,58 @@ import zipfile from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -METADATA_HEADER = "Evolve Version,System Name,CPU,GPU,OS Name,OS Version,GPU Driver Version,Rendering Backend" -SCORES_HEADER = "Ray Tracing,Rasterization,Compute,Work Graphs" -PER_FRAME_HEADER = ( - "Loop Index,Frame,Benchmark Time (ns),Ray Tracing (ns),Rasterization (ns),Compute (ns),Energy (W),Clock Speed (Mhz)" -) - - -def scope(start_ms: float, end_ms: float) -> dict: - return {"start": int(start_ms * 1e6), "end": int(end_ms * 1e6)} - - -def frame(seq_s: float, buffers: list[dict[str, list[dict]]], metrics: dict | None = None) -> dict: - return { - "sequence_time_ns": int(seq_s * 1e9), - "command_buffer_timings": { - f"Command Buffer {index}": {"cpu_submit_time": 0, "gpu_submit_time": 0, "scope_timings": scopes} - for index, scopes in enumerate(buffers) - }, - "cpu_timings": {"DriverApiCall": [1000]}, - "metrics": metrics, - } +from evolve_charts.categories import workload_variants + +# WorkloadCategory variant -> `frames` column. +FRAME_COLUMNS = { + "RayTracing": "gpu_ray_tracing_ns", + "AccelerationStructureBuilds": "gpu_acceleration_structure_builds_ns", + "Rasterization": "gpu_rasterization_ns", + "Compute": "gpu_compute_ns", + "Workgraphs": "gpu_workgraphs_ns", + "Nrc": "gpu_nrc_ns", + "UpscalingPerformance": "gpu_upscaler_performance_ns", +} +METRICS_SCHEMA = { + "simulation_time_ns": pa.uint64(), + "sample_time_ns": pa.uint64(), + "gpu_usage_percent": pa.float64(), + "gpu_clock_mhz": pa.uint32(), + "vram_clock_mhz": pa.uint32(), + "gpu_power_w": pa.float64(), + "board_power_w": pa.float64(), + "gpu_energy_j": pa.float64(), + "soc_power_w": pa.float64(), + "soc_energy_j": pa.float64(), + "voltage_mv": pa.uint32(), + "vram_usage_mib": pa.uint32(), + "fan_rpm": pa.uint32(), + "fan_percent": pa.uint32(), + "edge_temperature_c": pa.float64(), + "hotspot_temperature_c": pa.float64(), +} + + +def scope(start_ms: float, end_ms: float) -> tuple[int, int]: + return int(start_ms * 1e6), int(end_ms * 1e6) + + +def frame( + seq_s: float, + buffers: list[dict[str, list[tuple[int, int]]]], + metrics: dict | None = None, + driver_ns: int = 1000, + flip: float | None = None, + ssim: float | None = None, +) -> dict: + """One frame: per command buffer, the scopes of every pass; and the sensor sample taken at this frame.""" + return {"seq_s": seq_s, "buffers": buffers, "metrics": metrics, "driver_ns": driver_ns, "flip": flip, "ssim": ssim} def simple_frames(shadow_ms: float, count: int = 20) -> list[dict]: @@ -51,45 +78,99 @@ def simple_frames(shadow_ms: float, count: int = 20) -> list[dict]: "made-up-pass": [scope(base + 62, base + 62.1)], } ], - {"clock_speed_in_mhz": 2500 + index, "board_power_usage_in_w": 200.0, "fan_speed": {"Rpm": 1500}}, + {"gpu_clock_mhz": 2500 + index, "board_power_w": 200.0, "fan_rpm": 1500}, ) ) return frames -def combined_csv(gpu: str, shadow_ms: float, count: int = 20, scores: str = "4000,5000,6000,0") -> str: - """`evolve_results.csv`: system information, scores and per-frame workload times on every row.""" - rows = [f"{METADATA_HEADER},{SCORES_HEADER},{PER_FRAME_HEADER}"] - for index in range(count): - ray_tracing = int((shadow_ms + (50 if index == 0 else 0)) * 1e6) - rows.append( - f"15.10,host,CPU X,{gpu},Windows,11,1.2.3,Dx12,{scores}," - f"1,{index},{int(index * 0.5e9)},{ray_tracing},,{int(0.5e6)},200.0,{2500 + index}" - ) - return "\n".join(rows) + "\n" +def _parquet(columns: dict[str, pa.Array]) -> bytes: + buffer = io.BytesIO() + pq.write_table(pa.table(columns), buffer) + return buffer.getvalue() -def make_export( - loops: list[list[dict]] | None, +def make_recording( + frames: list[dict], gpu: str = "Test GPU", - truncated: bool = False, - results_csv: str | None = None, + variant: str = "RayTracingInline", + platform: str = "Desktop", + version: tuple[int, int] = (17, 0), + with_scopes: bool = True, ) -> bytes: - """An Evolve Export zip; `loops=None` leaves out deep analysis like exports without Professional.""" + """A `.evolve` file the way Evolve writes it: per-frame workload times summed from the scopes, like Evolve.""" + workload_ns = {column: [] for column in FRAME_COLUMNS.values()} + scope_rows = {"frame": [], "command_buffer": [], "name": [], "start_ns": [], "len_ns": []} + metric_rows = {name: [] for name in METRICS_SCHEMA} + for index, spec in enumerate(frames): + totals = dict.fromkeys(FRAME_COLUMNS.values(), 0) + for buffer, scopes in enumerate(spec["buffers"]): + for name, timings in scopes.items(): + for start, end in timings: + for workload in workload_variants(name): + if workload in FRAME_COLUMNS: + totals[FRAME_COLUMNS[workload]] += end - start + if with_scopes: + for key, value in zip(scope_rows, [index, buffer, name, start, end - start]): + scope_rows[key].append(value) + for column, total in totals.items(): + workload_ns[column].append(total or None) + if spec["metrics"] is not None: + sample = {"simulation_time_ns": int(spec["seq_s"] * 1e9), "sample_time_ns": int(spec["seq_s"] * 1e9)} + sample.update(spec["metrics"]) + for name in METRICS_SCHEMA: + metric_rows[name].append(sample.get(name)) + + seq_ns = [int(spec["seq_s"] * 1e9) for spec in frames] + tables = { + "frames": { + "frame": pa.array(range(len(frames)), pa.uint32()), + "simulation_time_ns": pa.array(seq_ns, pa.uint64()), + **{column: pa.array(values, pa.uint64()) for column, values in workload_ns.items()}, + "cpu_driver_ns": pa.array([spec["driver_ns"] for spec in frames], pa.uint64()), + "flip": pa.array([spec["flip"] for spec in frames], pa.float32()), + "ssim": pa.array([spec["ssim"] for spec in frames], pa.float32()), + }, + "metrics": {name: pa.array(values, METRICS_SCHEMA[name]) for name, values in metric_rows.items()}, + "gpu_scopes": { + "frame": pa.array(scope_rows["frame"], pa.uint32()), + "command_buffer": pa.array(scope_rows["command_buffer"], pa.uint8()), + "name": pa.array(scope_rows["name"], pa.string()).dictionary_encode(), + "start_ns": pa.array(scope_rows["start_ns"], pa.uint64()), + "len_ns": pa.array(scope_rows["len_ns"], pa.uint64()), + }, + "cpu_scopes": { + "frame": pa.array(range(len(frames)), pa.uint32()), + "thread": pa.array(["main"] * len(frames)).dictionary_encode(), + "name": pa.array(["ExecuteCommandLists"] * len(frames)).dictionary_encode(), + "start_ns": pa.array(seq_ns, pa.int64()), + "len_ns": pa.array([spec["driver_ns"] for spec in frames], pa.uint64()), + }, + } + client_data = { + "evolve_version": {"major": version[0], "minor": version[1]}, + "run_info": {"variant": variant}, + "platform": platform, + "rendering_backend": "Dx12", + "driver_info": {"device_name": gpu, "version": {"major": 1, "minor": 2, "patch": 3}}, + "system_info": { + "host_name": "host", + "cpu_name": "CPU X ", + "os_info": {"os_type": "Windows", "version": {"Semantic": [10, 0, 26200]}}, + }, + } + buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w") as archive: - if loops is not None: - text = json.dumps( - [{"loop_index": index + 1, "per_frame_results": frames} for index, frames in enumerate(loops)] - ) - archive.writestr("deep_analysis.json", text[:-1] if truncated else text) - shadow_ms = 2.0 if "1" in gpu else 1.0 - archive.writestr("evolve_results.csv", results_csv if results_csv is not None else combined_csv(gpu, shadow_ms)) + archive.writestr("datakit.json", json.dumps({"format": "datakit", "version": 1})) + for name, columns in tables.items(): + archive.writestr(f"tables/{name}.parquet", _parquet(columns)) + archive.writestr("documents/client_data", json.dumps(client_data)) return buffer.getvalue() @pytest.fixture def two_gpus(tmp_path: Path) -> Path: - (tmp_path / "gen1.zip").write_bytes(make_export([simple_frames(2.0)], gpu="Gen 1 GPU")) - (tmp_path / "gen2.zip").write_bytes(make_export([simple_frames(1.0)], gpu="Gen 2 GPU")) + (tmp_path / "gen1.evolve").write_bytes(make_recording(simple_frames(2.0), gpu="Gen 1 GPU")) + (tmp_path / "gen2.evolve").write_bytes(make_recording(simple_frames(1.0), gpu="Gen 2 GPU")) return tmp_path diff --git a/tests/test_loader.py b/tests/test_loader.py index 04dedf0..d6d691b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,67 +1,52 @@ import io +import json import zipfile import pytest -from conftest import PER_FRAME_HEADER, combined_csv, frame, make_export, scope, simple_frames +from conftest import frame, make_recording, scope, simple_frames -from evolve_charts.categories import workloads -from evolve_charts.loader import ExportError, load_export +from evolve_charts.categories import is_utility, workloads +from evolve_charts.loader import LoadError, load_recording -def test_full_export(): - result = load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "gen1.zip") +def test_full_recording(): + result = load_recording(make_recording(simple_frames(2.0), gpu="Gen 1 GPU"), "gen1.evolve") assert result.gpu == "Gen 1 GPU" - assert result.metadata["Evolve Version"] == "15.10" - assert result.metadata["Rendering Backend"] == "Dx12" - # A score of 0 means Evolve didn't calculate it for this benchmark. - assert result.scores == {"Ray Tracing": 4000, "Rasterization": 5000, "Compute": 6000} - assert result.contents == ["Scores", "Per frame", "Deep analysis"] + assert result.metadata == { + "Evolve Version": "17.0", + "Benchmark": "RayTracingInline", + "System Name": "host", + "CPU": "CPU X", + "GPU": "Gen 1 GPU", + "OS Name": "Windows", + "OS Version": "10.0.26200", + "GPU Driver Version": "1.2.3", + "Rendering Backend": "Dx12", + } + assert result.contents == ["Scores", "Per frame", "Passes"] assert set(result.passes["pass"]) == {"shadow-raytrace-inline", "shadow_filter_pass_0"} assert result.warnings == [] -def test_per_frame_workloads_come_from_the_csv(): - result = load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "gen1.zip") +def test_per_frame_workloads_come_from_the_frames_table(): + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") frame_3 = result.workloads.set_index("frame").loc[3] + assert frame_3["seq_s"] == pytest.approx(1.5) assert frame_3["Raytracing"] == pytest.approx(2.0) assert frame_3["Compute"] == pytest.approx(0.5) - assert "Rasterization" not in result.workloads # an all-empty column + assert "Rasterization" not in result.workloads # no frame has any -def test_export_without_deep_analysis(): - result = load_export(make_export(None, gpu="Gen 1 GPU"), "gen1.zip") +def test_recording_without_pass_timings(): + result = load_recording(make_recording(simple_frames(2.0), with_scopes=False), "x.evolve") assert result.contents == ["Scores", "Per frame"] assert result.passes is None assert result.frames["gpu_clock_mhz"].iloc[2] == 2502 assert result.frames["board_power_w"].iloc[2] == 200.0 -def test_standalone_csv_exports(): - scores = load_export(b"Loop,Ray Tracing,Compute\n1,4000,6000\n2,4200,0\n", "scores.csv") - assert scores.scores == {"Ray Tracing": 4100, "Compute": 6000} - assert scores.contents == ["Scores"] - assert any("system information" in warning for warning in scores.warnings) - - per_frame = load_export(f"{PER_FRAME_HEADER}\n1,0,0,1000000,,500000,150,2600\n".encode(), "frametimes.csv") - assert per_frame.contents == ["Per frame"] - assert per_frame.workloads["Raytracing"].item() == pytest.approx(1.0) - - combined = load_export(combined_csv("Gen 2 GPU", 1.0).encode(), "evolve_results.csv") - assert combined.gpu == "Gen 2 GPU" - assert combined.contents == ["Scores", "Per frame"] - - -def test_standalone_deep_analysis_derives_workloads(): - data = zipfile.ZipFile(io.BytesIO(make_export([simple_frames(2.0)]))).read("deep_analysis.json") - result = load_export(data, "deep_analysis.json") - assert result.contents == ["Per frame", "Deep analysis"] - frame_3 = result.workloads.set_index("frame").loc[3] - assert frame_3["Raytracing"] == pytest.approx(2.0) - assert frame_3["Compute"] == pytest.approx(0.5) - - def test_uncategorized_scopes_are_left_out_but_counted(): - result = load_export(make_export([simple_frames(2.0)]), "x.zip") + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") assert result.left_out == {"made-up-pass": pytest.approx(0.1 * 20)} assert result.frames["left_out_ms"].tolist() == pytest.approx([0.1] * 20, abs=1e-5) # The GPU frame span still covers all GPU work, including what's left out. @@ -69,14 +54,14 @@ def test_uncategorized_scopes_are_left_out_but_counted(): def test_repeated_scopes_in_a_frame_are_summed(): - passes = load_export(make_export([simple_frames(2.0)]), "x.zip").passes + passes = load_recording(make_recording(simple_frames(2.0)), "x.evolve").passes filter_ms = passes[(passes["pass"] == "shadow_filter_pass_0") & (passes["frame"] == 3)]["ms"] assert filter_ms.item() == pytest.approx(0.5) def test_nested_scopes_count_once_in_totals(): nested = frame(0.0, [{"Build GBuffers": [scope(0, 10)], "shadow_prepare": [scope(2, 4)]}]) - result = load_export(make_export([[nested]]), "x.zip") + result = load_recording(make_recording([nested]), "x.evolve") passes = result.passes.set_index("pass") assert passes.at["shadow_prepare", "ms"] == pytest.approx(2.0) assert passes.at["shadow_prepare", "top_ms"] == 0.0 @@ -86,36 +71,39 @@ def test_nested_scopes_count_once_in_totals(): def test_frame_span_and_overlap_across_command_buffers(): overlapping = frame(0.0, [{"Build GBuffers": [scope(0, 6)]}, {"shadow_prepare": [scope(4, 10)]}]) - result = load_export(make_export([[overlapping]]), "x.zip") + result = load_recording(make_recording([overlapping]), "x.evolve") assert result.frames["gpu_sum_ms"].item() == pytest.approx(12.0) assert result.frames["gpu_span_ms"].item() == pytest.approx(10.0) -def test_metrics_null_and_fan_units(): +def test_each_frame_gets_the_latest_sensor_sample(): frames = [ - frame(0.0, [{"Build GBuffers": [scope(0, 1)]}], None), - frame(1.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_speed": {"Percent": 40}, "edge_temperature_in_c": 60.0}), - frame(2.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_speed": {"Rpm": 1200}, "clock_speed_in_mhz": None}), + frame(0.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_percent": 40, "edge_temperature_c": 60.0}), + frame(1.0, [{"Build GBuffers": [scope(0, 1)]}], None), + frame(2.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_rpm": 1200, "vram_usage_mib": 4096}), ] - result = load_export(make_export([frames]), "x.zip") - assert result.frames["fan_pct"].tolist()[1] == 40 - assert result.frames["fan_rpm"].tolist()[2] == 1200 - assert result.frames["gpu_clock_mhz"].isna().all() + sensors = load_recording(make_recording(frames), "x.evolve").frames + assert sensors["fan_pct"].tolist()[:2] == [40, 40] + assert sensors["edge_temp_c"].tolist()[1] == 60.0 + assert sensors["fan_rpm"].tolist()[2] == 1200 + assert sensors["vram_used_mb"].tolist()[2] == 4096 + assert sensors["gpu_clock_mhz"].isna().all() -def test_interrupted_multi_loop_run_is_recovered(): - result = load_export(make_export([simple_frames(2.0, 5), simple_frames(2.0, 5)], truncated=True), "x.zip") - assert result.frames["loop"].nunique() == 2 - assert any("interrupted" in warning for warning in result.warnings) +def test_unreadable_inputs(): + with pytest.raises(LoadError, match="isn't an Evolve run file"): + load_recording(b"not a zip", "broken.evolve") + def container(manifest: dict) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("datakit.json", json.dumps(manifest)) + return buffer.getvalue() -def test_unreadable_inputs(): - with pytest.raises(ExportError, match="not a zip"): - load_export(b"{}", "broken.zip") - with pytest.raises(ExportError, match="isn't an Evolve"): - load_export(b"a,b\n1,2\n", "other.csv") - with pytest.raises(ExportError, match="use an Evolve export"): - load_export(b"", "picture.png") + with pytest.raises(LoadError, match="newer Evolve"): + load_recording(container({"format": "datakit", "version": 2}), "future.evolve") + with pytest.raises(LoadError, match="no frames table"): + load_recording(container({"format": "datakit", "version": 1}), "empty.evolve") def test_categories(): @@ -125,11 +113,11 @@ def test_categories(): assert workloads("not-in-evolve") == [] assert workloads("shadow-raytrace-inline") == ["Raytracing"] assert workloads("vol-ris-initial-sampling") == ["Raytracing", "Compute"] + # Renamed passes still count, so runs from older Evolve versions compare. + assert workloads("channel_animation") == ["Compute"] def test_utility_passes(): - from evolve_charts.categories import is_utility - assert is_utility("Copy buffer to buffer") assert is_utility("egui_render") # Uncategorized by Evolve's pass bucketing, but the work of a workload category of their own. diff --git a/tests/test_scores.py b/tests/test_scores.py new file mode 100644 index 0000000..6742406 --- /dev/null +++ b/tests/test_scores.py @@ -0,0 +1,60 @@ +import numpy as np +import pandas as pd +import pytest +from conftest import frame, make_recording, scope, simple_frames + +from evolve_charts import scores +from evolve_charts.loader import load_recording +from evolve_charts.score_catalog import MULTIPLIERS + + +def test_gpu_driver_and_energy_scores(): + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") + desktop = MULTIPLIERS["Desktop"]["RayTracingInline"] + # Average per frame over every frame, the warm-up frame included. + ray_tracing_s = (20 * 2.0 + 50.0) / 20 / 1e3 + assert result.scores["Ray Tracing"] == int(desktop["RayTracing"] / ray_tracing_s) + assert result.scores["Compute"] == int(desktop["Compute"] / 0.5e-3) + assert result.scores["Driver"] == int(desktop["Driver"] / 1e-6) + energy = np.float32(10_600) * np.exp(np.float32(-desktop["Energy"]) * np.float32(200.0)) + assert result.scores["Energy Consumption"] == int(energy) + # In Evolve's order; workloads without GPU time have no score. + assert list(result.scores) == ["Ray Tracing", "Compute", "Driver", "Energy Consumption"] + + +def test_benchmarks_only_have_their_own_scores(): + result = load_recording(make_recording(simple_frames(2.0), variant="Nrc"), "x.evolve") + assert result.scores == {} + + +def test_phone_driver_multiplier_depends_on_the_evolve_version(): + assert scores.multiplier("Driver", "RayTracingInline", "Phone", (16, 1)) == 553.17 + assert scores.multiplier("Driver", "RayTracingInline", "Phone", (17, 0)) == 40.0 + assert scores.multiplier("Driver", "RayTracingInline", "Desktop", (16, 1)) == 11.93 + + +def test_energy_counter_wins_over_sampled_power(): + metrics = pd.DataFrame( + { + "sample_time_ns": [0, 2_000_000_000], + "gpu_energy_j": [100.0, 500.0], + "board_power_w": [999.0, 999.0], + "gpu_power_w": [None, None], + } + ) + assert scores._average_power_w(metrics) == pytest.approx(200.0) + + +def test_image_quality_scores(): + frames = [ + frame(index * 0.5, [{"Fsr execute": [scope(0, 1)]}], flip=0.3156 + index * 0.001, ssim=0.777) + for index in range(20) + ] + result = load_recording(make_recording(frames, variant="Upscaling"), "x.evolve") + # SSIM at its calibrated best scores 10000. + assert result.scores["Image Quality 2"] == pytest.approx(10_000, abs=1) + # FLIP pools the mean with the worst 10% of frames, half and half, between its best and worst. + flip = [0.3156 + index * 0.001 for index in range(20)] + pooled = 0.5 * np.mean(flip) + 0.5 * np.mean(sorted(flip)[-2:]) + assert result.scores["Image Quality 1"] == pytest.approx(10_000 * (0.4255 - pooled) / (0.4255 - 0.3156), abs=1) + assert "Upscaler Performance" in result.scores diff --git a/tests/test_stats.py b/tests/test_stats.py index 2c8722b..d788a18 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,15 +1,15 @@ import pytest -from conftest import make_export, simple_frames +from conftest import make_recording, simple_frames from evolve_charts import stats -from evolve_charts.loader import load_export +from evolve_charts.loader import load_recording @pytest.fixture def results(): return { - "Gen 1": load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "gen1.zip"), - "Gen 2": load_export(make_export([simple_frames(1.0)], gpu="Gen 2 GPU"), "gen2.zip"), + "Gen 1": load_recording(make_recording(simple_frames(2.0), gpu="Gen 1 GPU"), "gen1.evolve"), + "Gen 2": load_recording(make_recording(simple_frames(1.0), gpu="Gen 2 GPU"), "gen2.evolve"), } @@ -42,13 +42,13 @@ def test_passes_of_a_workload_category(results): assert passes.at[("Gen 1", "shadow_filter_pass_0"), "workload"] == "Compute" -def test_results_without_deep_analysis_are_left_out_of_pass_views(): +def test_results_without_pass_timings_are_left_out_of_pass_views(): results = { - "Full": load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "a.zip"), - "Scores only": load_export(make_export(None, gpu="Gen 2 GPU"), "b.zip"), + "Full": load_recording(make_recording(simple_frames(2.0), gpu="Gen 1 GPU"), "a.evolve"), + "No passes": load_recording(make_recording(simple_frames(1.0), gpu="Gen 2 GPU", with_scopes=False), "b.evolve"), } assert set(stats.summary(results, 1.0, "pass")["result"]) == {"Full"} - assert set(stats.summary(results, 1.0, "workload")["result"]) == {"Full", "Scores only"} + assert set(stats.summary(results, 1.0, "workload")["result"]) == {"Full", "No passes"} def test_passes_missing_in_some_frames_count_as_zero(results): @@ -62,8 +62,9 @@ def test_passes_missing_in_some_frames_count_as_zero(results): def test_scores_table(results): table = stats.scores_table(results) - assert table.index.tolist() == ["Ray Tracing", "Rasterization", "Compute"] - assert table.at["Compute", "Gen 2"] == 6000 + assert table.index.tolist() == ["Ray Tracing", "Compute", "Driver", "Energy Consumption"] + assert table.at["Compute", "Gen 2"] == table.at["Compute", "Gen 1"] + assert table.at["Ray Tracing", "Gen 2"] > table.at["Ray Tracing", "Gen 1"] def test_comparison_table_and_key_numbers(results): @@ -77,10 +78,10 @@ def test_comparison_table_and_key_numbers(results): ) -def test_timeline_averages_loops_and_smooths(): - result = load_export(make_export([simple_frames(2.0), simple_frames(4.0)]), "x.zip") +def test_timeline_smooths(): + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") raw = stats.timeline(result, 1.0, "pass", ["shadow-raytrace-inline"], 0.0) - assert raw["shadow-raytrace-inline"].tolist() == pytest.approx([3.0] * len(raw)) + assert raw["shadow-raytrace-inline"].tolist() == pytest.approx([2.0] * len(raw)) smooth = stats.timeline(result, 1.0, "pass", ["shadow-raytrace-inline"], 2.0) assert len(smooth) == len(raw) assert len(stats.timeline(result, 1.0, "workload", ["Raytracing"], 0.0)) > 0 diff --git a/tests/test_ui.py b/tests/test_ui.py index 6a93aca..fd188e6 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -1,6 +1,6 @@ from pathlib import Path -from conftest import make_export +from conftest import make_recording, simple_frames from streamlit.testing.v1 import AppTest ENTRY = str(Path(__file__).resolve().parent.parent / "evolve-charts.py") @@ -20,12 +20,12 @@ def run(monkeypatch, folder) -> AppTest: - monkeypatch.setenv("EVOLVE_CHARTS_EXPORTS", str(folder)) + monkeypatch.setenv("EVOLVE_CHARTS_RESULTS", str(folder)) return AppTest.from_file(ENTRY, default_timeout=60).run() def page(monkeypatch, folder, name: str) -> AppTest: - monkeypatch.setenv("EVOLVE_CHARTS_EXPORTS", str(folder)) + monkeypatch.setenv("EVOLVE_CHARTS_RESULTS", str(folder)) app = AppTest.from_string(PAGE_SCRIPT.format(page=name), default_timeout=60).run() assert not app.exception, name return app @@ -55,16 +55,15 @@ def test_every_page_renders_with_two_results(two_gpus, monkeypatch): assert any("Compute: 0.50 ms per frame on Gen 1 GPU" in markdown.value for markdown in app.markdown) -def test_results_without_deep_analysis(tmp_path, monkeypatch): - (tmp_path / "scores_only.zip").write_bytes(make_export(None, gpu="Gen 1 GPU")) - (tmp_path / "scores.csv").write_text("Loop,Ray Tracing,Compute\n1,4000,6000\n") +def test_results_without_pass_timings(tmp_path, monkeypatch): + (tmp_path / "no_passes.evolve").write_bytes(make_recording(simple_frames(2.0), with_scopes=False)) app = run(monkeypatch, tmp_path) assert not app.exception visit_every_page(monkeypatch, tmp_path) def test_bad_file_shows_error(tmp_path, monkeypatch): - (tmp_path / "broken.zip").write_bytes(b"not a zip") + (tmp_path / "broken.evolve").write_bytes(b"not a zip") app = run(monkeypatch, tmp_path) assert not app.exception - assert "not a zip" in app.error[0].value + assert "isn't an Evolve run file" in app.error[0].value diff --git a/tools/package.py b/tools/package.py index 1ba8e37..4f7a9b0 100644 --- a/tools/package.py +++ b/tools/package.py @@ -24,7 +24,7 @@ "evolve-charts.py", "evolve-charts.py.lock", ".streamlit/config.toml", - "exports/README.txt", + "results/README.txt", ] FOLDERS = ["evolve_charts", "static"] EXECUTABLE = {"Start Evolve Charts.command", "start-evolve-charts.sh"} @@ -42,7 +42,7 @@ The first start downloads Python and the libraries the tool uses (about 150 MB) and needs an internet connection. Keep the window that opens running while you use the tool. -To load results automatically, put Evolve export files in the "exports" folder. +To load results automatically, put .evolve files in the "results" folder. """ diff --git a/tools/sync_categories.py b/tools/sync_categories.py index 064473c..8bb179c 100644 --- a/tools/sync_categories.py +++ b/tools/sync_categories.py @@ -18,7 +18,7 @@ DEFAULT_SOURCE = REPO_ROOT.parent / "evolve/crates/evolve-frame-bucketing/src/lib.rs" OUTPUT = REPO_ROOT / "evolve_charts/scope_catalog.py" -ARM = re.compile(r"Self::(\w+) => vec!\[(.*?)\n\s*\],", re.DOTALL) +ARM = re.compile(r"Self::(\w+) => vec!\[(.*?)\],", re.DOTALL) NAME = re.compile(r'Self::(\w+) => "([^"]*)",') TOKEN = re.compile(r'^\s*//.*$|get_or_intern_static\(\s*"(?P[^"]+)"\s*\)', re.MULTILINE) @@ -28,8 +28,14 @@ def _impl(source: str, category: str) -> str: return source[start : source.index("fn translated(self)", start)] +def _deprecated(source: str) -> str: + """Pass names Evolve renamed or removed. Evolve still counts them, so older exports stay comparable.""" + start = source.index("fn pipelines_list_deprecated") + return source[start : source.index("impl Category for WorkloadCategory", start)] + + def _lists(body: str) -> dict[str, list[str]]: - lists = body[: body.index("fn name(self)")] + lists = body[: body.index("fn name(self)")] if "fn name(self)" in body else body return { arm.group(1): [token.group("name") for token in TOKEN.finditer(arm.group(2)) if token.group("name")] for arm in ARM.finditer(lists) @@ -44,10 +50,11 @@ def _names(body: str) -> dict[str, str]: def parse(source: str) -> tuple[dict, dict, list[str]]: workloads_body = _impl(source, "WorkloadCategory") workloads: dict[str, list[str]] = {} - for workload, scopes in _lists(workloads_body).items(): - for scope in scopes: - if workload not in workloads.setdefault(scope, []): - workloads[scope].append(workload) + for lists in [_lists(workloads_body), _lists(_deprecated(source))]: + for workload, scopes in lists.items(): + for scope in scopes: + if workload not in workloads.setdefault(scope, []): + workloads[scope].append(workload) uncategorized = _lists(_impl(source, "BucketPassCategory")).get("Uncategorized", []) return workloads, _names(workloads_body), uncategorized @@ -77,7 +84,7 @@ def main() -> None: "}", "", ] - OUTPUT.write_text("\n".join(lines), encoding="utf-8") + OUTPUT.write_text("\n".join(lines), encoding="utf-8", newline="\n") print(f"Wrote {len(workloads)} scope names to {OUTPUT.relative_to(REPO_ROOT)}") diff --git a/tools/sync_scores.py b/tools/sync_scores.py new file mode 100644 index 0000000..3c9b05f --- /dev/null +++ b/tools/sync_scores.py @@ -0,0 +1,131 @@ +"""Regenerate evolve_charts/score_catalog.py from Evolve's score calculation source. + +Usage (from the repository root): + + python tools/sync_scores.py [path/to/evolve] + +`.evolve` files hold the measurements, not the scores: Evolve calculates the scores whenever it reads a +run. This copies what that calculation needs from the Rust source: the calibrated multiplier of every +score per benchmark and platform (`crates/evolve-highscore-api/src/score.rs`), the score names, and which +workload category each GPU score times (`crates/evolve-datakit-scores/src/lib.rs`). +""" + +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_EVOLVE = REPO_ROOT.parent / "evolve" +SCORE_RS = "crates/evolve-highscore-api/src/score.rs" +DATAKIT_SCORES_RS = "crates/evolve-datakit-scores/src/lib.rs" +OUTPUT = REPO_ROOT / "evolve_charts/score_catalog.py" + +COMMENT = re.compile(r"//[^\n]*") +VARIANT_ARM = re.compile(r"((?:BenchmarkVariant::\w+\s*\|?\s*)+)=>") +SCORE_ARM = re.compile(r"((?:Self::\w+\s*\|?\s*)+)=>\s*(return None|[0-9_.]+)") +SOME_ARM = re.compile(r"((?:BenchmarkVariant::\w+\s*\|?\s*)+)=>\s*Some\(([0-9_.]+)\)") +F32_CONST = re.compile(r"const (\w+): f32 = ([0-9_.]+);") + + +def _function(source: str, name: str) -> str: + start = source.index(f"fn {name}(") + following = re.compile(r"\n\s*(?:pub(?:\([\w:]+\))? )?(?:const )?fn ").search(source, start + 1) + return COMMENT.sub("", source[start : following.start() if following else len(source)]) + + +def _number(text: str) -> float: + return float(text.replace("_", "")) + + +def _variants(text: str) -> list[str]: + return re.findall(r"BenchmarkVariant::(\w+)", text) + + +def _multipliers(source: str, function: str) -> dict[str, dict[str, float]]: + """Benchmark variant -> score -> multiplier; a score the variant doesn't have is absent.""" + body = _function(source, function) + arms = list(VARIANT_ARM.finditer(body)) + table: dict[str, dict[str, float]] = {} + for arm, following in zip(arms, [*arms[1:], None]): + text = body[arm.end() : following.start() if following else len(body)] + scores = { + score: _number(value) + for names, value in SCORE_ARM.findall(text) + if value != "return None" + for score in re.findall(r"Self::(\w+)", names) + } + for variant in _variants(arm.group(1)): + table[variant] = scores + if not table: + raise SystemExit(f"Found no multipliers in `{function}`; has {SCORE_RS} changed shape?") + return table + + +def parse(score_rs: str, datakit_scores_rs: str) -> dict: + platforms = { + "Desktop": _multipliers(score_rs, "desktop_score_multipliers"), + "Phone": _multipliers(score_rs, "mobile_score_multipliers"), + } + handheld = _function(score_rs, "handheld_score_multipliers") + if "self.desktop_score_multipliers(variant)" not in handheld: + raise SystemExit("Handheld multipliers are no longer the desktop ones; teach this tool to read them.") + platforms["Handheld"] = platforms["Desktop"] + + enum = score_rs[score_rs.index("pub enum Score {") :] + order = re.findall(r"^\s*(\w+),$", enum[: enum.index("}")], re.MULTILINE) + name_body = _function(score_rs, "name") + names = dict(re.findall(r'Self::(\w+) => "([^"]+)"', name_body)) + + version = re.search(r"DRIVER_EXCLUDES_PRESENT: EvolveVersion = EvolveVersion::new\((\d+), (\d+)\)", score_rs) + phone_driver = { + variant: _number(value) + for variants, value in SOME_ARM.findall(_function(score_rs, "mobile_driver_multiplier_with_present")) + for variant in _variants(variants) + } + energy = re.search(r"\(([0-9_.]+) \* \(-m as f32 \* avg\)\.exp\(\)\)", score_rs) + gpu_scores = re.findall(r"\(\s*WorkloadCategory::(\w+),\s*Score::(\w+),?\s*\)", datakit_scores_rs) + if not (version and energy and gpu_scores and phone_driver): + raise SystemExit("Couldn't find every score constant; has the Rust source changed shape?") + + return { + "SCORE_ORDER": order, + "SCORE_NAMES": {score: names[score] for score in order}, + "GPU_SCORES": dict(gpu_scores), + "MULTIPLIERS": platforms, + "DRIVER_EXCLUDES_PRESENT": (int(version.group(1)), int(version.group(2))), + "PHONE_DRIVER_WITH_PRESENT": phone_driver, + "ENERGY_SCALE": _number(energy.group(1)), + "IMAGE_QUALITY": {name: _number(value) for name, value in F32_CONST.findall(score_rs)}, + } + + +COMMENTS = { + "SCORE_ORDER": "Score variants, in the order Evolve lists them", + "SCORE_NAMES": "Score variant -> the name Evolve shows", + "GPU_SCORES": "WorkloadCategory variant -> the Score of its GPU time", + "MULTIPLIERS": "Platform -> BenchmarkVariant -> Score -> multiplier; a score a benchmark doesn't have is absent", + "DRIVER_EXCLUDES_PRESENT": "Phone Driver scores of runs before this Evolve version use PHONE_DRIVER_WITH_PRESENT", + "PHONE_DRIVER_WITH_PRESENT": "BenchmarkVariant -> phone Driver multiplier from before DRIVER_EXCLUDES_PRESENT", + "ENERGY_SCALE": "Energy score = ENERGY_SCALE * exp(-multiplier * average power in W)", + "IMAGE_QUALITY": "FLIP and SSIM pooling and normalization constants", +} + + +def main() -> None: + evolve = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_EVOLVE + catalog = parse( + (evolve / SCORE_RS).read_text(encoding="utf-8"), (evolve / DATAKIT_SCORES_RS).read_text(encoding="utf-8") + ) + lines = [ + "# Generated by tools/sync_scores.py from evolve/crates/evolve-highscore-api/src/score.rs", + "# and evolve/crates/evolve-datakit-scores/src/lib.rs. Do not edit by hand.", + "", + ] + for name, value in catalog.items(): + lines += [f"# {COMMENTS[name]}", f"{name} = {value!r}", ""] + OUTPUT.write_text("\n".join(lines), encoding="utf-8", newline="\n") + print(f"Wrote {sum(len(v) for v in catalog['MULTIPLIERS'].values())} benchmark multiplier sets to {OUTPUT.name}") + + +if __name__ == "__main__": + main() From aa150cce613def07be5f7b036542e1a6f36f322b Mon Sep 17 00:00:00 2001 From: u64max Date: Mon, 9 Nov 2026 10:36:11 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A7=B9=20Drop=20what=20only=20reading?= =?UTF-8?q?=20exports=20needed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One .evolve file is one run with every table, so the loop column, the optional per-frame tables and the Per frame card entry go. The Android guide and script now pull the .evolve files instead of CLI exports. Co-Authored-By: Claude Opus 5.5 --- docs/Android.md | 28 ++++++++++++---------------- evolve_charts/loader.py | 27 ++++++++++----------------- evolve_charts/stats.py | 19 ++++++++----------- evolve_charts/ui.py | 22 ++++++++-------------- scripts/run_on_android.py | 7 +++---- tests/test_loader.py | 4 ++-- tools/sync_categories.py | 2 +- 7 files changed, 44 insertions(+), 65 deletions(-) diff --git a/docs/Android.md b/docs/Android.md index 312d261..bc04d17 100644 --- a/docs/Android.md +++ b/docs/Android.md @@ -1,21 +1,17 @@ -## Automating `evolve` benchmark run on Android +## Automating `evolve` benchmark runs on Android 1. Make sure the app is installed. -2. Use relative paths in `--export*` parameters, so that `evolve` has access to write them relative to `/sdcard/Android/data/nl.traverse_research.evolve/files/`. Subfolders are supported, these will automatically created. - - It is recommended to use subfolders with the current date and time, as `evolve` will refuse to start and overwrite any export files that aleady exist. -3. Use the `python ./scripts/run_on_android.py` script in this folder to start Evolve, like a desktop run. Common parameters to export all data from a `run-custom` are: +2. Use the `python ./scripts/run_on_android.py` script in this folder to start Evolve with command-line + arguments, like a desktop run: ```sh - ts=$(date '+%F-%T') - python ./scripts/run_on_android.py run-custom --export-scores $ts/scores.csv --export-per-frame $ts/frames.csv --export-deep-analysis $ts/deep-analysis.json + python ./scripts/run_on_android.py run-custom --benchmark ray-tracing-inline ``` -4. When the benchmark finishes, run the following commands to pull the resulting files to your desktop: - ```sh - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts/deep-analysis.json - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts/scores.csv - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts/frames.csv - ``` - Alternatively, pull the entire timestamped folder: - ```sh - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts +3. Evolve saves every run as a `.evolve` file in + `/sdcard/Android/data/nl.traverse_research.evolve/files/Documents/results/`. When the benchmark finishes, + pull them to your desktop: + ```sh + adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/Documents/results evolve-results ``` - These paths can also be found directly on the phone by navigating its internal storage using MTP (when `USB for file transfer` is enabled via the notification on the phone). + The folder can also be found on the phone by browsing its internal storage over MTP (when + `USB for file transfer` is enabled via the notification on the phone). +4. Load the `.evolve` files into Evolve Charts. diff --git a/evolve_charts/loader.py b/evolve_charts/loader.py index d6c5537..f0443b3 100644 --- a/evolve_charts/loader.py +++ b/evolve_charts/loader.py @@ -71,12 +71,12 @@ class Result: metadata: dict[str, str] = field(default_factory=dict) # Score name -> score. Scores the benchmark doesn't have are absent. scores: dict[str, float] = field(default_factory=dict) - # One row per (loop, frame): GPU milliseconds per workload category, one column each. - workloads: pd.DataFrame | None = None - # One row per (loop, frame): sensor readings, and the GPU time sum and span of the passes. - frames: pd.DataFrame | None = None - # One row per (loop, frame, pass): ms is the pass' own GPU time in that frame; top_ms only counts - # instances not nested inside another scope, so summing it never double-counts. + # One row per frame: GPU milliseconds per workload category, one column each. + workloads: pd.DataFrame = field(default_factory=pd.DataFrame) + # One row per frame: sensor readings, and the GPU time sum and span of the passes. + frames: pd.DataFrame = field(default_factory=pd.DataFrame) + # One row per (frame, pass): ms is the pass' own GPU time in that frame; top_ms only counts instances not + # nested inside another scope, so summing it never double-counts. None when the run has no pass timings. passes: pd.DataFrame | None = None # Total GPU milliseconds per pass that Evolve doesn't assign to a workload category. left_out: dict[str, float] = field(default_factory=dict) @@ -90,13 +90,7 @@ def gpu(self) -> str: def contents(self) -> list[str]: """What this result contains, in the words the tool uses.""" return [ - name - for name, present in [ - ("Scores", bool(self.scores)), - ("Per frame", self.workloads is not None), - ("Passes", self.passes is not None), - ] - if present + name for name, present in [("Scores", bool(self.scores)), ("Passes", self.passes is not None)] if present ] @@ -209,7 +203,7 @@ def _os_version(version: dict | str | None) -> str: def _base(frames: pd.DataFrame) -> pd.DataFrame: - return pd.DataFrame({"loop": 1, "frame": frames["frame"].astype(int), "seq_s": frames["simulation_time_ns"] / 1e9}) + return pd.DataFrame({"frame": frames["frame"].astype(int), "seq_s": frames["simulation_time_ns"] / 1e9}) def _workloads(frames: pd.DataFrame) -> pd.DataFrame: @@ -273,6 +267,5 @@ def _passes(gpu_scopes: pd.DataFrame, frames: pd.DataFrame) -> tuple[pd.DataFram passes = kept.groupby(["frame", "pass"], as_index=False, sort=True)[["ms", "top_ms"]].sum() totals["gpu_sum_ms"] = passes.groupby("frame")["top_ms"].sum() seq_s = frames.set_index("frame")["simulation_time_ns"] / 1e9 - passes.insert(0, "loop", 1) - passes.insert(2, "seq_s", passes["frame"].map(seq_s)) - return passes[["loop", "frame", "seq_s", "pass", "ms", "top_ms"]], totals, left_out + passes.insert(1, "seq_s", passes["frame"].map(seq_s)) + return passes[["frame", "seq_s", "pass", "ms", "top_ms"]], totals, left_out diff --git a/evolve_charts/stats.py b/evolve_charts/stats.py index 368424f..49ef37e 100644 --- a/evolve_charts/stats.py +++ b/evolve_charts/stats.py @@ -32,7 +32,7 @@ def has_level(result: Result, level: Level) -> bool: if level == "workload": - return result.workloads is not None and len(result.workloads.columns) > 3 + return any(name in result.workloads for name in WORKLOAD_ORDER) return result.passes is not None @@ -47,14 +47,14 @@ def per_frame(result: Result, skip_s: float, level: Level, within: Within = None """GPU milliseconds per frame, one column per item; frames where an item didn't run count as 0.""" if level == "workload": table = result.workloads[result.workloads["seq_s"] >= skip_s] - return table.set_index(["loop", "frame"]).drop(columns="seq_s") + return table.set_index("frame").drop(columns="seq_s") passes = result.passes[result.passes["seq_s"] >= skip_s] if within is not None: _, name = within passes = passes[passes["pass"].map(lambda scope: name in workloads(scope))] - wide = passes.pivot_table(index=["loop", "frame"], columns="pass", values="ms", aggfunc="sum") - frame_index = trimmed_frames(result, skip_s).set_index(["loop", "frame"]).index + wide = passes.pivot_table(index="frame", columns="pass", values="ms", aggfunc="sum") + frame_index = trimmed_frames(result, skip_s).set_index("frame").index return wide.reindex(frame_index).fillna(0.0) @@ -126,24 +126,21 @@ def frame_totals(results: dict[str, Result], skip_s: float) -> pd.DataFrame: def timeline( result: Result, skip_s: float, level: Level, items: list[str], smooth_s: float, within: Within = None ) -> pd.DataFrame: - """Per-frame milliseconds over benchmark time, averaged over loops by frame index.""" + """Per-frame milliseconds over benchmark time.""" wide = per_frame(result, skip_s, level, within).reindex(columns=items, fill_value=0.0) - source = result.workloads if level == "workload" else trimmed_frames(result, skip_s) - wide["seq_s"] = source.set_index(["loop", "frame"])["seq_s"] - return _smooth(wide.groupby(level="frame").mean().set_index("seq_s"), smooth_s) + wide["seq_s"] = result.workloads.set_index("frame")["seq_s"] + return _smooth(wide.set_index("seq_s"), smooth_s) def telemetry_timeline(result: Result, skip_s: float, smooth_s: float) -> pd.DataFrame: frames = trimmed_frames(result, skip_s) columns = [column for column in TELEMETRY if frames[column].notna().any()] - return _smooth(frames.groupby("frame")[["seq_s", *columns]].mean().set_index("seq_s"), smooth_s) + return _smooth(frames.set_index("seq_s")[columns], smooth_s) def telemetry_summary(results: dict[str, Result], skip_s: float) -> pd.DataFrame: rows = [] for label, result in results.items(): - if result.frames is None: - continue frames = trimmed_frames(result, skip_s) row = {"result": label} for column, (name, unit) in TELEMETRY.items(): diff --git a/evolve_charts/ui.py b/evolve_charts/ui.py index ec4bcb7..e7c3d14 100644 --- a/evolve_charts/ui.py +++ b/evolve_charts/ui.py @@ -98,7 +98,7 @@ HOW_IT_WORKS = f""" **Per frame.** A pass' time is the sum of all its executions in a frame; frames where it didn't run count as -0. Averages cover every frame of every loop; the first {WARM_UP_S:g} s are skipped because they contain +0. Averages cover every frame of the run; the first {WARM_UP_S:g} s are skipped because they contain one-off setup work. Lines over time are smoothed over {SMOOTHING_S:g} s. **Left out.** Passes Evolve doesn't assign to a workload category (copies, clears, the user interface, …) @@ -409,14 +409,10 @@ def _result_card(result: Result, label: str, color: str) -> None: " Β· ".join(filter(None, [meta.get("Rendering Backend"), version])), " Β· ".join(filter(None, [driver, os_name])), ] - per_frame = result.frames if result.frames is not None else result.workloads - if per_frame is not None: - frames = per_frame[per_frame["seq_s"] >= WARM_UP_S] - duration = frames["seq_s"].max() - frames["seq_s"].min() if len(frames) else 0 - lines.append(f"{len(frames):,} frames Β· {duration // 60:.0f}:{duration % 60:02.0f} min") - contents = "
".join( - f"{'βœ“' if name in result.contents else 'βœ—'} {name}" for name in ["Scores", "Per frame", "Passes"] - ) + frames = result.frames[result.frames["seq_s"] >= WARM_UP_S] + duration = frames["seq_s"].max() - frames["seq_s"].min() if len(frames) else 0 + lines.append(f"{len(frames):,} frames Β· {duration // 60:.0f}:{duration % 60:02.0f} min") + contents = "
".join(f"{'βœ“' if name in result.contents else 'βœ—'} {name}" for name in ["Scores", "Passes"]) body = "
".join(line for line in lines if line) st.markdown(f'
{body}
{contents}
', unsafe_allow_html=True) st.caption(result.filename) @@ -649,9 +645,7 @@ def _sensors_page() -> None: context = _context() _title("GPU sensors", "Clocks, power and temperature during the run. Falling clocks can mean throttling.") timelines = { - label: stats.telemetry_timeline(result, WARM_UP_S, SMOOTHING_S) - for label, result in context.results.items() - if result.frames is not None + label: stats.telemetry_timeline(result, WARM_UP_S, SMOOTHING_S) for label, result in context.results.items() } if _missing({label for label, timeline in timelines.items() if len(timeline.columns)}, "GPU sensor data"): return @@ -758,9 +752,9 @@ def _per_frame_long(results: dict[str, Result], skip_s: float) -> pd.DataFrame: passes = result.passes[result.passes["seq_s"] >= skip_s] frames.append( passes.assign(result=label, workload=passes["pass"].map(lambda scope: ", ".join(workloads(scope))))[ - ["result", "loop", "frame", "seq_s", "workload", "pass", "ms"] + ["result", "frame", "seq_s", "workload", "pass", "ms"] ] ) if not frames: - return pd.DataFrame(columns=["result", "loop", "frame", "benchmark_time_s", "workload", "pass", "ms"]) + return pd.DataFrame(columns=["result", "frame", "benchmark_time_s", "workload", "pass", "ms"]) return pd.concat(frames).rename(columns={"seq_s": "benchmark_time_s"}) diff --git a/scripts/run_on_android.py b/scripts/run_on_android.py index 75ee6cc..260a3d2 100644 --- a/scripts/run_on_android.py +++ b/scripts/run_on_android.py @@ -10,7 +10,7 @@ def run_command(command): def main(): id = "nl.traverse_research.evolve" - path = "/sdcard/Android/data/nl.traverse_research.evolve/files" + results = "/sdcard/Android/data/nl.traverse_research.evolve/files/Documents/results" # Get Evolve UID packages = run_command(["adb", "shell", "pm", "list", "packages", "-U", id]) @@ -38,9 +38,8 @@ def main(): sys.exit(1) moment = datetime.now().strftime("%Y%m%d_%H%M%S") - pull_suggestion = f"evolve-results-{moment}" - - print(f"Started Evolve on Android. When the benchmark finishes, manually pull the specified files from {path} using adb pull or MTP.") + print("Started Evolve on Android. When the benchmark finishes, pull its .evolve files with:") + print(f" adb pull {results} evolve-results-{moment}") print("\nStarting logcat in 4 seconds") time.sleep(4) diff --git a/tests/test_loader.py b/tests/test_loader.py index d6d691b..f1192cd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -23,7 +23,7 @@ def test_full_recording(): "GPU Driver Version": "1.2.3", "Rendering Backend": "Dx12", } - assert result.contents == ["Scores", "Per frame", "Passes"] + assert result.contents == ["Scores", "Passes"] assert set(result.passes["pass"]) == {"shadow-raytrace-inline", "shadow_filter_pass_0"} assert result.warnings == [] @@ -39,7 +39,7 @@ def test_per_frame_workloads_come_from_the_frames_table(): def test_recording_without_pass_timings(): result = load_recording(make_recording(simple_frames(2.0), with_scopes=False), "x.evolve") - assert result.contents == ["Scores", "Per frame"] + assert result.contents == ["Scores"] assert result.passes is None assert result.frames["gpu_clock_mhz"].iloc[2] == 2502 assert result.frames["board_power_w"].iloc[2] == 200.0 diff --git a/tools/sync_categories.py b/tools/sync_categories.py index 8bb179c..e5cdb31 100644 --- a/tools/sync_categories.py +++ b/tools/sync_categories.py @@ -29,7 +29,7 @@ def _impl(source: str, category: str) -> str: def _deprecated(source: str) -> str: - """Pass names Evolve renamed or removed. Evolve still counts them, so older exports stay comparable.""" + """Pass names Evolve renamed or removed. Evolve still counts them, so older runs stay comparable.""" start = source.index("fn pipelines_list_deprecated") return source[start : source.index("impl Category for WorkloadCategory", start)]