diff --git a/.gitignore b/.gitignore index e8d6f5c..d44216a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ __pycache__/ .venv/ .pytest_cache/ -exports/* -!exports/README.txt +results/* +!results/README.txt dist/ VERSION requirements.txt diff --git a/README.md b/README.md index 08c2e8a..9e8cf45 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # πŸ“Š Evolve Charts -**Turn Evolve deep analysis exports into publication-ready charts** +**Turn Evolve benchmark runs into publication-ready charts** [![Banner](./docs/images/banner.png)](https://traverseresearch.nl) @@ -29,29 +29,28 @@ your web browser, and **everything stays on your computer**. Nothing is uploaded the libraries the tool uses (about 150 MB). This needs an internet connection, takes a few minutes and only happens once. Keep the window that opens running while you use the tool; close it when you're done. -3. **Export your results from Evolve.** After a benchmark run, click **Export** on the score screen. Do - this on every GPU or setting you want to compare, using the same benchmark on each. Scores and - per-frame data are always included; deep analysis (per-pass timings) needs an Evolve Professional - licence. -4. **Add your results.** Your browser opens Evolve Charts. Drag the export files onto *Add Evolve - exports*, or put them in the `exports` folder so they load automatically every time. +3. **Run your benchmarks in Evolve.** Evolve saves every run as a `.evolve` file, in + `Documents\Evolve\results` on Windows. Run on every GPU or setting you want to compare, using the + same benchmark on each. +4. **Add your results.** Your browser opens Evolve Charts. Drag the `.evolve` files onto *Add Evolve + runs*, or put them in the `results` folder so they load automatically every time. 5. **Name your results** on their cards (for example "RTX 4070 Ti" or "Gen 1"). These names appear in every chart. -To update, download the zip again and replace the old folder (keep your `exports` folder if you use it). +To update, download the zip again and replace the old folder (keep your `results` folder if you use it). ## πŸ—ΊοΈ What's in the tool -The tool reads every Evolve export: the **Export** zip from the score screen, and the files Evolve's -command line writes (`evolve_results.csv`, `scores.csv`, the per-frame CSV and `deep_analysis.json`). -It has a page per question, in the bar at the top: +The tool reads the `.evolve` file Evolve saves for every run: the measurements of every frame and pass, +the GPU sensor readings and the system information. It calculates the scores from those the way Evolve +does. It has a page per question, in the bar at the top: | Page | What it shows | Needs | |---|---|---| -| **Results** | Load and name exports; GPU, driver, OS, Evolve version and what each file contains | | +| **Results** | Load and name runs; GPU, driver, OS, Evolve version, benchmark and what each file contains | | | **Scores** | Evolve's scores side by side, with the workload category each one scores, and the differences | scores | -| **GPU time** | Where the GPU spends its time each frame, per workload category; then one workload category over the run and the passes it's made of | per-frame data; the passes need deep analysis | -| **Passes** | Every render pass in one searchable table; tick passes to plot them over the run | deep analysis | +| **GPU time** | Where the GPU spends its time each frame, per workload category; then one workload category over the run and the passes it's made of | per-frame data; the passes need pass timings | +| **Passes** | Every render pass in one searchable table; tick passes to plot them over the run | pass timings | | **GPU sensors** | Clocks, power, temperature and more during the run, to spot throttling | per-frame data | | **Export** | CSV files for Excel, Google Sheets, Datawrapper and Flourish | | @@ -78,13 +77,13 @@ drivers and Evolve version. **Help** explains how the numbers are made. because a GPU can idle between passes or run passes at the same time. - **Whiskers** on bar charts show the range covering 90% of frames (5th to 95th percentile). - Only compare results from the **same Evolve version and benchmark settings**. The tool warns you - when the Evolve versions or graphics APIs differ. + when the benchmarks, Evolve versions or graphics APIs differ. ## ❓ Troubleshooting -- **A result is missing from *Passes***: its export has no deep analysis, which - needs an Evolve Professional licence. Its card shows βœ— Deep analysis; scores and workload - categories still work. +- **A result is missing from *Passes***: its run has no pass timings. Its card shows βœ— Passes; scores + and workload categories still work. +- **"was saved by a newer Evolve"**: download the latest Evolve Charts. - **The browser doesn't open**: open yourself while the black window is open. - **Anything else**: send a screenshot of the black window to Traverse Research. @@ -93,16 +92,18 @@ drivers and Evolve version. **Help** explains how the numbers are made. - Run without the launcher: `uv run evolve-charts.py`. Dependencies are declared inline ([PEP 723](https://peps.python.org/pep-0723/)), so no virtual environment is needed. - Code lives in `evolve_charts/`: - - `loader.py` reads export zips into tidy tables. + - `loader.py` reads `.evolve` files (a zip of Parquet tables and a JSON document) into tidy tables. + - `scores.py` calculates Evolve's scores from those tables. - `categories.py` describes Evolve's workload categories and which pass belongs to which. - `stats.py` computes the summaries. - `charts.py` builds the Plotly figures. - `ui.py` is the Streamlit page. - The look follows Evolve's `crates/evolve/src/ui/styling.rs`: theme colours and the Spartan font are set in `.streamlit/config.toml`, and the fonts, logo and icon live in `static/`. Chart colours are in `charts.py`. -- Which pass belongs to which workload category comes from Evolve's Rust source. After pass names - change in Evolve, regenerate the list with `python tools/sync_categories.py` (it reads - `../evolve/crates/evolve-frame-bucketing/src/lib.rs` by default). +- Which pass belongs to which workload category, and the score multipliers, come from Evolve's Rust + source. After pass names or score calibrations change in Evolve, regenerate them with + `python tools/sync_categories.py` and `python tools/sync_scores.py` (both read `../evolve` by default), + then `uvx ruff format evolve_charts`. - Dependencies are locked in `evolve-charts.py.lock`; refresh it with `uv lock --script evolve-charts.py --upgrade`. - Tests: `uv export --script evolve-charts.py --no-hashes > requirements.txt`, then `uv venv && uv pip install -r requirements.txt pytest && uv run --no-project pytest`. CI runs the same. diff --git a/docs/Android.md b/docs/Android.md index 312d261..bc04d17 100644 --- a/docs/Android.md +++ b/docs/Android.md @@ -1,21 +1,17 @@ -## Automating `evolve` benchmark run on Android +## Automating `evolve` benchmark runs on Android 1. Make sure the app is installed. -2. Use relative paths in `--export*` parameters, so that `evolve` has access to write them relative to `/sdcard/Android/data/nl.traverse_research.evolve/files/`. Subfolders are supported, these will automatically created. - - It is recommended to use subfolders with the current date and time, as `evolve` will refuse to start and overwrite any export files that aleady exist. -3. Use the `python ./scripts/run_on_android.py` script in this folder to start Evolve, like a desktop run. Common parameters to export all data from a `run-custom` are: +2. Use the `python ./scripts/run_on_android.py` script in this folder to start Evolve with command-line + arguments, like a desktop run: ```sh - ts=$(date '+%F-%T') - python ./scripts/run_on_android.py run-custom --export-scores $ts/scores.csv --export-per-frame $ts/frames.csv --export-deep-analysis $ts/deep-analysis.json + python ./scripts/run_on_android.py run-custom --benchmark ray-tracing-inline ``` -4. When the benchmark finishes, run the following commands to pull the resulting files to your desktop: - ```sh - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts/deep-analysis.json - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts/scores.csv - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts/frames.csv - ``` - Alternatively, pull the entire timestamped folder: - ```sh - adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/$ts +3. Evolve saves every run as a `.evolve` file in + `/sdcard/Android/data/nl.traverse_research.evolve/files/Documents/results/`. When the benchmark finishes, + pull them to your desktop: + ```sh + adb pull /sdcard/Android/data/nl.traverse_research.evolve/files/Documents/results evolve-results ``` - These paths can also be found directly on the phone by navigating its internal storage using MTP (when `USB for file transfer` is enabled via the notification on the phone). + The folder can also be found on the phone by browsing its internal storage over MTP (when + `USB for file transfer` is enabled via the notification on the phone). +4. Load the `.evolve` files into Evolve Charts. diff --git a/evolve-charts.py b/evolve-charts.py index 8c79bee..b77f5b1 100644 --- a/evolve-charts.py +++ b/evolve-charts.py @@ -4,9 +4,10 @@ # "streamlit>=1.65", # "plotly>=6", # "pandas>=2.2", +# "pyarrow>=18", # ] # /// -"""Evolve Charts: turn Evolve deep analysis exports into charts. +"""Evolve Charts: turn Evolve benchmark runs (.evolve files) into charts. Start it by double-clicking `Start Evolve Charts.bat` (Windows) or `start-evolve-charts.sh` (macOS/Linux), or run `uv run evolve-charts.py`. A page opens in your browser; nothing leaves diff --git a/evolve-charts.py.lock b/evolve-charts.py.lock index d0e408f..cd2ffb0 100644 --- a/evolve-charts.py.lock +++ b/evolve-charts.py.lock @@ -6,10 +6,10 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version < '3.12' and sys_platform == 'win32'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version < '3.12' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] @@ -17,6 +17,7 @@ resolution-markers = [ requirements = [ { name = "pandas", specifier = ">=2.2" }, { name = "plotly", specifier = ">=6" }, + { name = "pyarrow", specifier = ">=18" }, { name = "streamlit", specifier = ">=1.65" }, ] diff --git a/evolve_charts/categories.py b/evolve_charts/categories.py index a5a624b..3b23ba2 100644 --- a/evolve_charts/categories.py +++ b/evolve_charts/categories.py @@ -23,6 +23,11 @@ WORKLOAD_ORDER = list(WORKLOAD_DESCRIPTIONS) +def workload_variants(scope: str) -> list[str]: + """The `WorkloadCategory` variants the pass counts towards, as named in Evolve's source.""" + return WORKLOADS.get(scope, []) + + def workloads(scope: str) -> list[str]: """The workload categories the pass is part of, in Evolve's order; empty if Evolve leaves it uncategorized. diff --git a/evolve_charts/loader.py b/evolve_charts/loader.py index ade7bc6..f0443b3 100644 --- a/evolve_charts/loader.py +++ b/evolve_charts/loader.py @@ -1,14 +1,18 @@ -"""Reads every Evolve export into tidy pandas tables. +"""Reads Evolve's `.evolve` run files into tidy pandas tables. -Evolve exports three kinds of data, alone or bundled in the score screen's Export zip: +Evolve saves every benchmark run as a `.evolve` file (in `Documents/Evolve/results` on Windows). It's a +zip, called a datakit recording, of: -- **scores** (`scores.csv`): one score per workload category per loop, higher is better. -- **per frame** (`frametimes.csv`, also inside `evolve_results.csv`): GPU time per workload - category, plus GPU sensor readings, for every frame. -- **deep analysis** (`deep_analysis.json`, Evolve Professional): the GPU time of every render pass - in every frame. +- `datakit.json`: the format and its version. +- `tables/.parquet`: the measurements, one Parquet table each: + - `frames`: per frame, the GPU time of every workload category and the driver time. + - `gpu_scopes`: the GPU start and duration of every pass in every frame. + - `cpu_scopes`: the duration of every driver call. + - `metrics`: GPU sensor readings, sampled a few times per second. + - `command_buffers`: when each command buffer was submitted. +- `documents/client_data`: JSON with the system, driver and benchmark settings. -`evolve_results.csv` combines system information, scores and per-frame data in one file. +The scores aren't in the file; they're calculated from the tables the way Evolve does (`scores.py`). """ import hashlib @@ -16,66 +20,47 @@ import json import zipfile from dataclasses import dataclass, field -from pathlib import PurePath +import numpy as np import pandas as pd +import pyarrow.parquet as pq -from .categories import WORKLOAD_ORDER, workloads +from . import scores +from .categories import workloads -DEEP_ANALYSIS = "deep_analysis.json" +DATAKIT_VERSION = 1 +TABLES = ["frames", "gpu_scopes", "cpu_scopes", "metrics"] -METADATA_COLUMNS = [ - "Evolve Version", - "System Name", - "CPU", - "GPU", - "OS Name", - "OS Version", - "GPU Driver Version", - "Rendering Backend", -] - -# Per-frame CSV column -> workload category name (as Evolve shows it). +# `frames` column -> workload category name (as Evolve shows it). WORKLOAD_COLUMNS = { - "Ray Tracing (ns)": "Raytracing", - "Acceleration Structure Build (ns)": "Acceleration Structure Builds", - "Rasterization (ns)": "Rasterization", - "Compute (ns)": "Compute", - "Workgraphs (ns)": "Workgraphs", - "NRC (ns)": "NRC", - "Upscaling Performance (ns)": "Upscaling", + "gpu_ray_tracing_ns": "Raytracing", + "gpu_acceleration_structure_builds_ns": "Acceleration Structure Builds", + "gpu_rasterization_ns": "Rasterization", + "gpu_compute_ns": "Compute", + "gpu_workgraphs_ns": "Workgraphs", + "gpu_nrc_ns": "NRC", + "gpu_upscaler_performance_ns": "Upscaling", } -# Sensor columns in `Result.frames`, filled from deep analysis (GpuMetrics fields) or the per-frame CSV. -METRIC_FIELDS = { - "usage_percentage": "gpu_usage_pct", - "clock_speed_in_mhz": "gpu_clock_mhz", - "vram_clock_speed_in_mhz": "vram_clock_mhz", - "power_usage_in_w": "gpu_power_w", - "board_power_usage_in_w": "board_power_w", - "soc_power_usage_in_w": "soc_power_w", - "voltage_in_mv": "voltage_mv", - "vram_usage_in_mb": "vram_used_mb", - "edge_temperature_in_c": "edge_temp_c", - "hotspot_temperature_in_c": "hotspot_temp_c", -} -FAN_FIELDS = {"Rpm": "fan_rpm", "Percent": "fan_pct"} -METRIC_CSV_COLUMNS = { - # Evolve writes board power into the column it calls "Energy". - "Energy (W)": "board_power_w", - "Gpu Usage (%)": "gpu_usage_pct", - "Clock Speed (Mhz)": "gpu_clock_mhz", - "Vram Clock Speed (Mhz)": "vram_clock_mhz", - "Gpu Voltage (mV)": "voltage_mv", - "Fan Speed (rpm)": "fan_rpm", - "Fan Speed (%)": "fan_pct", - "Edge Temperature (C)": "edge_temp_c", - "Hot Spot Temperature (C)": "hotspot_temp_c", +# `metrics` column -> sensor column in `Result.frames`. +METRIC_COLUMNS = { + "gpu_usage_percent": "gpu_usage_pct", + "gpu_clock_mhz": "gpu_clock_mhz", + "vram_clock_mhz": "vram_clock_mhz", + "gpu_power_w": "gpu_power_w", + "board_power_w": "board_power_w", + "soc_power_w": "soc_power_w", + "voltage_mv": "voltage_mv", + "vram_usage_mib": "vram_used_mb", + "fan_rpm": "fan_rpm", + "fan_percent": "fan_pct", + "edge_temperature_c": "edge_temp_c", + "hotspot_temperature_c": "hotspot_temp_c", } -SENSOR_COLUMNS = list(dict.fromkeys([*METRIC_FIELDS.values(), *FAN_FIELDS.values(), *METRIC_CSV_COLUMNS.values()])) +SENSOR_COLUMNS = list(METRIC_COLUMNS.values()) -class ExportError(Exception): +class LoadError(Exception): """A problem with an input file, worded for the person using the tool.""" @@ -84,16 +69,16 @@ class Result: id: str filename: str metadata: dict[str, str] = field(default_factory=dict) - # Score name -> score, averaged over loops. Scores Evolve didn't produce for this run are absent. + # Score name -> score. Scores the benchmark doesn't have are absent. scores: dict[str, float] = field(default_factory=dict) - # One row per (loop, frame): GPU milliseconds per workload category, one column each. - workloads: pd.DataFrame | None = None - # One row per (loop, frame): sensor readings, and from deep analysis the GPU time sum and span. - frames: pd.DataFrame | None = None - # Deep analysis only. One row per (loop, frame, pass): ms is the pass' own GPU time in that frame; - # top_ms only counts instances not nested inside another scope, so summing it never double-counts. + # One row per frame: GPU milliseconds per workload category, one column each. + workloads: pd.DataFrame = field(default_factory=pd.DataFrame) + # One row per frame: sensor readings, and the GPU time sum and span of the passes. + frames: pd.DataFrame = field(default_factory=pd.DataFrame) + # One row per (frame, pass): ms is the pass' own GPU time in that frame; top_ms only counts instances not + # nested inside another scope, so summing it never double-counts. None when the run has no pass timings. passes: pd.DataFrame | None = None - # Deep analysis only: total GPU milliseconds per pass that Evolve doesn't assign to a workload category. + # Total GPU milliseconds per pass that Evolve doesn't assign to a workload category. left_out: dict[str, float] = field(default_factory=dict) warnings: list[str] = field(default_factory=list) @@ -105,133 +90,49 @@ def gpu(self) -> str: def contents(self) -> list[str]: """What this result contains, in the words the tool uses.""" return [ - name - for name, present in [ - ("Scores", bool(self.scores)), - ("Per frame", self.workloads is not None), - ("Deep analysis", self.passes is not None), - ] - if present + name for name, present in [("Scores", bool(self.scores)), ("Passes", self.passes is not None)] if present ] -def load_export(data: bytes, filename: str) -> Result: +def load_recording(data: bytes, filename: str) -> Result: result = Result(id=hashlib.sha256(data).hexdigest()[:12], filename=filename) - suffix = PurePath(filename).suffix.lower() - if suffix == ".zip": - _read_zip(result, data) - elif suffix == ".csv": - _read_csv(result, data, filename) - elif suffix == ".json": - _read_deep_analysis(result, data, filename) - else: - raise ExportError(f"{filename}: use an Evolve export (.zip, .csv or deep analysis .json).") - - if not result.contents: - raise ExportError(f"{filename} contains no Evolve scores, per-frame data or deep analysis.") - if result.workloads is None and result.passes is not None: - result.workloads = _workloads_from_passes(result.passes, result.frames) - if not result.metadata: - result.warnings.append(f"{filename} has no system information (GPU, driver); it's labelled by file name.") - return result - - -def _read_zip(result: Result, data: bytes) -> None: - try: - archive = zipfile.ZipFile(io.BytesIO(data)) - except zipfile.BadZipFile: - raise ExportError(f"{result.filename} is not a zip file.") from None - members = {PurePath(name).name: name for name in archive.namelist()} - - # The combined CSV holds everything the separate CSVs do, plus system information. - for name in ["evolve_results.csv", "frametimes.csv", "scores.csv"]: - if name in members: - _read_csv(result, archive.read(members[name]), f"{result.filename}/{name}") - if DEEP_ANALYSIS in members: - _read_deep_analysis(result, archive.read(members[DEEP_ANALYSIS]), result.filename) - - -def _read_csv(result: Result, data: bytes, name: str) -> None: - try: - table = pd.read_csv(io.BytesIO(data), encoding="utf-8-sig", dtype=dict.fromkeys(METADATA_COLUMNS, str)) - except (pd.errors.ParserError, pd.errors.EmptyDataError, UnicodeDecodeError): - raise ExportError(f"{name} can't be read as a CSV file.") from None - columns = list(table.columns) - - if "Evolve Version" in columns and not result.metadata: - first = table.iloc[0] - result.metadata = {key: str(first[key]) for key in METADATA_COLUMNS if key in table and pd.notna(first[key])} - if columns[:1] == ["Loop"] or "Evolve Version" in columns: - loop = "Loop" if "Loop" in columns else "Loop Index" - score_columns = columns[1:] if loop == "Loop" else columns[len(_metadata_prefix(columns)) : columns.index(loop)] - if not result.scores: - result.scores = _scores(table.drop_duplicates(loop), score_columns) - if {"Loop Index", "Frame", "Benchmark Time (ns)"} <= set(columns) and result.workloads is None: - result.workloads, sensors = _per_frame(table) - if result.frames is None: - result.frames = sensors - if not columns or not (result.scores or result.workloads is not None or "Evolve Version" in columns): - raise ExportError(f"{name} isn't an Evolve scores or per-frame export.") - - -def _metadata_prefix(columns: list[str]) -> list[str]: - return [column for column in columns if column in METADATA_COLUMNS] - - -def _scores(per_loop: pd.DataFrame, columns: list[str]) -> dict[str, float]: - scores = {} - for column in columns: - values = pd.to_numeric(per_loop[column], errors="coerce") - # Evolve writes 0 for scores it didn't calculate for this benchmark (e.g. Work Graphs). - values = values[values > 0] - if len(values): - scores[column] = float(values.mean()) - return scores - - -def _per_frame(table: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: - base = pd.DataFrame( - { - "loop": table["Loop Index"].astype(int), - "frame": table["Frame"].astype(int), - "seq_s": table["Benchmark Time (ns)"] / 1e9, - } + tables, client_data = _read_container(data, filename) + + run_info = client_data.get("run_info") or {} + variant = run_info.get("variant") + if not variant: + raise LoadError(f"{filename} doesn't say which benchmark it ran.") + platform = client_data.get("platform") or "Desktop" + version = client_data.get("evolve_version") + version = (version["major"], version["minor"]) if version else None + result.metadata = _metadata(client_data, variant) + + frames = tables["frames"] + gpu_scopes = tables["gpu_scopes"] + if not len(frames): + raise LoadError(f"{filename} has no frames; the run didn't finish.") + result.workloads = _workloads(frames) + metrics = tables["metrics"] + sensors = _sensors(frames, metrics) + result.scores = scores.calculate( + variant, + platform, + version, + len(frames), + gpu_scopes, + tables["cpu_scopes"]["len_ns"], + frames["flip"], + frames["ssim"], + metrics, ) - workloads = base.copy() - for column, name in WORKLOAD_COLUMNS.items(): - if column in table and table[column].notna().any(): - # Evolve leaves the cell empty when a workload took no time in that frame. - workloads[name] = table[column].fillna(0) / 1e6 - sensors = base.copy() - for column, name in METRIC_CSV_COLUMNS.items(): - sensors[name] = pd.to_numeric(table[column], errors="coerce") if column in table else float("nan") - for name in SENSOR_COLUMNS: - if name not in sensors: - sensors[name] = float("nan") - return workloads, sensors - - -def _read_deep_analysis(result: Result, data: bytes, filename: str) -> None: - text = data.decode("utf-8") - try: - loops = json.loads(text) - except json.JSONDecodeError: - # Evolve only closes the array after the last loop, so an interrupted run misses the final "]". - try: - loops = json.loads(text.rstrip().rstrip(",") + "]") - except json.JSONDecodeError: - raise ExportError(f"{filename}: {DEEP_ANALYSIS} is damaged and can't be read.") from None - result.warnings.append(f"{filename}: the run was interrupted; using the {len(loops)} completed loop(s).") - if not isinstance(loops, list) or (loops and "per_frame_results" not in loops[0]): - raise ExportError(f"{filename} isn't an Evolve deep analysis file.") - - passes, frames, left_out = _tabulate(loops) + + passes, totals, left_out = _passes(gpu_scopes, frames) + result.frames = sensors.join(totals, on="frame") + result.frames[["gpu_sum_ms", "left_out_ms"]] = result.frames[["gpu_sum_ms", "left_out_ms"]].fillna(0.0) if passes.empty: - result.warnings.append( - f"{filename}: its deep analysis has no GPU work that Evolve assigns to a workload category." - ) - return - result.passes, result.frames, result.left_out = passes, frames, left_out + result.warnings.append(f"{filename} has no GPU work that Evolve assigns to a workload category.") + return result + result.passes, result.left_out = passes, left_out nested = passes["ms"].sum() - passes["top_ms"].sum() if nested > 0: @@ -239,96 +140,132 @@ def _read_deep_analysis(result: Result, data: bytes, filename: str) -> None: f"{filename}: {nested / passes['ms'].sum():.1%} of measured GPU time is nested inside other passes. " "Totals only count it once." ) + return result -def _tabulate(loops: list[dict]) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, float]]: - # Pass name -> whether Evolve assigns it to a workload category. - categorized: dict[str, bool] = {} - left_out: dict[str, float] = {} - pass_rows = [] - frame_rows = [] - for loop in loops: - loop_index = loop["loop_index"] - for frame_index, frame in enumerate(loop["per_frame_results"]): - seq_s = frame["sequence_time_ns"] / 1e9 - own, top = {}, {} - left_out_ms = 0.0 - first_start, last_end = None, None - for command_buffer in frame["command_buffer_timings"].values(): - scopes = [] - for name, timings in command_buffer["scope_timings"].items(): - if name not in categorized: - categorized[name] = bool(workloads(name)) - for timing in timings: - start, end = timing["start"], timing["end"] - first_start = start if first_start is None else min(first_start, start) - last_end = end if last_end is None else max(last_end, end) - if not categorized[name]: - left_out[name] = left_out.get(name, 0.0) + (end - start) / 1e6 - left_out_ms += (end - start) / 1e6 - else: - scopes.append((start, end, name)) - - # Sorted by start, longest first: a scope ending before the current outer one is nested in it. - scopes.sort(key=lambda scope: (scope[0], -scope[1])) - outer_end = None - for start, end, name in scopes: - duration = (end - start) / 1e6 - own[name] = own.get(name, 0.0) + duration - if outer_end is None or end > outer_end: - top[name] = top.get(name, 0.0) + duration - outer_end = end - - for name, ms in own.items(): - pass_rows.append((loop_index, frame_index, seq_s, name, ms, top.get(name, 0.0))) - - row = { - "loop": loop_index, - "frame": frame_index, - "seq_s": seq_s, - "gpu_sum_ms": sum(top.values()), - "gpu_span_ms": (last_end - first_start) / 1e6 if first_start is not None else float("nan"), - "left_out_ms": left_out_ms, - } - row.update(_metrics(frame.get("metrics"))) - frame_rows.append(row) - - passes = pd.DataFrame(pass_rows, columns=["loop", "frame", "seq_s", "pass", "ms", "top_ms"]) - - frames = pd.DataFrame(frame_rows) - for column in SENSOR_COLUMNS: - if column not in frames: - frames[column] = float("nan") - return passes, frames, left_out - - -def _workloads_from_passes(passes: pd.DataFrame, frames: pd.DataFrame) -> pd.DataFrame: - """Per-frame GPU time per workload category, the way Evolve sums it for its per-frame export.""" - membership = pd.DataFrame( - [(name, workload) for name in passes["pass"].unique() for workload in workloads(name)], - columns=["pass", "workload"], - ) - if membership.empty: - return frames[["loop", "frame", "seq_s"]].copy() - per_workload = passes.merge(membership, on="pass").pivot_table( - index=["loop", "frame"], columns="workload", values="ms", aggfunc="sum" +def _read_container(data: bytes, filename: str) -> tuple[dict[str, pd.DataFrame], dict]: + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + manifest = json.loads(archive.read("datakit.json")) + except (zipfile.BadZipFile, KeyError, json.JSONDecodeError): + raise LoadError( + f"{filename} isn't an Evolve run file. Use the .evolve files Evolve saves for every run." + ) from None + if manifest.get("format") != "datakit": + raise LoadError(f"{filename} isn't an Evolve run file.") + if manifest.get("version") != DATAKIT_VERSION: + raise LoadError(f"{filename} was saved by a newer Evolve. Download the latest Evolve Charts to read it.") + + names = set(archive.namelist()) + tables = {} + for name in TABLES: + path = f"tables/{name}.parquet" + if path not in names: + raise LoadError(f"{filename} has no {name} table; it was saved by an Evolve version this tool can't read.") + tables[name] = pq.read_table(io.BytesIO(archive.read(path))).to_pandas() + try: + client_data = json.loads(archive.read("documents/client_data")) + except (KeyError, json.JSONDecodeError): + raise LoadError(f"{filename} has no readable system information.") from None + return tables, client_data + + +def _metadata(client_data: dict, variant: str) -> dict[str, str]: + system = client_data.get("system_info") or {} + os_info = system.get("os_info") or {} + driver = client_data.get("driver_info") or {} + version = client_data.get("evolve_version") + driver_version = driver.get("version") + metadata = { + "Evolve Version": f"{version['major']}.{version['minor']}" if version else "", + "Benchmark": variant, + "System Name": system.get("host_name") or "", + "CPU": (system.get("cpu_name") or "").strip(), + "GPU": driver.get("device_name") or "", + "OS Name": os_info.get("os_type") or "", + "OS Version": _os_version(os_info.get("version")), + "GPU Driver Version": ( + f"{driver_version['major']}.{driver_version['minor']}.{driver_version['patch']}" if driver_version else "" + ), + "Rendering Backend": client_data.get("rendering_backend") or "", + } + return {key: value for key, value in metadata.items() if value} + + +def _os_version(version: dict | str | None) -> str: + """The `os_info` crate's version: {"Semantic": [10, 0, 26200]}, {"Rolling": ...}, {"Custom": ...}.""" + if isinstance(version, dict): + for kind, value in version.items(): + if kind == "Semantic": + return ".".join(str(part) for part in value) + return str(value) if value else kind + return "" if version in (None, "Unknown") else str(version) + + +def _base(frames: pd.DataFrame) -> pd.DataFrame: + return pd.DataFrame({"frame": frames["frame"].astype(int), "seq_s": frames["simulation_time_ns"] / 1e9}) + + +def _workloads(frames: pd.DataFrame) -> pd.DataFrame: + table = _base(frames) + for column, name in WORKLOAD_COLUMNS.items(): + if column in frames and frames[column].notna().any(): + # Empty when a workload took no time in that frame. + table[name] = frames[column].fillna(0) / 1e6 + return table + + +def _sensors(frames: pd.DataFrame, metrics: pd.DataFrame) -> pd.DataFrame: + """Each frame gets the latest sensor sample taken at or before it, like Evolve's own exports.""" + table = _base(frames) + sample_times = metrics["simulation_time_ns"].to_numpy(dtype=float) if len(metrics) else np.array([]) + rows = np.searchsorted(sample_times, frames["simulation_time_ns"].to_numpy(dtype=float), side="right") - 1 + valid = rows >= 0 + if len(metrics): + valid &= ~np.isnan(sample_times[np.clip(rows, 0, None)]) + for column, name in METRIC_COLUMNS.items(): + values = np.full(len(frames), np.nan) + if column in metrics and len(metrics): + readings = pd.to_numeric(metrics[column], errors="coerce").to_numpy(dtype=float) + values[valid] = readings[rows[valid]] + table[name] = values + return table + + +def _passes(gpu_scopes: pd.DataFrame, frames: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, float]]: + """Per (frame, pass) GPU time, and per frame the summed and spanned GPU time and what was left out.""" + scopes = pd.DataFrame( + { + "frame": gpu_scopes["frame"].astype(int), + "command_buffer": gpu_scopes["command_buffer"].astype(int), + "pass": gpu_scopes["name"].astype(str), + "start": gpu_scopes["start_ns"].astype("int64"), + "len": gpu_scopes["len_ns"].astype("int64"), + } ) - table = frames[["loop", "frame", "seq_s"]].join(per_workload, on=["loop", "frame"]) - present = [name for name in WORKLOAD_ORDER if name in table] - return table[["loop", "frame", "seq_s", *present]].fillna({name: 0.0 for name in present}) - - -def _metrics(metrics: dict | None) -> dict[str, float]: - if not metrics: - return {} - values = {} - for key, column in METRIC_FIELDS.items(): - value = metrics.get(key) - if isinstance(value, (int, float)): - values[column] = float(value) - fan = metrics.get("fan_speed") - if isinstance(fan, dict): - for unit, value in fan.items(): - if unit in FAN_FIELDS and isinstance(value, (int, float)): - values[FAN_FIELDS[unit]] = float(value) - return values + scopes["end"] = scopes["start"] + scopes["len"] + categorized = {name: bool(workloads(name)) for name in scopes["pass"].unique()} + scopes["categorized"] = scopes["pass"].map(categorized).astype(bool) + + totals = scopes.groupby("frame").agg(first_start=("start", "min"), last_end=("end", "max")) + totals["gpu_span_ms"] = (totals["last_end"] - totals["first_start"]) / 1e6 + totals = totals[["gpu_span_ms"]] + + left = scopes[~scopes["categorized"]] + totals["left_out_ms"] = left.groupby("frame")["len"].sum() / 1e6 + left_out = (left.groupby("pass")["len"].sum() / 1e6).to_dict() + + # Sorted by start, longest first: a scope ending no later than every earlier one in its command buffer is + # nested inside one of them. + kept = scopes[scopes["categorized"]] + kept = kept.iloc[np.lexsort((-kept["end"], kept["start"], kept["command_buffer"], kept["frame"]))] + earlier_end = kept.groupby(["frame", "command_buffer"])["end"].cummax() + earlier_end = earlier_end.groupby([kept["frame"], kept["command_buffer"]]).shift(1) + kept = kept.assign(ms=kept["len"] / 1e6, top_ms=np.where(earlier_end.isna() | (kept["end"] > earlier_end), 1, 0)) + kept["top_ms"] *= kept["ms"] + + passes = kept.groupby(["frame", "pass"], as_index=False, sort=True)[["ms", "top_ms"]].sum() + totals["gpu_sum_ms"] = passes.groupby("frame")["top_ms"].sum() + seq_s = frames.set_index("frame")["simulation_time_ns"] / 1e9 + passes.insert(1, "seq_s", passes["frame"].map(seq_s)) + return passes[["frame", "seq_s", "pass", "ms", "top_ms"]], totals, left_out diff --git a/evolve_charts/scope_catalog.py b/evolve_charts/scope_catalog.py index b5a71ce..ba19564 100644 --- a/evolve_charts/scope_catalog.py +++ b/evolve_charts/scope_catalog.py @@ -163,7 +163,8 @@ "update_bones": ["Compute"], "apply_vertex_bones": ["Compute"], "reduce_aabbs": ["Compute"], - "channel_animation": ["Compute"], + "channel_animation_batched": ["Compute"], + "terrain-tiles": ["Compute"], "visualize-hierarchy-transforms": ["Compute"], "override_transforms": ["Compute"], "calculate_world_transforms": ["Compute"], @@ -254,7 +255,6 @@ "fused-mlp-forward-training-64": ["Compute"], "fused-mlp-forward-training-16": ["Compute"], "fused-mlp-backward-64": ["Compute"], - "fused-mlp-backward-32": ["Compute"], "fused-mlp-backward-16": ["Compute"], "sky-cubemap": ["Compute"], "clear-counters": ["Compute"], @@ -264,17 +264,11 @@ "Clear texture pipeline": ["Compute"], "generate_paths": ["Workgraphs"], "process_bounce": ["Workgraphs"], - "fused-mlp-forward-inference-coopvec-48-32-4": ["Nrc"], - "fused-mlp-forward-inference-coopvec-32-32-8": ["Nrc"], - "fused-mlp-forward-inference-coopvec-32-64-8": ["Nrc"], - "fused-mlp-forward-training-coopvec-48-32-4": ["Nrc"], - "fused-mlp-forward-training-coopvec-32-32-8": ["Nrc"], - "fused-mlp-forward-training-coopvec-32-64-8": ["Nrc"], - "fused-mlp-backward-coopvec-48-32-4": ["Nrc"], - "fused-mlp-backward-coopvec-32-32-8": ["Nrc"], - "fused-mlp-backward-coopvec-32-64-8": ["Nrc"], - "fused-mlp-forward-inference-coopmat-32-32-8": ["Nrc"], + "nrc-mlp-high-forward-inference-coopmat-nv-amd": ["Nrc"], + "nrc-mlp-high-forward-inference-coopmat-arm": ["Nrc"], + "nrc-mlp-high-forward-inference-coopmat-qcom": ["Nrc"], "fused-mlp-forward-training-32": ["Nrc"], + "fused-mlp-backward-32": ["Nrc"], "Fsr execute": ["UpscalingPerformance"], "Xess execute": ["UpscalingPerformance"], "DLSS evaluate": ["UpscalingPerformance"], @@ -317,6 +311,80 @@ "box_downsample": ["Uncategorized"], "crop_center": ["Uncategorized"], "fallback-upscaler": ["Uncategorized"], + "init_path_tracing_vbuffer_refactored": ["Compute"], + "shading_refactored": ["Compute"], + "neubtf_shading_refactored": ["Compute"], + "resolve_refactored": ["Compute"], + "fill_indirect_args_refactored": ["Compute"], + "fill_shading_neubtf_indirect_args_refactored": ["Compute"], + "water-compositing": ["Compute"], + "resolve_aabbs": ["Compute"], + "reset_aabbs": ["Compute"], + "channel_animation": ["Compute"], + "draw_mesh_selection": ["Uncategorized"], + "breda-text pipeline": ["Uncategorized"], + "begin render pass": ["Uncategorized"], + "render pass": ["Uncategorized"], + "upscaling-raster-pipeline-sample": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_0": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_1": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_2": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_3": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_4": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_5": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_6": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_7": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_8": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_9": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_10": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_11": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_12": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_13": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_14": ["Uncategorized"], + "upscaling-raster-pipeline-sample-easu_15": ["Uncategorized"], + "draw_mesh_selection_0": ["Uncategorized"], + "breda-manipulators-action-triangles_0": ["Uncategorized"], + "breda-manipulators-draw-triangles_2": ["Uncategorized"], + "breda-manipulators-action-lines_0": ["Uncategorized"], + "breda-manipulators-draw-lines_2": ["Uncategorized"], + "vbuffer-mesh-parts-prefix-sum-hierarchy": ["Rasterization"], + "vbuffer-mesh-parts-prefix-sum": ["Rasterization"], + "vbuffer-cull-instances": ["Rasterization"], + "vbuffer-raster-buckets-prefix-sum-hierarchy": ["Rasterization"], + "vbuffer-raster-buckets-prefix-sum": ["Rasterization"], + "vbuffer-build-instanced-draw-calls": ["Rasterization"], + "vbuffer-group-instances": ["Rasterization"], + "vbuffer": ["Rasterization"], + "vbuffer-indirect_0": ["Rasterization"], + "vbuffer-indirect_1": ["Rasterization"], + "vbuffer-indirect_2": ["Rasterization"], + "vbuffer-with-alpha-test": ["Rasterization"], + "vbuffer-indirect-with-alpha-test": ["Rasterization"], + "vbuffer-indirect-with-alpha-test_0": ["Rasterization"], + "vbuffer-indirect-with-alpha-test_1": ["Rasterization"], + "vbuffer-indirect-with-alpha-test_2": ["Rasterization"], + "particle-alpha-blend": ["Rasterization"], + "particle-alpha-blend-indirect": ["Rasterization"], + "particle-alpha-blend-indirect_0": ["Rasterization"], + "particle-alpha-blend-indirect_1": ["Rasterization"], + "particle-alpha-blend-indirect_2": ["Rasterization"], + "alpha-blend": ["Rasterization"], + "alpha-blend-indirect_0": ["Rasterization"], + "alpha-blend-indirect_1": ["Rasterization"], + "alpha-blend-indirect_2": ["Rasterization"], + "decal": ["Rasterization"], + "decal-indirect_0": ["Rasterization"], + "decal-indirect_1": ["Rasterization"], + "decal-indirect_2": ["Rasterization"], + "build-gbuffers_0": ["Rasterization"], + "trace-validation-rays": ["RayTracing"], + "trace-cache-diffuse-rays": ["RayTracing"], + "water-shading-inline": ["RayTracing"], + "water-shading-pipeline": ["RayTracing"], + "trace_rays_refactored": ["RayTracing"], + "trace_shadow_rays_refactored": ["RayTracing"], + "transmission-ray-tracing-pipeline": ["RayTracing"], + "transmission-ray-tracing-inline": ["RayTracing"], } # variant -> the name Evolve shows @@ -369,7 +437,6 @@ "nrc-pt-apply-nrc", "nrc-wfpt-apply-nrc", "fused-mlp-backward-64", - "fused-mlp-backward-32", "fused-mlp-backward-16", "fused-mlp-forward-inference-64", "fused-mlp-forward-inference-32", diff --git a/evolve_charts/score_catalog.py b/evolve_charts/score_catalog.py new file mode 100644 index 0000000..b63e105 --- /dev/null +++ b/evolve_charts/score_catalog.py @@ -0,0 +1,198 @@ +# Generated by tools/sync_scores.py from evolve/crates/evolve-highscore-api/src/score.rs +# and evolve/crates/evolve-datakit-scores/src/lib.rs. Do not edit by hand. + +# Score variants, in the order Evolve lists them +SCORE_ORDER = [ + "RayTracing", + "AccelerationStructureBuilds", + "Rasterization", + "Compute", + "Workgraphs", + "Nrc", + "Driver", + "Energy", + "UpscalerImageQuality1", + "UpscalerImageQuality2", + "UpscalerPerformance", +] + +# Score variant -> the name Evolve shows +SCORE_NAMES = { + "RayTracing": "Ray Tracing", + "AccelerationStructureBuilds": "Acceleration Structure", + "Rasterization": "Rasterization", + "Compute": "Compute", + "Workgraphs": "Work Graphs", + "Nrc": "NRC", + "Driver": "Driver", + "Energy": "Energy Consumption", + "UpscalerImageQuality1": "Image Quality 1", + "UpscalerImageQuality2": "Image Quality 2", + "UpscalerPerformance": "Upscaler Performance", +} + +# WorkloadCategory variant -> the Score of its GPU time +GPU_SCORES = { + "RayTracing": "RayTracing", + "AccelerationStructureBuilds": "AccelerationStructureBuilds", + "Rasterization": "Rasterization", + "Compute": "Compute", + "Workgraphs": "Workgraphs", + "Nrc": "Nrc", + "UpscalingPerformance": "UpscalerPerformance", +} + +# Platform -> BenchmarkVariant -> Score -> multiplier; a score a benchmark doesn't have is absent +MULTIPLIERS = { + "Desktop": { + "UpscalingQualcommToggleMode": {}, + "UpscalingQualcommBackToBack": {}, + "UpscalingQualcommPowerEfficiency": {}, + "Upscaling": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0, "UpscalerPerformance": 1.0049}, + "UpscalingQualcommImageQuality": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0}, + "Nrc": {"Nrc": 30.0}, + "RayTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "RayTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "PathTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 127.0, + "Compute": 190.0, + "Driver": 12.12, + "Energy": 0.003, + }, + "PathTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 615.0, + "Compute": 28.24, + "Driver": 14.59, + "Energy": 0.003, + }, + "PathTracingWorkGraph": { + "AccelerationStructureBuilds": 5.43, + "Driver": 54.63, + "Energy": 0.003, + "Workgraphs": 400.0, + }, + }, + "Phone": { + "UpscalingQualcommToggleMode": {}, + "UpscalingQualcommBackToBack": {}, + "UpscalingQualcommPowerEfficiency": {}, + "Upscaling": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0, "UpscalerPerformance": 10.0}, + "UpscalingQualcommImageQuality": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0}, + "Nrc": {}, + "RayTracingInline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 290.83, + "Rasterization": 42.76, + "Compute": 237.28, + "Driver": 40.0, + "Energy": 19350.0, + }, + "RayTracingPipeline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 290.83, + "Rasterization": 42.76, + "Compute": 237.28, + "Driver": 40.0, + "Energy": 19350.0, + }, + "PathTracingInline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 993.0, + "Compute": 487.0, + "Driver": 41.0, + "Energy": 19350.0, + }, + "PathTracingPipeline": { + "AccelerationStructureBuilds": 130.32, + "RayTracing": 1948.0, + "Compute": 276.24, + "Driver": 38.5, + "Energy": 19350.0, + }, + "PathTracingWorkGraph": {"AccelerationStructureBuilds": 130.32, "Workgraphs": 400.0}, + }, + "Handheld": { + "UpscalingQualcommToggleMode": {}, + "UpscalingQualcommBackToBack": {}, + "UpscalingQualcommPowerEfficiency": {}, + "Upscaling": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0, "UpscalerPerformance": 1.0049}, + "UpscalingQualcommImageQuality": {"UpscalerImageQuality1": 375.0, "UpscalerImageQuality2": 10000.0}, + "Nrc": {"Nrc": 30.0}, + "RayTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "RayTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 28.83, + "Rasterization": 10.76, + "Compute": 60.28, + "Driver": 11.93, + "Energy": 0.003, + }, + "PathTracingInline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 127.0, + "Compute": 190.0, + "Driver": 12.12, + "Energy": 0.003, + }, + "PathTracingPipeline": { + "AccelerationStructureBuilds": 5.43, + "RayTracing": 615.0, + "Compute": 28.24, + "Driver": 14.59, + "Energy": 0.003, + }, + "PathTracingWorkGraph": { + "AccelerationStructureBuilds": 5.43, + "Driver": 54.63, + "Energy": 0.003, + "Workgraphs": 400.0, + }, + }, +} + +# Phone Driver scores of runs before this Evolve version use PHONE_DRIVER_WITH_PRESENT +DRIVER_EXCLUDES_PRESENT = (17, 0) + +# BenchmarkVariant -> phone Driver multiplier from before DRIVER_EXCLUDES_PRESENT +PHONE_DRIVER_WITH_PRESENT = { + "RayTracingInline": 553.17, + "RayTracingPipeline": 553.17, + "PathTracingInline": 1224.0, + "PathTracingPipeline": 2040.0, +} + +# Energy score = ENERGY_SCALE * exp(-multiplier * average power in W) +ENERGY_SCALE = 10600.0 + +# FLIP and SSIM pooling and normalization constants +IMAGE_QUALITY = { + "WORST_CASE_FRACTION": 0.1, + "WORST_CASE_POOL_WEIGHT": 0.5, + "BEST_FLIP": 0.3156, + "WORST_FLIP": 0.4255, + "BEST_SSIM": 0.777, + "WORST_SSIM": 0.6102, +} diff --git a/evolve_charts/scores.py b/evolve_charts/scores.py new file mode 100644 index 0000000..e7c5f1b --- /dev/null +++ b/evolve_charts/scores.py @@ -0,0 +1,123 @@ +"""Evolve's scores, calculated from a run's tables the way Evolve does (`evolve-datakit-scores`). + +A `.evolve` file holds the measurements, not the scores. The calibrated multipliers come from +`score_catalog.py`, generated from Evolve's Rust source by `tools/sync_scores.py`. +""" + +import math + +import numpy as np +import pandas as pd + +from .categories import workload_variants +from .score_catalog import ( + DRIVER_EXCLUDES_PRESENT, + ENERGY_SCALE, + GPU_SCORES, + IMAGE_QUALITY, + MULTIPLIERS, + PHONE_DRIVER_WITH_PRESENT, + SCORE_NAMES, + SCORE_ORDER, +) + + +def multiplier(score: str, variant: str, platform: str, version: tuple[int, int] | None = None) -> float | None: + if score == "Driver" and platform == "Phone" and (version is None or version < DRIVER_EXCLUDES_PRESENT): + return PHONE_DRIVER_WITH_PRESENT.get(variant) + return MULTIPLIERS.get(platform, {}).get(variant, {}).get(score) + + +def calculate( + variant: str, + platform: str, + version: tuple[int, int] | None, + num_frames: int, + gpu_scopes: pd.DataFrame, + cpu_scope_ns: pd.Series, + flip: pd.Series, + ssim: pd.Series, + metrics: pd.DataFrame, +) -> dict[str, float]: + """Score name -> score, for every score the benchmark has on this platform, in Evolve's order. + + `gpu_scopes` has a `name` and `len_ns` column; `cpu_scope_ns` holds the duration of every driver call. + """ + has = MULTIPLIERS.get(platform, {}).get(variant, {}) + scores: dict[str, int] = {} + + def per_frame(score: str, total_ns: int) -> int | None: + factor = multiplier(score, variant, platform, version) + if factor is None or total_ns == 0 or num_frames == 0: + return None + return _u32(factor / (round(total_ns / num_frames) / 1e9)) + + if any(score in has for score in GPU_SCORES.values()): + per_name = gpu_scopes.groupby("name", observed=True)["len_ns"].sum() + totals = dict.fromkeys(GPU_SCORES, 0) + for name, total in per_name.items(): + for workload in workload_variants(name): + if workload in totals: + totals[workload] += int(total) + for workload, score in GPU_SCORES.items(): + if (value := per_frame(score, totals[workload])) is not None: + scores[score] = value + + if "Driver" in has and (value := per_frame("Driver", int(cpu_scope_ns.sum()))) is not None: + scores["Driver"] = value + + for score, errors in [("UpscalerImageQuality1", flip), ("UpscalerImageQuality2", ssim)]: + if score in has and (value := _image_quality(variant, score, errors.dropna().to_numpy())) is not None: + scores[score] = value + + if "Energy" in has and (power := _average_power_w(metrics)) is not None: + energy = np.float32(ENERGY_SCALE) * np.exp(np.float32(-has["Energy"]) * power) + scores["Energy"] = _u32(float(energy)) + + return {SCORE_NAMES[score]: float(scores[score]) for score in SCORE_ORDER if score in scores} + + +def _u32(value: float) -> int: + """Rust's saturating `as u32`.""" + return 0 if math.isnan(value) else int(min(max(value, 0), 2**32 - 1)) + + +def _f32_sum(values: np.ndarray) -> np.float32: + """Summed one value at a time in f32, like Rust's `iter().sum::()`.""" + return np.cumsum(values.astype(np.float32), dtype=np.float32)[-1] + + +def _average_power_w(metrics: pd.DataFrame) -> np.float32 | None: + """With an energy counter in the first and last samples, the energy used over the time between + them; otherwise the mean sampled board power, or GPU power for a sample without it.""" + if metrics.empty: + return None + energy = metrics["gpu_energy_j"] + if pd.notna(energy.iloc[0]) and pd.notna(energy.iloc[-1]): + joules = np.float32(energy.iloc[-1] - energy.iloc[0]) + elapsed_ns = max(int(metrics["sample_time_ns"].iloc[-1]) - int(metrics["sample_time_ns"].iloc[0]), 0) + return joules / np.float32(elapsed_ns / 1e9) + power = metrics["board_power_w"].fillna(metrics["gpu_power_w"]).dropna().to_numpy() + return _f32_sum(power) / np.float32(len(power)) if len(power) else None + + +def _image_quality(variant: str, score: str, errors: np.ndarray) -> int | None: + if not len(errors): + return None + if variant == "UpscalingQualcommImageQuality": + return 0 + errors = errors.astype(np.float32) + worse_is_higher = score == "UpscalerImageQuality1" + mean = _f32_sum(errors) / np.float32(len(errors)) + ordered = np.sort(errors) + count = max(math.ceil(np.float32(len(errors)) * np.float32(IMAGE_QUALITY["WORST_CASE_FRACTION"])), 1) + worst = ordered[len(ordered) - count :] if worse_is_higher else ordered[:count] + weight = np.float32(IMAGE_QUALITY["WORST_CASE_POOL_WEIGHT"]) + pooled = (np.float32(1) - weight) * mean + weight * (_f32_sum(worst) / np.float32(len(worst))) + if worse_is_higher: + best, worst_value = np.float32(IMAGE_QUALITY["BEST_FLIP"]), np.float32(IMAGE_QUALITY["WORST_FLIP"]) + normalized = (worst_value - pooled) / (worst_value - best) + else: + best, worst_value = np.float32(IMAGE_QUALITY["BEST_SSIM"]), np.float32(IMAGE_QUALITY["WORST_SSIM"]) + normalized = (pooled - worst_value) / (best - worst_value) + return _u32(float(max(normalized, np.float32(0)) * np.float32(10_000))) diff --git a/evolve_charts/stats.py b/evolve_charts/stats.py index 2c96087..49ef37e 100644 --- a/evolve_charts/stats.py +++ b/evolve_charts/stats.py @@ -32,7 +32,7 @@ def has_level(result: Result, level: Level) -> bool: if level == "workload": - return result.workloads is not None and len(result.workloads.columns) > 3 + return any(name in result.workloads for name in WORKLOAD_ORDER) return result.passes is not None @@ -47,21 +47,21 @@ def per_frame(result: Result, skip_s: float, level: Level, within: Within = None """GPU milliseconds per frame, one column per item; frames where an item didn't run count as 0.""" if level == "workload": table = result.workloads[result.workloads["seq_s"] >= skip_s] - return table.set_index(["loop", "frame"]).drop(columns="seq_s") + return table.set_index("frame").drop(columns="seq_s") passes = result.passes[result.passes["seq_s"] >= skip_s] if within is not None: _, name = within passes = passes[passes["pass"].map(lambda scope: name in workloads(scope))] - wide = passes.pivot_table(index=["loop", "frame"], columns="pass", values="ms", aggfunc="sum") - frame_index = trimmed_frames(result, skip_s).set_index(["loop", "frame"]).index + wide = passes.pivot_table(index="frame", columns="pass", values="ms", aggfunc="sum") + frame_index = trimmed_frames(result, skip_s).set_index("frame").index return wide.reindex(frame_index).fillna(0.0) def summary(results: dict[str, Result], skip_s: float, level: Level, within: Within = None) -> pd.DataFrame: """One row per (result, item) with mean/median/p5/p95 milliseconds per frame. - Results without data for the level (e.g. no deep analysis) are left out. + Results without data for the level (e.g. no pass timings) are left out. """ rows = [] for label, result in results.items(): @@ -106,7 +106,7 @@ def ordered(items, level: Level) -> list[str]: def frame_totals(results: dict[str, Result], skip_s: float) -> pd.DataFrame: - """Deep analysis only: GPU time per frame of all categorized passes, what was left out, and the span.""" + """Pass timings only: GPU time per frame of all categorized passes, what was left out, and the span.""" rows = [] for label, result in results.items(): if result.passes is None: @@ -126,24 +126,21 @@ def frame_totals(results: dict[str, Result], skip_s: float) -> pd.DataFrame: def timeline( result: Result, skip_s: float, level: Level, items: list[str], smooth_s: float, within: Within = None ) -> pd.DataFrame: - """Per-frame milliseconds over benchmark time, averaged over loops by frame index.""" + """Per-frame milliseconds over benchmark time.""" wide = per_frame(result, skip_s, level, within).reindex(columns=items, fill_value=0.0) - source = result.workloads if level == "workload" else trimmed_frames(result, skip_s) - wide["seq_s"] = source.set_index(["loop", "frame"])["seq_s"] - return _smooth(wide.groupby(level="frame").mean().set_index("seq_s"), smooth_s) + wide["seq_s"] = result.workloads.set_index("frame")["seq_s"] + return _smooth(wide.set_index("seq_s"), smooth_s) def telemetry_timeline(result: Result, skip_s: float, smooth_s: float) -> pd.DataFrame: frames = trimmed_frames(result, skip_s) columns = [column for column in TELEMETRY if frames[column].notna().any()] - return _smooth(frames.groupby("frame")[["seq_s", *columns]].mean().set_index("seq_s"), smooth_s) + return _smooth(frames.set_index("seq_s")[columns], smooth_s) def telemetry_summary(results: dict[str, Result], skip_s: float) -> pd.DataFrame: rows = [] for label, result in results.items(): - if result.frames is None: - continue frames = trimmed_frames(result, skip_s) row = {"result": label} for column, (name, unit) in TELEMETRY.items(): @@ -155,7 +152,7 @@ def telemetry_summary(results: dict[str, Result], skip_s: float) -> pd.DataFrame def scores_table(results: dict[str, Result]) -> pd.DataFrame: - """Scores as rows (in the order Evolve exports them), results as columns.""" + """Scores as rows (in Evolve's order), results as columns.""" names = list(dict.fromkeys(name for result in results.values() for name in result.scores)) return pd.DataFrame( {label: [result.scores.get(name) for name in names] for label, result in results.items() if result.scores}, @@ -201,7 +198,7 @@ def wide_for_spreadsheets(item_summary: pd.DataFrame, value: str = "mean_ms") -> def _smooth(by_time: pd.DataFrame, smooth_s: float) -> pd.DataFrame: - """Centred rolling mean over `smooth_s` seconds of benchmark time; exports differ a lot in frames per second.""" + """Centred rolling mean over `smooth_s` seconds of benchmark time; runs differ a lot in frames per second.""" if smooth_s <= 0 or len(by_time) < 2: return by_time step = pd.Series(by_time.index).diff().median() diff --git a/evolve_charts/ui.py b/evolve_charts/ui.py index ba1b9fe..e7c3d14 100644 --- a/evolve_charts/ui.py +++ b/evolve_charts/ui.py @@ -2,7 +2,7 @@ Started by `evolve-charts.py`. Pages: -- Results: load and name exports, see what each contains and how Evolve's measurements relate. +- Results: load and name `.evolve` run files, see what each contains and how Evolve's measurements relate. - Scores, GPU time, Passes, GPU sensors: the analysis, with the loaded results and the "compare against" choice in a bar at the top. - Export: CSV files for spreadsheets and newsroom chart tools. @@ -18,7 +18,7 @@ from . import charts, stats from .categories import WORKLOAD_DESCRIPTIONS, WORKLOAD_ORDER, is_utility, workloads -from .loader import ExportError, Result, load_export +from .loader import LoadError, Result, load_recording APP_DIR = Path(__file__).resolve().parent.parent STATIC_DIR = APP_DIR / "static" @@ -98,7 +98,7 @@ HOW_IT_WORKS = f""" **Per frame.** A pass' time is the sum of all its executions in a frame; frames where it didn't run count as -0. Averages cover every frame of every loop; the first {WARM_UP_S:g} s are skipped because they contain +0. Averages cover every frame of the run; the first {WARM_UP_S:g} s are skipped because they contain one-off setup work. Lines over time are smoothed over {SMOOTHING_S:g} s. **Left out.** Passes Evolve doesn't assign to a workload category (copies, clears, the user interface, …) @@ -145,11 +145,11 @@ def main() -> None: @st.cache_data(show_spinner=False, max_entries=32) def _load_file(path: str, modified: float) -> Result: - return load_export(Path(path).read_bytes(), Path(path).name) + return load_recording(Path(path).read_bytes(), Path(path).name) -def _exports_dir() -> Path: - return Path(os.environ.get("EVOLVE_CHARTS_EXPORTS", Path(__file__).resolve().parent.parent / "exports")) +def _results_dir() -> Path: + return Path(os.environ.get("EVOLVE_CHARTS_RESULTS", Path(__file__).resolve().parent.parent / "results")) def _store() -> dict: @@ -164,8 +164,8 @@ def _on_upload() -> None: for upload in st.session_state.get(f"uploader-{store['upload_key']}") or []: try: with st.spinner(f"Reading {upload.name}…"): - result = load_export(upload.getvalue(), upload.name) - except ExportError as error: + result = load_recording(upload.getvalue(), upload.name) + except LoadError as error: store["errors"][upload.name] = str(error) continue store["errors"].pop(upload.name, None) @@ -178,16 +178,16 @@ def _on_upload() -> None: def _loaded_results() -> dict[str, Result]: store = _store() loaded: list[Result] = [] - folder = _exports_dir() + folder = _results_dir() paths = sorted(folder.iterdir()) if folder.is_dir() else [] for path in paths: - if path.suffix.lower() not in {".zip", ".csv", ".json"}: + if path.suffix.lower() != ".evolve": continue try: with st.spinner(f"Reading {path.name}…"): loaded.append(_load_file(str(path), path.stat().st_mtime)) store["errors"].pop(path.name, None) - except ExportError as error: + except LoadError as error: store["errors"][path.name] = str(error) loaded += store["uploaded"].values() @@ -294,7 +294,7 @@ def _missing(present: set[str], what: str) -> bool: st.info(f"None of the loaded results contain {what}. See the Results page for what each file contains.") return True if missing: - st.caption(f"Not included, because their exports contain no {what}: {', '.join(missing)}.") + st.caption(f"Not included, because their runs have no {what}: {', '.join(missing)}.") return False @@ -329,19 +329,19 @@ def _results_page() -> None: results = st.session_state["results"] _title( "Results", - "Load Evolve exports to compare them. Everything stays on this computer.", + "Load Evolve runs to compare them. Everything stays on this computer.", ) columns = st.columns(CARDS_PER_ROW) with columns[0].container(border=True, height="stretch"): st.file_uploader( - "**Add Evolve exports**", - type=["zip", "csv", "json"], + "**Add Evolve runs**", + type=["evolve"], accept_multiple_files=True, key=f"uploader-{store['upload_key']}", on_change=_on_upload, - help="The .zip from Export on Evolve's score screen, or the CSV and JSON files Evolve's command line " - "exports. Files in the `exports` folder next to this tool load automatically.", + help="The .evolve file Evolve saves for every run, in Documents\\Evolve\\results on Windows. " + "Files in the `results` folder next to this tool load automatically.", ) for error in store["errors"].values(): st.error(error) @@ -351,9 +351,9 @@ def _results_page() -> None: """ **How to start** -1. In Evolve, finish a benchmark and click **Export** on the score screen. +1. Run a benchmark in Evolve. Evolve saves every run as a **.evolve** file in *Documents\\Evolve\\results*. 2. Repeat on every GPU or setting you want to compare, with the same benchmark. -3. Drop the files in the box on the left. +3. Drop the .evolve files in the box on the left. """ ) colors = charts.result_colors(list(results)) @@ -366,7 +366,11 @@ def _results_page() -> None: if results: problems = [] - for key, what in [("Evolve Version", "Evolve versions"), ("Rendering Backend", "graphics APIs")]: + for key, what in [ + ("Benchmark", "benchmarks"), + ("Evolve Version", "Evolve versions"), + ("Rendering Backend", "graphics APIs"), + ]: values = {result.metadata.get(key, "unknown") for result in results.values()} if len(values) > 1: problems.append(f"different {what} ({', '.join(sorted(values))})") @@ -405,14 +409,10 @@ def _result_card(result: Result, label: str, color: str) -> None: " Β· ".join(filter(None, [meta.get("Rendering Backend"), version])), " Β· ".join(filter(None, [driver, os_name])), ] - per_frame = result.frames if result.frames is not None else result.workloads - if per_frame is not None: - frames = per_frame[per_frame["seq_s"] >= WARM_UP_S] - duration = frames["seq_s"].max() - frames["seq_s"].min() if len(frames) else 0 - lines.append(f"{len(frames):,} frames Β· {duration // 60:.0f}:{duration % 60:02.0f} min") - contents = "
".join( - f"{'βœ“' if name in result.contents else 'βœ—'} {name}" for name in ["Scores", "Per frame", "Deep analysis"] - ) + frames = result.frames[result.frames["seq_s"] >= WARM_UP_S] + duration = frames["seq_s"].max() - frames["seq_s"].min() if len(frames) else 0 + lines.append(f"{len(frames):,} frames Β· {duration // 60:.0f}:{duration % 60:02.0f} min") + contents = "
".join(f"{'βœ“' if name in result.contents else 'βœ—'} {name}" for name in ["Scores", "Passes"]) body = "
".join(line for line in lines if line) st.markdown(f'
{body}
{contents}
', unsafe_allow_html=True) st.caption(result.filename) @@ -542,7 +542,7 @@ def _gpu_time_page() -> None: inside = stats.summary(context.results, WARM_UP_S, "pass", within=("workload", item)) inside = inside[inside["mean_ms"] > 0] if inside.empty: - st.caption("Load exports with deep analysis to see the passes this workload category is made of.") + st.caption("These runs have no pass timings to show the passes this workload category is made of.") return ranked = inside.groupby("item")["mean_ms"].max().sort_values(ascending=False) items = ranked.index[:TOP_PASSES].tolist() @@ -572,12 +572,12 @@ def _passes_page() -> None: context = _context() _title("Passes", "Single blocks of GPU work, named as in Evolve. Tick passes to follow them over the run.") summary = stats.summary(context.results, WARM_UP_S, "pass") - if _missing(set(summary["result"]), "deep analysis"): + if _missing(set(summary["result"]), "pass timings"): return labels = [label for label in context.labels if label in set(summary["result"])] baseline = context.baseline if context.baseline in labels else labels[0] if baseline != context.baseline: - st.caption(f"Compared against {baseline}: {context.baseline} has no deep analysis.") + st.caption(f"Compared against {baseline}: {context.baseline} has no pass timings.") table = stats.comparison_table(summary, labels, baseline) # Copies, clears and other support work still count towards their workload category, but aren't listed. utility = table["item"].map(is_utility) @@ -645,9 +645,7 @@ def _sensors_page() -> None: context = _context() _title("GPU sensors", "Clocks, power and temperature during the run. Falling clocks can mean throttling.") timelines = { - label: stats.telemetry_timeline(result, WARM_UP_S, SMOOTHING_S) - for label, result in context.results.items() - if result.frames is not None + label: stats.telemetry_timeline(result, WARM_UP_S, SMOOTHING_S) for label, result in context.results.items() } if _missing({label for label, timeline in timelines.items() if len(timeline.columns)}, "GPU sensor data"): return @@ -754,9 +752,9 @@ def _per_frame_long(results: dict[str, Result], skip_s: float) -> pd.DataFrame: passes = result.passes[result.passes["seq_s"] >= skip_s] frames.append( passes.assign(result=label, workload=passes["pass"].map(lambda scope: ", ".join(workloads(scope))))[ - ["result", "loop", "frame", "seq_s", "workload", "pass", "ms"] + ["result", "frame", "seq_s", "workload", "pass", "ms"] ] ) if not frames: - return pd.DataFrame(columns=["result", "loop", "frame", "benchmark_time_s", "workload", "pass", "ms"]) + return pd.DataFrame(columns=["result", "frame", "benchmark_time_s", "workload", "pass", "ms"]) return pd.concat(frames).rename(columns={"seq_s": "benchmark_time_s"}) diff --git a/exports/README.txt b/exports/README.txt deleted file mode 100644 index 350b906..0000000 --- a/exports/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -Put Evolve export .zip files in this folder and Evolve Charts loads them automatically. -You can also drag them into the upload box in the tool instead. diff --git a/results/README.txt b/results/README.txt new file mode 100644 index 0000000..bbb40bc --- /dev/null +++ b/results/README.txt @@ -0,0 +1,3 @@ +Put .evolve files in this folder and Evolve Charts loads them automatically. +Evolve saves one for every benchmark run, in Documents\Evolve\results on Windows. +You can also drag them into the upload box in the tool instead. diff --git a/scripts/run_on_android.py b/scripts/run_on_android.py index 75ee6cc..260a3d2 100644 --- a/scripts/run_on_android.py +++ b/scripts/run_on_android.py @@ -10,7 +10,7 @@ def run_command(command): def main(): id = "nl.traverse_research.evolve" - path = "/sdcard/Android/data/nl.traverse_research.evolve/files" + results = "/sdcard/Android/data/nl.traverse_research.evolve/files/Documents/results" # Get Evolve UID packages = run_command(["adb", "shell", "pm", "list", "packages", "-U", id]) @@ -38,9 +38,8 @@ def main(): sys.exit(1) moment = datetime.now().strftime("%Y%m%d_%H%M%S") - pull_suggestion = f"evolve-results-{moment}" - - print(f"Started Evolve on Android. When the benchmark finishes, manually pull the specified files from {path} using adb pull or MTP.") + print("Started Evolve on Android. When the benchmark finishes, pull its .evolve files with:") + print(f" adb pull {results} evolve-results-{moment}") print("\nStarting logcat in 4 seconds") time.sleep(4) diff --git a/tests/conftest.py b/tests/conftest.py index 4b34ecd..41e2807 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,31 +4,58 @@ import zipfile from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -METADATA_HEADER = "Evolve Version,System Name,CPU,GPU,OS Name,OS Version,GPU Driver Version,Rendering Backend" -SCORES_HEADER = "Ray Tracing,Rasterization,Compute,Work Graphs" -PER_FRAME_HEADER = ( - "Loop Index,Frame,Benchmark Time (ns),Ray Tracing (ns),Rasterization (ns),Compute (ns),Energy (W),Clock Speed (Mhz)" -) - - -def scope(start_ms: float, end_ms: float) -> dict: - return {"start": int(start_ms * 1e6), "end": int(end_ms * 1e6)} - - -def frame(seq_s: float, buffers: list[dict[str, list[dict]]], metrics: dict | None = None) -> dict: - return { - "sequence_time_ns": int(seq_s * 1e9), - "command_buffer_timings": { - f"Command Buffer {index}": {"cpu_submit_time": 0, "gpu_submit_time": 0, "scope_timings": scopes} - for index, scopes in enumerate(buffers) - }, - "cpu_timings": {"DriverApiCall": [1000]}, - "metrics": metrics, - } +from evolve_charts.categories import workload_variants + +# WorkloadCategory variant -> `frames` column. +FRAME_COLUMNS = { + "RayTracing": "gpu_ray_tracing_ns", + "AccelerationStructureBuilds": "gpu_acceleration_structure_builds_ns", + "Rasterization": "gpu_rasterization_ns", + "Compute": "gpu_compute_ns", + "Workgraphs": "gpu_workgraphs_ns", + "Nrc": "gpu_nrc_ns", + "UpscalingPerformance": "gpu_upscaler_performance_ns", +} +METRICS_SCHEMA = { + "simulation_time_ns": pa.uint64(), + "sample_time_ns": pa.uint64(), + "gpu_usage_percent": pa.float64(), + "gpu_clock_mhz": pa.uint32(), + "vram_clock_mhz": pa.uint32(), + "gpu_power_w": pa.float64(), + "board_power_w": pa.float64(), + "gpu_energy_j": pa.float64(), + "soc_power_w": pa.float64(), + "soc_energy_j": pa.float64(), + "voltage_mv": pa.uint32(), + "vram_usage_mib": pa.uint32(), + "fan_rpm": pa.uint32(), + "fan_percent": pa.uint32(), + "edge_temperature_c": pa.float64(), + "hotspot_temperature_c": pa.float64(), +} + + +def scope(start_ms: float, end_ms: float) -> tuple[int, int]: + return int(start_ms * 1e6), int(end_ms * 1e6) + + +def frame( + seq_s: float, + buffers: list[dict[str, list[tuple[int, int]]]], + metrics: dict | None = None, + driver_ns: int = 1000, + flip: float | None = None, + ssim: float | None = None, +) -> dict: + """One frame: per command buffer, the scopes of every pass; and the sensor sample taken at this frame.""" + return {"seq_s": seq_s, "buffers": buffers, "metrics": metrics, "driver_ns": driver_ns, "flip": flip, "ssim": ssim} def simple_frames(shadow_ms: float, count: int = 20) -> list[dict]: @@ -51,45 +78,99 @@ def simple_frames(shadow_ms: float, count: int = 20) -> list[dict]: "made-up-pass": [scope(base + 62, base + 62.1)], } ], - {"clock_speed_in_mhz": 2500 + index, "board_power_usage_in_w": 200.0, "fan_speed": {"Rpm": 1500}}, + {"gpu_clock_mhz": 2500 + index, "board_power_w": 200.0, "fan_rpm": 1500}, ) ) return frames -def combined_csv(gpu: str, shadow_ms: float, count: int = 20, scores: str = "4000,5000,6000,0") -> str: - """`evolve_results.csv`: system information, scores and per-frame workload times on every row.""" - rows = [f"{METADATA_HEADER},{SCORES_HEADER},{PER_FRAME_HEADER}"] - for index in range(count): - ray_tracing = int((shadow_ms + (50 if index == 0 else 0)) * 1e6) - rows.append( - f"15.10,host,CPU X,{gpu},Windows,11,1.2.3,Dx12,{scores}," - f"1,{index},{int(index * 0.5e9)},{ray_tracing},,{int(0.5e6)},200.0,{2500 + index}" - ) - return "\n".join(rows) + "\n" +def _parquet(columns: dict[str, pa.Array]) -> bytes: + buffer = io.BytesIO() + pq.write_table(pa.table(columns), buffer) + return buffer.getvalue() -def make_export( - loops: list[list[dict]] | None, +def make_recording( + frames: list[dict], gpu: str = "Test GPU", - truncated: bool = False, - results_csv: str | None = None, + variant: str = "RayTracingInline", + platform: str = "Desktop", + version: tuple[int, int] = (17, 0), + with_scopes: bool = True, ) -> bytes: - """An Evolve Export zip; `loops=None` leaves out deep analysis like exports without Professional.""" + """A `.evolve` file the way Evolve writes it: per-frame workload times summed from the scopes, like Evolve.""" + workload_ns = {column: [] for column in FRAME_COLUMNS.values()} + scope_rows = {"frame": [], "command_buffer": [], "name": [], "start_ns": [], "len_ns": []} + metric_rows = {name: [] for name in METRICS_SCHEMA} + for index, spec in enumerate(frames): + totals = dict.fromkeys(FRAME_COLUMNS.values(), 0) + for buffer, scopes in enumerate(spec["buffers"]): + for name, timings in scopes.items(): + for start, end in timings: + for workload in workload_variants(name): + if workload in FRAME_COLUMNS: + totals[FRAME_COLUMNS[workload]] += end - start + if with_scopes: + for key, value in zip(scope_rows, [index, buffer, name, start, end - start]): + scope_rows[key].append(value) + for column, total in totals.items(): + workload_ns[column].append(total or None) + if spec["metrics"] is not None: + sample = {"simulation_time_ns": int(spec["seq_s"] * 1e9), "sample_time_ns": int(spec["seq_s"] * 1e9)} + sample.update(spec["metrics"]) + for name in METRICS_SCHEMA: + metric_rows[name].append(sample.get(name)) + + seq_ns = [int(spec["seq_s"] * 1e9) for spec in frames] + tables = { + "frames": { + "frame": pa.array(range(len(frames)), pa.uint32()), + "simulation_time_ns": pa.array(seq_ns, pa.uint64()), + **{column: pa.array(values, pa.uint64()) for column, values in workload_ns.items()}, + "cpu_driver_ns": pa.array([spec["driver_ns"] for spec in frames], pa.uint64()), + "flip": pa.array([spec["flip"] for spec in frames], pa.float32()), + "ssim": pa.array([spec["ssim"] for spec in frames], pa.float32()), + }, + "metrics": {name: pa.array(values, METRICS_SCHEMA[name]) for name, values in metric_rows.items()}, + "gpu_scopes": { + "frame": pa.array(scope_rows["frame"], pa.uint32()), + "command_buffer": pa.array(scope_rows["command_buffer"], pa.uint8()), + "name": pa.array(scope_rows["name"], pa.string()).dictionary_encode(), + "start_ns": pa.array(scope_rows["start_ns"], pa.uint64()), + "len_ns": pa.array(scope_rows["len_ns"], pa.uint64()), + }, + "cpu_scopes": { + "frame": pa.array(range(len(frames)), pa.uint32()), + "thread": pa.array(["main"] * len(frames)).dictionary_encode(), + "name": pa.array(["ExecuteCommandLists"] * len(frames)).dictionary_encode(), + "start_ns": pa.array(seq_ns, pa.int64()), + "len_ns": pa.array([spec["driver_ns"] for spec in frames], pa.uint64()), + }, + } + client_data = { + "evolve_version": {"major": version[0], "minor": version[1]}, + "run_info": {"variant": variant}, + "platform": platform, + "rendering_backend": "Dx12", + "driver_info": {"device_name": gpu, "version": {"major": 1, "minor": 2, "patch": 3}}, + "system_info": { + "host_name": "host", + "cpu_name": "CPU X ", + "os_info": {"os_type": "Windows", "version": {"Semantic": [10, 0, 26200]}}, + }, + } + buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w") as archive: - if loops is not None: - text = json.dumps( - [{"loop_index": index + 1, "per_frame_results": frames} for index, frames in enumerate(loops)] - ) - archive.writestr("deep_analysis.json", text[:-1] if truncated else text) - shadow_ms = 2.0 if "1" in gpu else 1.0 - archive.writestr("evolve_results.csv", results_csv if results_csv is not None else combined_csv(gpu, shadow_ms)) + archive.writestr("datakit.json", json.dumps({"format": "datakit", "version": 1})) + for name, columns in tables.items(): + archive.writestr(f"tables/{name}.parquet", _parquet(columns)) + archive.writestr("documents/client_data", json.dumps(client_data)) return buffer.getvalue() @pytest.fixture def two_gpus(tmp_path: Path) -> Path: - (tmp_path / "gen1.zip").write_bytes(make_export([simple_frames(2.0)], gpu="Gen 1 GPU")) - (tmp_path / "gen2.zip").write_bytes(make_export([simple_frames(1.0)], gpu="Gen 2 GPU")) + (tmp_path / "gen1.evolve").write_bytes(make_recording(simple_frames(2.0), gpu="Gen 1 GPU")) + (tmp_path / "gen2.evolve").write_bytes(make_recording(simple_frames(1.0), gpu="Gen 2 GPU")) return tmp_path diff --git a/tests/test_loader.py b/tests/test_loader.py index 04dedf0..f1192cd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,67 +1,52 @@ import io +import json import zipfile import pytest -from conftest import PER_FRAME_HEADER, combined_csv, frame, make_export, scope, simple_frames +from conftest import frame, make_recording, scope, simple_frames -from evolve_charts.categories import workloads -from evolve_charts.loader import ExportError, load_export +from evolve_charts.categories import is_utility, workloads +from evolve_charts.loader import LoadError, load_recording -def test_full_export(): - result = load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "gen1.zip") +def test_full_recording(): + result = load_recording(make_recording(simple_frames(2.0), gpu="Gen 1 GPU"), "gen1.evolve") assert result.gpu == "Gen 1 GPU" - assert result.metadata["Evolve Version"] == "15.10" - assert result.metadata["Rendering Backend"] == "Dx12" - # A score of 0 means Evolve didn't calculate it for this benchmark. - assert result.scores == {"Ray Tracing": 4000, "Rasterization": 5000, "Compute": 6000} - assert result.contents == ["Scores", "Per frame", "Deep analysis"] + assert result.metadata == { + "Evolve Version": "17.0", + "Benchmark": "RayTracingInline", + "System Name": "host", + "CPU": "CPU X", + "GPU": "Gen 1 GPU", + "OS Name": "Windows", + "OS Version": "10.0.26200", + "GPU Driver Version": "1.2.3", + "Rendering Backend": "Dx12", + } + assert result.contents == ["Scores", "Passes"] assert set(result.passes["pass"]) == {"shadow-raytrace-inline", "shadow_filter_pass_0"} assert result.warnings == [] -def test_per_frame_workloads_come_from_the_csv(): - result = load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "gen1.zip") +def test_per_frame_workloads_come_from_the_frames_table(): + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") frame_3 = result.workloads.set_index("frame").loc[3] + assert frame_3["seq_s"] == pytest.approx(1.5) assert frame_3["Raytracing"] == pytest.approx(2.0) assert frame_3["Compute"] == pytest.approx(0.5) - assert "Rasterization" not in result.workloads # an all-empty column + assert "Rasterization" not in result.workloads # no frame has any -def test_export_without_deep_analysis(): - result = load_export(make_export(None, gpu="Gen 1 GPU"), "gen1.zip") - assert result.contents == ["Scores", "Per frame"] +def test_recording_without_pass_timings(): + result = load_recording(make_recording(simple_frames(2.0), with_scopes=False), "x.evolve") + assert result.contents == ["Scores"] assert result.passes is None assert result.frames["gpu_clock_mhz"].iloc[2] == 2502 assert result.frames["board_power_w"].iloc[2] == 200.0 -def test_standalone_csv_exports(): - scores = load_export(b"Loop,Ray Tracing,Compute\n1,4000,6000\n2,4200,0\n", "scores.csv") - assert scores.scores == {"Ray Tracing": 4100, "Compute": 6000} - assert scores.contents == ["Scores"] - assert any("system information" in warning for warning in scores.warnings) - - per_frame = load_export(f"{PER_FRAME_HEADER}\n1,0,0,1000000,,500000,150,2600\n".encode(), "frametimes.csv") - assert per_frame.contents == ["Per frame"] - assert per_frame.workloads["Raytracing"].item() == pytest.approx(1.0) - - combined = load_export(combined_csv("Gen 2 GPU", 1.0).encode(), "evolve_results.csv") - assert combined.gpu == "Gen 2 GPU" - assert combined.contents == ["Scores", "Per frame"] - - -def test_standalone_deep_analysis_derives_workloads(): - data = zipfile.ZipFile(io.BytesIO(make_export([simple_frames(2.0)]))).read("deep_analysis.json") - result = load_export(data, "deep_analysis.json") - assert result.contents == ["Per frame", "Deep analysis"] - frame_3 = result.workloads.set_index("frame").loc[3] - assert frame_3["Raytracing"] == pytest.approx(2.0) - assert frame_3["Compute"] == pytest.approx(0.5) - - def test_uncategorized_scopes_are_left_out_but_counted(): - result = load_export(make_export([simple_frames(2.0)]), "x.zip") + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") assert result.left_out == {"made-up-pass": pytest.approx(0.1 * 20)} assert result.frames["left_out_ms"].tolist() == pytest.approx([0.1] * 20, abs=1e-5) # The GPU frame span still covers all GPU work, including what's left out. @@ -69,14 +54,14 @@ def test_uncategorized_scopes_are_left_out_but_counted(): def test_repeated_scopes_in_a_frame_are_summed(): - passes = load_export(make_export([simple_frames(2.0)]), "x.zip").passes + passes = load_recording(make_recording(simple_frames(2.0)), "x.evolve").passes filter_ms = passes[(passes["pass"] == "shadow_filter_pass_0") & (passes["frame"] == 3)]["ms"] assert filter_ms.item() == pytest.approx(0.5) def test_nested_scopes_count_once_in_totals(): nested = frame(0.0, [{"Build GBuffers": [scope(0, 10)], "shadow_prepare": [scope(2, 4)]}]) - result = load_export(make_export([[nested]]), "x.zip") + result = load_recording(make_recording([nested]), "x.evolve") passes = result.passes.set_index("pass") assert passes.at["shadow_prepare", "ms"] == pytest.approx(2.0) assert passes.at["shadow_prepare", "top_ms"] == 0.0 @@ -86,36 +71,39 @@ def test_nested_scopes_count_once_in_totals(): def test_frame_span_and_overlap_across_command_buffers(): overlapping = frame(0.0, [{"Build GBuffers": [scope(0, 6)]}, {"shadow_prepare": [scope(4, 10)]}]) - result = load_export(make_export([[overlapping]]), "x.zip") + result = load_recording(make_recording([overlapping]), "x.evolve") assert result.frames["gpu_sum_ms"].item() == pytest.approx(12.0) assert result.frames["gpu_span_ms"].item() == pytest.approx(10.0) -def test_metrics_null_and_fan_units(): +def test_each_frame_gets_the_latest_sensor_sample(): frames = [ - frame(0.0, [{"Build GBuffers": [scope(0, 1)]}], None), - frame(1.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_speed": {"Percent": 40}, "edge_temperature_in_c": 60.0}), - frame(2.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_speed": {"Rpm": 1200}, "clock_speed_in_mhz": None}), + frame(0.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_percent": 40, "edge_temperature_c": 60.0}), + frame(1.0, [{"Build GBuffers": [scope(0, 1)]}], None), + frame(2.0, [{"Build GBuffers": [scope(0, 1)]}], {"fan_rpm": 1200, "vram_usage_mib": 4096}), ] - result = load_export(make_export([frames]), "x.zip") - assert result.frames["fan_pct"].tolist()[1] == 40 - assert result.frames["fan_rpm"].tolist()[2] == 1200 - assert result.frames["gpu_clock_mhz"].isna().all() + sensors = load_recording(make_recording(frames), "x.evolve").frames + assert sensors["fan_pct"].tolist()[:2] == [40, 40] + assert sensors["edge_temp_c"].tolist()[1] == 60.0 + assert sensors["fan_rpm"].tolist()[2] == 1200 + assert sensors["vram_used_mb"].tolist()[2] == 4096 + assert sensors["gpu_clock_mhz"].isna().all() -def test_interrupted_multi_loop_run_is_recovered(): - result = load_export(make_export([simple_frames(2.0, 5), simple_frames(2.0, 5)], truncated=True), "x.zip") - assert result.frames["loop"].nunique() == 2 - assert any("interrupted" in warning for warning in result.warnings) +def test_unreadable_inputs(): + with pytest.raises(LoadError, match="isn't an Evolve run file"): + load_recording(b"not a zip", "broken.evolve") + def container(manifest: dict) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("datakit.json", json.dumps(manifest)) + return buffer.getvalue() -def test_unreadable_inputs(): - with pytest.raises(ExportError, match="not a zip"): - load_export(b"{}", "broken.zip") - with pytest.raises(ExportError, match="isn't an Evolve"): - load_export(b"a,b\n1,2\n", "other.csv") - with pytest.raises(ExportError, match="use an Evolve export"): - load_export(b"", "picture.png") + with pytest.raises(LoadError, match="newer Evolve"): + load_recording(container({"format": "datakit", "version": 2}), "future.evolve") + with pytest.raises(LoadError, match="no frames table"): + load_recording(container({"format": "datakit", "version": 1}), "empty.evolve") def test_categories(): @@ -125,11 +113,11 @@ def test_categories(): assert workloads("not-in-evolve") == [] assert workloads("shadow-raytrace-inline") == ["Raytracing"] assert workloads("vol-ris-initial-sampling") == ["Raytracing", "Compute"] + # Renamed passes still count, so runs from older Evolve versions compare. + assert workloads("channel_animation") == ["Compute"] def test_utility_passes(): - from evolve_charts.categories import is_utility - assert is_utility("Copy buffer to buffer") assert is_utility("egui_render") # Uncategorized by Evolve's pass bucketing, but the work of a workload category of their own. diff --git a/tests/test_scores.py b/tests/test_scores.py new file mode 100644 index 0000000..6742406 --- /dev/null +++ b/tests/test_scores.py @@ -0,0 +1,60 @@ +import numpy as np +import pandas as pd +import pytest +from conftest import frame, make_recording, scope, simple_frames + +from evolve_charts import scores +from evolve_charts.loader import load_recording +from evolve_charts.score_catalog import MULTIPLIERS + + +def test_gpu_driver_and_energy_scores(): + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") + desktop = MULTIPLIERS["Desktop"]["RayTracingInline"] + # Average per frame over every frame, the warm-up frame included. + ray_tracing_s = (20 * 2.0 + 50.0) / 20 / 1e3 + assert result.scores["Ray Tracing"] == int(desktop["RayTracing"] / ray_tracing_s) + assert result.scores["Compute"] == int(desktop["Compute"] / 0.5e-3) + assert result.scores["Driver"] == int(desktop["Driver"] / 1e-6) + energy = np.float32(10_600) * np.exp(np.float32(-desktop["Energy"]) * np.float32(200.0)) + assert result.scores["Energy Consumption"] == int(energy) + # In Evolve's order; workloads without GPU time have no score. + assert list(result.scores) == ["Ray Tracing", "Compute", "Driver", "Energy Consumption"] + + +def test_benchmarks_only_have_their_own_scores(): + result = load_recording(make_recording(simple_frames(2.0), variant="Nrc"), "x.evolve") + assert result.scores == {} + + +def test_phone_driver_multiplier_depends_on_the_evolve_version(): + assert scores.multiplier("Driver", "RayTracingInline", "Phone", (16, 1)) == 553.17 + assert scores.multiplier("Driver", "RayTracingInline", "Phone", (17, 0)) == 40.0 + assert scores.multiplier("Driver", "RayTracingInline", "Desktop", (16, 1)) == 11.93 + + +def test_energy_counter_wins_over_sampled_power(): + metrics = pd.DataFrame( + { + "sample_time_ns": [0, 2_000_000_000], + "gpu_energy_j": [100.0, 500.0], + "board_power_w": [999.0, 999.0], + "gpu_power_w": [None, None], + } + ) + assert scores._average_power_w(metrics) == pytest.approx(200.0) + + +def test_image_quality_scores(): + frames = [ + frame(index * 0.5, [{"Fsr execute": [scope(0, 1)]}], flip=0.3156 + index * 0.001, ssim=0.777) + for index in range(20) + ] + result = load_recording(make_recording(frames, variant="Upscaling"), "x.evolve") + # SSIM at its calibrated best scores 10000. + assert result.scores["Image Quality 2"] == pytest.approx(10_000, abs=1) + # FLIP pools the mean with the worst 10% of frames, half and half, between its best and worst. + flip = [0.3156 + index * 0.001 for index in range(20)] + pooled = 0.5 * np.mean(flip) + 0.5 * np.mean(sorted(flip)[-2:]) + assert result.scores["Image Quality 1"] == pytest.approx(10_000 * (0.4255 - pooled) / (0.4255 - 0.3156), abs=1) + assert "Upscaler Performance" in result.scores diff --git a/tests/test_stats.py b/tests/test_stats.py index 2c8722b..d788a18 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,15 +1,15 @@ import pytest -from conftest import make_export, simple_frames +from conftest import make_recording, simple_frames from evolve_charts import stats -from evolve_charts.loader import load_export +from evolve_charts.loader import load_recording @pytest.fixture def results(): return { - "Gen 1": load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "gen1.zip"), - "Gen 2": load_export(make_export([simple_frames(1.0)], gpu="Gen 2 GPU"), "gen2.zip"), + "Gen 1": load_recording(make_recording(simple_frames(2.0), gpu="Gen 1 GPU"), "gen1.evolve"), + "Gen 2": load_recording(make_recording(simple_frames(1.0), gpu="Gen 2 GPU"), "gen2.evolve"), } @@ -42,13 +42,13 @@ def test_passes_of_a_workload_category(results): assert passes.at[("Gen 1", "shadow_filter_pass_0"), "workload"] == "Compute" -def test_results_without_deep_analysis_are_left_out_of_pass_views(): +def test_results_without_pass_timings_are_left_out_of_pass_views(): results = { - "Full": load_export(make_export([simple_frames(2.0)], gpu="Gen 1 GPU"), "a.zip"), - "Scores only": load_export(make_export(None, gpu="Gen 2 GPU"), "b.zip"), + "Full": load_recording(make_recording(simple_frames(2.0), gpu="Gen 1 GPU"), "a.evolve"), + "No passes": load_recording(make_recording(simple_frames(1.0), gpu="Gen 2 GPU", with_scopes=False), "b.evolve"), } assert set(stats.summary(results, 1.0, "pass")["result"]) == {"Full"} - assert set(stats.summary(results, 1.0, "workload")["result"]) == {"Full", "Scores only"} + assert set(stats.summary(results, 1.0, "workload")["result"]) == {"Full", "No passes"} def test_passes_missing_in_some_frames_count_as_zero(results): @@ -62,8 +62,9 @@ def test_passes_missing_in_some_frames_count_as_zero(results): def test_scores_table(results): table = stats.scores_table(results) - assert table.index.tolist() == ["Ray Tracing", "Rasterization", "Compute"] - assert table.at["Compute", "Gen 2"] == 6000 + assert table.index.tolist() == ["Ray Tracing", "Compute", "Driver", "Energy Consumption"] + assert table.at["Compute", "Gen 2"] == table.at["Compute", "Gen 1"] + assert table.at["Ray Tracing", "Gen 2"] > table.at["Ray Tracing", "Gen 1"] def test_comparison_table_and_key_numbers(results): @@ -77,10 +78,10 @@ def test_comparison_table_and_key_numbers(results): ) -def test_timeline_averages_loops_and_smooths(): - result = load_export(make_export([simple_frames(2.0), simple_frames(4.0)]), "x.zip") +def test_timeline_smooths(): + result = load_recording(make_recording(simple_frames(2.0)), "x.evolve") raw = stats.timeline(result, 1.0, "pass", ["shadow-raytrace-inline"], 0.0) - assert raw["shadow-raytrace-inline"].tolist() == pytest.approx([3.0] * len(raw)) + assert raw["shadow-raytrace-inline"].tolist() == pytest.approx([2.0] * len(raw)) smooth = stats.timeline(result, 1.0, "pass", ["shadow-raytrace-inline"], 2.0) assert len(smooth) == len(raw) assert len(stats.timeline(result, 1.0, "workload", ["Raytracing"], 0.0)) > 0 diff --git a/tests/test_ui.py b/tests/test_ui.py index 6a93aca..fd188e6 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -1,6 +1,6 @@ from pathlib import Path -from conftest import make_export +from conftest import make_recording, simple_frames from streamlit.testing.v1 import AppTest ENTRY = str(Path(__file__).resolve().parent.parent / "evolve-charts.py") @@ -20,12 +20,12 @@ def run(monkeypatch, folder) -> AppTest: - monkeypatch.setenv("EVOLVE_CHARTS_EXPORTS", str(folder)) + monkeypatch.setenv("EVOLVE_CHARTS_RESULTS", str(folder)) return AppTest.from_file(ENTRY, default_timeout=60).run() def page(monkeypatch, folder, name: str) -> AppTest: - monkeypatch.setenv("EVOLVE_CHARTS_EXPORTS", str(folder)) + monkeypatch.setenv("EVOLVE_CHARTS_RESULTS", str(folder)) app = AppTest.from_string(PAGE_SCRIPT.format(page=name), default_timeout=60).run() assert not app.exception, name return app @@ -55,16 +55,15 @@ def test_every_page_renders_with_two_results(two_gpus, monkeypatch): assert any("Compute: 0.50 ms per frame on Gen 1 GPU" in markdown.value for markdown in app.markdown) -def test_results_without_deep_analysis(tmp_path, monkeypatch): - (tmp_path / "scores_only.zip").write_bytes(make_export(None, gpu="Gen 1 GPU")) - (tmp_path / "scores.csv").write_text("Loop,Ray Tracing,Compute\n1,4000,6000\n") +def test_results_without_pass_timings(tmp_path, monkeypatch): + (tmp_path / "no_passes.evolve").write_bytes(make_recording(simple_frames(2.0), with_scopes=False)) app = run(monkeypatch, tmp_path) assert not app.exception visit_every_page(monkeypatch, tmp_path) def test_bad_file_shows_error(tmp_path, monkeypatch): - (tmp_path / "broken.zip").write_bytes(b"not a zip") + (tmp_path / "broken.evolve").write_bytes(b"not a zip") app = run(monkeypatch, tmp_path) assert not app.exception - assert "not a zip" in app.error[0].value + assert "isn't an Evolve run file" in app.error[0].value diff --git a/tools/package.py b/tools/package.py index 1ba8e37..4f7a9b0 100644 --- a/tools/package.py +++ b/tools/package.py @@ -24,7 +24,7 @@ "evolve-charts.py", "evolve-charts.py.lock", ".streamlit/config.toml", - "exports/README.txt", + "results/README.txt", ] FOLDERS = ["evolve_charts", "static"] EXECUTABLE = {"Start Evolve Charts.command", "start-evolve-charts.sh"} @@ -42,7 +42,7 @@ The first start downloads Python and the libraries the tool uses (about 150 MB) and needs an internet connection. Keep the window that opens running while you use the tool. -To load results automatically, put Evolve export files in the "exports" folder. +To load results automatically, put .evolve files in the "results" folder. """ diff --git a/tools/sync_categories.py b/tools/sync_categories.py index 064473c..e5cdb31 100644 --- a/tools/sync_categories.py +++ b/tools/sync_categories.py @@ -18,7 +18,7 @@ DEFAULT_SOURCE = REPO_ROOT.parent / "evolve/crates/evolve-frame-bucketing/src/lib.rs" OUTPUT = REPO_ROOT / "evolve_charts/scope_catalog.py" -ARM = re.compile(r"Self::(\w+) => vec!\[(.*?)\n\s*\],", re.DOTALL) +ARM = re.compile(r"Self::(\w+) => vec!\[(.*?)\],", re.DOTALL) NAME = re.compile(r'Self::(\w+) => "([^"]*)",') TOKEN = re.compile(r'^\s*//.*$|get_or_intern_static\(\s*"(?P[^"]+)"\s*\)', re.MULTILINE) @@ -28,8 +28,14 @@ def _impl(source: str, category: str) -> str: return source[start : source.index("fn translated(self)", start)] +def _deprecated(source: str) -> str: + """Pass names Evolve renamed or removed. Evolve still counts them, so older runs stay comparable.""" + start = source.index("fn pipelines_list_deprecated") + return source[start : source.index("impl Category for WorkloadCategory", start)] + + def _lists(body: str) -> dict[str, list[str]]: - lists = body[: body.index("fn name(self)")] + lists = body[: body.index("fn name(self)")] if "fn name(self)" in body else body return { arm.group(1): [token.group("name") for token in TOKEN.finditer(arm.group(2)) if token.group("name")] for arm in ARM.finditer(lists) @@ -44,10 +50,11 @@ def _names(body: str) -> dict[str, str]: def parse(source: str) -> tuple[dict, dict, list[str]]: workloads_body = _impl(source, "WorkloadCategory") workloads: dict[str, list[str]] = {} - for workload, scopes in _lists(workloads_body).items(): - for scope in scopes: - if workload not in workloads.setdefault(scope, []): - workloads[scope].append(workload) + for lists in [_lists(workloads_body), _lists(_deprecated(source))]: + for workload, scopes in lists.items(): + for scope in scopes: + if workload not in workloads.setdefault(scope, []): + workloads[scope].append(workload) uncategorized = _lists(_impl(source, "BucketPassCategory")).get("Uncategorized", []) return workloads, _names(workloads_body), uncategorized @@ -77,7 +84,7 @@ def main() -> None: "}", "", ] - OUTPUT.write_text("\n".join(lines), encoding="utf-8") + OUTPUT.write_text("\n".join(lines), encoding="utf-8", newline="\n") print(f"Wrote {len(workloads)} scope names to {OUTPUT.relative_to(REPO_ROOT)}") diff --git a/tools/sync_scores.py b/tools/sync_scores.py new file mode 100644 index 0000000..3c9b05f --- /dev/null +++ b/tools/sync_scores.py @@ -0,0 +1,131 @@ +"""Regenerate evolve_charts/score_catalog.py from Evolve's score calculation source. + +Usage (from the repository root): + + python tools/sync_scores.py [path/to/evolve] + +`.evolve` files hold the measurements, not the scores: Evolve calculates the scores whenever it reads a +run. This copies what that calculation needs from the Rust source: the calibrated multiplier of every +score per benchmark and platform (`crates/evolve-highscore-api/src/score.rs`), the score names, and which +workload category each GPU score times (`crates/evolve-datakit-scores/src/lib.rs`). +""" + +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_EVOLVE = REPO_ROOT.parent / "evolve" +SCORE_RS = "crates/evolve-highscore-api/src/score.rs" +DATAKIT_SCORES_RS = "crates/evolve-datakit-scores/src/lib.rs" +OUTPUT = REPO_ROOT / "evolve_charts/score_catalog.py" + +COMMENT = re.compile(r"//[^\n]*") +VARIANT_ARM = re.compile(r"((?:BenchmarkVariant::\w+\s*\|?\s*)+)=>") +SCORE_ARM = re.compile(r"((?:Self::\w+\s*\|?\s*)+)=>\s*(return None|[0-9_.]+)") +SOME_ARM = re.compile(r"((?:BenchmarkVariant::\w+\s*\|?\s*)+)=>\s*Some\(([0-9_.]+)\)") +F32_CONST = re.compile(r"const (\w+): f32 = ([0-9_.]+);") + + +def _function(source: str, name: str) -> str: + start = source.index(f"fn {name}(") + following = re.compile(r"\n\s*(?:pub(?:\([\w:]+\))? )?(?:const )?fn ").search(source, start + 1) + return COMMENT.sub("", source[start : following.start() if following else len(source)]) + + +def _number(text: str) -> float: + return float(text.replace("_", "")) + + +def _variants(text: str) -> list[str]: + return re.findall(r"BenchmarkVariant::(\w+)", text) + + +def _multipliers(source: str, function: str) -> dict[str, dict[str, float]]: + """Benchmark variant -> score -> multiplier; a score the variant doesn't have is absent.""" + body = _function(source, function) + arms = list(VARIANT_ARM.finditer(body)) + table: dict[str, dict[str, float]] = {} + for arm, following in zip(arms, [*arms[1:], None]): + text = body[arm.end() : following.start() if following else len(body)] + scores = { + score: _number(value) + for names, value in SCORE_ARM.findall(text) + if value != "return None" + for score in re.findall(r"Self::(\w+)", names) + } + for variant in _variants(arm.group(1)): + table[variant] = scores + if not table: + raise SystemExit(f"Found no multipliers in `{function}`; has {SCORE_RS} changed shape?") + return table + + +def parse(score_rs: str, datakit_scores_rs: str) -> dict: + platforms = { + "Desktop": _multipliers(score_rs, "desktop_score_multipliers"), + "Phone": _multipliers(score_rs, "mobile_score_multipliers"), + } + handheld = _function(score_rs, "handheld_score_multipliers") + if "self.desktop_score_multipliers(variant)" not in handheld: + raise SystemExit("Handheld multipliers are no longer the desktop ones; teach this tool to read them.") + platforms["Handheld"] = platforms["Desktop"] + + enum = score_rs[score_rs.index("pub enum Score {") :] + order = re.findall(r"^\s*(\w+),$", enum[: enum.index("}")], re.MULTILINE) + name_body = _function(score_rs, "name") + names = dict(re.findall(r'Self::(\w+) => "([^"]+)"', name_body)) + + version = re.search(r"DRIVER_EXCLUDES_PRESENT: EvolveVersion = EvolveVersion::new\((\d+), (\d+)\)", score_rs) + phone_driver = { + variant: _number(value) + for variants, value in SOME_ARM.findall(_function(score_rs, "mobile_driver_multiplier_with_present")) + for variant in _variants(variants) + } + energy = re.search(r"\(([0-9_.]+) \* \(-m as f32 \* avg\)\.exp\(\)\)", score_rs) + gpu_scores = re.findall(r"\(\s*WorkloadCategory::(\w+),\s*Score::(\w+),?\s*\)", datakit_scores_rs) + if not (version and energy and gpu_scores and phone_driver): + raise SystemExit("Couldn't find every score constant; has the Rust source changed shape?") + + return { + "SCORE_ORDER": order, + "SCORE_NAMES": {score: names[score] for score in order}, + "GPU_SCORES": dict(gpu_scores), + "MULTIPLIERS": platforms, + "DRIVER_EXCLUDES_PRESENT": (int(version.group(1)), int(version.group(2))), + "PHONE_DRIVER_WITH_PRESENT": phone_driver, + "ENERGY_SCALE": _number(energy.group(1)), + "IMAGE_QUALITY": {name: _number(value) for name, value in F32_CONST.findall(score_rs)}, + } + + +COMMENTS = { + "SCORE_ORDER": "Score variants, in the order Evolve lists them", + "SCORE_NAMES": "Score variant -> the name Evolve shows", + "GPU_SCORES": "WorkloadCategory variant -> the Score of its GPU time", + "MULTIPLIERS": "Platform -> BenchmarkVariant -> Score -> multiplier; a score a benchmark doesn't have is absent", + "DRIVER_EXCLUDES_PRESENT": "Phone Driver scores of runs before this Evolve version use PHONE_DRIVER_WITH_PRESENT", + "PHONE_DRIVER_WITH_PRESENT": "BenchmarkVariant -> phone Driver multiplier from before DRIVER_EXCLUDES_PRESENT", + "ENERGY_SCALE": "Energy score = ENERGY_SCALE * exp(-multiplier * average power in W)", + "IMAGE_QUALITY": "FLIP and SSIM pooling and normalization constants", +} + + +def main() -> None: + evolve = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_EVOLVE + catalog = parse( + (evolve / SCORE_RS).read_text(encoding="utf-8"), (evolve / DATAKIT_SCORES_RS).read_text(encoding="utf-8") + ) + lines = [ + "# Generated by tools/sync_scores.py from evolve/crates/evolve-highscore-api/src/score.rs", + "# and evolve/crates/evolve-datakit-scores/src/lib.rs. Do not edit by hand.", + "", + ] + for name, value in catalog.items(): + lines += [f"# {COMMENTS[name]}", f"{name} = {value!r}", ""] + OUTPUT.write_text("\n".join(lines), encoding="utf-8", newline="\n") + print(f"Wrote {sum(len(v) for v in catalog['MULTIPLIERS'].values())} benchmark multiplier sets to {OUTPUT.name}") + + +if __name__ == "__main__": + main()