diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d5eaec8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,59 @@ +name: tests + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + python: + name: Python ${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Tests and doctests, from the source + run: python -m unittest discover -s tests -t . -v + - name: The example builds, checks and has a page + run: | + PYTHONPATH=src python -m sentier_diagnostics example --out build/example + PYTHONPATH=src python -m sentier_diagnostics check build/example/diagnostics.json build/example/p01-fix-1.json + PYTHONPATH=src python -m sentier_diagnostics check examples/diagnostics.json examples/p01-fix-1.json + PYTHONPATH=src python -m sentier_diagnostics summary examples/diagnostics.json + + package: + name: The package, installed + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Build, install into a fresh environment, run from it + run: | + uv build --out-dir dist + uv venv --python 3.12 /tmp/installed + uv pip install --python /tmp/installed/bin/python dist/*.whl + /tmp/installed/bin/sentier-diagnostics --version + /tmp/installed/bin/sentier-diagnostics example --out /tmp/example + /tmp/installed/bin/sentier-diagnostics check /tmp/example/diagnostics.json + test -f /tmp/example/page/index.html + /tmp/installed/bin/python -m unittest discover -s tests -t . + + page: + name: The page script + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Acceptance cases against the script + run: node --test tests/js/page.test.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..962bcdb --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +build/ +dist/ +__pycache__/ +*.egg-info/ +.venv/ +.claude/launch.json diff --git a/GLOSSARY.md b/GLOSSARY.md new file mode 100644 index 0000000..6043bc0 --- /dev/null +++ b/GLOSSARY.md @@ -0,0 +1,31 @@ +# Glossary + +The words this repository uses, each in one line. [SPEC.md](SPEC.md) says the rest. + +- **Item**: one thing that has results, such as a product or a dataset. Its file calls it by its own word (`item.one`). +- **Indicator**: one impact category, named by an exact id. +- **Reference unit**: the unit an item's result is per (kg, MJ, machine hour). Floors are kept apart per unit. +- **State**: one set of results. The first may be the published results; the others are builds. +- **Step**: from one state to the next, with its kind, claim, tolerance and noise. +- **Step 0**: the reproduction of the published results. The start, never the target. A chain that does not start from the published results has no step 0: its steps are numbered from 1. +- **Chain**: the states in order and the steps between them. +- **Pair**: one item in one indicator, before and after a step. +- **Class**: what a pair is, by one rule: agrees, within noise, differs, beyond 10x, zero reference, zero result, sign change, near zero, missing. +- **Moved**: a pair of class differs, beyond 10x, zero reference, zero result or sign change. Only these have a direction. +- **Direction**: up where the result after is larger as a signed number, down otherwise. +- **Band**: the direction and the factor between the two sizes of a pair that differs. +- **Floor**: the size below which a result counts as near zero; one per indicator and reference unit. +- **Tolerance**: how far two results may differ, relative to the larger, and still agree. +- **Noise**: how far two builds of one state differ, measured. +- **Size filter**: a reader's choice on the page that hides small changes. It changes no count. +- **Claim**: what a step says it moves; its **scope entries** say which indicators and items, which way, how far, and whether they **must** or **may** move. +- **Verdict**: how a pair of a step holds against the step's claim. +- **Attribution**: the split of an item's whole difference into the steps' differences. +- **Explanation**: text about a step, an indicator or an item, with its **origin** (person, code or model) and its **finding** (a reason, or none found). It changes no count. +- **Breakdown**: what one item's result is made of, before and after one step. +- **Tier**: whether a file lists items (`items`) or holds counts only (`counts`). +- **Audience**: whether a file's writer says it may be published (`public`) or not (`local`). +- **Listed items**: the items a file holds, the worst first, up to `listed`. +- **Matrix**: every item of a local file, listed or not, in a second file the page loads after it has drawn. +- **Body**: the page as one element a site holds in its own page; its style stays inside it. +- **Drawer**: the panel of one item, at the right of the page, opened from a row. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..572ac5c --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,10 @@ +# Maintainers + +| Role | Who | +|---|---| +| Maintainer | Manuel Klarmann | +| Second maintainer | Laurenz Bougan (GitHub: lbougan), named by Manuel Klarmann on 16 November 2026 | + +The tool must be possible to run, review and release without any one person +([docs/runbook.md](docs/runbook.md)). Changes go through a pull request that someone other +than its author reviews and merges. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..4c954bb --- /dev/null +++ b/NOTICE @@ -0,0 +1,34 @@ +sentier-diagnostics +Copyright (c) 2026 Départ de Sentier. MIT licence: see LICENSE. + +Parts of this package are adapted from sentier-bafu +(https://github.com/sentier-dev/sentier-bafu): the classes of a pair of results, the +claims and verdicts, the bands and the floor of `scripts/runrecord/before_after.py` and +`scripts/runrecord/review.py`, and the drawing of a value in `site/assets/before-after.js`, +as they stood at commit 9460128 (the same at a6a8df5). They were rewritten for the format sentier-diagnostics/1; +no file was copied whole. Their notice: + + MIT License + + Copyright (c) 2026 sentier.dev + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +Nothing is taken from the private repositories of the backtest dashboards. Their +defects are described in SPEC.md, in words, as acceptance cases. diff --git a/README.md b/README.md index e1dfb7a..e08f4da 100644 --- a/README.md +++ b/README.md @@ -3,20 +3,59 @@ What changed, before and after, in the impact assessment results of a life cycle inventory database, and why. -This repository will hold one open tool for the Sentier projects (sentier-bafu, -sentier-bafu-agribalyse, sentier-agribalyse and others): a file format for results -compared before and after a change, one rule for counting what moved, the attribution -of a difference to the steps that made it, and a static page that shows it. +One open tool for the Sentier projects (sentier-bafu, sentier-bafu-agribalyse, +sentier-agribalyse and others): -The published results of a database are checked once, at the start: a faithful import -reproduces them. Every later step (a fix, a contribution, a changed mapping) is compared -with the step before it and with what it claimed to move. The page shows the whole way -from the published results to the current build, and what no step explains. +- a file format, `sentier-diagnostics/1`, and a strict validator; +- one rule for counting what moved, with named classes for zeros, sign changes, near-zero + and missing results, so that nothing is blanked, zeroed or called agreement; +- a chain of steps from the published results to the current build, each held against + what it claimed to move, with the whole difference split between the steps (exactly, up + to the rounding the page shows) and what nothing explains named; +- a page that shows it: a frameless body a site holds in its own page, with a style + scoped to it that takes the site's tokens, or a page of its own with no build step, no + server and no network; a local file can carry a matrix of every item, which the page + loads after it has drawn. -No data is held here: no inventory, no characterisation factors, no results of a -licensed database. Scoring and the readers of each database stay in the projects that -use this tool. +The published results are checked once, at the start (step 0). Every later step is +compared with the step before it. -Status: being specified. Not ready for use. +No data is held here: no inventory, no characterisation factor, no result of a licensed +database. Scoring and the readers of each database stay in the projects that use this. + +## Try it + +```bash +PYTHONPATH=src python3 -m sentier_diagnostics example --out build/example +``` + +Then open `build/example/page/index.html`, or `build/example/local/page/index.html` for a +local file whose page loads the matrix of every item. The example is made up. + +In a site's own page: + +```python +from sentier_diagnostics import page + +html = page.body(data, title=False) # one
, the file inside +assets = page.assets() # diagnostics.js and diagnostics.css, to copy +``` + +Install it in a project, pinned to a tag: + +```bash +uv add "sentier-diagnostics @ git+https://github.com/sentier-dev/sentier-diagnostics@v0.1.0" +``` + +Python 3.10 or later, standard library only. + +## Read more + +- [SPEC.md](SPEC.md): the questions, the format, the rule, the views, the acceptance cases. +- [docs/runbook.md](docs/runbook.md): tests, releases, reviews, for a maintainer alone. +- [docs/migration.md](docs/migration.md): how the existing dashboards move here and retire. +- [GLOSSARY.md](GLOSSARY.md), [MAINTAINERS.md](MAINTAINERS.md), [NOTICE](NOTICE). + +Status: first version, under review. The tag `v0.1.0` does not exist yet. Licence: MIT, see [LICENSE](LICENSE). diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..20d5d03 --- /dev/null +++ b/SPEC.md @@ -0,0 +1,774 @@ +# sentier-diagnostics: specification + +Version 1 of the format, `sentier-diagnostics/1`. Written 16 November 2026. This file says +what the tool answers, what its files hold, how it counts and what it shows. The code in +`src/sentier_diagnostics/` follows it, and every rule here has a test (see +[Acceptance cases](#11-acceptance-cases)). Where the code and this text disagree, one of +them is wrong: open an issue. + +Contents: 1 Purpose · 2 Questions · 3 The chain · 4 The format · 5 The counting rule · +6 Claims and verdicts · 7 Attribution · 8 Explanations · 9 Breakdowns · 10 Views · +11 Acceptance cases · 12 Out of scope · 13 Decision: seed or rewrite · 14 What the +projects asked for · 15 Open questions + +## 1. Purpose + +One open tool, for every Sentier project, that answers: **what changed in the impact +assessment results of a database, from its published results to the current build, and +why.** + +It replaces three tools that answered parts of this question and disagreed with each +other: + +- the backtest dashboards by Laurenz Bougan: one React page per database, five copies that + have drifted apart, one of them (ADEME) deployed behind a login; +- the before/after page of sentier-bafu, versions 1 and 2; +- the trial dashboard of sentier-bafu-agribalyse. + +Three rules from Manuel Klarmann (16 November 2026) frame it: + +1. Reproducing a database's published results is **step 0**. It is checked once, at the + start. It is never the target of a later step. +2. The final view is **a chain of steps** from the published results to the current + build. Each step's share of the whole difference is named, and so is what nothing + explains. The split is exact (up to the rounding of floating point, which the page + shows: section 7). +3. The tool must be possible to trial, establish and maintain **without Laurenz**. The + code is open; **no data goes into this repository**. + +## 2. The questions it answers + +| Question | Where the answer is | +|---|---| +| Does our computation reproduce the published results? | Step 0: its counts against its tolerance, the published rounding. What moves beyond it is the residual the chain starts from (section 3) | +| What did each later step move? | Each step's counts by class, direction and band | +| Did each step move what it said it would, and only that? | Each step's verdicts against its claim | +| Which step carries how much of the whole difference? | Attribution: the steps' differences add up to the whole, per item, up to rounding | +| What does no step explain? | Steps marked `explained: false`, and the count of items they moved | +| Which items are worst, and why? | The worst-N list, the grid, the item view with its breakdown and explanations | +| What could not be judged? | Missing results (counted by side), near-zero pairs, items not attributable | + +It does not say whether a result is right. Coming closer to the published results after +step 0 is counted as movement, never as better (see [A22](#11-acceptance-cases)). + +## 3. The chain + +A **state** is one set of results: an item's result per indicator. The first state may be +the published results (`kind: "published"`); every other state is a build (`kind: "build"`). +A state's `identity` names what makes it: commit, release, machine, and the solver, since +two solvers on one package differ by about 3e-7. + +A **step** goes from one state to the next. Its kind is free text that the writer +chooses (`reproduction`, `fix`, `mapping`, `contribution`, `rebuild`, ...). A step has: + +- a **claim** (optional): what it says it moves (section 6); +- a **tolerance**: how far two results may differ and still agree; +- a **noise** (optional): how far two builds of one state differ, measured; +- `explained`: `false` for a step that no cause explains (a rebuild on another machine, + a change nobody can name). Its moves are counted as what nothing explains. + +**Step 0 needs no claim.** Its tolerance (the published rounding, or a declared one) does +the judging, and what moves beyond it is the residual of the reproduction: the distance the +chain starts from, not a regression. The page and the summary say so. A claim "nothing +moves" is allowed, but it adds nothing: every value of the residual then reads "not claimed". + +A plain before/after comparison is a chain of two states and one step. The first real one +will be sentier-bafu's **located factors** (a flow that names a country takes that +country's factor), with the claim "water use and land use may move; nothing else moves": +one scope entry, water use and land use, every item, `may`. Any other indicator that moves +is "not claimed". The claim was first written for water use alone; sentier-bafu's first +run of this tool on its own results (16 November 2026) showed land use moving too, for 5 +datasets, read as "not claimed". The tool found what the claim left out, which is its job. +sentier-bafu then took the list from its build, not by hand: its install report names the +methods that actually take a country's own factor (`coverage.located_by_method`), and on +BAFU:2026 v1 those are water use and land use. Its file builds the scope from that list, +so the claim stays right when the factors change. `tests/fixtures/located-factors/` is +that step, made up, and a test holds what it shows. + +**Order matters.** Where two steps touch the same results, each one's share depends on the +order they were applied in. The file keeps the order; the page says so. + +**One chain per reference.** A database with two references (sentier-bafu-agribalyse has +ADEME's published results on ecoinvent and its own build on ecoinvent) gets one file per +reference. A joint view is an open question (section 15). + +## 4. The format + +### 4.1 Versions + +A file says `"format": "sentier-diagnostics/1"`. The reader is strict: it refuses any key +it does not know and any other format, and says which format it read +(`format 'sentier-diagnostics/2': this reader knows sentier-diagnostics/1 only`). So every +change to the keys, and every change to what a class means, is a new major version. There +are no minor versions. The package version (`0.1.0`) is separate; a later package may read +several majors. + +Files of `sentier/before-after/1` and `/2` (sentier-bafu) are not read. Section 13 says +why, and [docs/migration.md](docs/migration.md) says what happens to them. + +### 4.2 The file + +One JSON object. Numbers are finite: `NaN` and `Infinity` are refused when writing and +when reading. Results are written as computed, never rounded. All text is checked: no +control characters, bounded lengths. + +| Key | What it holds | +|---|---| +| `format` | `"sentier-diagnostics/1"` | +| `tier` | `"items"`: the file lists items with their results. `"counts"`: counts only, no item anywhere (no item list, no item in a claim, no explanation of an item) | +| `audience` | `"local"` (the default) or `"public"`: whether its writer says it may be published. The tool checks only that it is one of the two; a site's own check decides what it accepts. A page of a local file says so at its top | +| `title`, `about` | Short text; `about` may be null | +| `item` | `{"one": "product", "many": "products"}`: the word for an item | +| `indicator` | Optional. `{"one": "impact category", "many": "impact categories"}`: the word for an indicator, which the page and the summary use; null or absent for "indicator" | +| `indicators` | `[{id, name, short, unit}]`. An indicator is matched by its exact id, never by a prefix | +| `rule` | The floors and the tolerance of the whole way (section 5) | +| `states` | `[{id, label, about, kind, identity, join}]` in order. `identity` is a short table of texts (commit, release, machine); `join` is what the reader counted when it matched its rows to items: `{compared, unmatched, ambiguous, note}`, or null | +| `steps` | `[{id, from, to, label, about, kind, explained, link, claim, tolerance, noise}]`, one fewer than the states, each from one state to the next | +| `counts` | `{steps: {step: {indicator: block}}, total: {indicator: block}}` | +| `attribution` | `{indicator: {dominant: {step: n}, not_attributable: n, spread: n, unexplained_moved: n}}` (section 7) | +| `items` | The listed items, worst first (section 10.4) | +| `items_count` | `{all, qualifying, listed, not_listed}` | +| `explanations` | Section 8 | +| `notes` | Up to 20 texts | +| `made_by` | `"sentier-diagnostics 0.1.0"` | +| `matrix` | Optional. `{"file": name, "items": n}`: the full matrix of every item, a second file beside this one (section 4.5); null or absent for none. Only a local file of items names one, and `items` is `items_count.all` | + +A **block** counts one indicator in one step, or over the whole way: + +```json +{"pairs": 2847, + "classes": {"agrees": 2790, "within noise": 3, "differs": 40, "beyond 10x": 1, + "zero reference": 2, "zero result": 0, "sign change": 1, "near zero": 6, "missing": 4}, + "missing": {"before": 1, "after": 3, "both": 0}, + "directions": {"up": 30, "down": 14}, + "bands": {"down, factor over 10": 0, "...": 0, "up, factor over 10": 1}, + "verdicts": {"as claimed": 35, "...": 0} , + "closer": 12, "further": 9} +``` + +`verdicts` is null for a step without a claim and for the whole way. `closer` and +`further` count, for a step after step 0, the moved values that came closer to or went +further from the published results. They are null for step 0 and the whole way, null for +every step of a chain whose first state is not the published results (there is nothing to +come closer to), and no page shows them as better or worse. + +An **item** holds `id, name, place, group, unit` (its reference unit), `values` (per +indicator, one result per state, `null` where missing), and per indicator one class, +direction and band per step plus one for the whole way, the verdicts per step (or null +when no step has a claim), and `breakdowns` (`{step or "total": file name}`, or null). + +### 4.3 The validator + +`sentier_diagnostics.fileformat.errors(data)` returns every fault it finds, and nothing for +a file a page can show. `sentier-diagnostics check FILE ...` runs it and exits 1 on any +fault. Besides the shape, it **works every listed value out again**: class, direction and +band from the values with the file's own floor and tolerance, and the verdict from the +claim where the claim's list is in the file. It checks that counts add up (classes to +pairs, bands to what differs, up and down to what moved, missing by side to missing, +verdicts to pairs), that the items are listed worst first and that `items_count` says how +many were left out. A file that disagrees with its own rule is refused, not shown. + +It also **works the counts out again from the listed items.** Only a pair worth a look +adds to the classes that moved, to missing (by side), to directions, bands, the verdicts +"as claimed", "off claim", "not claimed" and "claimed, not moved", to `closer` and +`further`, and to the attribution; and every item with such a pair is worth a look. So +where nothing worth a look is left out (`not_listed` 0), these counts must equal what the +listed items hold, and where some are left out, no count may be smaller than what the +listed items hold. What only quiet pairs add to (agrees, within noise, near zero, and the +verdicts of those) is held by the sums alone: a file that invents agreeing items it does +not list cannot be caught from the file. Further: a claim whose lists are all in the file +has the size of their union as its `reach`, `"*"` entries aside; the floors come from the first state, +and a floor's basis agrees with its count (`median` from results, `none` is 0 from none, +`declared` from none); `closer` and `further` are counted for a step after step 0 from +published results and are null everywhere else. + +### 4.4 Limits + +A page holds the whole file: 12 MB at most, which the writer reaches by listing fewer +items (`listed`, 200 by default). Up to 100 indicators, 30 states, 5,000 listed items, +500 explanations and 20 notes; a claim has up to 50 scope entries, and an entry may list as +many items as a file may (5,000); a state's identity has up to 12 entries. Ids are up to +100 characters, labels 80, texts 600 and an explanation's text 2,000. The limits are in +`fileformat.py`, and a refusal names the limit it met. + +### 4.5 The matrix of every item + +A file lists the items worth a look, up to `listed`. A project looking at its own build +wants every item: `chain.build_with_matrix` writes them in a second file of format +`sentier-diagnostics-matrix/1`, which the page loads after it has drawn. Only a local file +of items has one; a public file keeps its listed items only. A reader that writes CSV tables +names it in its description (`"matrix": "matrix.json"`), and `sentier-diagnostics build` +writes it beside the file. + +```json +{"format": "sentier-diagnostics-matrix/1", + "of": {"title": "...", "states": ["a", "b", "c"], "indicators": ["x:climate", "x:water"], "steps": ["s0", "s1"]}, + "items": [["p01", "Wheat", "FR", "crops", "kg"], ...], + "values": [[0.52, 0.53, 0.47, 1.2, 1.2, null], ...], + "classes": ["=Dd===", ...], + "verdicts": ["a.==", ...]} +``` + +One row per item, in the order of the listed items (worst first): `[id, name, place, +group, unit]`; the results per indicator and state, indicator after indicator; per +indicator one code per step and one for the whole way; and, where a step has a claim, per +indicator one verdict code per step (`.` for a step without a claim; `verdicts` is null when +no step has one). The class codes are `=` agrees, `~` within noise, `0` near zero, `-` +missing, and for a pair that moved its class in upper case up and lower case down: `D` +differs, `B` beyond 10x, `R` zero reference, `T` zero result, `S` sign change. The verdict +codes are `a` as claimed, `o` off claim, `n` not claimed, `c` claimed, not moved, `~` within +noise, `=` unchanged, `?` not judged. A matrix holds no band: it follows from the results. + +`matrix.errors(m)` checks the shape and never raises. `matrix.against(m, data)` works every +code out again with the file's floors, tolerances and noise (and every verdict where the +claim's lists are in the file), and holds the file to it: with every item at hand, its +counts, attribution, `items_count.all` and `qualifying` must be what the matrix holds, +exactly, and every listed item must be in the matrix with the same name and results. +`sentier-diagnostics check` does both when the matrix lies beside its file. A matrix holds +up to 200,000 items and 64 MB. + +## 5. The counting rule + +### 5.1 One class per pair + +A **pair** is one item in one indicator, before and after a step. It gets exactly one +class. The classes are tried in this order; the first that holds is the class: + +| Class | Holds when | Moved? | +|---|---|---| +| missing | a side has no result. Never read as zero | no | +| near zero | both sides lie within the floor | no | +| agrees | they share a sign and differ by no more than the tolerance, relative to the larger size | no | +| within noise | they share a sign and differ by more, but by no more than the step's measured noise | no | +| zero reference | the side before lies within the floor, the side after does not | yes | +| zero result | the side after lies within the floor, the side before does not | yes | +| sign change | both lie beyond the floor and have different signs | yes | +| beyond 10x | one size is more than ten times the other (exactly ten is not beyond) | yes | +| differs | any other difference | yes | + +Agreement is tried before the floor's own classes, so a pair that lies on both sides of the +floor and agrees (0.010000001 and 0.009999999 at a floor of 0.01) agrees: crossing the floor +by solver noise is not a change (A26). A pair beyond the floor on both sides is classed as +before; only a pair that crosses the floor by no more than the tolerance or the noise is +affected. + +No class blanks a value, sets it to zero or calls it agreement. Both results are always +kept. A ratio is taken only where both sides lie beyond the floor and share a sign. How far +two sizes lie apart is the |log10| of their factor; where the factor is too large or too +small for a float (1e-300 and 1e300), the difference of their logarithms, in Python and on +the page alike (A27). + +### 5.2 The floor + +One floor per **indicator and reference unit**: `floor_factor` (0.01) times the median +size of the non-zero results of the first state in that group. Results per kilogram and +results per machine hour are never pooled. A writer may declare floors instead; the file +says for each floor its value, its basis (`median`, `declared`, or `none` where no result +was there to take one from) and how many results it was taken from. + +Why the median and not the largest value: land use in AGRIBALYSE has a median about +4e-12 of its largest result, so a floor relative to the largest hides almost every item. +Why not a floor per value from the gross sum of its terms (sentier-agribalyse's proposal, +`max(|expected|, 1e-6 x gross)`): it needs the terms, which only a breakdown holds, and a +breakdown is often local. A project that has them can declare its floors. See section 15. + +### 5.3 Tolerance and noise + +The **tolerance** says when two results agree. Default 1e-5, relative to the larger size: +two runners differ by up to 2.8e-7 (measured in sentier-bafu) and two solvers by about +3e-7 (sentier-agribalyse), so differences of 1e-6 agree. Step 0 against published results takes +the rounding of what was published (`basis: "published rounding"`, e.g. 5e-4 for four +significant digits). The whole way is counted with the largest tolerance of its steps. + +The **noise** is measured, never assumed: the largest relative difference between two +builds of one state, per indicator, over the items whose results lie beyond the floor in +both builds (the floors of the chain's first state). A pair within the floor is left out: +a residue of 4e-16 in one build and 0 in the other would otherwise make the noise 1.0 and +put every move below a factor of 2 "within noise" (found by sentier-bafu: it hid a +correction by a factor of 580). An indicator with no such pair has no noise in the file, +and the page and the summary say "not measured" for it, never "identical". A noise above +1 % (`chain.LARGE_NOISE`) is called large on the page and in the summary, which give its +size. A step with measured noise has a class "within noise" for differences no larger than +it. Two words, two jobs: +tolerance (agreement by rule) and noise (measured instability). A third, the reader's +**size filter**, hides small changes on the page and changes no count (section 10.5). + +One number per indicator is coarse: a few unstable items make every smaller difference +of that indicator "within noise". Two builds that are identical are a finding, shown as +"identical", and never as "not measured". Large noise is a defect to fix (sentier-agribalyse's +random identifiers made water use differ by up to 21 % between two builds), not a width +to absorb. Noise per item is in section 15. + +### 5.4 Direction, bands and colour + +**Direction** is up where the result after is larger as a signed number, down otherwise, +and only for a pair that moved. Two results below zero that go further below zero went +**down**, although the second is larger in size. + +**Bands** split a pair that differs or is beyond 10x by its direction and by the factor +between its two sizes: 1-1.1, 1.1-1.25, 1.25-2, 2-10, over 10. Ten bands, the same +distance from 1 on both sides, so that half and double look alike. + +**Colour follows the direction and nothing else**: up blue, down orange (a pair readable +with the common colour-vision deficiencies). The strength of the tint follows the band. A +pair without a ratio (zero reference, zero result, sign change) is hatched. Within noise +is grey and hatched, near zero is grey, missing is red italic text. + +## 6. Claims and verdicts + +A step may say what it moves. A claim is text, who wrote it (section 8), and a scope: + +```json +{"text": "Wheat and rice go down in climate change, by 30 % at most.", + "by": {"kind": "person", "name": "A. Author", "model": null, "run_by": null}, + "reach": 2, + "scope": [{"indicators": ["x:climate"], "items": ["p01", "p05"], + "direction": "down", "largest": 0.3, "expect": "must"}]} +``` + +- `indicators`: `"*"` or a list of ids. `items`: `"*"`, a list, or null where the list + stays with its writer. Every listed id is an item of the chain: `chain.build` refuses a + claim that names an item no state holds (a typo would otherwise be a claim about nothing). +- `reach`: how many items the claim's lists name together, `"*"` entries aside. Where every + list is in the file the validator checks it; where a list stays private it is the only + trace of that list. It is null where no entry lists items. +- `direction`: `up`, `down` or `any`. `largest`: the largest change relative to the result + before, or null. +- `expect`: `must` (each covered pair must move) or `may` (the default: it may move). + Only a `must` entry can be unmet. A scope a program worked out (everything downstream of + a change) says what may move. +- A claim with no entry says that nothing moves. +- A pair whose result before lies within the floor (a zero reference) has no size for a + relative `largest` to be relative to: it is off claim under an entry with `largest`, and + an entry without `largest` judges it by direction alone. + +Each pair of a step with a claim gets one verdict. They are tried in this order; the first +that holds is the verdict: + +| Verdict | When | +|---|---| +| not judged | it is missing: nothing can be said of it | +| claimed, not moved | a `must` entry covers it, and it agrees, is within noise or is near zero: it did not move beyond the tolerance, the noise or the floor | +| unchanged | it agrees | +| within noise | it differs by no more than the noise | +| not judged | it is near zero | +| not claimed | it moved, and no entry covers it | +| as claimed | it moved, and an entry it is held to fits its direction and size | +| off claim | it moved, and no entry it is held to fits | + +A pair that moved is held to the `must` entries that cover it, and only where none does to +the `may` entries. So a `may any` entry beside a `must down` entry cannot make a move up +"as claimed": the `must` entry says which way the pair must go. + +A claim's list may stay private (`claim_lists=False`): the verdicts are worked out by the +writer, kept in the file and marked on the page as not checkable from the file. + +## 7. Attribution + +For an item with a result in every state, the whole difference is the sum of the steps' +differences: `last − first = Σ (state k+1 − state k)`. The split is exact by construction, +up to the rounding of floating point; it needs one build per state, which is what a chain +is. Per indicator the file counts, among the items that moved over the whole way: + +- `dominant`: for each step, the items whose largest difference comes from that step, + among the steps in which the item moved. A step in which it agrees, is within noise or + is near zero carries none of it, however large its difference. Where two steps move it + by the same amount, the earlier one carries it; +- `spread`: items that moved over the whole way while no single step moved them: each + step stayed within its tolerance, its noise or the floor, and only together do they + move it. No step carries such an item; +- `not_attributable`: items that have no result in some state between. Their difference + cannot be split, and the file says how many; +- `unexplained_moved`: items that moved in a step marked `explained: false`, counted + whether or not they moved over the whole way. + +So `dominant` over the steps, `spread` and `not_attributable` add up to the items that +moved over the whole way. + +The item view shows each step's share of the item's whole difference, signed, the share of +the steps nothing explains, and the **sum of the shares it shows**: in floating point that +sum may differ from 100 % in its last digits, and the page prints what it computed instead +of a fixed 100 %. Where the steps' differences, taken by size, add up to more than ten +times the whole difference, the steps **cancel**: their shares would be large, of both +signs, and partly rounding. The page then prints no shares and says how many times the +whole difference the steps add up to. It also says why it prints none for an item that +did not move, that is missing in a state between, or that is missing at either end. + +A finer split inside a step (by link, by flow) is a breakdown (section 9). sentier-bafu- +agribalyse can split step 2 by link exactly because LCA is linear, but the ecoinvent unit +scores it needs are licensed, so that split stays local. + +## 8. Explanations + +An explanation is text about a step, an indicator, an item, or a combination, with: + +- `by`, required: who wrote it. `{"kind": "person", "name": ...}`, `{"kind": "code", + "name": ...}` (a computed reason), or `{"kind": "model", "model": ..., "run_by": ...}` + (generated text: the model and who ran it are both required). The page labels each one + by its origin, and a model's text reads "A claim, not checked by a person". Nothing + generated is shown as a fact. +- `finding`: `"reason"` when it gives one, `"none found"` when someone looked and found + none. An item with no explanation has none given. These are three different states, and + the page keeps them apart. +- `source` (`{link, doc}`) and `not_checked` (what the writer did not check), both optional. + +An explanation is a claim: it never changes a count, a class or a verdict. A file carries +no status, no "accepted by" and no approval of any kind (sentier-bafu-agribalyse ADR 0010): +the validator refuses such keys, and refuses an entry of a state's `identity` whose name +says status, accepted or approved, since an identity holds free names. + +## 9. Breakdowns + +A breakdown says what one item's result is made of, before and after one step: format +`sentier-diagnostics-breakdown/1`, one file per item and step, named in the item's +`breakdowns`. Per indicator: `total` (before, after), `parts` (label, kind, before, +after), `rest` and `parts_not_listed`. + +- Parts are chosen by the **larger of their two sides**, so a part that appears only after + the step is listed. +- What is not listed is summed into `rest`, and the count is written: never cut short + silently. The page says what share of the total the first N parts make, on both sides. +- Parts and rest add up to the total on both sides (to 1e-9 of the size), and the totals + are the item's results in the two states. +- A breakdown is of the item that names it and of the step (or the whole way) it is named + for. `check` refuses one named by another item or for another step, saying which item + and step named it and which it is of; the page does not draw it under the item that + named it. +- `check` refuses a breakdown that disagrees with its diagnostics file, and says so when a + named breakdown is not beside it. No fallback. +- Parts keep their sign; bars are drawn from the signed value. + +Whether a breakdown may be published is its writer's decision. sentier-bafu publishes none; +sentier-agribalyse's sums over ecoinvent processes and may not be published. + +## 10. Views + +**One body for a site and for a page of its own.** `page.body(data)` is the diagnostics as +one element, `
`, with the file +(and its breakdowns) inside it as JSON: no ``, `` or ``, and no id, so a +site may hold it anywhere, even twice. The site loads `diagnostics.js` and +`diagnostics.css` from `page.assets()`; the script mounts into every such element (a site +that draws it later marks it `="manual"` and calls `SentierDiagnostics.mount(element, +{data})`). `data-sd-base` names where the files it loads lie, and `title=False` leaves the +heading to the site. The script sets the document's title only on a page of its own. + +**The style is scoped.** Every rule of `diagnostics.css` is under `.sentier-diagnostics`; +it sets no font, nothing on `:root`, `html` or `body`, and resets inside its root what a +site does to bare tables (borders, phone layouts that turn rows into blocks). Its colours +are `--sd-*` properties that take the site's tokens where the site sets them +(`--sd-ink: var(--ink, #1d232a)`, and so on for `--muted`, `--surface`, `--paper-deep`, +`--line`, `--line-strong`, `--index`, `--fail`, `--radius`, `--t-*`, `--s*`); up, down and +noise keep colours of their own (amber up, teal down), so that a direction never takes a +site's accent. A test reads the sheet and holds this. + +**A host lets the body be wide.** The matrix needs the page's width, and a site that caps +the children of an article at a reading measure (`.prose > * { max-width: 44rem }`) caps +the body with them, which leaves a 25-indicator grid a few columns wide. A host lifts that +cap for the body, as sentier-bafu does: +`.prose > .sentier-diagnostics { max-width: none }` (and the same for any wrapper of the +page that holds a measure). + +`sentier-diagnostics page FILE --out FOLDER` writes a page of its own: `index.html` with +the body in it, the two assets, and `standalone.css`, which sets the tokens and font a site +would. It opens from disk; no build step, no server, no network, no font or script from +elsewhere. Where the file names a matrix that lies beside it, the page writes it as +`matrix.js`, which loads from disk too; a site serving `matrix.json` needs nothing else, as +the page fetches it over HTTP. Served over HTTP the page can also read `?file=...`, and it +opens a file a reader chooses (read in the browser, sent nowhere). **The page works out no +class, no direction and no count of its own.** It reads them from the file, which the +validator has checked, and only adds the file's counts up: the classes that moved into +"moved", and a step's counts over its indicators. The one number it works out itself is +the median and percentiles of a distribution (10.3), and only from every item. A file read by `?file=` or chosen by a +reader has not passed through the validator: the page checks its shape only, says so above +everything it draws, and refuses a file it cannot draw instead of drawing part of it. A +matrix is held to the file it says it is of (title, states, indicators, steps, number of +items, the length of every row) before a row of it is drawn; its codes are checked by +`sentier-diagnostics check`, not again in the browser. Whatever the file says, only an +`https://` address becomes a link. + +The text a page renders comes from the file or from words chosen to be neutral: it avoids +stage, triage, ledger, layer, upstream, provider and candidate (a test holds this). + +**The page is a matrix page.** From the top: the header (10.1); a row of chips that +chooses the comparison, the whole way first and then each step with its number, the values +that moved in it, a `!` where nothing explains it and a mark for each verdict worth a look +(the arrow keys move between them); under the chips one line for the chosen step: the +states it goes between, what it is, its claim with who made it, what the claim found, its +tolerance and its noise; one row of controls (name, group, class, verdict, the size +filter) with the count and the legend in the row under it; then the matrix (10.5), which +opens an item in a drawer (10.6); directly under it, always open, the distributions of the +chosen comparison (10.3). The address keeps the chosen step and an open item +(`#step=&item=`), so that a reviewer can send someone to exactly that view. Below +them, each closed until a reader opens it: the steps and the whole way (10.2), the worst +values (10.4), the explanations and the rule (10.7). A host's page never offers to open a +file; it says only why nothing is drawn. + +The line of the chosen step names every verdict on a move with its count, "as claimed" +first ("40 values moved as claimed, 2 off claim ◐"), and "Nothing moved, as claimed" only +where every count is zero. It gives them weight, not colour: a verdict is not an approval +(ADR 0010, A20). + +### 10.1 Header +The title, the file's `about`, and one quiet line: how many items and indicators, how many +it holds of how many worth a look, and, for a local file, that its writer has not said it +may be published (a quiet part of the line, as a site's own notice, not a box). The states +in order, with their identity and join counts, head the steps (10.2). + +### 10.2 Steps and the whole way +Steps are numbered from 0 where the first state is `published`: step 0 is then the +reproduction (rule 1 of section 1). A chain that starts from a build of its own has no +reproduction, and its steps are numbered from 1. + +One row per step: kind, explained or not, the claim and its origin, moved, up, down, the +verdicts, missing before and after, tolerance and noise. Then the whole way per indicator: +every class, up, down, the step that carries most of the difference (section 7), spread +over steps, not attributable, and moved by what nothing explains. + +### 10.3 Distributions +One card per indicator, in the order of the matrix's columns (10.5), for the comparison the +chips have chosen: a histogram of the file's own counts, from left to right a sign change, +the bands down from far to near, agrees and within noise (two bars, the band around 1), +the bands up from near to far, then zero reference, zero result, near zero and missing. +Every count is in its bar's title. + +**Bar heights: each card to its own tallest bar by default**, with one scale for every card +(the share of all values) as a switch beside the legend. A step that moves a few values +of thousands moves them in most indicators by less than a pixel on one scale, and the +reviewer looks for exactly those; so each card says what its full height is ("tallest bar: +7", or on one scale "full height: 300 of 11 947"), and a bar of one value is never drawn as +large as a bar of thousands without that number beside it. An indicator in which nothing +moved and nothing is missing is quietened, as its column is. + +Under each card, how many moved and are missing, and **the median, 10th and 90th +percentile of after ÷ before: the one statistic the page works out itself.** It is taken +only where the page holds every item (the matrix is loaded, or the file lists them all), +over the values with a ratio (both results neither zero nor missing, of one sign), and says +over how many ("Median of 11 893 ratios", and how many have none). From the listed items +alone it gives none and says why: they are the furthest apart, and their median would say +more than the whole holds. The three percentiles share one precision: the fewest digits +(three at least, six at most) at which every two that differ read differently, so that no +rounded number stands as exact beside a longer one. Where six still read a percentile as the +median, it says "just under the median" or "just over the median" instead of two equal +numbers, and the numbers left take the digits they need. + +A card's title sorts the matrix by its indicator; a heading of the matrix marks its card. + +### 10.4 Worst +The N worst values of one comparison (a step or the whole way), ordered by class (sign +change, zero reference, zero result, beyond 10x, differs, missing), then by how far the two +sizes lie apart (`|log10(after/before)|`, or the relative difference where there is no +ratio). The page always says how many it shows of how many, and how many items worth a +look the file does not hold. + +### 10.5 Matrix +Items by indicator for the comparison the chips have chosen. **The indicators in which that +comparison moved most come first**, so that what a step did is in view without scrolling; +an indicator in which it moved nothing and nothing is missing comes last, quietened, and +keeps its column (a missing value counts with what moved: it is a finding, not stillness). +Choosing another comparison sorts the grid furthest-first again, since a new comparison +has a new order. In it: each cell with a ratio shows the result after +divided by the result before in three significant digits (`1.00`, `0.984`, `2.27`, +`1.01e−3`), agreeing pairs included, so that a column reads as numbers; a cell without one +says what the class says (from 0, to 0, + → −, missing, ≈ 0). The class shows as the tint +and a mark, never in place of the number: an agreeing cell has no tint, one within noise +is marked `~`, one beyond 10x `»`. A `1.00` with a tint is a pair that moved by less than +the rounding shows; its title holds both results exactly. The cell's colour is its +direction's and its strength the band's; for a matrix row, which holds no band, the +band is named from the results as the rule names it (a test holds the two equal). The ratio +and the shading are a display: they change no class. The listed items, or every item once +the matrix is loaded, a dot marking the listed ones; the page says which it holds and why +(none named, loading, loaded, or the reason it was not loaded). **The grid opens with the +items furthest apart**: by the largest factor between the two results over every +indicator, among the values that moved and have a ratio; then the items whose moves have +none (from or to zero, a sign change), by their worst value in the order of 10.4; then +those that did not move. A heading sorts by how far the two lie apart in that indicator, +the furthest first (the order of 10.4), or by name, and a second click goes back; +filters by text, group, class and verdict; 300 rows at a time with "show more". The +reader's **size filter** ("size at least ×1.1 ... ×10") hides smaller changes here and in +the worst list; zeros, sign changes and missing values always show, and the page says the +filter changes no count. + +### 10.6 Item +A drawer at the right (the whole window on a phone), opened from a row of the matrix or the +worst list, closed with Close or Escape. First **why it differs**: the explanations about +the item, then those about a step or an indicator in which it moved (said to be so, as they +may not be about it), or "none given" when the file holds neither; the file holds every +explanation, so this "none" is never said over a part of a list. Then per indicator, the +one whose whole way lies furthest apart first: both sides in every state, the class, +verdict and signed share of each step, the share nothing explains and the sum of the shares +(or why there are none: section 7). Then its breakdowns where they are in the file and fit +the item and step. + +### 10.7 Explanations and rule +Every explanation with its origin badge; the rule with its tolerances, noise, floors and +notes. + +Counts are also printed as Markdown by `sentier-diagnostics summary FILE`, for a pull +request or a reader without JavaScript. + +## 11. Acceptance cases + +Each defect found in the tools this replaces is a test. The pair cases are in +`tests/acceptance/cases.json`, shared by the Python rule (`tests/test_acceptance.py`) and +the page script (`tests/js/page.test.js`), so the two cannot drift apart unnoticed. + +| Case | Defect or rule | Found in | Test | +|---|---|---|---| +| A1 | Two results below zero going further below were coloured as higher | before/after v1 | cases A1-*, `test_a01`, node "tone comes from the direction" | +| A2 | "Moved" was recounted by the page without its floor | before/after v1 | case A2-floor, `test_a02`, node "takes the class from the file" | +| A3 | 1e-6 solver noise was counted as a change | before/after v1 | cases A3-*, `test_a03` | +| A4 | Differences above 1000 % were blanked | backtest | cases A4-*, `test_a04` | +| A5 | Near-zero pairs were set to agreement | backtest | cases A5-*, `test_a05` | +| A6 | References of different units were pooled in one floor | backtest | `test_a06` | +| A7 | A missing cell was shown as 0.00 %; a missing value was turned into 0 and drawn as a match | backtest page and dashboard | cases A7-*, `test_a07`, node "missing side" | +| A8 | Reference columns were read by position | backtest | `test_a08` | +| A9 | Generated outlier notes were unlabelled and began from the premise that the deviation was not a calculation bug | backtest | `test_a09`, node "a text by a model is drawn as a claim" | +| A10 | Zero and sign change are separate classes, never a ratio | rule | cases A10-*, `test_a10` | +| A11 | No rounding before the ratio | trial dashboard, backtest | case A11, `test_a11` | +| A12 | NaN and infinity are refused, in tables, in builds and in files | rule | `test_a12` | +| A13 | A missing indicator column is refused | rule | `test_a13` | +| A14 | A breakdown that does not match its item, or does not add up, is refused | rule | `test_a14` | +| A15 | Breakdown parts keep their sign | rule | `test_a15` | +| A16 | Nothing is cut short silently: lists and breakdowns say what they left out | trial dashboard | `test_a16` | +| A17 | A file of counts names no item | sentier-bafu-agribalyse | `test_a17` | +| A18 | The split is exact up to rounding, the page prints the sum it computed, and an item it cannot split is counted | rule, review of PR 2 | `test_a18`, node "A18" | +| A19 | Only `must` can be unmet, and it is unmet by a pair that stayed within the tolerance, the noise or the floor; empty scope means nothing moves; a zero reference is never within `largest`; a private claim list | sentier-bafu, sentier-agribalyse, review of PR 2 | `test_a19`, `test_a19b`, `Verdicts` | +| A20 | No status or acceptance field anywhere | sentier-bafu-agribalyse ADR 0010 | `test_a20` | +| A21 | Distributions: each card says what its full height is; one scale for every card is a switch; percentiles only from every item, over the values with a ratio, with their number | trial dashboard, review of PR 21 | node "one scale", "a median of after ÷ before only from every item", "over how many ratios", "just under or over the median" | +| A22 | Coming closer to the reference is never called better | ADR 0009, sentier-bafu | `test_a22`, node "nothing drawn calls a move better or worse" | +| A23 | An indicator is matched by its exact id, not a prefix | backtest | `test_a23` | +| A24 | Results on one side only are counted, by side | sentier-bafu | `test_a24` | +| A25 | A file is local unless its writer says otherwise | sentier-bafu, sentier-bafu-agribalyse | `test_a25` | +| A26 | A pair on both sides of the floor that agrees was a zero result, and listed second worst | review of PR 2 | cases A26-* | +| A27 | A factor too large for a float: Python and the page ordered the pair differently, the page wrote "×Infinity" | review of PR 2 | case A27 | + +Also tested: the validator refuses a class, direction or verdict that contradicts the +values, items out of order, counts that do not add up, unknown keys and other formats +(`tests/test_modules.py`); it refuses counts, attribution, `reach` and floors that +disagree with the listed items (`CountsAgainstItems`, one test per edit sentier-bafu's +review made by hand); each of its checks, broken on its own, names its own fault +(`EachRuleOnItsOwn`), so that a check another one backs up cannot go missing unseen; it +returns faults and never raises, whatever type a value has +(a fuzz test of every value in the example); a file is read strictly (no `NaN`, no key +twice), and a table's numbers are plain ASCII decimals; `build` and `check` end in one +line on ordinary mistakes, never a traceback; the summary escapes the file's texts in +Markdown; noise is taken beyond the floor, and a scope entry read from tables without +`expect` may move; the page draws the example whole under Node with a small stand-in for +the DOM (`tests/js/dom.js`), loads nothing from elsewhere and writes no markup from data; +the page takes the matrix the Python writes and refuses one of another file, loads it as a +script or over HTTP after it has drawn, draws 300 rows at a time, sorts by a heading, and +opens the drawer with "why it differs" first (node tests); `matrix.against` finds a changed +code, direction, result, verdict or count (`tests/test_matrix_page.py`); `page.body` is one +root with no frame and no id, and its data cannot end the element that holds it; every +selector of the style is inside the root, with no font and nothing on `:root` or `body`; +the committed examples (the public file, and the local file with its matrix in +`examples/local/`) are what the code makes, the public one holds every class and every verdict, and no +tracked file carries an identifier-shaped string (`tests/test_repository.py`). + +## 12. Out of scope + +- **Scoring.** The tool takes results; it computes no LCA. The scorers stay with the build + tools (Laurenz's in his backtest, sentier-bafu's in its release build). +- **Readers of a database or a reference.** Each project reads its own results and + references, with its own fail-closed column mapping, and writes CSV tables or calls the + Python API. `tables.py` reads only those tables, by column name. +- **Any data.** No inventory, no characterisation factor, no result of a licensed or + embargoed database, no ecoinvent identifier, nothing from a partner. The example and the + fixtures are made up. A test refuses identifier-shaped strings and data folders. +- **Deciding what may be published.** `audience` and `tier` are the writer's declaration; + the tool checks their form and a site checks their meaning. +- **Approval.** No field says that something was accepted. + +## 13. Decision: seed or rewrite + +**The code of sentier-bafu's before/after version 2 is the seed. The format is rewritten, +under a new name.** + +The seed is sound where it matters: classification in Python written into the file, a +strict reader that refuses unknown keys and non-finite numbers, the floor per indicator from +the median, symmetric bands, the direction from signed values, the claim and verdict model, +and an example that is the contract. Those were kept, adapted, and credited in `NOTICE`. + +The format could not stay `sentier/before-after/2`, because what its words mean changes: + +1. it compares two states, and the chain needs any number with attribution; +2. a stated claim was always "must" (about 4,900 false "claimed, not moved" values in + sentier-agribalyse's identifier fix); `must` and `may` are now explicit; +3. missing results were dropped without a count; they are now a class, counted by side; +4. near zero, zero reference and zero result are new classes, and the floor is per unit; +5. the list `moved` also held items that did not move. + +A reader of version 2 would read these files and draw wrong conclusions without an error. +A new major under the name of this repository makes that impossible. Old files stay +readable by the old page at its commit; [docs/migration.md](docs/migration.md) says how +each view moves. + +The backtest dashboards are not a seed: their code is in private repositories that also +hold licensed data, and their MIT licence names Laurenz personally. Nothing was taken from +them. Their defects are described here, in words, as acceptance cases A4 to A9 and A23. + +## 14. What the projects asked for + +On 16 November 2026 the three sessions of sentier-bafu, sentier-agribalyse and +sentier-bafu-agribalyse wrote their requirements. What became of each: + +| Asked for | By | In version 1 | +|---|---|---| +| `must` and `may` per scope entry, default `may` | bafu, agribalyse | yes | +| Counts of results on one side only | bafu | yes: `missing` by side | +| A materiality threshold apart from noise | bafu | yes, as the reader's size filter; it changes no count (section 5.3) | +| Decide `largest` where before is zero | bafu, agribalyse | decided (section 6); an absolute cap is open | +| More than one reference | bafu, agribalyse | one file per reference; a joint view is open | +| Closer and further not at the head of a later step | bafu, ADR 0009 | yes: counted, never shown as better | +| A chain of more than two states, each step's share | all three | yes | +| `listed` named and counted apart from what moved | bafu | yes: `items_count` | +| The page fetches its data as a file | bafu | yes: the data inside the body, or `window.SENTIER_DIAGNOSTICS` from a site's own `data.js` (`page.as_script`), or `?file=` | +| Every item, not only the listed ones, on a local page | bafu | yes: the matrix (section 4.5), loaded after the page is drawn | +| Noise per item | bafu | no: later | +| The number of digits written | bafu | values are written unrounded; a rounded source is a tolerance basis | +| Symmetric bands, floor of 1 % of the median per indicator | bafu | yes, and per unit | +| No fixed prose such as "only the data differs" | bafu | yes: states carry their own `about` and `identity` | +| A test that the script agrees with the Python rule | bafu | yes: shared cases | +| A formatter target at the lowest Python | bafu | yes: `py310`, CI runs 3.10 to 3.14 | +| Standard library only; assets as package data with their paths | bafu, bafu-agribalyse | yes: `page.assets()` | +| A page without a frame, with a list of tokens for a site's frame | bafu, bafu-agribalyse | yes: `page.body`, a scoped style whose `--sd-*` take the site's tokens (section 10) | +| The grid as the trial dashboard reads: a ratio in every cell, a strong tint, the furthest apart first, compact columns | bafu (from Manuel, 17 November 2026) | yes (section 10.5); 25 indicators still scroll sideways at a laptop's width | +| The look of the trial dashboard: a sortable grid, a drawer for an item | bafu-agribalyse | yes: its look and structure; none of its values, and not its two definitions of agreement, its hand-written labels or a "none" over a top-N list (section 10) | +| Reading version 1 and 2 files, or a converter | bafu | no: see section 13 and the migration plan | +| A tier a site check can refuse on | bafu, bafu-agribalyse | yes: `tier` (items or counts) and `audience` (public or local) | +| Join accounting per reference | bafu-agribalyse | yes: `join` per state | +| Reasons as data: given, none found, none given | bafu-agribalyse | yes: `finding`; "does not explain it" when a reason points the other way is later | +| Explanations labelled by origin | bafu-agribalyse, ADR 0009/0010 | yes | +| Breakdown on both sides, "the first N cover X %", tied to its run | bafu-agribalyse | yes, tied by the states it names | +| Annotations on a part (link quality band, open cases) | bafu-agribalyse | no: later | +| Placeholder share per item | bafu-agribalyse | no: later, derivable from a breakdown's part kinds | +| The comparison of characterisation factors | agribalyse | no: a reference-side explanation; later | +| Per-product table of all categories, distributions, drill-down, written reasons | agribalyse (from the backtest) | yes: grid, distributions, item view, explanations | +| Run identity on the page | agribalyse | yes: each state's `identity` | +| Indicators not hardcoded | agribalyse | yes: from the file | +| The file's own word for an indicator ("impact category") | bafu | yes: the optional `indicator` | +| No step 0 in a chain that does not start from the published results | bafu | yes: its steps are numbered from 1 (section 10.2) | +| Readable without JavaScript for the counts; 375 px wide | bafu-agribalyse | 375 px: yes, no sideways scroll of the page (tables scroll inside their frame; checked in a browser). Without JavaScript: the summary is Markdown; a no-script page is later | + +## 15. Open questions + +For Manuel, unless he names someone else: + +1. **The second maintainer.** Answered by Manuel on 16 November 2026: Laurenz Bougan. + Rule 3 still holds: the runbook is written so that either can work alone. +2. **What may be public.** Per-item results against ADEME (the ADEME dashboard holds + exactly these behind a login), the cap of 200 listed items in sentier-bafu, and + breakdowns. The tool does not decide; each project's site check does. +3. **A per-value floor from the gross sum of terms** (section 5.2), for projects that have + breakdowns. It would be a declared floor per pair, which the format cannot hold yet. +4. **Noise per item**, and an absolute cap for `largest` beside the relative one (useful + for climate change from land use, where results are often near zero). The case for noise + per item, from sentier-agribalyse's first use on 16 November 2026: two builds of + upstream main differ in 228 of 2,451 products, by up to 21 % in water use. One noise + number per indicator then puts 1,507 values of step 0 and 152 of step 1 "within noise" + because of those 228. +5. **A joint view of two references.** +6. **Laurenz's backtest.** Its scorer and join accounting are the stronger half of what it + did, and the migration of the ADEME dashboard goes with him. sentier-agribalyse relays + that Manuel wants his dashboard built on, not replaced; this plan retires its page and + keeps his scorer. Which of the two Manuel means is for him and Laurenz to settle. diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 0000000..39378f5 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,175 @@ +# Migration plan + +How each existing view of "what changed" moves to sentier-diagnostics and is retired, in +order, and what each project keeps. Written 16 November 2026. Nothing here has started: +each move is a pull request in the project that moves, reviewed and merged by a person. + +## Order, and why + +1. **The trial dashboard of sentier-bafu-agribalyse.** The smallest and the newest; its + requirements shaped the format, and it has no outside readers. +2. **The before/after page of sentier-bafu.** Public, read by others, with old files on its + site; it moves once the trial has shown the package works from a pinned release. +3. **Laurenz Bougan's backtest copies** that are not deployed. Only after Manuel has cleared + ownership with Laurenz: the code is in private repositories that also hold licensed data, + and its licence names him personally. +4. **The deployed ADEME dashboard, last and together with Laurenz.** It is the one people + use; it moves when the others have run for a while, and with its author. + +No view is retired before its replacement has run beside it and the differences between +the two are explained in writing (the parity step of each move). + +## How a project depends on this + +A package, never a copied file. Standard library only, Python 3.10 or later. + +- **By tag** in the project's lock file (sentier-bafu-agribalyse, with `uv`): + `sentier-diagnostics @ git+https://github.com/sentier-dev/sentier-diagnostics@v0.1.0`. +- **By commit** in one pins file (sentier-bafu's `scripts/release_build/pins.toml`). +- **Tags are never moved or deleted.** A fresh clone runs `uv sync --offline` from its cache, + so a moved tag would break it silently. A fix is a new tag. +- A site copies the page's files from `sentier_diagnostics.page.assets()` and writes the + data with `page.as_script("SENTIER_DIAGNOSTICS", data)`, or calls `page.write`. + +The first tag, `v0.1.0`, is made by a person after the first pull request is merged. + +## 1. The trial dashboard (sentier-bafu-agribalyse) + +**What it is.** `trial_dashboard.py` writes one HTML page with about 600 lines of +JavaScript inside Python strings; `trial.py` and `trial_categories.py` count agreement in +two ways that disagree. + +**Steps.** + +1. Pin `v0.1.0`. Write a function that turns a trial's results into a chain: the ADEME + published results (state 0, `kind: "published"`), the strict BAFU build, then one + state per accepted amendment. Step 1 of the trial's own chain (AGRIBALYSE rebuilt on + ecoinvent) is sentier-agribalyse's build; include it where its results may be read + locally, otherwise the chain starts at the published results and says so in `about`. +2. Its ADEME reader keeps mapping each heading to a method and unit through its explicit + table (fail-closed) and writes the results as CSV by indicator id, or passes them to + `chain.build`. The counts of compared, unmatched and ambiguous rows go into the state's + `join`. +3. `trial-notes.json` becomes explanations: curated reasons `by: person`, computed counts + `by: code`, blind readings by an agent `by: model` with who ran them. A note that looked + and found nothing is `finding: "none found"`. +4. `trial_explain.py` writes breakdowns (`sentier-diagnostics-breakdown/1`), local only. +5. Two files per trial: a **local** one with items (`audience: "local"`, `tier: "items"`), + and a **public** one of counts (`tier: "counts"`, `audience: "public"`), which its site + check accepts because it names no item. +6. **Parity.** On one trial build, put the old dashboard's counts beside the new file's. + Every difference must be one of the rule changes in SPEC section 13 (the floor per unit, + near zero as its own class, missing counted). Write the table into the pull request. +7. Remove the dashboard's page code and the second definition of agreement. Keep the + readers. + +**Keeps.** Its ADEME reader and column table, its mapping and link quality, its notes and +their checks against the documents they cite, its site frame and its site check. + +**Waits for.** Link quality bands and open cases on a breakdown part, and the placeholder +share per item (SPEC section 14, later). Until then the trial page shows them beside the +diagnostics page, not inside it. + +## 2. The before/after page (sentier-bafu) + +**What it is.** `sentier/before-after/1` and `/2` files, written by +`scripts/release_build/before_after_account.py`, shown by `scripts/sitebuild/beforeafter.py`, +`review.py` and `site/assets/before-after.js` on the release site. + +**The first step it shows** is the located factors (a flow that names a country takes that +country's factor), compared before and after with the claim "water use and land use may +move; nothing else moves". `tests/fixtures/located-factors/description.json` is its shape +with made-up values: two states, one step of kind `method`, one scope entry +`{"indicators": [water use, land use], "items": "*", "direction": "any", "expect": "may"}`. +A value of another indicator that moves is "not claimed", which is the finding to look at. + +The claim was first "water use moves; nothing else moves". sentier-bafu's first run on its +own results (16 November 2026) found land use moving for 5 datasets, read as "not claimed". +The scope is now built from the build: the install report's +`coverage.located_by_method` names the methods that actually take a country's own factor +(on BAFU:2026 v1, water use with 181 flows of a country and land use with 2), and +`before_after_account.py` writes the scope from it, so it follows the factors when they +change. + +**Steps.** + +1. Pin a commit of this repository in `pins.toml`. +2. `before_after_account.py` writes `sentier-diagnostics/1` through `chain.build`: BAFU's + release (published), then the import fixes, each patch and each contribution, each a + state. Where a step has not been built on its own yet, the file has fewer states and the + step's `about` says what it holds; the steps' shares are exact only for steps that were + built. A claim derived from reach is `expect: "may"`; the claim's list stays private + (`claim_lists=False`), as today. +3. Measure noise: keep the scores of an earlier build of the same state, or build one + state twice in a run, and pass them as the step's noise. Until then the file says "not + measured", as now. +4. `build_site.py` copies `page.assets()` into the site and writes `data.js` beside them. + The page keeps its own tokens until a frame-less body exists (SPEC section 14). +5. **Parity.** For build 22, put the version 2 file and the new file side by side: the + same pairs, with each class of version 2 mapped to its class here. List every pair that + changes class and why (the floor per unit, `must`/`may`, missing counted). +6. **Old files.** Builds whose inputs are kept are written again with the new writer. Older + artifacts stay readable by the old page at commit `a6a8df5`, which stays in the fork's + history; their site entries link to that page. No converter: a version 2 file lists + only the items that moved and does not count missing results, so a converted file would + look complete and not be. +7. Remove `scripts/runrecord/before_after.py`, `review.py` and `example.py`, + `scripts/sitebuild/beforeafter.py`, `review.py` and `text.py`, + `site/assets/before-after.js`, `before-after.css` and `review.js`, and + `docs/before-after.md` (link to this repository's SPEC instead). The tests + `ClassTests`, `ExampleTests`, `ClaimTests`, `VersionTests`, `IndexTests`, `FrameTests` and + `TextTests` are covered here by the acceptance cases; `RecordTests` and `PageTests` stay. + +**Keeps.** `before_after_account.py` (the writer, with its refusal when its "not claimed" +count differs from the comparison's), `compare_builds.py` and its refusals, `last_run.py`, +`build_site.py`, `sitebuild/lastrun.py`, `runrecord/summary.py` and `transcript.py`, +`shared_files.py`, and its scoring. The cap on listed datasets (200) and what may be public +stay its decision. + +## 3. Laurenz Bougan's backtest copies + +**Before anything:** Manuel clears with Laurenz who owns the code and under what licence. +Nothing is read from those repositories into this one before that. + +**Steps, per copy.** + +1. His scorer stays where it is. Beside its outputs it writes two CSV tables per state + (results by item and indicator id; items with their reference unit) and a short + `description.json`; `sentier-diagnostics build` makes the file. Reference columns are + then read by name (fixing A8), missing cells stay missing (A7), nothing is blanked (A4) + or set to agreement (A5), and floors are per unit (A6). +2. `outlier_reasons.json` becomes explanations `by: model` with the model and who ran it + (A9). The premise "the deviation is not a calculation bug" goes; a reason says what it + rests on and what it did not check. +3. **Parity**, as above, against his table for one run. +4. Retire the copy's page. Keep the scorer and its join accounting. + +**Waits for.** The comparison of characterisation factors (his `cf_comparison.csv` panel) +has no place here yet; it is a reference-side explanation (SPEC section 14). Until it does, +that panel stays on his page, linked from the diagnostics page's notes. + +## 4. The deployed ADEME dashboard + +Last, and with Laurenz. It sits behind a login and shows per-product results against ADEME. + +1. Manuel decides whether per-product differences against ADEME may be public. Until then + the public file is counts only; the logged-in deployment can serve a local file with + items. +2. Run the new page beside the old one on the same run for an agreed time, with Laurenz. +3. Switch the deployment to the new page; keep the old one read-only for a while. +4. Retire it. + +## What each project keeps + +| Project | Keeps | Moves here | +|---|---|---| +| sentier-bafu | its scoring, `compare_builds.py`, the writer `before_after_account.py`, the run record and site frame, what may be public | counting, claims, the format, the page | +| sentier-bafu-agribalyse | its ADEME reader and column table, mapping and link quality, notes and their checks, its site check | counting, the format, the page; its reasons as explanations | +| sentier-agribalyse | the build, its reference file, its scoring, its noise fix | the chain's format and page | +| Laurenz's backtests | the scorer and its join with the reference | the page, the counting rule | + +## When it is done + +Every view above is retired, each project depends on a tagged release, and this +repository has a second maintainer who can cut a release without the first +([runbook](runbook.md)). diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..b634040 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,128 @@ +# Runbook + +For whoever maintains this repository, with nobody else to ask. Everything below runs on a +fresh clone with Python 3.10 or later and, for the page tests, Node 20 or later. No package +is needed beyond the standard library. + +## Run the tests + +```bash +python3 -m unittest discover -s tests -t . +``` + +```bash +node --test tests/js/page.test.js +``` + +The first runs the unit tests, the acceptance cases and the doctests of every module. The +second runs the same acceptance cases against the page script. CI +(`.github/workflows/ci.yml`) runs both, on Python 3.10 to 3.14, and also builds the +package, installs it into a fresh environment and runs it from there. + +## See the example + +```bash +PYTHONPATH=src python3 -m sentier_diagnostics example --out build/example +``` + +Open `build/example/page/index.html` in a browser. The example is made up: ten products, +three indicators, five states, one value of every class and one of every verdict. The same +file is committed as `examples/diagnostics.json`; a test fails when the code would write +something else. After a deliberate change, write it again: + +```bash +PYTHONPATH=src python3 -c "from pathlib import Path; from sentier_diagnostics import chain, breakdown, example; chain.write(example.build(), Path('examples/diagnostics.json')); breakdown.write(example.wheat_breakdown(), Path('examples/p01-fix-1.json'))" +``` + +and read the diff: it names the items that changed, one per line. + +## Make a file from a project's results + +Without Python code: one CSV table of results per state (`item` and one column per +indicator id, in any order), one table of items (`item`, `unit`, and optionally `name`, +`place`, `group`), and a description. `tests/fixtures/tables/` is a complete example. + +```bash +sentier-diagnostics build description.json --out diagnostics.json +``` + +```bash +sentier-diagnostics check diagnostics.json +``` + +```bash +sentier-diagnostics page diagnostics.json --out site/diagnostics +``` + +```bash +sentier-diagnostics summary diagnostics.json +``` + +From Python, `chain.build(...)` takes the same and returns the file, checked. `check` exits 1 +and prints every fault; a fault means the file disagrees with its own rule, and the page +would refuse it. + +## Review a pull request + +1. CI is green on every Python. +2. A change to the rule (`rule.py`, `claims.py`) comes with a case in + `tests/acceptance/cases.json` or a test in `tests/test_acceptance.py`, and SPEC.md says + the same as the code. +3. A change to what a file holds is a new format (below), or it is refused. +4. The page still works out nothing: `diagnostics.js` reads classes, directions, bands and + verdicts from the file. It sets text with `textContent` only. +5. No data: no results of a real database, no identifier of a dataset or flow, nothing from + a partner. `tests/test_repository.py` refuses identifier-shaped strings and data folders, + but it cannot see everything; read the diff. + +A person reviews and merges. Nobody merges their own pull request. + +## Change the format + +The reader refuses what it does not know, so every change to the keys or to what a class +means is a new major version: + +1. `fileformat.FORMAT` becomes `sentier-diagnostics/2`; the page's `FORMAT` too. +2. Decide whether the package reads `/1` as well. If it does, keep the old checks in their + own function and test both. +3. SPEC.md gets the new version's section; the old one stays, marked as such. +4. Tell the projects that use it (see [migration.md](migration.md)): each moves when it is + ready, on its own pin. + +## Make a release + +After a merge to `main`, on the merge commit: + +1. `__version__` in `src/sentier_diagnostics/__init__.py` is the new version. +2. Tag it `vX.Y.Z` and push the tag. **Never move or delete a tag**: projects pin them, and + their offline installs read them from a cache. +3. Write what changed in the release notes on GitHub, and which format it writes. + +## Where things are + +| File | What it does | +|---|---| +| `src/sentier_diagnostics/rule.py` | The counting rule: classes, direction, bands, floors, order worst first | +| `claims.py` | Claims, scope entries, verdicts | +| `chain.py` | Builds a file from states, steps and results; attribution; writing and reading | +| `fileformat.py` | The validator | +| `breakdown.py` | Breakdowns of one item, and their check against a file | +| `tables.py`, `from_tables.py` | Reading CSV tables by column name; building from a description | +| `summary.py` | The Markdown summary | +| `page.py`, `page/` | The static page and its writer | +| `cli.py` | The command | +| `example.py` | The made-up example | +| `SPEC.md` | What all of it means | + +## When something looks wrong + +- **A value is in the wrong class.** Find its pair in the file, then run + `rule.judge(before, after, floor=..., tolerance=..., noise=...)` with the floor and + tolerance the file's `rule` and step give. If the rule is wrong, add a case first. +- **The page shows nothing.** Open the browser's console; the page says why it refused a + file. `sentier-diagnostics check` says the same at more length. +- **Everything of an indicator is "within noise".** Its noise is one number, the largest + difference between two builds: one unstable item widens it for all. Look at the two + builds, not at the tolerance. +- **A breakdown is "not with this page".** It is named in the file and was not beside it + when the page was written. Whether to publish it is its writer's decision. diff --git a/examples/diagnostics.json b/examples/diagnostics.json new file mode 100644 index 0000000..ebc727d --- /dev/null +++ b/examples/diagnostics.json @@ -0,0 +1,1112 @@ +{ + "format": "sentier-diagnostics/1", + "tier": "items", + "audience": "public", + "title": "A made-up chain: published results to the current build", + "about": "Made-up results of ten items in three indicators. Nobody's data.", + "item": { + "one": "product", + "many": "products" + }, + "indicators": [ + { + "id": "x:climate", + "name": "Climate change", + "short": "Climate", + "unit": "kg CO2 eq" + }, + { + "id": "x:water", + "name": "Water use", + "short": "Water", + "unit": "m3" + }, + { + "id": "x:tox", + "name": "Human toxicity", + "short": "Toxicity", + "unit": "CTUh" + } + ], + "rule": { + "floor_factor": 0.01, + "floors_from": "published", + "beyond": 10.0, + "total_tolerance": 0.0005, + "floors": [ + { + "indicator": "x:climate", + "unit": "MJ", + "value": 0.0003, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:climate", + "unit": "h", + "value": 0.42, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:climate", + "unit": "kg", + "value": 0.0049, + "basis": "median", + "n": 8 + }, + { + "indicator": "x:water", + "unit": "MJ", + "value": 4.000000000000001e-06, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:water", + "unit": "h", + "value": 0.009000000000000001, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:water", + "unit": "kg", + "value": 0.011000000000000001, + "basis": "median", + "n": 7 + }, + { + "indicator": "x:tox", + "unit": "MJ", + "value": 3e-11, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:tox", + "unit": "h", + "value": 0.0, + "basis": "none", + "n": 0 + }, + { + "indicator": "x:tox", + "unit": "kg", + "value": 1.95e-09, + "basis": "median", + "n": 8 + } + ] + }, + "states": [ + { + "id": "published", + "label": "Published", + "about": "What the publisher released.", + "kind": "published", + "identity": { + "release": "1.0" + }, + "join": { + "compared": 10, + "unmatched": 1, + "ambiguous": 0, + "note": "One published row names no item of the build." + } + }, + { + "id": "repro", + "label": "Recomputed", + "about": null, + "kind": "build", + "identity": { + "commit": "0000001" + }, + "join": null + }, + { + "id": "fix-1", + "label": "With fix 1", + "about": null, + "kind": "build", + "identity": { + "commit": "0000002" + }, + "join": null + }, + { + "id": "rebuilt", + "label": "Rebuilt", + "about": null, + "kind": "build", + "identity": { + "commit": "0000002", + "machine": "second" + }, + "join": null + }, + { + "id": "mapping-2", + "label": "With mapping 2", + "about": null, + "kind": "build", + "identity": { + "commit": "0000003" + }, + "join": null + } + ], + "steps": [ + { + "id": "step-0", + "from": "published", + "to": "repro", + "label": "Reproduction", + "about": "The released data computed again with the same method.", + "kind": "reproduction", + "explained": true, + "link": null, + "claim": { + "text": "Nothing moves beyond the rounding of the published results.", + "by": { + "kind": "code", + "name": "reproduction rule", + "model": null, + "run_by": null + }, + "reach": null, + "scope": [] + }, + "tolerance": { + "value": 0.0005, + "basis": "published rounding", + "note": "four significant digits" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "fix-1", + "from": "repro", + "to": "fix-1", + "label": "Fertiliser amounts of two cereals", + "about": "The nitrogen amounts of wheat and rice were per hectare, not per kilogram.", + "kind": "fix", + "explained": true, + "link": "https://example.org/findings/fix-1", + "claim": { + "text": "Wheat and rice go down in climate change, by 30 % at most. Nothing else moves.", + "by": { + "kind": "person", + "name": "A. Author", + "model": null, + "run_by": null + }, + "reach": 2, + "scope": [ + { + "indicators": [ + "x:climate" + ], + "items": [ + "p01", + "p05" + ], + "direction": "down", + "largest": 0.3, + "expect": "must" + } + ] + }, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "rebuilt", + "from": "fix-1", + "to": "rebuilt", + "label": "Built again on another machine", + "about": "The same state, built on a second machine.", + "kind": "rebuild", + "explained": false, + "link": null, + "claim": null, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "mapping-2", + "from": "rebuilt", + "to": "mapping-2", + "label": "Mapping change 2", + "about": "Five items take another background dataset.", + "kind": "mapping", + "explained": true, + "link": null, + "claim": { + "text": "Items whose supply chain reaches the changed links may move in water use; maize must.", + "by": { + "kind": "code", + "name": "reach of the changed links", + "model": null, + "run_by": null + }, + "reach": 4, + "scope": [ + { + "indicators": [ + "x:water" + ], + "items": [ + "p02", + "p04", + "p07", + "p10" + ], + "direction": "any", + "largest": null, + "expect": "may" + }, + { + "indicators": [ + "x:water" + ], + "items": [ + "p02" + ], + "direction": "up", + "largest": 0.5, + "expect": "must" + } + ] + }, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": true, + "between": [ + "fix-1", + "rebuilt" + ], + "by": { + "x:climate": 0.0, + "x:water": 0.02439024390243894, + "x:tox": 0.0 + }, + "note": "One pair of builds of one state." + } + } + ], + "counts": { + "steps": { + "step-0": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 10, + "not judged": 0 + }, + "closer": null, + "further": null + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 9, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 9, + "not judged": 1 + }, + "closer": null, + "further": null + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 1, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 2 + }, + "closer": null, + "further": null + } + }, + "fix-1": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 3, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 2 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 1, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 1, + "off claim": 1, + "not claimed": 1, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 0 + }, + "closer": 0, + "further": 3 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 10, + "not judged": 0 + }, + "closer": 0, + "further": 0 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 8, + "not judged": 2 + }, + "closer": 0, + "further": 0 + } + }, + "rebuilt": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 0 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 1 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 0 + } + }, + "mapping-2": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 1, + "sign change": 1, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 1, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 2, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 1 + }, + "closer": 0, + "further": 2 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 1, + "differs": 0, + "beyond 10x": 1, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 1 + }, + "verdicts": { + "as claimed": 1, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 1, + "within noise": 1, + "unchanged": 6, + "not judged": 1 + }, + "closer": 0, + "further": 1 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 1, + "zero result": 0, + "sign change": 0, + "near zero": 1, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 2, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 1 + }, + "closer": 0, + "further": 2 + } + } + }, + "total": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 4, + "within noise": 0, + "differs": 3, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 1, + "sign change": 1, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 2, + "down": 3 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 1, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": null, + "further": null + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 6, + "within noise": 0, + "differs": 2, + "beyond 10x": 1, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 3, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 1 + }, + "verdicts": null, + "closer": null, + "further": null + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 6, + "within noise": 0, + "differs": 2, + "beyond 10x": 0, + "zero reference": 1, + "zero result": 0, + "sign change": 0, + "near zero": 1, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 2, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": null, + "further": null + } + } + }, + "attribution": { + "x:climate": { + "dominant": { + "step-0": 0, + "fix-1": 3, + "rebuilt": 0, + "mapping-2": 2 + }, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0 + }, + "x:water": { + "dominant": { + "step-0": 0, + "fix-1": 0, + "rebuilt": 1, + "mapping-2": 1 + }, + "not_attributable": 1, + "spread": 0, + "unexplained_moved": 1 + }, + "x:tox": { + "dominant": { + "step-0": 1, + "fix-1": 0, + "rebuilt": 0, + "mapping-2": 2 + }, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0 + } + }, + "items": [ + {"id": "p08", "name": "Compost, used as fertiliser", "place": "CH", "group": "waste", "unit": "kg", "values": {"x:climate": [-0.5, -0.5, -0.5, -0.5, 0.2], "x:water": [0.05, 0.05, 0.05, 0.05, 0.05], "x:tox": [-2e-06, -2e-06, -2e-06, -2e-06, -4e-06]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "sign change", "sign change"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "differs", "differs"]}, "directions": {"x:climate": [null, null, null, "up", "up"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "down", "down"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "down, factor 1.25-2", "down, factor 1.25-2"]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not claimed"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "not claimed"]}, "breakdowns": null}, + {"id": "p09", "name": "Machine hour, harvesting", "place": "CH", "group": "services", "unit": "h", "values": {"x:climate": [42.0, 42.00042, 42.00042, 42.00042, 42.00042], "x:water": [0.9, 0.9, 0.9, 0.9, 0.9], "x:tox": [0.0, 0.0, 0.0, 0.0, 4e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["near zero", "near zero", "near zero", "zero reference", "zero reference"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "up", "up"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["not judged", "not judged", null, "not claimed"]}, "breakdowns": null}, + {"id": "p07", "name": "Heat from wood chips", "place": "CH", "group": "energy", "unit": "MJ", "values": {"x:climate": [0.03, 0.03, 0.03, 0.03, 0.0], "x:water": [0.0004, 0.0004, 0.0004, 0.0004, 0.0004], "x:tox": [3e-09, 3e-09, 3e-09, 3e-09, 3e-09]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "zero result", "zero result"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, null, null, "down", "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not claimed"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p10", "name": "Tomato, greenhouse", "place": "NL", "group": "crops", "unit": "kg", "values": {"x:climate": [1.3, 1.3, 1.3, 1.3, 1.3], "x:water": [0.02, 0.02, 0.02, 0.02, 0.5], "x:tox": [2.1e-07, 2.3e-07, 2.3e-07, 2.3e-07, 2.3e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "beyond 10x", "beyond 10x"], "x:tox": ["differs", "agrees", "agrees", "agrees", "differs"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, "up", "up"], "x:tox": ["up", null, null, null, "up"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, "up, factor over 10", "up, factor over 10"], "x:tox": ["up, factor 1-1.1", null, null, null, "up, factor 1-1.1"]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "as claimed"], "x:tox": ["not claimed", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p05", "name": "Rice", "place": "IT", "group": "crops", "unit": "kg", "values": {"x:climate": [1.9, 1.9, 0.95, 0.95, 0.95], "x:water": [3.4, 3.4, 3.4, 3.4, 3.4], "x:tox": [1e-12, 1e-12, 1e-12, 1e-12, 3e-12]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["near zero", "near zero", "near zero", "near zero", "near zero"]}, "directions": {"x:climate": [null, "down", null, null, "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "down, factor 1.25-2", null, null, "down, factor 1.25-2"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "off claim", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["not judged", "not judged", null, "not judged"]}, "breakdowns": null}, + {"id": "p03", "name": "Barley", "place": "DE", "group": "crops", "unit": "kg", "values": {"x:climate": [0.48, 0.47999, 0.575988, 0.575988, 0.575988], "x:water": [1.1, 1.1, 1.1, null, 1.3], "x:tox": [1.9e-07, 1.9e-07, 1.9e-07, 1.9e-07, 1.9e-07]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "missing", "missing", "differs"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, "up", null, null, "up"], "x:water": [null, null, null, null, "up"], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "up, factor 1.1-1.25", null, null, "up, factor 1.1-1.25"], "x:water": [null, null, null, null, "up, factor 1.1-1.25"], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "not claimed", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "not judged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p01", "name": "Wheat", "place": "FR", "group": "crops", "unit": "kg", "values": {"x:climate": [0.52, 0.520013, 0.468012, 0.468012, 0.468012], "x:water": [1.2, 1.2, 1.2, 1.2000001, 1.2000001], "x:tox": [2e-07, 2.0000002e-07, 2.0000002e-07, 2.0000002e-07, 2.0000002e-07]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, "down", null, null, "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "down, factor 1.1-1.25", null, null, "down, factor 1.1-1.25"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "as claimed", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": {"fix-1": "p01-fix-1.json"}}, + {"id": "p04", "name": "Oats", "place": "DE", "group": "crops", "unit": "kg", "values": {"x:climate": [0.45, 0.45, 0.45, 0.45, 0.45], "x:water": [1.0, 1.0, 1.0, 1.025, 1.0455], "x:tox": [1.7e-07, 1.7e-07, 1.7e-07, 1.7e-07, 1.7e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "differs", "within noise", "differs"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, "up", null, "up"], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, "up, factor 1-1.1", null, "up, factor 1-1.1"], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "within noise"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p06", "name": "Rye", "place": "PL", "group": "crops", "unit": "kg", "values": {"x:climate": [0.43, 0.43, 0.43, 0.43, null], "x:water": [null, 1.15, 1.15, 1.15, 1.15], "x:tox": [2.1e-07, 2.1e-07, 2.1e-07, 2.1e-07, 2.1e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "missing", "missing"], "x:water": ["missing", "agrees", "agrees", "agrees", "missing"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not judged"], "x:water": ["not judged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p02", "name": "Maize", "place": "FR", "group": "crops", "unit": "kg", "values": {"x:climate": [0.41, 0.41, 0.41, 0.41, 0.41], "x:water": [2.5, 2.500025, 2.500025, 2.500025, 2.500025], "x:tox": [1.8e-07, 1.8e-07, 1.8e-07, 1.8e-07, 1.8e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "claimed, not moved"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null} + ], + "items_count": { + "all": 10, + "qualifying": 10, + "listed": 10, + "not_listed": 0 + }, + "explanations": [ + { + "about": { + "step": "fix-1", + "indicator": "x:climate", + "item": null + }, + "finding": "reason", + "text": "The fertiliser amounts were per hectare; divided by the yield they are per kilogram, which lowers the nitrogen field emissions of both cereals.", + "by": { + "kind": "person", + "name": "A. Author", + "model": null, + "run_by": null + }, + "source": { + "link": null, + "doc": "docs/findings/fix-1.md" + }, + "not_checked": null + }, + { + "about": { + "step": null, + "indicator": "x:tox", + "item": "p05" + }, + "finding": "reason", + "text": "The toxicity results of rice lie far below one hundredth of the median result of the items measured per kilogram: they are counted as near zero.", + "by": { + "kind": "code", + "name": "floor rule", + "model": null, + "run_by": null + }, + "source": null, + "not_checked": null + }, + { + "about": { + "step": "mapping-2", + "indicator": "x:water", + "item": "p10" + }, + "finding": "reason", + "text": "The new mapping links tomatoes to a greenhouse heated with fresh water from a well, which multiplies the water use.", + "by": { + "kind": "model", + "name": null, + "model": "example-model-1", + "run_by": "example maintainers (agent-run)" + }, + "source": null, + "not_checked": "Nobody compared the greenhouse's water balance with a second source." + }, + { + "about": { + "step": "step-0", + "indicator": "x:tox", + "item": "p08" + }, + "finding": "none found", + "text": "Compost's toxicity result fell to almost nothing in the reproduction. The recipe, the background datasets and the method were compared; none of them explains it.", + "by": { + "kind": "person", + "name": "B. Reviewer", + "model": null, + "run_by": null + }, + "source": null, + "not_checked": null + } + ], + "notes": [ + "A made-up example: nobody's data." + ], + "made_by": "sentier-diagnostics 0.1.0" +} diff --git a/examples/local/diagnostics.json b/examples/local/diagnostics.json new file mode 100644 index 0000000..def56d0 --- /dev/null +++ b/examples/local/diagnostics.json @@ -0,0 +1,1112 @@ +{ + "format": "sentier-diagnostics/1", + "tier": "items", + "audience": "local", + "title": "A made-up chain: published results to the current build", + "about": "Made-up results of ten items in three indicators. Nobody's data.", + "item": { + "one": "product", + "many": "products" + }, + "indicators": [ + { + "id": "x:climate", + "name": "Climate change", + "short": "Climate", + "unit": "kg CO2 eq" + }, + { + "id": "x:water", + "name": "Water use", + "short": "Water", + "unit": "m3" + }, + { + "id": "x:tox", + "name": "Human toxicity", + "short": "Toxicity", + "unit": "CTUh" + } + ], + "rule": { + "floor_factor": 0.01, + "floors_from": "published", + "beyond": 10.0, + "total_tolerance": 0.0005, + "floors": [ + { + "indicator": "x:climate", + "unit": "MJ", + "value": 0.0003, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:climate", + "unit": "h", + "value": 0.42, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:climate", + "unit": "kg", + "value": 0.0049, + "basis": "median", + "n": 8 + }, + { + "indicator": "x:water", + "unit": "MJ", + "value": 4.000000000000001e-06, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:water", + "unit": "h", + "value": 0.009000000000000001, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:water", + "unit": "kg", + "value": 0.011000000000000001, + "basis": "median", + "n": 7 + }, + { + "indicator": "x:tox", + "unit": "MJ", + "value": 3e-11, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:tox", + "unit": "h", + "value": 0.0, + "basis": "none", + "n": 0 + }, + { + "indicator": "x:tox", + "unit": "kg", + "value": 1.95e-09, + "basis": "median", + "n": 8 + } + ] + }, + "states": [ + { + "id": "published", + "label": "Published", + "about": "What the publisher released.", + "kind": "published", + "identity": { + "release": "1.0" + }, + "join": { + "compared": 10, + "unmatched": 1, + "ambiguous": 0, + "note": "One published row names no item of the build." + } + }, + { + "id": "repro", + "label": "Recomputed", + "about": null, + "kind": "build", + "identity": { + "commit": "0000001" + }, + "join": null + }, + { + "id": "fix-1", + "label": "With fix 1", + "about": null, + "kind": "build", + "identity": { + "commit": "0000002" + }, + "join": null + }, + { + "id": "rebuilt", + "label": "Rebuilt", + "about": null, + "kind": "build", + "identity": { + "commit": "0000002", + "machine": "second" + }, + "join": null + }, + { + "id": "mapping-2", + "label": "With mapping 2", + "about": null, + "kind": "build", + "identity": { + "commit": "0000003" + }, + "join": null + } + ], + "steps": [ + { + "id": "step-0", + "from": "published", + "to": "repro", + "label": "Reproduction", + "about": "The released data computed again with the same method.", + "kind": "reproduction", + "explained": true, + "link": null, + "claim": { + "text": "Nothing moves beyond the rounding of the published results.", + "by": { + "kind": "code", + "name": "reproduction rule", + "model": null, + "run_by": null + }, + "reach": null, + "scope": [] + }, + "tolerance": { + "value": 0.0005, + "basis": "published rounding", + "note": "four significant digits" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "fix-1", + "from": "repro", + "to": "fix-1", + "label": "Fertiliser amounts of two cereals", + "about": "The nitrogen amounts of wheat and rice were per hectare, not per kilogram.", + "kind": "fix", + "explained": true, + "link": "https://example.org/findings/fix-1", + "claim": { + "text": "Wheat and rice go down in climate change, by 30 % at most. Nothing else moves.", + "by": { + "kind": "person", + "name": "A. Author", + "model": null, + "run_by": null + }, + "reach": 2, + "scope": [ + { + "indicators": [ + "x:climate" + ], + "items": [ + "p01", + "p05" + ], + "direction": "down", + "largest": 0.3, + "expect": "must" + } + ] + }, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "rebuilt", + "from": "fix-1", + "to": "rebuilt", + "label": "Built again on another machine", + "about": "The same state, built on a second machine.", + "kind": "rebuild", + "explained": false, + "link": null, + "claim": null, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "mapping-2", + "from": "rebuilt", + "to": "mapping-2", + "label": "Mapping change 2", + "about": "Five items take another background dataset.", + "kind": "mapping", + "explained": true, + "link": null, + "claim": { + "text": "Items whose supply chain reaches the changed links may move in water use; maize must.", + "by": { + "kind": "code", + "name": "reach of the changed links", + "model": null, + "run_by": null + }, + "reach": 4, + "scope": [ + { + "indicators": [ + "x:water" + ], + "items": [ + "p02", + "p04", + "p07", + "p10" + ], + "direction": "any", + "largest": null, + "expect": "may" + }, + { + "indicators": [ + "x:water" + ], + "items": [ + "p02" + ], + "direction": "up", + "largest": 0.5, + "expect": "must" + } + ] + }, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": true, + "between": [ + "fix-1", + "rebuilt" + ], + "by": { + "x:climate": 0.0, + "x:water": 0.02439024390243894, + "x:tox": 0.0 + }, + "note": "One pair of builds of one state." + } + } + ], + "counts": { + "steps": { + "step-0": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 10, + "not judged": 0 + }, + "closer": null, + "further": null + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 9, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 9, + "not judged": 1 + }, + "closer": null, + "further": null + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 1, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 2 + }, + "closer": null, + "further": null + } + }, + "fix-1": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 3, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 2 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 1, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 1, + "off claim": 1, + "not claimed": 1, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 0 + }, + "closer": 0, + "further": 3 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 10, + "not judged": 0 + }, + "closer": 0, + "further": 0 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 8, + "not judged": 2 + }, + "closer": 0, + "further": 0 + } + }, + "rebuilt": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 0 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 1 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 0 + } + }, + "mapping-2": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 1, + "sign change": 1, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 1, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 2, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 1 + }, + "closer": 0, + "further": 2 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 1, + "differs": 0, + "beyond 10x": 1, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 1 + }, + "verdicts": { + "as claimed": 1, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 1, + "within noise": 1, + "unchanged": 6, + "not judged": 1 + }, + "closer": 0, + "further": 1 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 1, + "zero result": 0, + "sign change": 0, + "near zero": 1, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 2, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 1 + }, + "closer": 0, + "further": 2 + } + } + }, + "total": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 4, + "within noise": 0, + "differs": 3, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 1, + "sign change": 1, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 2, + "down": 3 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 1, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": null, + "further": null + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 6, + "within noise": 0, + "differs": 2, + "beyond 10x": 1, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 3, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 1 + }, + "verdicts": null, + "closer": null, + "further": null + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 6, + "within noise": 0, + "differs": 2, + "beyond 10x": 0, + "zero reference": 1, + "zero result": 0, + "sign change": 0, + "near zero": 1, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 2, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": null, + "further": null + } + } + }, + "attribution": { + "x:climate": { + "dominant": { + "step-0": 0, + "fix-1": 3, + "rebuilt": 0, + "mapping-2": 2 + }, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0 + }, + "x:water": { + "dominant": { + "step-0": 0, + "fix-1": 0, + "rebuilt": 1, + "mapping-2": 1 + }, + "not_attributable": 1, + "spread": 0, + "unexplained_moved": 1 + }, + "x:tox": { + "dominant": { + "step-0": 1, + "fix-1": 0, + "rebuilt": 0, + "mapping-2": 2 + }, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0 + } + }, + "items": [ + {"id": "p08", "name": "Compost, used as fertiliser", "place": "CH", "group": "waste", "unit": "kg", "values": {"x:climate": [-0.5, -0.5, -0.5, -0.5, 0.2], "x:water": [0.05, 0.05, 0.05, 0.05, 0.05], "x:tox": [-2e-06, -2e-06, -2e-06, -2e-06, -4e-06]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "sign change", "sign change"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "differs", "differs"]}, "directions": {"x:climate": [null, null, null, "up", "up"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "down", "down"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "down, factor 1.25-2", "down, factor 1.25-2"]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not claimed"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "not claimed"]}, "breakdowns": null}, + {"id": "p09", "name": "Machine hour, harvesting", "place": "CH", "group": "services", "unit": "h", "values": {"x:climate": [42.0, 42.00042, 42.00042, 42.00042, 42.00042], "x:water": [0.9, 0.9, 0.9, 0.9, 0.9], "x:tox": [0.0, 0.0, 0.0, 0.0, 4e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["near zero", "near zero", "near zero", "zero reference", "zero reference"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "up", "up"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["not judged", "not judged", null, "not claimed"]}, "breakdowns": null}, + {"id": "p07", "name": "Heat from wood chips", "place": "CH", "group": "energy", "unit": "MJ", "values": {"x:climate": [0.03, 0.03, 0.03, 0.03, 0.0], "x:water": [0.0004, 0.0004, 0.0004, 0.0004, 0.0004], "x:tox": [3e-09, 3e-09, 3e-09, 3e-09, 3e-09]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "zero result", "zero result"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, null, null, "down", "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not claimed"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p10", "name": "Tomato, greenhouse", "place": "NL", "group": "crops", "unit": "kg", "values": {"x:climate": [1.3, 1.3, 1.3, 1.3, 1.3], "x:water": [0.02, 0.02, 0.02, 0.02, 0.5], "x:tox": [2.1e-07, 2.3e-07, 2.3e-07, 2.3e-07, 2.3e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "beyond 10x", "beyond 10x"], "x:tox": ["differs", "agrees", "agrees", "agrees", "differs"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, "up", "up"], "x:tox": ["up", null, null, null, "up"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, "up, factor over 10", "up, factor over 10"], "x:tox": ["up, factor 1-1.1", null, null, null, "up, factor 1-1.1"]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "as claimed"], "x:tox": ["not claimed", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p05", "name": "Rice", "place": "IT", "group": "crops", "unit": "kg", "values": {"x:climate": [1.9, 1.9, 0.95, 0.95, 0.95], "x:water": [3.4, 3.4, 3.4, 3.4, 3.4], "x:tox": [1e-12, 1e-12, 1e-12, 1e-12, 3e-12]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["near zero", "near zero", "near zero", "near zero", "near zero"]}, "directions": {"x:climate": [null, "down", null, null, "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "down, factor 1.25-2", null, null, "down, factor 1.25-2"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "off claim", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["not judged", "not judged", null, "not judged"]}, "breakdowns": null}, + {"id": "p03", "name": "Barley", "place": "DE", "group": "crops", "unit": "kg", "values": {"x:climate": [0.48, 0.47999, 0.575988, 0.575988, 0.575988], "x:water": [1.1, 1.1, 1.1, null, 1.3], "x:tox": [1.9e-07, 1.9e-07, 1.9e-07, 1.9e-07, 1.9e-07]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "missing", "missing", "differs"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, "up", null, null, "up"], "x:water": [null, null, null, null, "up"], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "up, factor 1.1-1.25", null, null, "up, factor 1.1-1.25"], "x:water": [null, null, null, null, "up, factor 1.1-1.25"], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "not claimed", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "not judged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null} + ], + "items_count": { + "all": 10, + "qualifying": 10, + "listed": 6, + "not_listed": 4 + }, + "explanations": [ + { + "about": { + "step": "fix-1", + "indicator": "x:climate", + "item": null + }, + "finding": "reason", + "text": "The fertiliser amounts were per hectare; divided by the yield they are per kilogram, which lowers the nitrogen field emissions of both cereals.", + "by": { + "kind": "person", + "name": "A. Author", + "model": null, + "run_by": null + }, + "source": { + "link": null, + "doc": "docs/findings/fix-1.md" + }, + "not_checked": null + }, + { + "about": { + "step": null, + "indicator": "x:tox", + "item": "p05" + }, + "finding": "reason", + "text": "The toxicity results of rice lie far below one hundredth of the median result of the items measured per kilogram: they are counted as near zero.", + "by": { + "kind": "code", + "name": "floor rule", + "model": null, + "run_by": null + }, + "source": null, + "not_checked": null + }, + { + "about": { + "step": "mapping-2", + "indicator": "x:water", + "item": "p10" + }, + "finding": "reason", + "text": "The new mapping links tomatoes to a greenhouse heated with fresh water from a well, which multiplies the water use.", + "by": { + "kind": "model", + "name": null, + "model": "example-model-1", + "run_by": "example maintainers (agent-run)" + }, + "source": null, + "not_checked": "Nobody compared the greenhouse's water balance with a second source." + }, + { + "about": { + "step": "step-0", + "indicator": "x:tox", + "item": "p08" + }, + "finding": "none found", + "text": "Compost's toxicity result fell to almost nothing in the reproduction. The recipe, the background datasets and the method were compared; none of them explains it.", + "by": { + "kind": "person", + "name": "B. Reviewer", + "model": null, + "run_by": null + }, + "source": null, + "not_checked": null + } + ], + "notes": [ + "A made-up example: nobody's data." + ], + "made_by": "sentier-diagnostics 0.1.0", + "matrix": { + "file": "matrix.json", + "items": 10 + } +} diff --git a/examples/local/matrix.json b/examples/local/matrix.json new file mode 100644 index 0000000..7945f66 --- /dev/null +++ b/examples/local/matrix.json @@ -0,0 +1,50 @@ +{"format": "sentier-diagnostics-matrix/1", "of": {"title": "A made-up chain: published results to the current build", "states": ["published", "repro", "fix-1", "rebuilt", "mapping-2"], "indicators": ["x:climate", "x:water", "x:tox"], "steps": ["step-0", "fix-1", "rebuilt", "mapping-2"]}, + "items": [ + ["p08","Compost, used as fertiliser","CH","waste","kg"], + ["p09","Machine hour, harvesting","CH","services","h"], + ["p07","Heat from wood chips","CH","energy","MJ"], + ["p10","Tomato, greenhouse","NL","crops","kg"], + ["p05","Rice","IT","crops","kg"], + ["p03","Barley","DE","crops","kg"], + ["p01","Wheat","FR","crops","kg"], + ["p04","Oats","DE","crops","kg"], + ["p06","Rye","PL","crops","kg"], + ["p02","Maize","FR","crops","kg"] + ], + "values": [ + [-0.5,-0.5,-0.5,-0.5,0.2,0.05,0.05,0.05,0.05,0.05,-2e-06,-2e-06,-2e-06,-2e-06,-4e-06], + [42.0,42.00042,42.00042,42.00042,42.00042,0.9,0.9,0.9,0.9,0.9,0.0,0.0,0.0,0.0,4e-07], + [0.03,0.03,0.03,0.03,0.0,0.0004,0.0004,0.0004,0.0004,0.0004,3e-09,3e-09,3e-09,3e-09,3e-09], + [1.3,1.3,1.3,1.3,1.3,0.02,0.02,0.02,0.02,0.5,2.1e-07,2.3e-07,2.3e-07,2.3e-07,2.3e-07], + [1.9,1.9,0.95,0.95,0.95,3.4,3.4,3.4,3.4,3.4,1e-12,1e-12,1e-12,1e-12,3e-12], + [0.48,0.47999,0.575988,0.575988,0.575988,1.1,1.1,1.1,null,1.3,1.9e-07,1.9e-07,1.9e-07,1.9e-07,1.9e-07], + [0.52,0.520013,0.468012,0.468012,0.468012,1.2,1.2,1.2,1.2000001,1.2000001,2e-07,2.0000002e-07,2.0000002e-07,2.0000002e-07,2.0000002e-07], + [0.45,0.45,0.45,0.45,0.45,1.0,1.0,1.0,1.025,1.0455,1.7e-07,1.7e-07,1.7e-07,1.7e-07,1.7e-07], + [0.43,0.43,0.43,0.43,null,null,1.15,1.15,1.15,1.15,2.1e-07,2.1e-07,2.1e-07,2.1e-07,2.1e-07], + [0.41,0.41,0.41,0.41,0.41,2.5,2.500025,2.500025,2.500025,2.500025,1.8e-07,1.8e-07,1.8e-07,1.8e-07,1.8e-07] + ], + "classes": [ + "===SS========dd", + "==========000RR", + "===tt==========", + "========BBD===D", + "=d==d=====00000", + "=D==D==--D=====", + "=d==d==========", + "=======D~D=====", + "===---===-=====", + "===============" + ], + "verdicts": [ + "==.n==.===.n", + "==.===.=??.n", + "==.n==.===.=", + "==.===.an=.=", + "=o.===.=??.?", + "=n.===.?==.=", + "=a.===.===.=", + "==.===.~==.=", + "==.??=.===.=", + "==.===.c==.=" + ] +} diff --git a/examples/p01-fix-1.json b/examples/p01-fix-1.json new file mode 100644 index 0000000..3412bc8 --- /dev/null +++ b/examples/p01-fix-1.json @@ -0,0 +1,62 @@ +{ + "format": "sentier-diagnostics-breakdown/1", + "item": "p01", + "from": "repro", + "to": "fix-1", + "indicators": { + "x:climate": { + "unit": "kg CO2 eq", + "total": [ + 0.520013, + 0.468012 + ], + "parts": [ + { + "label": "Dinitrogen monoxide, field", + "kind": "elementary flow", + "values": [ + 0.18, + 0.128012 + ] + }, + { + "label": "Nitrogen fertiliser, production", + "kind": "background process", + "values": [ + 0.11, + 0.099 + ] + }, + { + "label": "Diesel, burned in machinery", + "kind": "background process", + "values": [ + 0.09, + 0.09 + ] + }, + { + "label": "Seed", + "kind": "background process", + "values": [ + 0.05, + 0.05 + ] + }, + { + "label": "Drying", + "kind": "background process", + "values": [ + 0.04, + 0.04 + ] + } + ], + "rest": [ + 0.050013, + 0.061 + ], + "parts_not_listed": 3 + } + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2eef880 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "sentier-diagnostics" +dynamic = ["version"] +description = "What changed, before and after, in the impact assessment results of a life cycle inventory database, and why." +readme = "README.md" +license = "MIT" +license-files = ["LICENSE", "NOTICE"] +authors = [{ name = "Départ de Sentier" }] +requires-python = ">=3.10" +# Standard library only, at run time and in the tests. Keep it so: the projects that use +# this run it with `uv run --no-project` on the Python their runners have. +dependencies = [] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering", +] + +[project.urls] +Repository = "https://github.com/sentier-dev/sentier-diagnostics" + +[project.scripts] +sentier-diagnostics = "sentier_diagnostics.cli:main" + +[tool.setuptools.dynamic] +version = { attr = "sentier_diagnostics.__version__" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +sentier_diagnostics = ["page/*.html", "page/*.js", "page/*.css"] + +[tool.ruff] +# The lowest Python this supports. A formatter set to a newer one rewrites +# `except (A, B):` as `except A, B:`, which Python 3.13 and older cannot read. +target-version = "py310" +line-length = 110 diff --git a/src/sentier_diagnostics/__init__.py b/src/sentier_diagnostics/__init__.py new file mode 100644 index 0000000..1961582 --- /dev/null +++ b/src/sentier_diagnostics/__init__.py @@ -0,0 +1,8 @@ +"""sentier-diagnostics: what changed in the impact assessment results, and why. + +A chain of results from a database's published ones to the current build, each step +counted with one rule and held against its claim, the whole difference attributed to the +steps, and a static page that shows it. Standard library only; no data. +""" + +__version__ = "0.1.0" diff --git a/src/sentier_diagnostics/__main__.py b/src/sentier_diagnostics/__main__.py new file mode 100644 index 0000000..dd8a8c9 --- /dev/null +++ b/src/sentier_diagnostics/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/src/sentier_diagnostics/breakdown.py b/src/sentier_diagnostics/breakdown.py new file mode 100644 index 0000000..e44131e --- /dev/null +++ b/src/sentier_diagnostics/breakdown.py @@ -0,0 +1,175 @@ +"""What the result of one item is made of, on both sides of a step: `sentier-diagnostics-breakdown/1`. + + {"format": "sentier-diagnostics-breakdown/1", "item": "p01", "from": "repro", "to": "fix-1", + "indicators": {"x:climate": { + "unit": "kg CO2 eq", + "total": [0.52, 0.47], + "parts": [{"label": "Fertiliser, nitrogen", "kind": "background process", + "values": [0.21, 0.16]}], + "rest": [0.31, 0.31], + "parts_not_listed": 14}}} + +A part is whatever the result is made of: an elementary flow, a process of the +background, the item's own flows. Its two values are its contribution to the result +before and after, in the indicator's unit. Parts are chosen by the larger of their two +sizes, so that a part that is large on one side only is listed. What is not listed is +summed in `rest` and counted in `parts_not_listed`: the listed parts and the rest add up +to the total, and the check refuses a breakdown where they do not. Nothing is cut short +without saying so. + +Whether a project may publish a breakdown is a question of its data's terms: it says +more about an inventory than a result does. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from pathlib import Path + +from . import rule +from .fileformat import MAX_INDICATORS, MAX_WORD, is_count, is_optional, is_text + +FORMAT = "sentier-diagnostics-breakdown/1" +MAX_PARTS = 200 +#: How closely the parts and the rest must add up to the total, relative to the largest +#: of the sizes involved: the sum of many products in floating point. +CLOSURE = 1e-9 + + +def _adds_up(parts: Iterable[float], total: float) -> bool: + parts = list(parts) + size = max([abs(total), *map(abs, parts)], default=0.0) + return abs(sum(parts) - total) <= CLOSURE * size + + +def errors(data) -> list[str]: + """Everything wrong with a breakdown; nothing when a page can show it.""" + if not isinstance(data, dict) or data.get("format") != FORMAT: + return [f"not a breakdown of format {FORMAT}"] + found = [f"a key this format does not know: {k}" for k in sorted(set(data) - {"format", "item", "from", "to", "indicators"})] + if not all(is_text(data.get(k), 100) for k in ("item", "from", "to")): + found.append("a breakdown names its item and the two states it compares") + indicators = data.get("indicators") + if not isinstance(indicators, dict) or not 0 < len(indicators) <= MAX_INDICATORS: + return [*found, f"`indicators` is not a table of 1 to {MAX_INDICATORS}"] + for key, entry in indicators.items(): + where = f"indicator {key!r}" + if not is_text(key, 100) or not isinstance(entry, dict) or set(entry) != {"unit", "total", "parts", "rest", "parts_not_listed"}: + found.append(f"{where}: not {{unit, total, parts, rest, parts_not_listed}}") + continue + pairs = [entry["total"], entry["rest"]] + if not is_optional(entry["unit"], MAX_WORD) or not is_count(entry["parts_not_listed"]): + found.append(f"{where}: `unit` is a short text and `parts_not_listed` a count") + parts = entry["parts"] + if not isinstance(parts, list) or len(parts) > MAX_PARTS: + found.append(f"{where}: `parts` is not a list of at most {MAX_PARTS}") + continue + for part in parts: + if not (isinstance(part, dict) and set(part) == {"label", "kind", "values"} + and is_text(part["label"], 300) and is_optional(part["kind"], MAX_WORD)): # fmt: skip + found.append(f"{where}: a part is not {{label, kind, values}}") + break + pairs.append(part["values"]) + if not all(isinstance(p, list) and len(p) == 2 and all(rule.is_value(v) for v in p) for p in pairs): + found.append(f"{where}: a total, rest or part is not [before, after] in finite numbers") + continue + if entry["rest"] != [0, 0] and entry["rest"] != [0.0, 0.0] and entry["parts_not_listed"] == 0: + found.append(f"{where}: a rest without parts that are not listed") + for side, name in ((0, "before"), (1, "after")): + values = [p["values"][side] for p in parts] + [entry["rest"][side]] + if not _adds_up(values, entry["total"][side]): + found.append(f"{where}: the parts and the rest do not add up to the total {name}") + return found + + +def build(item: str, before_state: str, after_state: str, + indicators: Mapping[str, tuple[str | None, Iterable[tuple[str, str | None, float, float]]]], + *, top: int = 20) -> dict: # fmt: skip + """A breakdown, checked. `indicators` is `{indicator: (unit, parts)}`, every part + `(label, kind, before, after)`; the `top` largest by the larger of their two sizes are + listed and the others summed into the rest.""" + out = {} + for key, (unit, parts) in indicators.items(): + parts = [(label, kind, float(b), float(a)) for label, kind, b, a in parts] + ranked = sorted(range(len(parts)), key=lambda i: (-max(abs(parts[i][2]), abs(parts[i][3])), i)) + chosen, left = ranked[:top], ranked[top:] + out[key] = { + "unit": unit, + "total": [sum(p[2] for p in parts), sum(p[3] for p in parts)], + "parts": [ + {"label": parts[i][0], "kind": parts[i][1], "values": [parts[i][2], parts[i][3]]} + for i in chosen + ], + "rest": [sum(parts[i][2] for i in left), sum(parts[i][3] for i in left)], + "parts_not_listed": len(left), + } + data = {"format": FORMAT, "item": item, "from": before_state, "to": after_state, "indicators": out} + found = errors(data) + if found: + raise ValueError("not a breakdown: " + "; ".join(found)) + return data + + +def write(data: Mapping, path: Path) -> Path: + found = errors(data) + if found: + raise ValueError("not a breakdown: " + "; ".join(found)) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=1, ensure_ascii=False, allow_nan=False) + "\n", encoding="utf-8") + return path + + +def compares(diagnostics: Mapping, key: str) -> tuple[str, str]: + """The two states a step of a diagnostics file compares, or the first and the last for + the whole way (`"total"`).""" + if key == "total": + return diagnostics["states"][0]["id"], diagnostics["states"][-1]["id"] + step = next(s for s in diagnostics["steps"] if s["id"] == key) + return step["from"], step["to"] + + +def namings(diagnostics: Mapping) -> list[tuple[str, str, str]]: + """Every breakdown a diagnostics file names: `(file name, item, step or "total")`.""" + return [ + (name, item["id"], key) + for item in diagnostics["items"] + for key, name in sorted((item["breakdowns"] or {}).items()) + ] + + +def against(data: Mapping, diagnostics: Mapping, named_by: tuple[str, str] | None = None) -> list[str]: + """Whether a breakdown agrees with a diagnostics file: it is the breakdown of the item + and step that name it (`named_by`, `(item, step or "total")`), and the totals of both + sides are the item's results in the two states it compares.""" + if named_by is not None: + item_id, key = named_by + want = compares(diagnostics, key) + if (data["item"], data["from"], data["to"]) != (item_id, *want): + said = ( + f"named by {item_id!r} for {key!r}, which compares {want[0]!r} with {want[1]!r}, " + f"it is the breakdown of {data['item']!r} from {data['from']!r} to {data['to']!r}" + ) + return [said] + states = [s["id"] for s in diagnostics["states"]] + if data["from"] not in states or data["to"] not in states: + return [f"breakdown of {data['item']!r}: compares states the file does not hold"] + item = next((i for i in diagnostics["items"] if i["id"] == data["item"]), None) + if item is None: + return [f"breakdown of {data['item']!r}: the file does not list this item"] + found = [] + first, second = states.index(data["from"]), states.index(data["to"]) + for key, entry in data["indicators"].items(): + if key not in item["values"]: + found.append(f"breakdown of {data['item']!r}: an indicator the file does not hold, {key!r}") + continue + for side, at in ((0, first), (1, second)): + result = item["values"][key][at] + total = entry["total"][side] + if result is None or not _adds_up([total], result): + found.append( + f"breakdown of {data['item']!r}, {key}: its total {total!r} is not the " + f"item's result {result!r} in state {states[at]!r}" + ) + return found diff --git a/src/sentier_diagnostics/chain.py b/src/sentier_diagnostics/chain.py new file mode 100644 index 0000000..905b12a --- /dev/null +++ b/src/sentier_diagnostics/chain.py @@ -0,0 +1,523 @@ +"""A chain of results, from the published ones to the current build, counted and attributed. + +A chain is an ordered list of states (sets of results over the same items and +indicators) and the steps between each state and the next. The first state is usually +what its publisher released, the second the same database computed again by us (step 0, +the reproduction), and every later state one change further. A plain before/after +comparison is a chain of two states and one step. + +Every step is counted with the one rule (`rule`), and judged against its claim where it +has one (`claims`). The whole way, first state against last, is counted too ("total"). + +**Attribution is exact.** For one item in one indicator the difference between the last +state and the first is the sum of the differences of the steps, because each step starts +where the one before ended. A step that no cause explains (`explained: false`: a rebuild +on another machine, or a change between two states that nobody named) keeps its share, +named as what nothing explains. Where an item has no result in one of the states, its +steps on either side of that state cannot be measured, and its total is counted as not +attributable instead of being split. + +`build` writes the whole file and checks it (`fileformat.errors`); `write` stores it. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from pathlib import Path + +from . import __version__, claims, fileformat, matrix, rule + +FILE = "diagnostics.json" + + +def tolerance(value=None, basis=None, note=None) -> dict: + """The tolerance of a step: below it, two results agree. Without a value, the default.""" + if value is None: + return { + "value": rule.DEFAULT_TOLERANCE, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds", + } + return {"value": float(value), "basis": basis or "declared", "note": note} + + +def noise(between=None, by=None, note=None) -> dict: + """What two builds of one state differ by, per indicator; not measured without `between`. + An indicator whose noise is None (`noise_of` found no pair to take it from) is left out: + its noise was not measured, which is not the same as two identical builds.""" + if not between: + return {"measured": False, "between": None, "by": None, "note": note} + found = {k: v for k, v in (by or {}).items() if v is not None} + return {"measured": True, "between": list(between), "by": found, "note": note} + + +#: Noise above this is large: it puts every smaller difference of its indicator "within +#: noise". The page and the summary say so; it is a defect to fix, not a width to absorb. +LARGE_NOISE = 1e-2 + + +def noise_of(first: Mapping[str, float | None], second: Mapping[str, float | None], *, + floor: float | Mapping[str, float]) -> float | None: # fmt: skip + """The noise of one indicator: the largest relative difference between two builds of one + state, over the items whose results lie beyond the floor in both. `first` and `second` + are `{item: result}`; `floor` is the floor of the indicator, or `{item: floor}` where + items of several units have floors of their own (`rule.floors_of`). None where no pair + lies beyond the floor: nothing was measured. + + A pair within the floor is left out because its relative difference says nothing: a + residue of 4e-16 in one build and 0 in the other would make the noise 1.0. + + >>> noise_of({"a": 1.0, "b": 2.0, "c": 5.0}, {"a": 1.0, "b": 2.5}, floor=0.0) + 0.2 + >>> noise_of({"a": 1.0, "z": 4e-16}, {"a": 1.0, "z": 0.0}, floor=0.01) + 0.0 + >>> noise_of({"z": 4e-16}, {"z": 0.0}, floor=0.01) is None + True + """ + def beyond(item, value): + level = floor.get(item, 0.0) if isinstance(floor, Mapping) else floor + return rule.is_value(value) and abs(value) > level + + shared = [k for k in first if beyond(k, first[k]) and beyond(k, second.get(k))] + if not shared: + return None + return max(rule.relative_difference(first[k], second[k]) for k in shared) + + +def step(id: str, label: str, *, kind: str, explained: bool = True, about: str | None = None, + link: str | None = None, claim: dict | None = None, tolerance: dict | None = None, + noise: dict | None = None) -> dict: # fmt: skip + """One step between two states, as `build` takes it.""" + return { + "id": id, + "label": label, + "about": about, + "kind": kind, + "explained": explained, + "link": link, + "claim": claim, + "tolerance": tolerance, + "noise": noise, + } + + +def state(id: str, label: str, *, kind: str = "build", about: str | None = None, + identity: Mapping[str, str | None] | None = None, join: Mapping | None = None) -> dict: # fmt: skip + """One state: a set of results. `kind` is "published" for the results its publisher released. + + `identity` is a short table of texts (commit, release, machine); an entry whose value is + None is left out, so that a writer can pass what it may not know. + + `join` is what the reader that made the state counted when it matched its rows to items: + `{"compared": n, "unmatched": n, "ambiguous": n, "note": text or None}`. + + >>> state("b", "Build", identity={"commit": "0e1f5be", "machine": None})["identity"] + {'commit': '0e1f5be'} + """ + if join is not None: + join = {"compared": join["compared"], "unmatched": join.get("unmatched", 0), + "ambiguous": join.get("ambiguous", 0), "note": join.get("note")} # fmt: skip + return {"id": id, "label": label, "about": about, "kind": kind, + "identity": {k: v for k, v in (identity or {}).items() if v is not None}, "join": join} # fmt: skip + + +def explanation(text: str, by: dict, *, step: str | None = None, indicator: str | None = None, + item: str | None = None, link: str | None = None, doc: str | None = None, + not_checked: str | None = None, finding: str = "reason") -> dict: # fmt: skip + """Why something moved, in words, with who wrote it (`claims.origin`). An explanation is + a claim: it never changes a count, a class or a verdict. + + `finding` is "reason" when the text gives one, and "none found" when someone looked and + found none. Where nothing is written, no one has said anything: a third state. + """ + source = {"link": link, "doc": doc} if (link or doc) else None + return { + "about": {"step": step, "indicator": indicator, "item": item}, + "finding": finding, + "text": text, + "by": by, + "source": source, + "not_checked": not_checked, + } + + +def _indicator(entry) -> dict: + if isinstance(entry, Mapping): + found = {k: entry.get(k) for k in ("id", "name", "short", "unit")} + else: + found = dict(zip(("id", "name", "short", "unit"), entry, strict=True)) + return found + + +def _values(values, states, indicators) -> dict: + """The results as `{state: {indicator: {item: value}}}`, refusing what is not a result.""" + found = {} + for one in states: + by_indicator = values.get(one["id"]) + if by_indicator is None: + raise ValueError(f"no results for the state {one['id']!r}") + unknown = set(by_indicator) - set(indicators) + if unknown: + raise ValueError(f"state {one['id']!r} has results of an unknown indicator: {sorted(unknown)}") + found[one["id"]] = {} + for indicator in indicators: + column = {} + for item, value in (by_indicator.get(indicator) or {}).items(): + if not isinstance(item, str): + raise ValueError(f"an item id is not a text: {item!r}") + if value is not None and not rule.is_value(value): + raise ValueError( + f"state {one['id']!r}, {indicator}, {item}: {value!r} is not a finite " + "number (a missing result is None, never zero and never NaN)" + ) + column[item] = None if value is None else float(value) + found[one["id"]][indicator] = column + return found + + +def build(title: str, **given) -> dict: + """A whole chain, counted, judged, attributed and checked: the file, without a matrix. + The keywords are those of `_build`, whose docstring says what each one holds.""" + return _build(title, **given)[0] + + +def build_with_matrix(title: str, *, matrix_file: str = "matrix.json", **given) -> tuple[dict, dict]: + """The file and its full matrix (`sentier_diagnostics.matrix`): every item with its + results and codes, the worst first, in a second file named `matrix_file` that the file + names. Only a local file of items has one.""" + data, found = _build(title, matrix_file=matrix_file, **given) + return data, found + + +def _build( + title: str, + *, + indicators: Iterable, + states: list[dict], + steps: list[dict], + values: Mapping[str, Mapping[str, Mapping[str, float | None]]], + units: Mapping[str, str | None] | None = None, + names: Mapping[str, Mapping] | None = None, + item_word: tuple[str, str] = ("item", "items"), + indicator_word: tuple[str, str] | None = None, + about: str | None = None, + floors: Mapping[tuple[str, str | None], float] | None = None, + floor_factor: float = rule.FLOOR_FACTOR, + listed: int = 200, + tier: str = "items", + audience: str = "local", + claim_lists: bool = True, + explanations: Iterable[dict] = (), + notes: Iterable[str] = (), + breakdowns: Mapping[str, Mapping[str, str]] | None = None, + matrix_file: str | None = None, +) -> tuple[dict, dict | None]: + """A whole chain, counted, judged, attributed and checked, and its matrix where + `matrix_file` names one. + + - `indicators`: `(id, name, short, unit)` or dicts with those keys. + - `states`: `state(...)`, in order; `steps`: `step(...)`, one fewer than the states. + - `values`: `{state id: {indicator id: {item id: result or None}}}`; an item absent from + a state's indicator is missing there. + - `units`: the reference unit of each item; floors are kept apart per unit. + - `names`: `{item id: {"name", "place", "group"}}`. + - `floors`: `{(indicator, unit): floor}` where the project sets them; otherwise each is + `floor_factor` times the median size of the first state's results of that group. + - `listed`: how many items go into the file with their results, the worst first. + - `tier`: "items", or "counts" for a file that holds counts only and no item. + - `claim_lists`: false to leave every claim's list of items out of the file (only its + size, `reach`, is written). The verdicts are still worked out here; a reader of the + file cannot work them out again, and the page says so. A file of counts holds no list. + - `audience`: "local" (the default: for the people who made it) or "public" (its writer + says it may be published). The tool checks only that it is one of the two. + - `breakdowns`: `{item id: {step id or "total": file name}}`. + - `matrix_file`: the name of the full matrix, for a local file of items; None for none. + """ + if matrix_file is not None and (tier != "items" or audience != "local"): + raise ValueError("only a local file of items has a matrix: a public file keeps the listed items only") + meta = [_indicator(i) for i in indicators] + ids = [i["id"] for i in meta] + if len(steps) != len(states) - 1: + raise ValueError("a chain has one step fewer than it has states") + units, names = dict(units or {}), dict(names or {}) + table = _values(values, states, ids) + order = [s["id"] for s in states] + items = sorted({item for s in order for i in ids for item in table[s][i]}) + for item in items: + units.setdefault(item, None) + # a claim names items of this chain: an id no state holds is a mistake, not a claim + for one in steps: + for entry in (one.get("claim") or {}).get("scope", ()): + if entry["items"] not in ("*", None): + unknown = set(entry["items"]) - set(items) + if unknown: + shown = ", ".join(sorted(unknown)[:5]) + (", ..." if len(unknown) > 5 else "") + raise ValueError( + f"step {one['id']!r}: its claim names {len(unknown)} items no state holds ({shown})" + ) + + # floors: one per indicator and reference unit, from the first state + computed = rule.floors_of(table[order[0]], units, floor_factor) + floor_table, floor_rows = {}, [] + for indicator in ids: + for unit in sorted({units[item] for item in items}, key=lambda u: (u is None, u or "")): + key = (indicator, unit) + if floors is not None and key in floors: + value, basis, n = float(floors[key]), "declared", 0 + else: + value, n = computed.get(key, (0.0, 0)) + basis = "median" if n else "none" + floor_table[key] = value + floor_rows.append({"indicator": indicator, "unit": unit, "value": value, + "basis": basis, "n": n}) # fmt: skip + + # the steps, normalised + out_steps = [] + for k, one in enumerate(steps): + found_claim = one.get("claim") + out_steps.append({ + "id": one["id"], + "from": order[k], + "to": order[k + 1], + "label": one["label"], + "about": one.get("about"), + "kind": one["kind"], + "explained": bool(one.get("explained", True)), + "link": one.get("link"), + "claim": claims.published(found_claim, tier == "items" and claim_lists) if found_claim else None, + "tolerance": one.get("tolerance") or tolerance(), + "noise": one.get("noise") or noise(), + }) # fmt: skip + total_tolerance = max(s["tolerance"]["value"] for s in out_steps) + published_first = states[0].get("kind") == "published" + + def judged(k, indicator, item): + """Class, direction, band and verdict of one item in one step (k) or the total (None).""" + first, last = (order[0], order[-1]) if k is None else (order[k], order[k + 1]) + before, after = table[first][indicator].get(item), table[last][indicator].get(item) + if k is None: + level, tol = None, total_tolerance + else: + said = out_steps[k]["noise"] + level = (said["by"] or {}).get(indicator) if said["measured"] else None + tol = out_steps[k]["tolerance"]["value"] + floor = floor_table[(indicator, units[item])] + kind, way, where = rule.judge(before, after, floor=floor, tolerance=tol, noise=level) + said_verdict = None + if k is not None and steps[k].get("claim"): + entries = claims.entries_for(steps[k]["claim"], indicator) + covering = [e for e in entries if claims.covers(e, item, indicator)] + said_verdict = claims.verdict(kind, before, after, way, covering) + return before, after, kind, way, where, said_verdict + + # every pair, once + found = { + (k, indicator, item): judged(k, indicator, item) + for k in [*range(len(out_steps)), None] + for indicator in ids + for item in items + } + + def block(k, indicator): + classes = dict.fromkeys(rule.CLASSES, 0) + missing = dict.fromkeys(fileformat.MISSING_SIDES, 0) + bands = dict.fromkeys(rule.BANDS, 0) + ways = dict.fromkeys(rule.DIRECTIONS, 0) + verdicts = dict.fromkeys(claims.VERDICTS, 0) if k is not None and steps[k].get("claim") else None + closer = further = None + if published_first and k not in (None, 0): + closer = further = 0 + for item in items: + before, after, kind, way, where, said = found[(k, indicator, item)] + classes[kind] += 1 + if kind == rule.MISSING: + missing["both" if before is None and after is None else "before" if before is None else "after"] += 1 + if where: + bands[where] += 1 + if way: + ways[way] += 1 + if verdicts is not None: + verdicts[said] += 1 + if closer is not None and kind in rule.MOVES: + reference = table[order[0]][indicator].get(item) + if reference is not None: + was, now = abs(before - reference), abs(after - reference) + closer += now < was + further += now > was + return {"pairs": len(items), "classes": classes, "missing": missing, "directions": ways, "bands": bands, + "verdicts": verdicts, "closer": closer, "further": further} # fmt: skip + + counts = { + "steps": {s["id"]: {i: block(k, i) for i in ids} for k, s in enumerate(out_steps)}, + "total": {i: block(None, i) for i in ids}, + } + + # attribution: which step carries the largest share of each item's whole difference + attribution = {} + for indicator in ids: + dominant = {s["id"]: 0 for s in out_steps} + not_attributable = spread = unexplained_moved = 0 + for item in items: + kinds = [found[(k, indicator, item)][2] for k in range(len(out_steps))] + if any(not s["explained"] and kinds[k] in rule.MOVES for k, s in enumerate(out_steps)): + unexplained_moved += 1 + if found[(None, indicator, item)][2] not in rule.MOVES: + continue + series = [table[s][indicator].get(item) for s in order] + if any(v is None for v in series): + not_attributable += 1 + continue + carrier = rule.largest_step(series, kinds) + if carrier is None: + spread += 1 + else: + dominant[out_steps[carrier]["id"]] += 1 + attribution[indicator] = {"dominant": dominant, "not_attributable": not_attributable, + "spread": spread, "unexplained_moved": unexplained_moved} # fmt: skip + + # the items worth a look, the worst first + def worth(item): + for indicator in ids: + for k in [*range(len(out_steps)), None]: + _, _, kind, _, _, said = found[(k, indicator, item)] + if kind not in rule.QUIET or said == claims.NOT_MOVED: + return True + return False + + def worst(item): + keys = [] + for indicator in ids: + for k in [*range(len(out_steps)), None]: + before, after, kind, *_ = found[(k, indicator, item)] + keys.append(rule.severity(before, after, kind)) + return (min(keys), item) + + qualifying = sorted((item for item in items if worth(item)), key=worst) + shown = qualifying[: max(0, listed)] if tier == "items" else [] + breakdowns = breakdowns or {} + any_claim = any(s.get("claim") for s in steps) + + def record_of(item): + about_item = names.get(item) or {} + record = { + "id": item, + "name": about_item.get("name") or item, + "place": about_item.get("place"), + "group": about_item.get("group"), + "unit": units[item], + "values": {i: [table[s][i].get(item) for s in order] for i in ids}, + "classes": {}, + "directions": {}, + "bands": {}, + "verdicts": {} if any_claim else None, + "breakdowns": dict(breakdowns[item]) if item in breakdowns else None, + } + for indicator in ids: + row = [found[(k, indicator, item)] for k in [*range(len(out_steps)), None]] + record["classes"][indicator] = [r[2] for r in row] + record["directions"][indicator] = [r[3] for r in row] + record["bands"][indicator] = [r[4] for r in row] + if any_claim: + record["verdicts"][indicator] = [r[5] for r in row[:-1]] + return record + + out_items = [record_of(item) for item in shown] + + data = { + "format": fileformat.FORMAT, + "tier": tier, + "audience": audience, + "title": title, + "about": about, + "item": {"one": item_word[0], "many": item_word[1]}, + **({"indicator": {"one": indicator_word[0], "many": indicator_word[1]}} if indicator_word else {}), + "indicators": meta, + "rule": { + "floor_factor": floor_factor, + "floors_from": order[0], + "beyond": rule.BEYOND_FACTOR, + "total_tolerance": total_tolerance, + "floors": floor_rows, + }, + "states": [ + {"id": s["id"], "label": s["label"], "about": s.get("about"), + "kind": s.get("kind", "build"), "identity": dict(s.get("identity") or {}), + "join": s.get("join")} + for s in states + ], # fmt: skip + "steps": out_steps, + "counts": counts, + "attribution": attribution, + "items": out_items, + "items_count": { + "all": len(items), + "qualifying": len(qualifying), + "listed": len(out_items), + "not_listed": len(qualifying) - len(out_items), + }, + "explanations": list(explanations), + "notes": list(notes), + "made_by": f"sentier-diagnostics {__version__}", + } + if matrix_file is not None: + data["matrix"] = {"file": matrix_file, "items": len(items)} + errors = fileformat.errors(data) + if errors: + raise ValueError("not a diagnostics file: " + "; ".join(errors[:10])) + if matrix_file is None: + return data, None + full = matrix.make(data, [record_of(item) for item in sorted(items, key=worst)]) + errors = matrix.errors(full) or matrix.against(full, data) + if errors: + raise ValueError("not a matrix of this file: " + "; ".join(errors[:10])) + return data, full + + +#: Stands in for the list of items while the rest of a file is written. A file's texts hold +#: no control character, so no text can be written as this. +_ITEMS_HERE = "\x00items\x00" + + +def dumps(data: Mapping) -> str: + """The text of a file: one line per listed item, so that a diff names the items. The keys + keep their order, whatever it is. + + >>> dumps({"items": [{"id": "p"}], "steps": [{"items": []}]}).splitlines()[:3] + ['{', ' "items": [', ' {"id": "p"}'] + """ + listed = data.get("items") or [] + head = json.dumps(dict(data) | {"items": _ITEMS_HERE}, indent=1, ensure_ascii=False, allow_nan=False) + marker = json.dumps(_ITEMS_HERE) + if head.count(marker) != 1: + raise ValueError("a file holds the list of items once, at its top level") + rows = ",\n".join(" " + json.dumps(i, ensure_ascii=False, allow_nan=False) for i in listed) + head = head.replace(marker, f"[\n{rows}\n ]" if listed else "[]") + return head + "\n" + + +def write(data: Mapping, path: Path) -> Path: + """Write a file, checked first; refuse one larger than a page holds.""" + errors = fileformat.errors(data) + if errors: + raise ValueError("not a diagnostics file: " + "; ".join(errors[:10])) + text = dumps(data) + size = len(text.encode("utf-8")) + if size > fileformat.MAX_BYTES: + raise ValueError( + f"the file would be {size} bytes, and a page holds {fileformat.MAX_BYTES} at most: " + "list fewer items (`listed`)" + ) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def read(path: Path) -> dict: + """A file, read and checked; raises ValueError with every error found.""" + data = fileformat.load_json(path) + errors = fileformat.errors(data) + if errors: + raise ValueError(f"{path}: " + "; ".join(errors[:10])) + return data diff --git a/src/sentier_diagnostics/claims.py b/src/sentier_diagnostics/claims.py new file mode 100644 index 0000000..56cdf60 --- /dev/null +++ b/src/sentier_diagnostics/claims.py @@ -0,0 +1,189 @@ +"""What a step claims to move, and how each pair it moved is held against that claim. + +A step of a chain (a fix, a mapping, a contribution, the import itself) may carry a +claim: which indicators and items it moves, in which direction, by how much at most, and +whether each must move or only may. A claim with no entry says that nothing moves. + + {"text": "Wheat and rice go down in climate change, by 30 % at most.", + "by": {"kind": "person", "name": "A. Author"}, + "reach": 2, + "scope": [{"indicators": ["x:climate"], "items": ["p01", "p05"], + "direction": "down", "largest": 0.3, "expect": "must"}]} + +`items` is `"*"` (every item), a list of ids, or null where the list stays with the +project that made it (a list it may not publish). `reach` is the number of items the +claim's lists name together, `"*"` entries aside; where a list stays private it is the +only trace of it. `largest` is relative to the size of the result before; a pair whose +result before lies within the floor (a zero reference) cannot be within it. + +Each pair of a step with a claim gets one verdict, tried in this order: + + not judged it is missing: nothing can be said of it + claimed, not moved an entry says it must move, and it agrees, is within noise or is + near zero: it did not move beyond the tolerance, the noise or the floor + unchanged it agrees, and no entry says it must move + within noise it moved by no more than two builds of one state differ + not judged it is near zero, and no entry says it must move + not claimed it moved, and no entry covers it + as claimed it moved, an entry covers it, in that entry's direction and size + off claim it moved, entries cover it, and none in its direction and size + +Where `must` entries cover a moved pair, it is held against them only: a `may` entry +beside them cannot excuse a move the `must` entries do not allow. + +A scope that a program worked out (every item whose supply chain reaches the change) +says what *may* move: it can be exceeded, never unmet. Its entries carry `"may"`. + +Adapted in part from `before_after.py` of sentier-bafu (MIT, Copyright (c) 2026 +sentier.dev); see NOTICE. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping + +from . import rule + +UNCHANGED = "unchanged" +NOISE = "within noise" +AS_CLAIMED = "as claimed" +OFF_CLAIM = "off claim" +NOT_CLAIMED = "not claimed" +NOT_MOVED = "claimed, not moved" +NOT_JUDGED = "not judged" +#: The verdicts, in the order a page lists them. +VERDICTS = (AS_CLAIMED, OFF_CLAIM, NOT_CLAIMED, NOT_MOVED, NOISE, UNCHANGED, NOT_JUDGED) +#: The verdicts a reviewer has to look at. +TO_LOOK_AT = frozenset({OFF_CLAIM, NOT_CLAIMED, NOT_MOVED}) + +MUST, MAY = "must", "may" +EXPECTS = (MUST, MAY) +WAYS = ("up", "down", "any") + + +def origin(kind: str, name: str | None = None, *, model: str | None = None, + run_by: str | None = None) -> dict: # fmt: skip + """Who wrote a claim or an explanation: a person (by name), code (the program or rule, + by name), or a model (its id, and who ran it).""" + return {"kind": kind, "name": name, "model": model, "run_by": run_by} + + +def scope(indicators="*", items="*", direction="any", largest=None, expect=MAY) -> dict: + """One entry of a claim. `items` may be a set the project keeps to itself: it is used to + judge, and `published` leaves it out of a file. A single text other than "*" is refused: + it would be read as a set of its characters.""" + if isinstance(items, str) and items != "*": + raise TypeError(f"the items of a scope are '*', a list of ids or None, not the text {items!r}") + if isinstance(indicators, str) and indicators != "*": + raise TypeError(f"the indicators of a scope are '*' or a list of ids, not the text {indicators!r}") + return { + "indicators": indicators if indicators == "*" else sorted(indicators), + "items": items if items == "*" or items is None else set(items), + "direction": direction, + "largest": largest, + "expect": expect, + } + + +def claim(text: str, entries: Iterable[dict] = (), *, by: dict | None = None, + reach: int | None = None) -> dict: # fmt: skip + """What a step is expected to move. Without an entry: nothing moves.""" + return {"text": text, "by": by, "reach": reach, "scope": list(entries)} + + +def reach_of(entries: Iterable[Mapping]) -> int | None: + """How many items the lists of a claim name together, `"*"` entries aside; None where a + list is not in hand or no entry lists items. + + >>> reach_of([scope(["x"], {"a", "b"}), scope(["y"], {"b"}), scope(["z"])]) + 2 + >>> reach_of([scope(["x"], None)]) is None, reach_of([scope()]) is None + (True, True) + """ + entries = list(entries) + if any(e["items"] is None for e in entries): + return None + listed = [set(e["items"]) for e in entries if e["items"] != "*"] + return len(set().union(*listed)) if listed else None + + +def published(found: dict, keep_items: bool = True) -> dict: + """A claim as a file holds it: lists sorted, and with `keep_items` false every list of + items left out (`items: null`), its size kept in `reach`.""" + reach, size = found.get("reach"), reach_of(found["scope"]) + if size is not None: + if reach is not None and reach != size: + raise ValueError(f"a claim's reach is {reach}, and its lists name {size} items") + reach = size + entries = [] + for entry in found["scope"]: + items = entry["items"] + if items not in ("*", None): + items = sorted(items) if keep_items else None + entries.append(entry | {"items": items}) + return {"text": found["text"], "by": found.get("by"), "reach": reach, "scope": entries} + + +def covers(entry: Mapping, item: str, indicator: str) -> bool: + """Whether a scope entry covers one item in one indicator. An entry whose list of items + is not in hand covers nothing here: judging needs the list.""" + if entry["indicators"] != "*" and indicator not in entry["indicators"]: + return False + items = entry["items"] + return items == "*" or (items is not None and item in items) + + +def fits(entry: Mapping, before: float, after: float, way: str | None, kind: str | None = None) -> bool: + """Whether a pair moved as an entry expects: its direction, and within its size. A zero + reference has no size to be relative to, so it is never within a `largest`.""" + if entry["direction"] != "any" and entry["direction"] != way: + return False + if entry["largest"] is None: + return True + if kind == rule.ZERO_REFERENCE: + return False + moved_by = rule.change(before, after) + return moved_by is not None and abs(moved_by) <= entry["largest"] + + +#: The classes of a pair that did not move beyond the tolerance, the noise or the floor. +STILL = frozenset({rule.AGREES, rule.WITHIN_NOISE, rule.NEAR_ZERO}) + + +def verdict(kind: str, before, after, way: str | None, covering: list[Mapping]) -> str: + """The verdict of one pair, given its class, its direction and the entries covering it. + + >>> down = scope(["x"], {"a"}, "down", 0.25, "must") + >>> verdict("differs", 4.0, 3.5, "down", [down]), verdict("differs", 4.0, 2.0, "down", [down]) + ('as claimed', 'off claim') + >>> verdict("agrees", 4.0, 4.0, None, [down]), verdict("agrees", 4.0, 4.0, None, [down | {"expect": "may"}]) + ('claimed, not moved', 'unchanged') + >>> verdict("differs", 1.0, 2.0, "up", []), verdict("near zero", 0.0, 0.0, None, [down]) + ('not claimed', 'claimed, not moved') + >>> verdict("missing", None, 1.0, None, [down]), verdict("differs", 4.0, 5.0, "up", [down, scope(["x"], {"a"})]) + ('not judged', 'off claim') + """ + if kind == rule.MISSING: + return NOT_JUDGED + musts = [e for e in covering if e["expect"] == MUST] + if kind in STILL: + if musts: + return NOT_MOVED + return {rule.AGREES: UNCHANGED, rule.WITHIN_NOISE: NOISE}.get(kind, NOT_JUDGED) + if not covering: + return NOT_CLAIMED + held_to = musts or covering + return AS_CLAIMED if any(fits(e, before, after, way, kind) for e in held_to) else OFF_CLAIM + + +def entries_for(found: Mapping | None, indicator: str) -> list[Mapping]: + """The entries of a claim that can cover anything in one indicator.""" + if not found: + return [] + return [e for e in found["scope"] if e["indicators"] == "*" or indicator in e["indicators"]] + + +def judgeable(found: Mapping | None) -> bool: + """Whether every list of items of a claim is in hand, so that its verdicts can be + worked out again from a file.""" + return bool(found) and all(e["items"] is not None for e in found["scope"]) diff --git a/src/sentier_diagnostics/cli.py b/src/sentier_diagnostics/cli.py new file mode 100644 index 0000000..9c732aa --- /dev/null +++ b/src/sentier_diagnostics/cli.py @@ -0,0 +1,130 @@ +"""The command `sentier-diagnostics`: check, build, summarise and show diagnostics files. + +sentier-diagnostics check FILE [FILE ...] exit 1 if any file is not one a page shows + (a matrix beside a file is checked against it) +sentier-diagnostics build DESCRIPTION --out FILE +sentier-diagnostics summary FILE +sentier-diagnostics page FILE --out FOLDER +sentier-diagnostics example --out FOLDER +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from . import __version__, breakdown, chain, example, fileformat, from_tables, matrix, page, summary + + +def check(paths) -> int: + failed = 0 + for path in paths: + try: + data = fileformat.load_json(path) + except ValueError as error: + print(f"FAIL {error}") + failed += 1 + continue + if isinstance(data, dict) and data.get("format") == breakdown.FORMAT: + errors = breakdown.errors(data) + elif isinstance(data, dict) and str(data.get("format", "")).startswith(matrix.FAMILY): + errors = matrix.errors(data) # against its file when the file is checked + else: + errors = fileformat.errors(data) + size = Path(path).stat().st_size + if size > fileformat.MAX_BYTES: + errors.append( + f"{size} bytes, and a page holds {fileformat.MAX_BYTES} at most: list fewer items" + ) + if not errors: + try: + found, absent = page.named_breakdowns(data, Path(path).parent) + except ValueError as error: + found, absent = {}, [] + errors.append(str(error)) + errors += page.breakdown_errors(data, found) + for name in absent: + print(f"note {path}: names the breakdown {name}, which is not beside it (not checked)") + if data.get("matrix"): + try: + if page.named_matrix(data, Path(path).parent) is None: + print( + f"note {path}: names the matrix {data['matrix']['file']}, which is not beside it (not checked)" + ) + except ValueError as error: + errors.append(str(error)) + if errors: + failed += 1 + print(f"FAIL {path}") + for line in errors: + print(f" {line}") + else: + print(f"ok {path}") + return 1 if failed else 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="sentier-diagnostics", description=__doc__.splitlines()[0]) + parser.add_argument("--version", action="version", version=f"sentier-diagnostics {__version__}") + commands = parser.add_subparsers(dest="command", required=True) + one = commands.add_parser("check", help="check diagnostics and breakdown files") + one.add_argument("files", nargs="+", type=Path) + one = commands.add_parser("build", help="build a diagnostics file from CSV tables") + one.add_argument("description", type=Path) + one.add_argument("--out", type=Path, required=True) + one = commands.add_parser("summary", help="print a diagnostics file as Markdown") + one.add_argument("file", type=Path) + one = commands.add_parser("page", help="write the static page of a diagnostics file") + one.add_argument("file", type=Path) + one.add_argument("--out", type=Path, required=True) + one = commands.add_parser("example", help="write the made-up example and its page") + one.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + + try: + if args.command == "check": + return check(args.files) + if args.command == "build": + data, full = from_tables.build_with_matrix(args.description) + chain.write(data, args.out) + if full is None: + print(f"wrote {args.out}") + return 0 + beside = matrix.write(full, Path(args.out).parent / data["matrix"]["file"]) + print(f"wrote {args.out} and its matrix {beside}") + return 0 + if args.command == "summary": + print(summary.text(chain.read(args.file)), end="") + return 0 + if args.command == "page": + data = chain.read(args.file) + found, absent = page.named_breakdowns(data, args.file.parent) + full = page.named_matrix(data, args.file.parent) + where = page.write(data, args.out, found, full) + for name in absent: + print(f"note: the breakdown {name} is not beside the file; the page says so") + if data.get("matrix") and full is None: + print( + f"note: the matrix {data['matrix']['file']} is not beside the file; the page shows the listed items and says so" + ) + print(f"wrote {where}") + return 0 + if args.command == "example": + data = example.build() + chain.write(data, args.out / chain.FILE) + one = example.wheat_breakdown() + breakdown.write(one, args.out / "p01-fix-1.json") + where = page.write(data, args.out / "page", {"p01-fix-1.json": one}) + local, full = example.build_local() + chain.write(local, args.out / "local" / chain.FILE) + matrix.write(full, args.out / "local" / local["matrix"]["file"]) + local_page = page.write(local, args.out / "local" / "page", None, full) + print( + f"wrote {args.out / chain.FILE} and {where}; the local example and its matrix: {local_page}" + ) + return 0 + except (ValueError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 2 diff --git a/src/sentier_diagnostics/example.py b/src/sentier_diagnostics/example.py new file mode 100644 index 0000000..e6719bf --- /dev/null +++ b/src/sentier_diagnostics/example.py @@ -0,0 +1,253 @@ +"""A made-up chain with every class and every verdict in it. Nobody's data. + +Five states of ten made-up items in three indicators: + + published what the publisher released (rounded to four significant digits) + repro step 0: the same database computed again by us + fix-1 a person's fix of two fertiliser amounts + rebuilt the same state built again on another machine (nothing explains it) + mapping-2 a changed mapping, whose reach a program worked out + +What it must show, at least once each: + + agrees most values in every step + within noise Oats, water use, in mapping-2: less than the rebuild differs + differs Wheat, climate change, in fix-1: down 10 % + beyond 10x Tomato, water use, in mapping-2: 25 times as much + zero reference Machine hour, toxicity, in mapping-2: from 0 to a result + zero result Heat, climate change, in mapping-2: from a result to 0 + sign change Compost, climate change, in mapping-2: from a credit to a burden + near zero Rice, toxicity: both results far below the floor of their unit + missing Rye, water use, in the published results; Rye, climate, in mapping-2; + Barley, water use, in the rebuilt state, so that its whole + difference cannot be split among the steps (not attributable) + + as claimed Wheat, climate, fix-1; Tomato, water, mapping-2 + off claim Rice, climate, fix-1: down 50 %, further than the claim's 30 % + not claimed Barley, climate, fix-1; Tomato, toxicity, step 0 + claimed, not moved Maize, water, mapping-2: the claim says it must move + within noise Oats, water, mapping-2 + not judged Rye: missing; Rice, toxicity: near zero + +Two results below zero that go further below zero (Compost, toxicity, mapping-2) went +down: the page colours them as down. +""" + +from __future__ import annotations + +from . import breakdown, chain, claims, rule + +CLIMATE, WATER, TOX = "x:climate", "x:water", "x:tox" +INDICATORS = [ + (CLIMATE, "Climate change", "Climate", "kg CO2 eq"), + (WATER, "Water use", "Water", "m3"), + (TOX, "Human toxicity", "Toxicity", "CTUh"), +] +ITEMS = { + "p01": ("Wheat", "kg", "FR", "crops"), + "p02": ("Maize", "kg", "FR", "crops"), + "p03": ("Barley", "kg", "DE", "crops"), + "p04": ("Oats", "kg", "DE", "crops"), + "p05": ("Rice", "kg", "IT", "crops"), + "p06": ("Rye", "kg", "PL", "crops"), + "p07": ("Heat from wood chips", "MJ", "CH", "energy"), + "p08": ("Compost, used as fertiliser", "kg", "CH", "waste"), + "p09": ("Machine hour, harvesting", "h", "CH", "services"), + "p10": ("Tomato, greenhouse", "kg", "NL", "crops"), +} +PUBLISHED = { + CLIMATE: {"p01": 0.5200, "p02": 0.4100, "p03": 0.4800, "p04": 0.4500, "p05": 1.900, + "p06": 0.4300, "p07": 0.03000, "p08": -0.5000, "p09": 42.00, "p10": 1.300}, + WATER: {"p01": 1.200, "p02": 2.500, "p03": 1.100, "p04": 1.000, "p05": 3.400, + "p06": None, "p07": 0.0004000, "p08": 0.05000, "p09": 0.9000, "p10": 0.02000}, + TOX: {"p01": 2.000e-7, "p02": 1.800e-7, "p03": 1.900e-7, "p04": 1.700e-7, "p05": 1.000e-12, + "p06": 2.100e-7, "p07": 3.000e-9, "p08": -2.000e-6, "p09": 0.0, "p10": 2.100e-7}, +} # fmt: skip + + +def _with(base, **changes): + """A copy of a set of results with some values changed: `{indicator: {item: value}}`.""" + found = {key: dict(column) for key, column in base.items()} + for key, column in changes.items(): + found[{"climate": CLIMATE, "water": WATER, "tox": TOX}[key]].update(column) + return found + + +# step 0: computed again; differs from the published results by their rounding, except +# Tomato's toxicity (an import difference nobody claimed) and Rye's water use, which the +# publisher did not release +REPRO = _with( + PUBLISHED, + climate={"p01": 0.520013, "p03": 0.479990, "p09": 42.00042}, + water={"p06": 1.150, "p02": 2.500025}, + tox={"p10": 2.300e-7, "p01": 2.0000002e-7}, +) +# fix-1: wheat 10 % lower, rice 50 % lower, barley 20 % higher +FIX_1 = _with(REPRO, climate={"p01": 0.468012, "p05": 0.95, "p03": 0.575988}) +# rebuilt elsewhere: oats' water use 2.5 % higher, the rest at solver noise; barley's water +# use failed to compute +REBUILT = _with(FIX_1, water={"p04": 1.025, "p01": 1.2000001, "p03": None}) +# mapping-2 +MAPPING_2 = _with( + REBUILT, + climate={"p07": 0.0, "p08": 0.2, "p06": None}, + water={"p04": 1.0455, "p10": 0.5, "p03": 1.3}, + tox={"p09": 4.0e-7, "p08": -4.0e-6, "p05": 3.0e-12}, +) +VALUES = { + "published": PUBLISHED, + "repro": REPRO, + "fix-1": FIX_1, + "rebuilt": REBUILT, + "mapping-2": MAPPING_2, +} + + +def steps() -> list[dict]: + # noise beyond the floors the chain counts with: those of its first state + units = {item: about[1] for item, about in ITEMS.items()} + floors = rule.floors_of(PUBLISHED, units) + noise = { + key: chain.noise_of(FIX_1[key], REBUILT[key], + floor={item: floors[(key, unit)][0] for item, unit in units.items()}) # fmt: skip + for key in FIX_1 + } + return [ + chain.step( + "step-0", + "Reproduction", + kind="reproduction", + about="The released data computed again with the same method.", + claim=claims.claim( + "Nothing moves beyond the rounding of the published results.", + by=claims.origin("code", "reproduction rule"), + ), + tolerance=chain.tolerance(5e-4, "published rounding", "four significant digits"), + ), + chain.step( + "fix-1", + "Fertiliser amounts of two cereals", + kind="fix", + about="The nitrogen amounts of wheat and rice were per hectare, not per kilogram.", + link="https://example.org/findings/fix-1", + claim=claims.claim( + "Wheat and rice go down in climate change, by 30 % at most. Nothing else moves.", + [claims.scope([CLIMATE], {"p01", "p05"}, "down", 0.3, claims.MUST)], + by=claims.origin("person", "A. Author"), + ), + ), + chain.step( + "rebuilt", + "Built again on another machine", + kind="rebuild", + explained=False, + about="The same state, built on a second machine.", + ), + chain.step( + "mapping-2", + "Mapping change 2", + kind="mapping", + about="Five items take another background dataset.", + claim=claims.claim( + "Items whose supply chain reaches the changed links may move in water use; " + "maize must.", + [ + claims.scope([WATER], {"p02", "p04", "p07", "p10"}, expect=claims.MAY), + claims.scope([WATER], {"p02"}, "up", 0.5, claims.MUST), + ], + by=claims.origin("code", "reach of the changed links"), + ), + noise=chain.noise(("fix-1", "rebuilt"), noise, "One pair of builds of one state."), + ), + ] + + +def explanations() -> list[dict]: + return [ + chain.explanation( + "The fertiliser amounts were per hectare; divided by the yield they are per " + "kilogram, which lowers the nitrogen field emissions of both cereals.", + claims.origin("person", "A. Author"), + step="fix-1", + indicator=CLIMATE, + doc="docs/findings/fix-1.md", + ), + chain.explanation( + "The toxicity results of rice lie far below one hundredth of the median result " + "of the items measured per kilogram: they are counted as near zero.", + claims.origin("code", "floor rule"), + indicator=TOX, + item="p05", + ), + chain.explanation( + "The new mapping links tomatoes to a greenhouse heated with fresh water from a " + "well, which multiplies the water use.", + claims.origin("model", model="example-model-1", run_by="example maintainers (agent-run)"), + step="mapping-2", + indicator=WATER, + item="p10", + not_checked="Nobody compared the greenhouse's water balance with a second source.", + ), + chain.explanation( + "Compost's toxicity result fell to almost nothing in the reproduction. The recipe, " + "the background datasets and the method were compared; none of them explains it.", + claims.origin("person", "B. Reviewer"), + step="step-0", + indicator=TOX, + item="p08", + finding="none found", + ), + ] + + +def build(**more) -> dict: + """The made-up chain, counted and checked.""" + return _made(chain.build, **more) + + +def build_local() -> tuple[dict, dict]: + """The same chain as a local file that lists the six worst products, and its full matrix + of all ten: what a project keeps on its own machine.""" + return _made(chain.build_with_matrix, audience="local", listed=6) + + +def _made(builder, **more): + return builder( + "A made-up chain: published results to the current build", + about="Made-up results of ten items in three indicators. Nobody's data.", + indicators=INDICATORS, + states=[ + chain.state("published", "Published", kind="published", + about="What the publisher released.", identity={"release": "1.0"}, + join={"compared": 10, "unmatched": 1, "ambiguous": 0, + "note": "One published row names no item of the build."}), + chain.state("repro", "Recomputed", identity={"commit": "0000001"}), + chain.state("fix-1", "With fix 1", identity={"commit": "0000002"}), + chain.state("rebuilt", "Rebuilt", identity={"commit": "0000002", "machine": "second"}), + chain.state("mapping-2", "With mapping 2", identity={"commit": "0000003"}), + ], # fmt: skip + steps=steps(), + values=VALUES, + units={k: v[1] for k, v in ITEMS.items()}, + names={k: {"name": v[0], "place": v[2], "group": v[3]} for k, v in ITEMS.items()}, + item_word=("product", "products"), + explanations=[e for e in explanations() if more.get("tier") != "counts" or e["about"]["item"] is None], + breakdowns={"p01": {"fix-1": "p01-fix-1.json"}}, + notes=["A made-up example: nobody's data."], + **{"audience": "public", **more}, + ) + + +def wheat_breakdown() -> dict: + """What wheat's climate change result is made of, before and after fix 1.""" + parts = [ + ("Nitrogen fertiliser, production", "background process", 0.110, 0.099), + ("Dinitrogen monoxide, field", "elementary flow", 0.180, 0.1280120), + ("Diesel, burned in machinery", "background process", 0.090, 0.090), + ("Seed", "background process", 0.050, 0.050), + ("Drying", "background process", 0.040, 0.040), + ("Transport to the mill", "background process", 0.030, 0.030), + ("Ammonia, field", "elementary flow", 0.0, 0.010), + ("Pesticides", "background process", 0.020013, 0.021), + ] + return breakdown.build("p01", "repro", "fix-1", {CLIMATE: ("kg CO2 eq", parts)}, top=5) diff --git a/src/sentier_diagnostics/fileformat.py b/src/sentier_diagnostics/fileformat.py new file mode 100644 index 0000000..e00b03d --- /dev/null +++ b/src/sentier_diagnostics/fileformat.py @@ -0,0 +1,878 @@ +"""The format of a diagnostics file, `sentier-diagnostics/1`, and its check. + +A file is data from a run, so it is checked before anything shows it. `errors(data)` +returns everything wrong with it; an empty list means a page can show it. The check +refuses a key the format does not name, a text that is too long or holds a control +character, a number that is not finite, an id the file does not hold, and a list that is +too long. It then works the classes, directions, bands and verdicts of every listed item +out again from the item's own results and the file's rule, and refuses a file whose +written answers differ: a writer that counts differently cannot pass. It checks that the +counts add up, and that no item that had to be listed is missing without being counted +as not listed. + +A file of another major version is refused with that reason. The reader refuses every key +it does not know, so any change to the keys is a new major version (SPEC 4.1). + +Parts of the shape check are adapted from `before_after.py` of sentier-bafu (MIT, +Copyright (c) 2026 sentier.dev); see NOTICE. +""" + +from __future__ import annotations + +import json +import re +import unicodedata +from collections.abc import Mapping +from pathlib import Path + +from . import claims, rule + +FORMAT = "sentier-diagnostics/1" +FAMILY = "sentier-diagnostics/" +TIERS = ("items", "counts") +AUDIENCES = ("public", "local") +FINDINGS = ("reason", "none found") +MISSING_SIDES = ("before", "after", "both") +JOIN_KEYS = {"compared", "unmatched", "ambiguous", "note"} +STATE_KINDS = ("published", "build") +TOLERANCE_BASES = ("default", "declared", "measured", "published rounding") +FLOOR_BASES = ("median", "declared", "none") +ORIGIN_KINDS = ("person", "code", "model") +TOTAL = "total" +#: Keys a file may leave out; each is null where it says nothing. +OPTIONAL_KEYS = {"matrix", "indicator"} + +#: The largest file a page reads: it holds the whole file. +MAX_BYTES = 12 * 1024 * 1024 +MAX_INDICATORS, MAX_STATES, MAX_ITEMS, MAX_SCOPE = 100, 30, 5000, 50 +#: An entry of a claim may list as many items as a file may. +MAX_SCOPE_ITEMS = MAX_ITEMS +MAX_EXPLANATIONS, MAX_NOTES, MAX_IDENTITY = 500, 20, 12 +MAX_ID, MAX_LABEL, MAX_WORD, MAX_TEXT, MAX_LONG = 100, 80, 40, 600, 2000 +FILE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\.json") + +KEYS = { + "format", "tier", "audience", "title", "about", "item", "indicators", "rule", "states", "steps", + "counts", "attribution", "items", "items_count", "explanations", "notes", "made_by", +} # fmt: skip +INDICATOR_KEYS = {"id", "name", "short", "unit"} +RULE_KEYS = {"floor_factor", "floors_from", "beyond", "total_tolerance", "floors"} +FLOOR_KEYS = {"indicator", "unit", "value", "basis", "n"} +STATE_KEYS = {"id", "label", "about", "kind", "identity", "join"} +STEP_KEYS = {"id", "from", "to", "label", "about", "kind", "explained", "link", "claim", + "tolerance", "noise"} # fmt: skip +CLAIM_KEYS = {"text", "by", "reach", "scope"} +SCOPE_KEYS = {"indicators", "items", "direction", "largest", "expect"} +ORIGIN_KEYS = {"kind", "name", "model", "run_by"} +BLOCK_KEYS = {"pairs", "classes", "missing", "directions", "bands", "verdicts", "closer", "further"} +ITEM_KEYS = {"id", "name", "place", "group", "unit", "values", "classes", "directions", + "bands", "verdicts", "breakdowns"} # fmt: skip +COUNT_KEYS = {"all", "qualifying", "listed", "not_listed"} +ATTRIBUTION_KEYS = {"dominant", "not_attributable", "spread", "unexplained_moved"} +EXPLANATION_KEYS = {"about", "finding", "text", "by", "source", "not_checked"} + +#: The verdicts a pair of each class can have. +ALLOWED_VERDICTS = { + rule.MISSING: {claims.NOT_JUDGED}, + rule.NEAR_ZERO: {claims.NOT_JUDGED, claims.NOT_MOVED}, + rule.AGREES: {claims.UNCHANGED, claims.NOT_MOVED}, + rule.WITHIN_NOISE: {claims.NOISE, claims.NOT_MOVED}, + **{kind: {claims.AS_CLAIMED, claims.OFF_CLAIM, claims.NOT_CLAIMED} for kind in rule.MOVES}, +} + + +# ------------------------------------------------------------------------ reading + + +def load_json(path) -> object: + """A JSON file, read strictly: `NaN` and `Infinity` are refused, and so is a key written + twice in one object (the last would win silently). Raises ValueError naming the file. + + >>> import tempfile, pathlib + >>> where = pathlib.Path(tempfile.mkdtemp()) / "twice.json" + >>> _ = where.write_text('{"a": 1, "a": 2}') + >>> load_json(where) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...twice.json: cannot be read (the key 'a' is written twice in one object) + """ + try: + return json.loads(Path(path).read_text(encoding="utf-8"), parse_constant=_refuse_constant, + object_pairs_hook=_refuse_twice) # fmt: skip + except (OSError, UnicodeDecodeError, json.JSONDecodeError, _Twice) as error: + raise ValueError(f"{path}: cannot be read ({error})") from None + + +class _Twice(ValueError): + pass + + +def _refuse_twice(pairs): + found = {} + for key, value in pairs: + if key in found: + raise _Twice(f"the key {key!r} is written twice in one object") + found[key] = value + return found + + +def _refuse_constant(constant): + raise json.JSONDecodeError(f"{constant} is not a number a file may hold", constant, 0) + + +# ------------------------------------------------------------------------ small checks + + +def is_text(value, limit: int = MAX_TEXT) -> bool: + """A text a page can show: not empty, not too long, no control character and no half of + a character (a lone surrogate, which cannot be written as UTF-8). + + >>> is_text("Wheat"), is_text("a\\nb"), is_text("half \\ud800"), is_text(" ") + (True, False, False, False) + """ + return ( + isinstance(value, str) + and 0 < len(value) <= limit + and bool(value.strip()) + and not any(unicodedata.category(c) in ("Cc", "Cs") for c in value) + ) + + +#: A word that says a state was approved: SPEC 8 allows no approval of any kind in a file, +#: and a state's free `identity` table is no way round it. +APPROVAL = re.compile(r"status|accept|approv", re.IGNORECASE) + + +def is_optional(value, limit: int = MAX_TEXT) -> bool: + return value is None or is_text(value, limit) + + +def is_count(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def is_share(value) -> bool: + return rule.is_value(value) and value >= 0 + + +def indicator_word(data: Mapping) -> dict: + """What the file calls an indicator, {one, many}: its own word, or "indicator". + + >>> indicator_word({"indicator": {"one": "impact category", "many": "impact categories"}})["many"] + 'impact categories' + >>> indicator_word({})["one"] + 'indicator' + """ + return data.get("indicator") or {"one": "indicator", "many": "indicators"} + + +def first_step(data: Mapping) -> int: + """The number of the first step: 0 where the chain starts from the published results (that + step checks they are reproduced), 1 otherwise. + + >>> first_step({"states": [{"kind": "published"}]}), first_step({"states": [{"kind": "build"}]}) + (0, 1) + """ + return 0 if data["states"][0]["kind"] == "published" else 1 + + +def is_link(value) -> bool: + return value is None or (is_text(value, 500) and value.startswith("https://")) + + +def origin_errors(found, where: str) -> list[str]: + """Who wrote a claim or an explanation.""" + if not isinstance(found, dict) or set(found) != ORIGIN_KEYS: + return [f"{where}: `by` is not {{kind, name, model, run_by}}"] + if found["kind"] not in ORIGIN_KINDS: + return [f"{where}: `by.kind` is not one of {', '.join(ORIGIN_KINDS)}"] + if not all(is_optional(found[k], MAX_LABEL) for k in ("name", "model", "run_by")): + return [f"{where}: `by` holds a text that is too long or not a text"] + if found["kind"] in ("person", "code") and found["name"] is None: + return [f"{where}: a {found['kind']} is named (`by.name`)"] + if found["kind"] == "model" and (found["model"] is None or found["run_by"] is None): + return [f"{where}: a text by a model names the model and who ran it"] + return [] + + +# ------------------------------------------------------------------------ the parts + + +def _indicators(data, found) -> list[str]: + indicators = data.get("indicators") + if not isinstance(indicators, list) or not 0 < len(indicators) <= MAX_INDICATORS: + found.append(f"`indicators` is not a list of 1 to {MAX_INDICATORS}") + return [] + ids = [] + for one in indicators: + if not isinstance(one, dict) or set(one) != INDICATOR_KEYS: + found.append("an indicator is not {id, name, short, unit}") + return [] + if not is_text(one["id"], MAX_ID) or not is_text(one["name"], MAX_ID): + found.append("an indicator has no `id` or `name` that is a short text") + return [] + if not is_optional(one["short"], MAX_WORD) or not is_optional(one["unit"], MAX_WORD): + found.append(f"indicator {one['id']!r}: `short` or `unit` is not a short text") + ids.append(one["id"]) + if len(set(ids)) != len(ids): + found.append("two indicators have the same id") + return [] + return ids + + +def _states(data, found) -> list[str]: + states = data.get("states") + if not isinstance(states, list) or not 2 <= len(states) <= MAX_STATES: + found.append(f"`states` is not a list of 2 to {MAX_STATES}") + return [] + ids = [] + for k, one in enumerate(states): + if not isinstance(one, dict) or set(one) != STATE_KEYS or not is_text(one["id"], MAX_ID): + found.append(f"a state is not {{{', '.join(sorted(STATE_KEYS))}}}") + return [] + where = f"state {one['id']!r}" + if not is_text(one["label"], MAX_LABEL) or not is_optional(one["about"]): + found.append(f"{where}: `label` or `about` is not a text") + if one["kind"] not in STATE_KINDS: + found.append(f"{where}: `kind` is not one of {', '.join(STATE_KINDS)}") + elif one["kind"] == "published" and k != 0: + found.append(f"{where}: only the first state can be the published results") + identity = one["identity"] + if not ( + isinstance(identity, dict) + and len(identity) <= MAX_IDENTITY + and all(is_text(k2, MAX_WORD) and is_text(v, 200) for k2, v in identity.items()) + ): + found.append(f"{where}: `identity` is not a table of at most {MAX_IDENTITY} short texts") + elif any(APPROVAL.search(k2) for k2 in identity): + found.append(f"{where}: `identity` names what makes a state, never a status or an approval (SPEC 8)") + join = one["join"] + if join is not None and not ( + isinstance(join, dict) + and set(join) == JOIN_KEYS + and all(is_count(join[k2]) for k2 in ("compared", "unmatched", "ambiguous")) + and is_optional(join["note"]) + ): + found.append(f"{where}: `join` is not {{compared, unmatched, ambiguous, note}}") + ids.append(one["id"]) + if len(set(ids)) != len(ids): + found.append("two states have the same id") + return [] + return ids + + +def claim_errors(found_claim, where, indicators, tier) -> list[str]: + """Everything wrong with a claim: its shape first, then whether its `reach` is the size of + its lists.""" + found = _claim_shape(found_claim, where, indicators, tier) + if found: + return found + size = claims.reach_of(found_claim["scope"]) + if size is not None and found_claim["reach"] != size: + found.append(f"{where}: the reach of a claim is {found_claim['reach']}, and its lists name {size} items") + return found + + +def _claim_shape(found_claim, where, indicators, tier) -> list[str]: + """What keeps a claim from being read: a page or the validator cannot judge with it.""" + if not isinstance(found_claim, dict) or set(found_claim) != CLAIM_KEYS: + return [f"{where}: `claim` is not {{text, by, reach, scope}}"] + found = [] + if not is_text(found_claim["text"]): + found.append(f"{where}: a claim says what it claims (`text`)") + if found_claim["by"] is not None: + found += origin_errors(found_claim["by"], where) + if found_claim["reach"] is not None and not is_count(found_claim["reach"]): + found.append(f"{where}: the reach of a claim is not a count") + entries = found_claim["scope"] + if not isinstance(entries, list) or len(entries) > MAX_SCOPE: + return [*found, f"{where}: the scope of a claim is not a list of at most {MAX_SCOPE}"] + for entry in entries: + if not isinstance(entry, dict) or set(entry) != SCOPE_KEYS: + return [*found, f"{where}: a scope entry is not {{{', '.join(sorted(SCOPE_KEYS))}}}"] + named, items = entry["indicators"], entry["items"] + if named != "*" and not ( + isinstance(named, list) and all(isinstance(i, str) for i in named) and set(named) <= set(indicators) + ): + found.append(f"{where}: a scope's indicators are not '*' or ids the file holds") + if items not in ("*", None): + if tier == "counts": + found.append(f"{where}: a file of counts lists no item in a claim") + elif not ( + isinstance(items, list) + and len(items) <= MAX_SCOPE_ITEMS + and all(is_text(i, MAX_ID) for i in items) + ): + found.append(f"{where}: the items of a scope are not at most {MAX_SCOPE_ITEMS} ids") + if entry["direction"] not in claims.WAYS: + found.append(f"{where}: a scope's direction is not one of {', '.join(claims.WAYS)}") + if entry["largest"] is not None and not is_share(entry["largest"]): + found.append(f"{where}: a scope's largest move is not a share") + if entry["expect"] not in claims.EXPECTS: + found.append(f"{where}: a scope says whether its values must or may move") + return found + + +def _steps(data, found, states, indicators, tier) -> list[dict]: + steps = data.get("steps") + if not isinstance(steps, list) or len(steps) != max(0, len(states) - 1): + found.append("`steps` is not a list of one step fewer than the states") + return [] + ids = [] + for k, one in enumerate(steps): + if not isinstance(one, dict) or set(one) != STEP_KEYS or not is_text(one["id"], MAX_ID): + found.append(f"a step is not {{{', '.join(sorted(STEP_KEYS))}}}") + return [] + where = f"step {one['id']!r}" + if one["id"] == TOTAL: + found.append(f"{where}: `total` names the whole way and is no step's id") + if (one["from"], one["to"]) != (states[k], states[k + 1]): + found.append(f"{where}: it does not lead from state {states[k]!r} to {states[k + 1]!r}") + if not is_text(one["label"], MAX_LABEL) or not is_optional(one["about"]): + found.append(f"{where}: `label` or `about` is not a text") + if not is_text(one["kind"], MAX_WORD) or not isinstance(one["explained"], bool): + found.append(f"{where}: a step has a `kind` and says whether a cause explains it") + if not is_link(one["link"]): + found.append(f"{where}: `link` is not an https address") + if one["claim"] is not None: + shape = _claim_shape(one["claim"], where, indicators, tier) + if shape: # nothing can be judged against a claim that cannot be read + found += shape + return [] + found += claim_errors(one["claim"], where, indicators, tier) + said = one["tolerance"] + if not ( + isinstance(said, dict) + and set(said) == {"value", "basis", "note"} + and is_share(said["value"]) + and said["basis"] in TOLERANCE_BASES + and is_optional(said["note"]) + ): + found.append(f"{where}: `tolerance` is not {{value, basis, note}}") + return [] + noise = one["noise"] + if not ( + isinstance(noise, dict) + and set(noise) == {"measured", "between", "by", "note"} + and isinstance(noise["measured"], bool) + and is_optional(noise["note"]) + ): + found.append(f"{where}: `noise` is not {{measured, between, by, note}}") + return [] + if noise["measured"]: + if not ( + isinstance(noise["between"], list) + and len(noise["between"]) == 2 + and all(is_text(n, MAX_LABEL) for n in noise["between"]) + and isinstance(noise["by"], dict) + and set(noise["by"]) <= set(indicators) + and all(is_share(v) for v in noise["by"].values()) + ): + found.append(f"{where}: measured noise names the two builds and a size per indicator") + return [] + elif noise["between"] is not None or noise["by"] is not None: + found.append(f"{where}: noise that was not measured has no builds and no sizes") + ids.append(one["id"]) + if len(set(ids)) != len(ids): + found.append("two steps have the same id") + return [] + return steps + + +def _rule(data, found, indicators, states, steps) -> dict | None: + said = data.get("rule") + if not isinstance(said, dict) or set(said) != RULE_KEYS: + found.append(f"`rule` is not {{{', '.join(sorted(RULE_KEYS))}}}") + return None + if not is_share(said["floor_factor"]): + found.append("`rule`: the floor factor is not a share") + if said["floors_from"] != states[0]: + found.append("`rule`: the floors come from the first state") + if said["beyond"] != rule.BEYOND_FACTOR: + found.append(f"`rule.beyond` is {rule.BEYOND_FACTOR:g}: one rule for every file") + if not is_share(said["total_tolerance"]): + found.append("`rule.total_tolerance` is not a share") + return None + if steps and said["total_tolerance"] != max(s["tolerance"]["value"] for s in steps): + found.append("`rule.total_tolerance` is not the largest tolerance of the steps") + floors = said["floors"] + if not isinstance(floors, list): + found.append("`rule.floors` is not a list") + return None + table = {} + for one in floors: + if not ( + isinstance(one, dict) + and set(one) == FLOOR_KEYS + and one["indicator"] in indicators + and is_optional(one["unit"], MAX_WORD) + and is_share(one["value"]) + and one["basis"] in FLOOR_BASES + and is_count(one["n"]) + ): + found.append("a floor is not {indicator, unit, value, basis, n} of a known indicator") + return None + key = (one["indicator"], one["unit"]) + if key in table: + found.append(f"two floors for {one['indicator']!r} in unit {one['unit']!r}") + basis, n, value = one["basis"], one["n"], one["value"] + if (basis == "none" and (n or value)) or (basis == "median" and not n) or (basis == "declared" and n): + found.append( + f"the floor of {one['indicator']!r} in unit {one['unit']!r}: basis {basis!r} with " + f"{n} results and the value {value!r} (a median is taken from results, none is 0 from " + "none, a declared floor from none)" + ) + table[key] = value + return table + + +def _block(block, where, step) -> list[str]: + if not isinstance(block, dict) or set(block) != BLOCK_KEYS: + return [f"{where}: counts are not {{{', '.join(sorted(BLOCK_KEYS))}}}"] + found = [] + if not is_count(block["pairs"]): + return [f"{where}: `pairs` is not a count"] + classes, bands, verdicts = block["classes"], block["bands"], block["verdicts"] + if not (isinstance(classes, dict) and set(classes) == set(rule.CLASSES) + and all(map(is_count, classes.values()))): # fmt: skip + return [f"{where}: `classes` is not a count for each class"] + if sum(classes.values()) != block["pairs"]: + found.append(f"{where}: the classes do not add up to the pairs") + missing = block["missing"] + if not (isinstance(missing, dict) and set(missing) == set(MISSING_SIDES) + and all(map(is_count, missing.values()))): # fmt: skip + return [*found, f"{where}: `missing` is not a count for before, after and both"] + if sum(missing.values()) != classes[rule.MISSING]: + found.append(f"{where}: missing before, after and on both sides do not add up to missing") + if not (isinstance(bands, dict) and set(bands) == set(rule.BANDS) + and all(map(is_count, bands.values()))): # fmt: skip + return [*found, f"{where}: `bands` is not a count for each band"] + if sum(bands.values()) != classes[rule.DIFFERS] + classes[rule.BEYOND]: + found.append(f"{where}: the bands do not add up to what differs and what is beyond 10x") + ways = block["directions"] + if not (isinstance(ways, dict) and set(ways) == set(rule.DIRECTIONS) + and all(map(is_count, ways.values()))): # fmt: skip + return [*found, f"{where}: `directions` is not a count for up and for down"] + if sum(ways.values()) != sum(classes[k] for k in rule.MOVES): + found.append(f"{where}: up and down do not add up to what moved") + for way in rule.DIRECTIONS: + banded = sum(n for name, n in bands.items() if name.startswith(way + ",")) + if banded > ways[way]: + found.append(f"{where}: more bands {way} than values that went {way}") + has_claim = step is not None and step["claim"] is not None + if has_claim: + if not (isinstance(verdicts, dict) and set(verdicts) == set(claims.VERDICTS) + and all(map(is_count, verdicts.values()))): # fmt: skip + return [*found, f"{where}: `verdicts` is not a count for each verdict"] + if sum(verdicts.values()) != block["pairs"]: + found.append(f"{where}: the verdicts do not add up to the pairs") + elif verdicts is not None: + found.append(f"{where}: verdicts without a claim") + for key in ("closer", "further"): + if block[key] is not None and not is_count(block[key]): + found.append(f"{where}: `{key}` is not a count") + moved = sum(classes[k] for k in rule.MOVES) + if is_count(block["closer"]) and is_count(block["further"]): + if block["closer"] + block["further"] > moved: + found.append(f"{where}: more came closer or went further than moved") + return found + + +def _counts(data, found, indicators, steps, total_pairs): + counts = data.get("counts") + if not isinstance(counts, dict) or set(counts) != {"steps", TOTAL}: + found.append("`counts` is not {steps, total}") + return {} + by_step = counts["steps"] + if not isinstance(by_step, dict) or set(by_step) != {s["id"] for s in steps}: + found.append("`counts.steps` does not hold every step") + return {} + published_first = data["states"][0]["kind"] == "published" + blocks = {} + for k, one in enumerate([*steps, None]): + key = one["id"] if one else TOTAL + table = by_step[key] if one else counts[TOTAL] + if not isinstance(table, dict) or set(table) != set(indicators): + found.append(f"counts of {key!r} do not hold every indicator") + return {} + # closer and further: only for a step after step 0, and only from published results + counted_closer = published_first and one is not None and k > 0 + for indicator in indicators: + where = f"counts of {key!r} in {indicator!r}" + errors = _block(table[indicator], where, one) + found += errors + if errors: + continue + if table[indicator]["pairs"] != total_pairs: + found.append(f"{where}: not every item is counted") + if counted_closer != (table[indicator]["closer"] is not None) or counted_closer != ( + table[indicator]["further"] is not None + ): + found.append( + f"{where}: `closer` and `further` are counted for a step after step 0 from " + "published results, and are null everywhere else" + ) + blocks[(key, indicator)] = table[indicator] + return blocks + + +def _attribution(data, found, indicators, steps, blocks): + said = data.get("attribution") + if not isinstance(said, dict) or set(said) != set(indicators): + found.append("`attribution` does not hold every indicator") + return + for indicator in indicators: + one = said[indicator] + where = f"attribution in {indicator!r}" + if not ( + isinstance(one, dict) + and set(one) == ATTRIBUTION_KEYS + and isinstance(one["dominant"], dict) + and set(one["dominant"]) == {s["id"] for s in steps} + and all(map(is_count, one["dominant"].values())) + and all(is_count(one[k]) for k in ("not_attributable", "spread", "unexplained_moved")) + ): + found.append(f"{where}: not {{dominant per step, not_attributable, spread, unexplained_moved}}") + continue + block = blocks.get((TOTAL, indicator)) + if block: + moved = sum(block["classes"][k] for k in rule.MOVES) + if sum(one["dominant"].values()) + one["not_attributable"] + one["spread"] != moved: + found.append(f"{where}: does not account for every item whose total moved") + + +def _items(data, found, indicators, states, steps, floors, tier, blocks): + listed = data.get("items") + said = data.get("items_count") + if not (isinstance(said, dict) and set(said) == COUNT_KEYS and all(map(is_count, said.values()))): + found.append("`items_count` is not {all, qualifying, listed, not_listed} in counts") + return [] + if said["qualifying"] != said["listed"] + said["not_listed"] or said["qualifying"] > said["all"]: + found.append("`items_count`: the items listed and not listed are not those worth a look") + if not isinstance(listed, list) or len(listed) > MAX_ITEMS or len(listed) != said["listed"]: + found.append(f"`items` is not a list of the {said['listed']} items said to be listed, " + f"and of at most {MAX_ITEMS}") # fmt: skip + return [] + if tier == "counts" and listed: + found.append("a file of counts holds no item") + return [] + nsteps = len(steps) + claims_by_step = [s["claim"] for s in steps] + any_claim = any(claims_by_step) + published_first = data["states"][0]["kind"] == "published" + seen, previous_key = set(), None + # what the listed items hold, counted as the blocks count it: every part of a block that + # only a pair worth a look can add to (`_LOUD`), and the attribution + tally: dict[tuple[str, str], dict] = {} + attributed = { + i: {"dominant": dict.fromkeys((s["id"] for s in steps), 0), "not_attributable": 0, "spread": 0, + "unexplained_moved": 0} + for i in indicators + } # fmt: skip + for item in listed: + if not isinstance(item, dict) or set(item) != ITEM_KEYS or not is_text(item["id"], MAX_ID): + found.append(f"an item is not {{{', '.join(sorted(ITEM_KEYS))}}}") + return [] + where = f"item {item['id']!r}" + if item["id"] in seen: + found.append(f"{where} is listed twice") + seen.add(item["id"]) + if not is_text(item["name"], 300): + found.append(f"{where}: `name` is not a text") + if not (is_optional(item["place"], 60) and is_optional(item["group"], MAX_LABEL) + and is_optional(item["unit"], MAX_WORD)): # fmt: skip + found.append(f"{where}: `place`, `group` or `unit` is not a short text") + return [] + tables = [item[k] for k in ("values", "classes", "directions", "bands")] + if not all(isinstance(t, dict) and set(t) == set(indicators) for t in tables): + found.append(f"{where}: results, classes, directions and bands are given for every indicator") + return [] + if any_claim != isinstance(item["verdicts"], dict) or ( + any_claim and set(item["verdicts"]) != set(indicators) + ): + found.append(f"{where}: verdicts are given for every indicator where, and only where, a step has a claim") + return [] + breakdowns = item["breakdowns"] + if breakdowns is not None and not ( + isinstance(breakdowns, dict) + and set(breakdowns) <= {s["id"] for s in steps} | {TOTAL} + and all(isinstance(f, str) and FILE_NAME.fullmatch(f) for f in breakdowns.values()) + ): + found.append(f"{where}: `breakdowns` names a step it does not know or a file that is a path") + worth, keys = False, [] + for indicator in indicators: + values = item["values"][indicator] + kinds, ways, bands = (item[k][indicator] for k in ("classes", "directions", "bands")) + if not (isinstance(values, list) and len(values) == len(states) + and all(v is None or rule.is_value(v) for v in values)): # fmt: skip + found.append(f"{where}, {indicator}: the results are not one finite number or null per state") + return [] + if not all(isinstance(x, list) and len(x) == nsteps + 1 for x in (kinds, ways, bands)): + found.append(f"{where}, {indicator}: classes, directions and bands are one per step and the total") + return [] + floor = floors.get((indicator, item["unit"])) + if floor is None: + found.append(f"{where}, {indicator}: the file holds no floor for unit {item['unit']!r}") + return [] + verdicts = item["verdicts"][indicator] if any_claim else None + if any_claim and not (isinstance(verdicts, list) and len(verdicts) == nsteps): + found.append(f"{where}, {indicator}: verdicts are one per step") + return [] + for k in [*range(nsteps), None]: + at = nsteps if k is None else k + name = TOTAL if k is None else steps[k]["id"] + before, after = (values[0], values[-1]) if k is None else (values[k], values[k + 1]) + if k is None: + tol, level = data["rule"]["total_tolerance"], None + else: + tol = steps[k]["tolerance"]["value"] + noise = steps[k]["noise"] + level = (noise["by"] or {}).get(indicator) if noise["measured"] else None + expected = rule.judge(before, after, floor=floor, tolerance=tol, noise=level) + if (kinds[at], ways[at], bands[at]) != expected: + found.append( + f"{where}, {indicator}, {name}: written as {kinds[at]!r}/{ways[at]!r}/" + f"{bands[at]!r}, the rule gives {expected[0]!r}/{expected[1]!r}/{expected[2]!r}" + ) + return [] + one = tally.setdefault((name, indicator), {}) + kind = kinds[at] + if kind not in rule.QUIET: + _add(one, ("classes", kind)) + if kind == rule.MISSING: + _add(one, ("missing", "both" if before is None and after is None + else "before" if before is None else "after")) # fmt: skip + if ways[at]: + _add(one, ("directions", ways[at])) + if bands[at]: + _add(one, ("bands", bands[at])) + if published_first and k not in (None, 0) and kind in rule.MOVES and values[0] is not None: + was, now = abs(before - values[0]), abs(after - values[0]) + one.setdefault(("closer",), 0) + one.setdefault(("further",), 0) + one[("closer",)] += now < was + one[("further",)] += now > was + keys.append(rule.severity(before, after, kind)) + worth = worth or kind not in rule.QUIET + if k is None or not any_claim: + continue + said_verdict = verdicts[k] + step_claim = claims_by_step[k] + if step_claim is None: + if said_verdict is not None: + found.append(f"{where}, {indicator}, {name}: a verdict without a claim") + continue + if not isinstance(said_verdict, str) or said_verdict not in ALLOWED_VERDICTS[kinds[at]]: + found.append(f"{where}, {indicator}, {name}: {kinds[at]!r} cannot be {said_verdict!r}") + return [] + worth = worth or said_verdict == claims.NOT_MOVED + if claims.judgeable(step_claim): + covering = [ + e for e in claims.entries_for(step_claim, indicator) + if claims.covers(e, item["id"], indicator) + ] # fmt: skip + again = claims.verdict(kinds[at], before, after, ways[at], covering) + if again != said_verdict: + found.append(f"{where}, {indicator}, {name}: written {said_verdict!r}, the claim gives {again!r}") + return [] + if said_verdict in _LOUD_VERDICTS: + _add(one, ("verdicts", said_verdict)) + # attribution of this item in this indicator, as `chain.build` makes it + if kinds[nsteps] in rule.MOVES: + if any(v is None for v in values): + attributed[indicator]["not_attributable"] += 1 + else: + carrier = rule.largest_step(values, kinds[:nsteps]) + if carrier is None: + attributed[indicator]["spread"] += 1 + else: + attributed[indicator]["dominant"][steps[carrier]["id"]] += 1 + if any(not steps[j]["explained"] and kinds[j] in rule.MOVES for j in range(nsteps)): + attributed[indicator]["unexplained_moved"] += 1 + if not worth: + found.append(f"{where}: listed, though nothing in it is worth a look") + key = (min(keys), item["id"]) + if previous_key is not None and key < previous_key: + found.append(f"{where}: the items are not listed the worst first") + previous_key = key + # Only a pair worth a look adds to these parts of a block, and every item with such a pair + # is worth a look. So nothing listed can be more than was counted, and with nothing left + # out (`not_listed` 0) the listed items hold exactly what was counted. + exact = said["not_listed"] == 0 + left_out = f"{said['not_listed']} said not to be listed" + for (name, indicator), block in blocks.items(): + one = tally.get((name, indicator), {}) + parts = [("classes", k) for k in rule.CLASSES if k not in rule.QUIET] + parts += [("missing", k) for k in MISSING_SIDES] + [("directions", k) for k in rule.DIRECTIONS] + parts += [("bands", k) for k in rule.BANDS] + if block["verdicts"] is not None: + parts += [("verdicts", k) for k in claims.VERDICTS if k in _LOUD_VERDICTS] + if block["closer"] is not None: + parts += [("closer",), ("further",)] + for part in parts: + counted = block[part[0]] if len(part) == 1 else block[part[0]][part[1]] + if not is_count(counted): + continue # `_counts` has said what is wrong with it + n = one.get(part, 0) + if n > counted or (exact and n != counted): + found.append( + f"counts of {name!r} in {indicator!r}: {counted} {' '.join(map(str, part))!r}, " + f"{n} in the listed items, {left_out}" + ) + said_attribution = data.get("attribution") + for indicator in indicators: + one = said_attribution.get(indicator) if isinstance(said_attribution, dict) else None + if not (isinstance(one, dict) and isinstance(one.get("dominant"), dict)): + continue # `_attribution` has said what is wrong with it + mine = attributed[indicator] + pairs = [(f"dominant {s}", mine["dominant"][s], one["dominant"].get(s)) for s in mine["dominant"]] + pairs += [(k, mine[k], one.get(k)) for k in ("not_attributable", "spread", "unexplained_moved")] + for label, n, counted in pairs: + if not is_count(counted): + continue + if n > counted or (exact and n != counted): + found.append(f"attribution in {indicator!r}: {counted} {label}, {n} in the listed items, {left_out}") + return [i["id"] for i in listed] + + +#: The verdicts only a pair worth a look can have. +_LOUD_VERDICTS = frozenset({claims.AS_CLAIMED, claims.OFF_CLAIM, claims.NOT_CLAIMED, claims.NOT_MOVED}) + + +def _add(tally: dict, key: tuple) -> None: + tally[key] = tally.get(key, 0) + 1 + + +def _explanations(data, found, indicators, steps, items, tier): + said = data.get("explanations") + if not isinstance(said, list) or len(said) > MAX_EXPLANATIONS: + found.append(f"`explanations` is not a list of at most {MAX_EXPLANATIONS}") + return + step_ids = {s["id"] for s in steps} | {TOTAL} + for k, one in enumerate(said): + where = f"explanation {k + 1}" + if not isinstance(one, dict) or set(one) != EXPLANATION_KEYS: + found.append(f"{where}: not {{{', '.join(sorted(EXPLANATION_KEYS))}}}") + continue + about = one["about"] + if not (isinstance(about, dict) and set(about) == {"step", "indicator", "item"}): + found.append(f"{where}: `about` is not {{step, indicator, item}}") + continue + if not all(is_optional(v, MAX_ID) for v in about.values()): + found.append(f"{where}: `about` names a step, an indicator and an item by their ids, or null") + continue + if all(v is None for v in about.values()): + found.append(f"{where}: it is about something: a step, an indicator or an item") + if about["step"] is not None and about["step"] not in step_ids: + found.append(f"{where}: about a step the file does not hold") + if about["indicator"] is not None and about["indicator"] not in indicators: + found.append(f"{where}: about an indicator the file does not hold") + if about["item"] is not None: + if tier == "counts": + found.append(f"{where}: a file of counts explains no single item") + elif about["item"] not in items: + found.append(f"{where}: about an item the file does not list") + if one["finding"] not in FINDINGS: + found.append(f"{where}: `finding` is not one of {', '.join(FINDINGS)}") + if not is_text(one["text"], MAX_LONG): + found.append(f"{where}: `text` is not a text of at most {MAX_LONG} characters") + if one["by"] is None: + found.append(f"{where}: every explanation says who wrote it (`by`)") + else: + found += origin_errors(one["by"], where) + source = one["source"] + if source is not None and not ( + isinstance(source, dict) + and set(source) == {"link", "doc"} + and is_link(source["link"]) + and is_optional(source["doc"], 200) + ): + found.append(f"{where}: `source` is not {{link, doc}}") + if not is_optional(one["not_checked"]): + found.append(f"{where}: `not_checked` is not a text") + + +# ------------------------------------------------------------------------ the whole file + + +#: How a fault begins where the check met a value it could not read and had to stop. The +#: check names the place of every wrong value it knows of; this is the backstop for the +#: rest, so that `errors` never raises (a test feeds every node values of other types). +STOPPED = "the check stopped at a value it cannot read" + + +def errors(data) -> list[str]: + """Everything wrong with a diagnostics file; nothing when a page can show it. Never + raises: a value of the wrong type is a fault like any other.""" + found: list[str] = [] + try: + return _errors(data, found) + except Exception as error: # noqa: BLE001 - the backstop of a check that must not raise + return [*found, f"{STOPPED} ({type(error).__name__}: {str(error)[:200]})"] + + +def _errors(data, found: list[str]) -> list[str]: + if not isinstance(data, Mapping): + return ["not a diagnostics file: not an object"] + said = data.get("format") + if said != FORMAT: + if isinstance(said, str) and said.startswith(FAMILY): + return [f"format {said!r}: this reader knows {FORMAT} only"] + return [f"not a diagnostics file of format {FORMAT}"] + found += [f"a key this format does not know: {key}" for key in sorted(set(data) - KEYS - OPTIONAL_KEYS)] + found += [f"a key this format requires: {key}" for key in sorted(KEYS - set(data))] + if found: + return found + tier = data["tier"] + if tier not in TIERS: + return [f"`tier` is not one of {', '.join(TIERS)}"] + if data["audience"] not in AUDIENCES: + return [f"`audience` is not one of {', '.join(AUDIENCES)}"] + if not is_text(data["title"], 120) or not is_optional(data["about"]): + found.append("`title` is not a short text, or `about` not a text") + item = data["item"] + if not (isinstance(item, dict) and set(item) == {"one", "many"} + and all(is_text(w, 30) for w in item.values())): # fmt: skip + found.append("`item` is not {one, many}") + word = data.get("indicator") + if word is not None and not (isinstance(word, dict) and set(word) == {"one", "many"} + and all(is_text(w, 30) for w in word.values())): # fmt: skip + found.append("`indicator` is not {one, many}, or null") + if not is_text(data["made_by"], 60): + found.append("`made_by` is not a short text") + notes = data["notes"] + if not isinstance(notes, list) or len(notes) > MAX_NOTES or not all(map(is_text, notes)): + found.append(f"`notes` is not a list of at most {MAX_NOTES} texts") + indicators = _indicators(data, found) + states = _states(data, found) + if not indicators or not states: + return found + steps = _steps(data, found, states, indicators, tier) + if not steps: + return found + floors = _rule(data, found, indicators, states, steps) + count = data["items_count"] + total_pairs = count.get("all") if isinstance(count, dict) else None + blocks = _counts(data, found, indicators, steps, total_pairs) + if floors is None or not blocks: + return found + _attribution(data, found, indicators, steps, blocks) + items = _items(data, found, indicators, states, steps, floors, tier, blocks) + _explanations(data, found, indicators, steps, set(items), tier) + _matrix(data, found) + return found + + +def _matrix(data, found) -> None: + """A matrix holds every item with its results: only a local file of items may name one.""" + named = data.get("matrix") + if named is None: + return + if not (isinstance(named, dict) and set(named) == {"file", "items"} and isinstance(named["file"], str) + and FILE_NAME.fullmatch(named["file"]) and is_count(named["items"])): # fmt: skip + found.append("`matrix` is not {file, items} with a file name, or null") + return + if data["tier"] != "items" or data["audience"] != "local": + found.append("only a local file of items names a matrix: a public file keeps the listed items only") + count = data["items_count"] + if isinstance(count, dict) and named["items"] != count.get("all"): + found.append(f"`matrix` says {named['items']} items, the file counts {count.get('all')}") diff --git a/src/sentier_diagnostics/from_tables.py b/src/sentier_diagnostics/from_tables.py new file mode 100644 index 0000000..538dd0a --- /dev/null +++ b/src/sentier_diagnostics/from_tables.py @@ -0,0 +1,295 @@ +"""Build a diagnostics file from CSV tables and a short JSON description, without Python. + + { + "title": "Our database, from the published results to main", + "item": {"one": "dataset", "many": "datasets"}, + "indicator": {"one": "impact category", "many": "impact categories"}, + "indicators": [{"id": "ef31:climate", "name": "Climate change", "short": "Climate", + "unit": "kg CO2 eq"}], + "items": "items.csv", + "states": [ + {"id": "published", "label": "Published", "kind": "published", "results": "published.csv"}, + {"id": "repro", "label": "Recomputed", "results": "repro.csv"} + ], + "steps": [ + {"id": "step-0", "label": "Reproduction", "kind": "reproduction", + "tolerance": {"value": 5e-4, "basis": "published rounding"}, + "claim": {"text": "Nothing moves.", "by": {"kind": "code", "name": "rule"}, "scope": []}} + ], + "listed": 200, + "tier": "items" + } + +A local file of items may also name its matrix of every item (`"matrix": "matrix.json"`, +with `"audience": "local"`): `build_with_matrix` then gives it, and `sentier-diagnostics +build` writes it beside the file. + +Paths are relative to the description. A step's `noise` is +`{"states": ["a", "b"]}` (two states of this chain that are builds of one state) or +`{"tables": [{"label": "...", "results": "a.csv"}, {"label": "...", "results": "b.csv"}]}`. +It is taken over the results beyond the floors of the chain's first state. A scope entry +without `expect` says what may move (`"may"`), as SPEC section 6 says. +""" + +from __future__ import annotations + +from pathlib import Path + +from . import chain, claims, fileformat, tables + +KEYS = {"title", "about", "item", "indicator", "indicators", "items", "states", "steps", "listed", "tier", "audience", + "claim_lists", "explanations", "notes", "floor_factor", "matrix"} # fmt: skip + + +def _origin(said): + if said is None: + return None + return claims.origin(said.get("kind"), said.get("name"), model=said.get("model"), + run_by=said.get("run_by")) # fmt: skip + + +def _text(value) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _table(value, keys) -> bool: + return isinstance(value, dict) and all(isinstance(k, str) for k in value) and set(keys) <= set(value) + + +def _number(value) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def description_errors(spec) -> list[str]: + """What keeps a description from being read, one sentence each, before any table is + opened. What the description says is checked again, in full, when the file is built.""" + if not isinstance(spec, dict): + return ["a description is a JSON object"] + found = [f"a key a description does not know: {k}" for k in sorted(set(spec) - KEYS)] + for key in ("title", "items"): + if not _text(spec.get(key)): + found.append(f"`{key}` is a text") + for key, check, says in ( + ("about", _text, "a text"), + ("tier", _text, "a text"), + ("audience", _text, "a text"), + ("listed", lambda v: isinstance(v, int) and not isinstance(v, bool) and v >= 0, "a whole number"), + ("claim_lists", lambda v: isinstance(v, bool), "true or false"), + ("floor_factor", _number, "a number"), + ( + "item", + lambda v: _table(v, ("one", "many")) and all(map(_text, v.values())), + "{one, many} in words", + ), + ( + "indicator", + lambda v: _table(v, ("one", "many")) and all(map(_text, v.values())), + "{one, many} in words", + ), + ("notes", lambda v: isinstance(v, list) and all(map(_text, v)), "a list of texts"), + ( + "matrix", + lambda v: isinstance(v, str) and fileformat.FILE_NAME.fullmatch(v) is not None, + "the plain name of a .json file", + ), + ): + if key in spec and spec[key] is not None and not check(spec[key]): + found.append(f"`{key}` is {says}") + indicators = spec.get("indicators") + if not ( + isinstance(indicators, list) and indicators and all(_table(i, ("id", "name")) for i in indicators) + ): + found.append("`indicators` is a list of {id, name, short, unit}") + states = spec.get("states") + if not (isinstance(states, list) and len(states) >= 2 + and all(_table(s, ("id", "label", "results")) and _text(s["results"]) for s in states)): # fmt: skip + found.append("`states` is a list of at least two {id, label, results}, `results` naming a table") + states = [] + known = {s["id"] for s in states if isinstance(s.get("id"), str)} + steps = spec.get("steps") + if not (isinstance(steps, list) and all(_table(s, ("id", "label", "kind")) for s in steps)): + return [*found, "`steps` is a list of {id, label, kind}"] + for one in steps: + where = f"step {one['id']!r}" + said = one.get("claim") + if said is not None: + if not (_table(said, ("text",)) and _text(said["text"])): + found.append(f"{where}: a claim says what it claims (`text`)") + elif not ( + isinstance(said.get("scope", []), list) + and all(isinstance(e, dict) for e in said.get("scope", [])) + ): + found.append(f"{where}: a claim's `scope` is a list of entries") + elif said.get("by") is not None and not isinstance(said["by"], dict): + found.append(f"{where}: a claim's `by` is {{kind, name}}") + said = one.get("tolerance") + if said is not None and not (_table(said, ("value",)) and _number(said["value"])): + found.append(f"{where}: a tolerance is {{value, basis, note}} with a number as its value") + said = one.get("noise") + if said is not None: + pair = said.get("states") if isinstance(said, dict) else None + tables_ = said.get("tables") if isinstance(said, dict) else None + if pair is not None: + if not ( + isinstance(pair, list) + and len(pair) == 2 + and all(isinstance(s, str) and s in known for s in pair) + ): + found.append(f"{where}: a noise's `states` names two states of this chain") + elif not (isinstance(tables_, list) and len(tables_) == 2 + and all(_table(t, ("label", "results")) and _text(t["results"]) for t in tables_)): # fmt: skip + found.append( + f"{where}: a noise is {{states: [a, b]}} or {{tables: [{{label, results}}, {{label, results}}]}}" + ) + explanations = spec.get("explanations", []) + if not (isinstance(explanations, list) and all(_table(e, ("text",)) for e in explanations)): + found.append("`explanations` is a list of {text, by, ...}") + return found + + +def build(path: Path) -> dict: + """The diagnostics file a description and its tables give, checked. Every mistake is a + ValueError that says what is wrong (`tables.TableError` for a table).""" + return build_with_matrix(path)[0] + + +def build_with_matrix(path: Path) -> tuple[dict, dict | None]: + """The diagnostics file, and its matrix of every item where the description names one + (None where it does not). Only a local file of items may name one.""" + path = Path(path) + spec = fileformat.load_json(path) + errors = description_errors(spec) + if errors: + raise ValueError(f"{path}: " + "; ".join(errors)) + here = path.parent + indicators = spec["indicators"] + ids = [i["id"] for i in indicators] + items = tables.read_items(here / spec["items"]) + values = {s["id"]: tables.read_results(here / s["results"], ids) for s in spec["states"]} + for state_id, by_indicator in values.items(): + unknown = {item for column in by_indicator.values() for item in column} - set(items) + if unknown: + raise ValueError( + f"state {state_id!r}: items without a row in {spec['items']} (and so without " + f"a reference unit): {', '.join(sorted(unknown)[:10])}" + ) + # an item of the items table that no results table holds is missing in every state: + # counted, never dropped + first = values[spec["states"][0]["id"]] + held = {item for by_indicator in values.values() for column in by_indicator.values() for item in column} + for item in sorted(set(items) - held): + for indicator in ids: + first[indicator][item] = None + # noise is taken beyond the floors the chain counts with: those of its first state + floor_factor = spec.get("floor_factor", chain.rule.FLOOR_FACTOR) + units = {k: v["unit"] for k, v in items.items()} + floors = chain.rule.floors_of(values[spec["states"][0]["id"]], units, floor_factor) + + def floor_by_item(indicator): + return {item: floors.get((indicator, unit), (0.0, 0))[0] for item, unit in units.items()} + + steps = [] + for one in spec["steps"]: + said_claim = one.get("claim") + found_claim = None + if said_claim is not None: + found_claim = claims.claim( + said_claim["text"], + [ + claims.scope( + e.get("indicators", "*"), + e.get("items", "*"), + e.get("direction", "any"), + e.get("largest"), + e.get("expect", claims.MAY), + ) + for e in said_claim.get("scope", []) + ], # fmt: skip + by=_origin(said_claim.get("by")), + reach=said_claim.get("reach"), + ) + said_noise = one.get("noise") + found_noise = None + if said_noise: + if "states" in said_noise: + a, b = said_noise["states"] + pair, labels = (values[a], values[b]), (a, b) + else: + first, second = said_noise["tables"] + pair = (tables.read_results(here / first["results"], ids), + tables.read_results(here / second["results"], ids)) # fmt: skip + labels = (first["label"], second["label"]) + found_noise = chain.noise( + labels, + {i: chain.noise_of(pair[0][i], pair[1][i], floor=floor_by_item(i)) for i in ids}, + said_noise.get("note"), + ) + said_tol = one.get("tolerance") + steps.append( + chain.step( + one["id"], + one["label"], + kind=one["kind"], + explained=one.get("explained", True), + about=one.get("about"), + link=one.get("link"), + claim=found_claim, + tolerance=chain.tolerance(said_tol["value"], said_tol.get("basis"), said_tol.get("note")) + if said_tol + else None, + noise=found_noise, + ) # fmt: skip + ) + explanations = [ + chain.explanation( + e["text"], + _origin(e.get("by")), + step=e.get("step"), + indicator=e.get("indicator"), + item=e.get("item"), + link=e.get("link"), + doc=e.get("doc"), + not_checked=e.get("not_checked"), + finding=e.get("finding", "reason"), + ) # fmt: skip + for e in spec.get("explanations", []) + ] + word = spec.get("item", {"one": "item", "many": "items"}) + named = spec.get("matrix") + if named is not None and ( + spec.get("audience", "local") != "local" or spec.get("tier", "items") != "items" + ): + raise ValueError( + f"{path}: only a local file of items names a matrix; a public file keeps the listed items only" + ) + given = dict( + about=spec.get("about"), + indicators=indicators, + states=[ + chain.state( + s["id"], + s["label"], + kind=s.get("kind", "build"), + about=s.get("about"), + identity=s.get("identity"), + join=s.get("join"), + ) # fmt: skip + for s in spec["states"] + ], + steps=steps, + values=values, + units=units, + names=items, + item_word=(word["one"], word["many"]), + indicator_word=(spec["indicator"]["one"], spec["indicator"]["many"]) if spec.get("indicator") else None, + listed=spec.get("listed", 200), + tier=spec.get("tier", "items"), + audience=spec.get("audience", "local"), + claim_lists=spec.get("claim_lists", True), + floor_factor=floor_factor, + explanations=explanations, + notes=spec.get("notes", []), + ) + if named is None: + return chain.build(spec["title"], **given), None + return chain.build_with_matrix(spec["title"], matrix_file=named, **given) diff --git a/src/sentier_diagnostics/matrix.py b/src/sentier_diagnostics/matrix.py new file mode 100644 index 0000000..f7c3e0d --- /dev/null +++ b/src/sentier_diagnostics/matrix.py @@ -0,0 +1,413 @@ +"""The full matrix of a local file, `sentier-diagnostics-matrix/1`: every item, not only the +listed ones. + +A diagnostics file lists the items worth a look, the worst first, up to `listed` (200 by +default), so that it stays small enough for a page to hold. A project looking at its own +build wants every item: the matrix is that, in a second file the page loads after it has +drawn. It is compact (one row per item, the indicators and states named once) and it holds +only what the rule gives: the results, and one code per pair for its class and direction, +and per step with a claim one code for its verdict. + +Only a local file may name a matrix (`"matrix": {"file": ..., "items": n}`); a public file +keeps the listed items only. `errors(m)` checks the shape; `against(m, data)` works every +code out again from the results and the diagnostics file's own rule, and holds the file's +counts, attribution and `items_count` to what the matrix holds: with every item at hand, +they must match exactly. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +from . import claims, fileformat, rule + +FORMAT = "sentier-diagnostics-matrix/1" +FAMILY = "sentier-diagnostics-matrix/" +KEYS = {"format", "of", "items", "values", "classes", "verdicts"} +OF_KEYS = {"title", "states", "indicators", "steps"} + +#: A matrix is loaded after the page is drawn, not held inside it: it may be larger. +MAX_BYTES = 64 * 1024 * 1024 +MAX_ITEMS = 200_000 + +#: One character per pair: its class, and for a pair that moved its direction (upper case +#: up, lower case down). +CODES = { + (rule.AGREES, None): "=", + (rule.WITHIN_NOISE, None): "~", + (rule.NEAR_ZERO, None): "0", + (rule.MISSING, None): "-", + (rule.DIFFERS, rule.UP): "D", + (rule.DIFFERS, rule.DOWN): "d", + (rule.BEYOND, rule.UP): "B", + (rule.BEYOND, rule.DOWN): "b", + (rule.ZERO_REFERENCE, rule.UP): "R", + (rule.ZERO_REFERENCE, rule.DOWN): "r", + (rule.ZERO_RESULT, rule.UP): "T", + (rule.ZERO_RESULT, rule.DOWN): "t", + (rule.SIGN_CHANGE, rule.UP): "S", + (rule.SIGN_CHANGE, rule.DOWN): "s", +} +DECODE = {code: pair for pair, code in CODES.items()} + +#: One character per pair and step with a claim: its verdict. "." for a step without one. +VERDICT_CODES = { + claims.AS_CLAIMED: "a", + claims.OFF_CLAIM: "o", + claims.NOT_CLAIMED: "n", + claims.NOT_MOVED: "c", + claims.NOISE: "~", + claims.UNCHANGED: "=", + claims.NOT_JUDGED: "?", +} +NO_CLAIM = "." +VERDICT_DECODE = {code: verdict for verdict, code in VERDICT_CODES.items()} + + +def code_of(kind: str, way: str | None) -> str: + """The code of a class and direction. + + >>> code_of("differs", "up"), code_of("agrees", None), code_of("sign change", "down") + ('D', '=', 's') + """ + return CODES[(kind, way)] + + +def make(data: Mapping, rows: list[dict]) -> dict: + """A matrix of a diagnostics file from its rows, as `chain.build_with_matrix` gives them: + `{id, name, place, group, unit, values: {indicator: [per state]}, classes, directions, + verdicts}` with classes, directions and verdicts per indicator as the file lists them.""" + ids = [i["id"] for i in data["indicators"]] + steps = data["steps"] + out = { + "format": FORMAT, + "of": { + "title": data["title"], + "states": [s["id"] for s in data["states"]], + "indicators": ids, + "steps": [s["id"] for s in steps], + }, + "items": [], + "values": [], + "classes": [], + "verdicts": [] if any(s["claim"] for s in steps) else None, + } + for row in rows: + out["items"].append([row["id"], row["name"], row["place"], row["group"], row["unit"]]) + out["values"].append([v for i in ids for v in row["values"][i]]) + out["classes"].append( + "".join( + code_of(kind, way) for i in ids for kind, way in zip(row["classes"][i], row["directions"][i]) + ) + ) + if out["verdicts"] is not None: + out["verdicts"].append( + "".join( + NO_CLAIM if said is None else VERDICT_CODES[said] + for i in ids + for said in row["verdicts"][i] + ) + ) + return out + + +def dumps(m: Mapping) -> str: + """The text of a matrix: one line per item, so that a diff names the items.""" + head = {k: v for k, v in m.items() if k not in ("items", "values", "classes", "verdicts")} + lines = [json.dumps(head, ensure_ascii=False, allow_nan=False)[:-1] + ","] + for key in ("items", "values", "classes", "verdicts"): + rows = m[key] + if rows is None: + lines.append(f' "{key}": null' + ("," if key != "verdicts" else "")) + continue + body = ",\n".join( + " " + json.dumps(r, ensure_ascii=False, allow_nan=False, separators=(",", ":")) for r in rows + ) + lines.append(f' "{key}": [\n{body}\n ]' + ("," if key != "verdicts" else "")) + return "\n".join(lines) + "\n}\n" + + +def write(m: Mapping, path) -> Path: + """Write a matrix, its shape checked first; refuse one larger than a page loads.""" + found = errors(m) + if found: + raise ValueError("not a matrix: " + "; ".join(found[:10])) + text = dumps(m) + size = len(text.encode("utf-8")) + if size > MAX_BYTES: + raise ValueError(f"the matrix would be {size} bytes, and a page loads {MAX_BYTES} at most") + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def read(path, data: Mapping) -> dict: + """A matrix, read and checked against its checked diagnostics file; raises ValueError + with every error found.""" + path = Path(path) + if path.stat().st_size > MAX_BYTES: + raise ValueError(f"{path}: larger than {MAX_BYTES} bytes") + m = fileformat.load_json(path) + found = errors(m) or against(m, data) + if found: + raise ValueError(f"{path}: " + "; ".join(found[:10])) + return m + + +def _is_row(row) -> bool: + return ( + isinstance(row, list) + and len(row) == 5 + and fileformat.is_text(row[0], fileformat.MAX_ID) + and fileformat.is_text(row[1], 300) + and fileformat.is_optional(row[2], 60) + and fileformat.is_optional(row[3], fileformat.MAX_LABEL) + and fileformat.is_optional(row[4], fileformat.MAX_WORD) + ) + + +def errors(m) -> list[str]: + """Everything wrong with the shape of a matrix; nothing for one a page can load. It never + raises: a value it cannot read is a fault like any other.""" + try: + return _errors(m) + except Exception: # noqa: BLE001 - a backstop: the checks above it name the faults they know + return [fileformat.STOPPED] + + +def _errors(m) -> list[str]: + if not isinstance(m, Mapping): + return ["not a matrix: not an object"] + said = m.get("format") + if said != FORMAT: + if isinstance(said, str) and said.startswith(FAMILY): + return [f"format {said!r}: this reader knows {FORMAT} only"] + return [f"not a matrix of format {FORMAT}"] + found = [f"a key this format does not know: {k}" for k in sorted(set(m) - KEYS)] + found += [f"a key this format requires: {k}" for k in sorted(KEYS - set(m))] + if found: + return found + of = m["of"] + if not ( + isinstance(of, dict) + and set(of) == OF_KEYS + and fileformat.is_text(of["title"], 120) + and all(isinstance(of[k], list) and all(fileformat.is_text(x, fileformat.MAX_ID) for x in of[k]) + for k in ("states", "indicators", "steps")) + ): # fmt: skip + return ["`of` is not {title, states, indicators, steps}"] + nstates, nind, nsteps = len(of["states"]), len(of["indicators"]), len(of["steps"]) + if nsteps != nstates - 1 or not nind: + return ["`of`: one step fewer than states, and at least one indicator"] + items = m["items"] + if not isinstance(items, list) or len(items) > MAX_ITEMS: + return [f"`items` is not a list of at most {MAX_ITEMS} items"] + n = len(items) + for key in ("values", "classes"): + if not isinstance(m[key], list) or len(m[key]) != n: + found.append(f"`{key}` is not one row per item") + has_claims = m["verdicts"] is not None + if has_claims and (not isinstance(m["verdicts"], list) or len(m["verdicts"]) != n): + found.append("`verdicts` is not one row per item, or null") + if found: + return found + seen = set() + for at, row in enumerate(items): + if not _is_row(row): + return [f"item {at}: not [id, name, place, group, unit]"] + if row[0] in seen: + found.append(f"item {row[0]!r} is in the matrix twice") + seen.add(row[0]) + values = m["values"][at] + if not (isinstance(values, list) and len(values) == nind * nstates + and all(v is None or rule.is_value(v) for v in values)): # fmt: skip + return [f"item {row[0]!r}: not one finite number or null per indicator and state"] + codes = m["classes"][at] + if not ( + isinstance(codes, str) and len(codes) == nind * (nsteps + 1) and all(c in DECODE for c in codes) + ): + return [f"item {row[0]!r}: not one class code per indicator, step and the whole way"] + if has_claims: + said = m["verdicts"][at] + if not (isinstance(said, str) and len(said) == nind * nsteps + and all(c in VERDICT_DECODE or c == NO_CLAIM for c in said)): # fmt: skip + return [f"item {row[0]!r}: not one verdict code per indicator and step"] + return found + + +def against(m: Mapping, data: Mapping) -> list[str]: + """Everything a checked matrix and its checked diagnostics file disagree on. Every code + is worked out again with the file's floors, tolerances and noise, and, where a claim's + lists are in the file, every verdict; the file's counts, attribution and `items_count` + must be what the matrix holds, exactly.""" + try: + return _against(m, data) + except Exception: # noqa: BLE001 - a backstop, as in `errors` + return [fileformat.STOPPED] + + +def _against(m: Mapping, data: Mapping) -> list[str]: + found = [] + named = data.get("matrix") + if not named: + return ["the diagnostics file names no matrix"] + ids = [i["id"] for i in data["indicators"]] + order = [s["id"] for s in data["states"]] + steps = data["steps"] + of = {"title": data["title"], "states": order, "indicators": ids, "steps": [s["id"] for s in steps]} + if m["of"] != of: + return ["it is the matrix of another file: its title, states, indicators or steps differ"] + n = len(m["items"]) + if named["items"] != n: + found.append(f"the diagnostics file says {named['items']} items, the matrix holds {n}") + if data["items_count"]["all"] != n: + found.append(f"the diagnostics file counts {data['items_count']['all']} items, the matrix holds {n}") + nstates, nsteps = len(order), len(steps) + floors = {(f["indicator"], f["unit"]): f["value"] for f in data["rule"]["floors"]} + published_first = data["states"][0]["kind"] == "published" + listed = {item["id"]: item for item in data["items"]} + + # what the counts and attribution must be, counted from every item + def empty(k): + return { + "pairs": 0, + "classes": dict.fromkeys(rule.CLASSES, 0), + "missing": dict.fromkeys(fileformat.MISSING_SIDES, 0), + "directions": dict.fromkeys(rule.DIRECTIONS, 0), + "bands": dict.fromkeys(rule.BANDS, 0), + "verdicts": dict.fromkeys(claims.VERDICTS, 0) if k is not None and steps[k]["claim"] else None, + "closer": 0 if published_first and k not in (None, 0) else None, + "further": 0 if published_first and k not in (None, 0) else None, + } + + counted = {(k, i): empty(k) for k in [*range(nsteps), None] for i in ids} + attributed = { + i: { + "dominant": {s["id"]: 0 for s in steps}, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0, + } + for i in ids + } + qualifying = 0 + for at, (item_id, name, place, group, unit) in enumerate(m["items"]): + where = f"item {item_id!r}" + flat, codes = m["values"][at], m["classes"][at] + said_verdicts = m["verdicts"][at] if m["verdicts"] is not None else None + worth = False + values_of = {} + for x, indicator in enumerate(ids): + values = flat[x * nstates : (x + 1) * nstates] + values_of[indicator] = values + floor = floors.get((indicator, unit)) + if floor is None: + return [f"{where}, {indicator}: the file holds no floor for unit {unit!r}"] + kinds = [] + for k in [*range(nsteps), None]: + at_k = nsteps if k is None else k + before, after = (values[0], values[-1]) if k is None else (values[k], values[k + 1]) + if k is None: + tol, level = data["rule"]["total_tolerance"], None + else: + tol = steps[k]["tolerance"]["value"] + noise = steps[k]["noise"] + level = (noise["by"] or {}).get(indicator) if noise["measured"] else None + kind, way, band = rule.judge(before, after, floor=floor, tolerance=tol, noise=level) + code = codes[x * (nsteps + 1) + at_k] + if code != code_of(kind, way): + step_name = "the whole way" if k is None else steps[k]["id"] + return [ + f"{where}, {indicator}, {step_name}: written as {DECODE[code]}, the rule gives {kind}/{way}" + ] + kinds.append(kind) + block = counted[(k, indicator)] + block["pairs"] += 1 + block["classes"][kind] += 1 + if kind == rule.MISSING: + side = ( + "both" + if before is None and after is None + else "before" + if before is None + else "after" + ) + block["missing"][side] += 1 + if way: + block["directions"][way] += 1 + if band: + block["bands"][band] += 1 + if block["closer"] is not None and kind in rule.MOVES and values[0] is not None: + was, now = abs(before - values[0]), abs(after - values[0]) + block["closer"] += now < was + block["further"] += now > was + worth = worth or kind not in rule.QUIET + if k is None: + continue + step_claim = steps[k]["claim"] + vcode = said_verdicts[x * nsteps + k] if said_verdicts is not None else NO_CLAIM + if step_claim is None: + if vcode != NO_CLAIM: + return [f"{where}, {indicator}, {steps[k]['id']}: a verdict without a claim"] + continue + said = VERDICT_DECODE.get(vcode) + if said is None or said not in fileformat.ALLOWED_VERDICTS[kind]: + return [f"{where}, {indicator}, {steps[k]['id']}: {kind!r} cannot be {said!r}"] + if claims.judgeable(step_claim): + covering = [ + e + for e in claims.entries_for(step_claim, indicator) + if claims.covers(e, item_id, indicator) + ] + again = claims.verdict(kind, before, after, way, covering) + if again != said: + return [ + f"{where}, {indicator}, {steps[k]['id']}: written {said!r}, the claim gives {again!r}" + ] + block["verdicts"][said] += 1 + worth = worth or said == claims.NOT_MOVED + # attribution, as `chain.build` makes it + one = attributed[indicator] + if any(not s["explained"] and kinds[j] in rule.MOVES for j, s in enumerate(steps)): + one["unexplained_moved"] += 1 + if kinds[nsteps] in rule.MOVES: + if any(v is None for v in values): + one["not_attributable"] += 1 + else: + carrier = rule.largest_step(values, kinds[:nsteps]) + if carrier is None: + one["spread"] += 1 + else: + one["dominant"][steps[carrier]["id"]] += 1 + qualifying += worth + if item_id in listed: + item = listed[item_id] + if [item["name"], item["place"], item["group"], item["unit"]] != [name, place, group, unit]: + found.append(f"{where}: its name, place, group or unit differ from the diagnostics file") + if item["values"] != values_of: + found.append(f"{where}: its results differ from the diagnostics file") + missing_listed = set(listed) - {row[0] for row in m["items"]} + if missing_listed: + found.append(f"{len(missing_listed)} listed items are not in the matrix") + if data["items_count"]["qualifying"] != qualifying: + found.append( + f"the diagnostics file counts {data['items_count']['qualifying']} items worth a look, " + f"the matrix holds {qualifying}" + ) + for (k, indicator), block in counted.items(): + name = fileformat.TOTAL if k is None else steps[k]["id"] + written = ( + data["counts"][fileformat.TOTAL][indicator] + if k is None + else data["counts"]["steps"][name][indicator] + ) + for key in block: + if block[key] != written[key]: + found.append(f"counts of {name!r} in {indicator!r}: `{key}` is not what the matrix holds") + for indicator in ids: + if attributed[indicator] != data["attribution"][indicator]: + found.append(f"attribution in {indicator!r} is not what the matrix holds") + return found diff --git a/src/sentier_diagnostics/page.py b/src/sentier_diagnostics/page.py new file mode 100644 index 0000000..25acf45 --- /dev/null +++ b/src/sentier_diagnostics/page.py @@ -0,0 +1,212 @@ +"""The page: a frameless body a site holds in its own page, or a page of its own. + +`body(data)` is the diagnostics as one element, `
`, with the +file inside it as JSON: no ``, no ``, no id. A site puts it where it wants and +loads `diagnostics.js` and `diagnostics.css` from `assets()`. The style is scoped to that +element and sets no font and nothing on `:root` or `body`; its colours take the site's tokens +(`--ink`, `--line`, `--surface` ...) where the site sets them. + +`write(data, folder)` writes a page of its own: `index.html` with the body in it, the two +assets and `standalone.css`, which sets the tokens and font a site would. It opens from disk +by a double click and needs no server and no network. A local file may name its matrix of +every item (`chain.build_with_matrix`); given it, `write` puts it beside the page as +`matrix.js`, which the page loads after it has drawn. + +The page shows what the file says. It works out no class, no direction and no count of +its own: it only adds the file's counts up (the classes that moved into "moved", a step's +counts over its indicators), divides an item's result after by its result before to show +their ratio, and splits one item's difference into the steps' shares. +""" + +from __future__ import annotations + +import html +import json +import re +import shutil +from collections.abc import Mapping +from importlib import resources +from pathlib import Path + +from . import breakdown, fileformat +from . import matrix as matrix_format + +#: What a site copies into its assets to hold the body. +ASSETS = ("diagnostics.js", "diagnostics.css") +#: What a page of its own adds: its frame and the template of the body. +STANDALONE = ("index.html", "standalone.css") +TEMPLATE = "body.html" +MATRIX_SCRIPT = "matrix.js" +_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*") +_ROOT = '
' + + +def _json(value) -> str: + """A value as JSON that no HTML parser reads as markup: ASCII only, and every `<` escaped, + so that no text of a file can end the element that holds it or open a comment.""" + text = json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")) + return text.replace("<", "\\u003c") + + +def as_script(name: str, value) -> str: + """A value as a script that assigns it (`window.NAME = ...;`), for a site that keeps the + data in a file of its own; the page reads `SENTIER_DIAGNOSTICS`, + `SENTIER_DIAGNOSTICS_BREAKDOWNS` and `SENTIER_DIAGNOSTICS_MATRIX`.""" + return f"window.{name} = {_json(value)};\n" + + +def _read(name: str) -> str: + return (resources.files(__package__) / "page" / name).read_text(encoding="utf-8") + + +def assets(standalone: bool = False) -> dict[str, Path]: + """The page's own files by name, as paths of the installed package: the script and the + style a site copies, and with `standalone` also the frame of a page of its own and the + template of the body.""" + source = resources.files(__package__) / "page" + names = ASSETS + ((*STANDALONE, TEMPLATE) if standalone else ()) + return {name: Path(str(source / name)) for name in names} + + +def _checked(data: Mapping, breakdowns: Mapping[str, Mapping]) -> None: + found = fileformat.errors(data) + if found: + raise ValueError("not a diagnostics file: " + "; ".join(found[:10])) + errors = breakdown_errors(data, breakdowns) + if errors: + raise ValueError("; ".join(errors)) + + +def body( + data: Mapping, + *, + breakdowns: Mapping[str, Mapping] | None = None, + base: str = "", + title: bool = True, + matrix: str | None = None, +) -> str: + """The diagnostics as one element for a site's page, the file and its breakdowns inside. + + `base` is put before the names of the files the page loads (breakdowns, the matrix), for + a site that keeps them elsewhere than the page. `title=False` leaves the heading to the + site. `matrix` names the file of the matrix the page loads after it has drawn, where it + is not the one the file names (a site that serves `matrix.json` needs none). + """ + return _body(data, breakdowns=breakdowns, base=base, title=title, matrix=matrix, standalone=False) + + +def _body(data, *, breakdowns, base, title, matrix, standalone) -> str: + breakdowns = dict(breakdowns or {}) + _checked(data, breakdowns) + if matrix is not None and not (_NAME.fullmatch(matrix) and ".." not in matrix): + raise ValueError(f"the matrix {matrix!r} is not a plain file name") + scripts = f'\n' + if breakdowns: + scripts += f'\n' + if len(scripts) > fileformat.MAX_BYTES: # ASCII: one byte per character + raise ValueError( + f"the page would hold {len(scripts)} bytes of data, and holds {fileformat.MAX_BYTES} at most: " + "list fewer items, and keep every item in the matrix" + ) + attributes = "" + if base: + attributes += f' data-sd-base="{html.escape(base, quote=True)}"' + if matrix: + attributes += f' data-sd-matrix="{html.escape(matrix, quote=True)}"' + if standalone: + attributes += " data-sd-standalone" + text = _read(TEMPLATE) + if _ROOT not in text or "" not in text: + raise RuntimeError("the template of the body has lost its root or its place for the data") + text = text.replace(_ROOT, _ROOT[:-1] + attributes + ">", 1) + if not title: + text = text.replace( + '

', '

{name}

\n{inner}", 1) + folder = Path(folder) + folder.mkdir(parents=True, exist_ok=True) + source = resources.files(__package__) / "page" + for one in (*ASSETS, "standalone.css"): + with resources.as_file(source / one) as path: + shutil.copyfile(path, folder / one) + (folder / "index.html").write_text(page, encoding="utf-8") + stale = folder / MATRIX_SCRIPT + if script is not None: + stale.write_text(script, encoding="utf-8") + elif stale.exists(): + stale.unlink() # the matrix of an earlier page: this one names none + return folder / "index.html" + + +def breakdown_errors(data: Mapping, breakdowns: Mapping[str, Mapping]) -> list[str]: + """Everything wrong with the breakdowns given for a checked diagnostics file: each one + alone, then against every item and step that names it (it must be theirs), or against + the file where nothing names it.""" + found, broken = [], set() + for name, one in breakdowns.items(): + errors = breakdown.errors(one) + if errors: + broken.add(name) + found += [f"{name}: {e}" for e in errors] + named = set() + for name, item, key in breakdown.namings(data): + if name in breakdowns and name not in broken: + named.add(name) + found += [f"{name}: {e}" for e in breakdown.against(breakdowns[name], data, (item, key))] + for name in sorted(set(breakdowns) - named - broken): + found += [f"{name}: {e}" for e in breakdown.against(breakdowns[name], data)] + return found + + +def named_breakdowns(data: Mapping, beside: Path) -> tuple[dict, list[str]]: + """The breakdowns a file names that lie beside it, read; and the names of those that + do not (a project may keep them to itself). A breakdown that cannot be read is refused + with its name.""" + found, absent = {}, [] + names = sorted({f for item in data["items"] for f in (item["breakdowns"] or {}).values()}) + for name in names: + path = Path(beside) / name + if not path.is_file(): + absent.append(name) + continue + found[name] = fileformat.load_json(path) + return found, absent + + +def named_matrix(data: Mapping, beside: Path) -> dict | None: + """The matrix a file names, read from beside it and checked against the file; None where + it names none or where the matrix is not there (a page then shows the listed items and + says so). A matrix that is there and not the file's is refused.""" + named = data.get("matrix") + if not named: + return None + path = Path(beside) / named["file"] + if not path.is_file(): + return None + return matrix_format.read(path, data) diff --git a/src/sentier_diagnostics/page/body.html b/src/sentier_diagnostics/page/body.html new file mode 100644 index 0000000..0c8b053 --- /dev/null +++ b/src/sentier_diagnostics/page/body.html @@ -0,0 +1,110 @@ +
+
+

Diagnostics

+

+

+ +
+ + + + + +

+ +
diff --git a/src/sentier_diagnostics/page/diagnostics.css b/src/sentier_diagnostics/page/diagnostics.css new file mode 100644 index 0000000..53d0361 --- /dev/null +++ b/src/sentier_diagnostics/page/diagnostics.css @@ -0,0 +1,243 @@ +/* sentier-diagnostics: the style of the diagnostics, inside a host's page or on their own. + Every rule is scoped to .sentier-diagnostics: nothing here touches the host's :root, body, + headings or tables outside it, and no font is set (the host's is inherited). Colours and + sizes are custom properties that take the host's tokens where it sets them (the shared + tokens of the Sentier sites) and fall back otherwise. Up, down and noise keep colours of + their own, so that a direction never takes a site's accent. No font, script or image is + loaded from anywhere else. */ +.sentier-diagnostics { + --sd-ink: var(--ink, #1d232a); + --sd-muted: var(--muted, #5b6672); + --sd-surface: var(--surface, #ffffff); + --sd-paper-deep: var(--paper-deep, #f3f5f7); + --sd-line: var(--line, #d9dee3); + --sd-line-strong: var(--line-strong, #b6bfc8); + --sd-accent: var(--index, #1d625e); + --sd-on-accent: var(--on-index, #ffffff); + --sd-fail: var(--fail, #a3262a); + --sd-radius: var(--radius, 6px); + --sd-gutter: var(--gutter, 16px); + --sd-t-xs: var(--t-xs, 0.75rem); + --sd-t-sm: var(--t-sm, 0.875rem); + --sd-t-md: var(--t-md, 1rem); + --sd-t-lg: var(--t-lg, 1.2rem); + --sd-s2: var(--s2, 0.5rem); + --sd-s3: var(--s3, 0.75rem); + --sd-s4: var(--s4, 1rem); + --sd-s5: var(--s5, 1.5rem); + --sd-s6: var(--s6, 2rem); + --sd-up: #b07a1b; /* went up: amber */ + --sd-down: #1d625e; /* went down: teal; the pair reads for colour-blind readers */ + --sd-noise: #8a939c; + --sd-drawer: clamp(22rem, 42vw, 52rem); + color: var(--sd-ink); + font-size: var(--sd-t-md); + line-height: 1.45; +} +@media (prefers-color-scheme: dark) { + .sentier-diagnostics { --sd-up: #d9a441; --sd-down: #5fa29d; --sd-noise: #77818b; } +} + +/* a class that sets `display` would otherwise show an element marked hidden */ +.sentier-diagnostics [hidden] { display: none !important; } +.sentier-diagnostics .sd-visually-hidden { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); } + +/* the head */ +.sentier-diagnostics .sd-head { margin: 0 0 var(--sd-s4); } +.sentier-diagnostics .sd-title { margin: 0.2rem 0; } +.sentier-diagnostics .sd-lead { margin: 0.3rem 0; max-width: 60rem; } +.sentier-diagnostics .sd-lead:empty { display: none; } +.sentier-diagnostics .sd-tier { margin: 0.2rem 0; } +/* a local file says so as a quiet part of the line, as a site's own notice would */ +.sentier-diagnostics .sd-local { font-weight: 700; color: var(--sd-ink); } +.sentier-diagnostics .sd-local::before { content: "· "; font-weight: 400; color: var(--sd-muted); } +.sentier-diagnostics .sd-quiet { color: var(--sd-muted); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-note { max-width: 70rem; color: var(--sd-muted); font-size: var(--sd-t-sm); line-height: 1.5; } +.sentier-diagnostics .sd-error { color: var(--sd-fail); } +.sentier-diagnostics .sd-plain { list-style: none; padding: 0; } +.sentier-diagnostics .sd-plain > li { margin: var(--sd-s2) 0; } +.sentier-diagnostics .sd-states { display: flex; flex-wrap: wrap; gap: 0.25rem 0.5rem; margin: 0.6rem 0; padding: 0; list-style: none; } +.sentier-diagnostics .sd-states > li { padding: 0.15rem 0.5rem; border: 1px solid var(--sd-line); border-radius: var(--sd-radius); background: var(--sd-paper-deep); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-states > li + li::before { content: "→ "; color: var(--sd-muted); } +.sentier-diagnostics .sd-banner { display: inline-block; margin: var(--sd-s2) 0; padding: 0.3rem 0.6rem; border: 1px solid var(--sd-fail); border-radius: var(--sd-radius); color: var(--sd-fail); } +.sentier-diagnostics .sd-made-by { margin-top: var(--sd-s6); } + +/* the comparison: a row of chips, the whole way first, then each step */ +.sentier-diagnostics .sd-chips { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sd-s2) var(--sd-s3); margin: var(--sd-s3) 0 var(--sd-s2); } +.sentier-diagnostics .sd-chips-label { font-size: var(--sd-t-xs); font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--sd-muted); } +.sentier-diagnostics .sd-chip-row { display: flex; flex-wrap: wrap; gap: 0.35rem; min-width: 0; } +.sentier-diagnostics .sd-chip { display: inline-flex; align-items: baseline; gap: 0.4rem; max-width: 24rem; min-width: 0; padding: 0.3rem 0.75rem; border: 1px solid var(--sd-line-strong); + border-radius: 999px; background: var(--sd-surface); color: var(--sd-ink); font: inherit; font-size: var(--sd-t-sm); line-height: 1.3; cursor: pointer; } +.sentier-diagnostics .sd-chip:hover { border-color: var(--sd-accent); } +.sentier-diagnostics .sd-chip:focus-visible { outline: 2px solid var(--sd-accent); outline-offset: 2px; } +.sentier-diagnostics .sd-chip[aria-pressed="true"] { border-color: var(--sd-accent); background: var(--sd-accent); color: var(--sd-on-accent); } +.sentier-diagnostics .sd-chip-no { font-weight: 800; } +.sentier-diagnostics .sd-chip-label { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 600; } +.sentier-diagnostics .sd-chip-count { font-size: var(--sd-t-xs); font-variant-numeric: tabular-nums; white-space: nowrap; opacity: 0.75; } +.sentier-diagnostics .sd-chip-flag { font-weight: 800; color: var(--sd-fail); } +.sentier-diagnostics .sd-chip[aria-pressed="true"] .sd-chip-flag { color: inherit; } +.sentier-diagnostics .sd-chip-mark { font-size: var(--sd-t-xs); } + +/* the line of the chosen step: what it compares, its claim and what the claim found, its rule */ +.sentier-diagnostics .sd-step { margin: 0 0 var(--sd-s3); padding: var(--sd-s2) var(--sd-s4); border-left: 3px solid var(--sd-accent); border-radius: 0 var(--sd-radius) var(--sd-radius) 0; background: var(--sd-paper-deep); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-step > p { margin: 0; } +.sentier-diagnostics .sd-step > p + p { margin-top: 0.25rem; } +.sentier-diagnostics .sd-claim-label { display: inline-block; margin-right: 0.5em; padding: 0 0.45em; border-radius: 3px; background: var(--sd-surface); font-size: var(--sd-t-xs); font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--sd-muted); } +/* what a claim found: weight, not colour; a verdict is not an approval (ADR 0010) */ +.sentier-diagnostics .sd-verdicts { font-weight: 700; } + +/* an indicator in which the chosen comparison moved nothing: last, and quiet */ +.sentier-diagnostics .sd-grid .sd-still { color: var(--sd-muted); opacity: 0.7; } +.sentier-diagnostics .sd-grid thead th.sd-still { font-weight: 400; } +.sentier-diagnostics .sd-grid-note { margin: 0 0 var(--sd-s2); } +.sentier-diagnostics .sd-matrix-said { margin: 0; font-size: var(--sd-t-xs); } +.sentier-diagnostics .sd-matrix-said:empty { display: none; } + +/* below the matrix: the rest of the file, each part closed until a reader opens it */ +.sentier-diagnostics .sd-section { margin-top: var(--sd-s4); border-top: 1px solid var(--sd-line); } +.sentier-diagnostics .sd-section:last-of-type { border-bottom: 1px solid var(--sd-line); } +.sentier-diagnostics .sd-section > summary { padding: var(--sd-s2) 0; cursor: pointer; } +.sentier-diagnostics .sd-section > summary > h3 { display: inline; margin: 0; font-size: var(--sd-t-lg); } +.sentier-diagnostics .sd-section[open] > summary { margin-bottom: var(--sd-s2); } +.sentier-diagnostics .sd-section + .sd-section { margin-top: 0; border-top: 0; } +.sentier-diagnostics .sd-section + .sd-section > summary { border-top: 1px solid var(--sd-line); } +/* the steps: the label and the claim have room, the numbers stay narrow */ +.sentier-diagnostics .sd-steps td:nth-child(2) { min-width: 15rem; } +.sentier-diagnostics .sd-steps td:nth-child(5) { min-width: 18rem; } +.sentier-diagnostics .sd-table td.sd-wide { min-width: 16rem; } +.sentier-diagnostics h4 { margin: var(--sd-s5) 0 var(--sd-s2); } + +/* fields, buttons */ +.sentier-diagnostics .sd-controls { display: flex; flex-wrap: wrap; align-items: flex-end; gap: var(--sd-s2) var(--sd-s4); margin: var(--sd-s3) 0; } +.sentier-diagnostics .sd-filters { display: flex; flex: 1 1 auto; flex-wrap: wrap; align-items: flex-end; gap: var(--sd-s2) var(--sd-s3); min-width: 0; } +/* the count and the legend on the right of the filters, as in a dashboard's control row */ +.sentier-diagnostics .sd-controls > .sd-status { flex: 1 1 100%; gap: 0.15rem var(--sd-s4); flex-direction: row; flex-wrap: wrap; align-items: baseline; justify-content: space-between; text-align: left; } +.sentier-diagnostics .sd-controls > .sd-status .sd-legend { margin: 0; } +.sentier-diagnostics .sd-field { display: flex; flex-direction: column; gap: 0.2rem; font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-field > span { font-size: var(--sd-t-xs); font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--sd-muted); } +.sentier-diagnostics .sd-grow { flex: 0 1 20rem; } +.sentier-diagnostics .sd-filters select { min-width: 7rem; } +.sentier-diagnostics select, .sentier-diagnostics input, .sentier-diagnostics .sd-button { + min-height: 2.25rem; padding: 0.25rem 0.5rem; border: 1px solid var(--sd-line-strong); border-radius: var(--sd-radius); + background: var(--sd-surface); color: var(--sd-ink); font: inherit; } +.sentier-diagnostics input[type="search"] { width: 100%; } +.sentier-diagnostics .sd-button { cursor: pointer; font-weight: 700; } +.sentier-diagnostics .sd-button:hover { border-color: var(--sd-accent); color: var(--sd-accent); } +.sentier-diagnostics select:focus-visible, .sentier-diagnostics input:focus-visible, .sentier-diagnostics .sd-button:focus-visible { outline: 2px solid var(--sd-accent); outline-offset: 1px; } + +/* tables: tables at every width, whatever the host does with bare ones */ +.sentier-diagnostics .sd-wrap { max-width: 100%; overflow-x: auto; } +.sentier-diagnostics table { display: table; width: auto; margin: 0; border: 0; border-collapse: collapse; background: none; font-size: var(--sd-t-sm); } +.sentier-diagnostics thead { display: table-header-group; position: static; width: auto; height: auto; overflow: visible; clip-path: none; } +.sentier-diagnostics tbody { display: table-row-group; } +.sentier-diagnostics tr { display: table-row; padding: 0; border: 0; } +.sentier-diagnostics th, .sentier-diagnostics td { display: table-cell; padding: 0.25rem 0.5rem; border: 0; border-bottom: 1px solid var(--sd-line); border-radius: 0; + text-align: left; vertical-align: top; letter-spacing: 0; text-transform: none; } +.sentier-diagnostics td + td::before { content: none; } +.sentier-diagnostics th { font-weight: 700; background: var(--sd-paper-deep); } +.sentier-diagnostics td.num, .sentier-diagnostics th.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } +.sentier-diagnostics tr.sd-row { cursor: pointer; } +.sentier-diagnostics tr.sd-row:hover > *, .sentier-diagnostics tr.sd-row:focus-visible > * { background: var(--sd-paper-deep); } +.sentier-diagnostics tr.sd-row:focus { outline: none; } +.sentier-diagnostics tr.sd-row:focus-visible > th { box-shadow: inset 3px 0 0 var(--sd-accent); } +.sentier-diagnostics .zero { color: var(--sd-muted); } + +/* the matrix: its own frame, scrolling both ways; the names stay on the left, the headings on top */ +.sentier-diagnostics .sd-status { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 0.2rem var(--sd-s5); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-count { margin: 0; font-weight: 700; font-variant-numeric: tabular-nums; } +.sentier-diagnostics .sd-grid-wrap { position: relative; max-height: max(24rem, calc(100vh - 6rem)); overflow: auto; border: 1px solid var(--sd-line); border-radius: var(--sd-radius); } +.sentier-diagnostics table.sd-grid { border-collapse: separate; border-spacing: 0; font-size: var(--sd-t-xs); line-height: 1.35; } +.sentier-diagnostics .sd-grid th, .sentier-diagnostics .sd-grid td { padding: 0.28rem 0.35rem; vertical-align: middle; } +.sentier-diagnostics .sd-grid thead th { position: sticky; top: 0; z-index: 2; vertical-align: bottom; border-bottom-color: var(--sd-line-strong); white-space: normal; min-width: 3.4rem; max-width: 6.5rem; overflow-wrap: break-word; hyphens: auto; font-size: 0.75rem; line-height: 1.2; } +.sentier-diagnostics .sd-grid thead th.sd-name { left: 0; z-index: 3; min-width: 12rem; } +.sentier-diagnostics .sd-grid thead th.sd-name, .sentier-diagnostics .sd-grid thead th.sd-group { max-width: none; overflow-wrap: normal; } +.sentier-diagnostics .sd-grid tbody th { position: sticky; left: 0; z-index: 1; max-width: 18rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; background: var(--sd-surface); font-weight: 600; } +.sentier-diagnostics .sd-grid tbody th .sd-listed { display: inline-block; width: 0.45rem; height: 0.45rem; margin-right: 0.35rem; border-radius: 50%; background: var(--sd-accent); vertical-align: 0.1em; } +.sentier-diagnostics .sd-grid td.sd-group { max-width: 8rem; overflow: hidden; text-overflow: ellipsis; color: var(--sd-muted); white-space: nowrap; } +.sentier-diagnostics .sd-sort { display: block; width: 100%; padding: 0; border: 0; background: none; color: inherit; font: inherit; font-weight: 700; text-align: right; cursor: pointer; } +.sentier-diagnostics .sd-name .sd-sort { text-align: left; } +.sentier-diagnostics .sd-sort:hover { color: var(--sd-accent); } +.sentier-diagnostics th[aria-sort] { box-shadow: inset 0 -3px 0 var(--sd-accent); } +.sentier-diagnostics tr[aria-current="true"] > th { box-shadow: inset 4px 0 0 var(--sd-accent); } +.sentier-diagnostics .sd-more { position: sticky; left: 0; margin: 0; padding: var(--sd-s3) var(--sd-s4); } + +/* a value: colour from its direction only, strength from how far the two lie apart */ +.sentier-diagnostics .cell { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } +.sentier-diagnostics .tone-up { background: color-mix(in srgb, var(--sd-up) calc(var(--s, 0.5) * 55%), var(--sd-surface)); } +.sentier-diagnostics .tone-down { background: color-mix(in srgb, var(--sd-down) calc(var(--s, 0.5) * 55%), var(--sd-surface)); } +.sentier-diagnostics .tone-up.no-ratio, .sentier-diagnostics .tone-down.no-ratio { background-image: repeating-linear-gradient(135deg, transparent 0 4px, rgba(127, 127, 127, 0.35) 4px 6px); font-weight: 700; } +.sentier-diagnostics .tone-noise { color: var(--sd-noise); background-image: repeating-linear-gradient(135deg, transparent 0 5px, rgba(127, 127, 127, 0.18) 5px 7px); } +.sentier-diagnostics .tone-near { color: var(--sd-muted); } +.sentier-diagnostics .tone-missing { color: var(--sd-fail); font-style: italic; } +.sentier-diagnostics .tone-none { color: var(--sd-muted); } +.sentier-diagnostics .mark { margin-left: 0.2em; font-size: 0.75em; } +.sentier-diagnostics .verdict-look { font-weight: 700; } + +.sentier-diagnostics .sd-legend { display: flex; flex-wrap: wrap; gap: 0.2rem 0.9rem; margin: 0.4rem 0; font-size: var(--sd-t-xs); color: var(--sd-muted); } +.sentier-diagnostics .sd-status > * { min-width: 0; } +.sentier-diagnostics .sd-legend > span { min-width: 0; overflow-wrap: anywhere; } +.sentier-diagnostics .sd-legend .key { display: inline-block; white-space: nowrap; min-width: 2.6em; margin-right: 0.3em; padding: 0 0.3em; border: 1px solid var(--sd-line); border-radius: 3px; text-align: center; color: var(--sd-ink); } + +/* the distributions: one card per indicator, as many in a row as fit, under the matrix */ +.sentier-diagnostics .sd-dists-part { margin: var(--sd-s6) 0 var(--sd-s5); } +.sentier-diagnostics .sd-dists-part > h3 { margin: 0 0 var(--sd-s2); font-size: var(--sd-t-lg); } +.sentier-diagnostics .sd-dist-controls { margin: var(--sd-s2) 0 0; align-items: center; } +.sentier-diagnostics .sd-dist-controls .sd-legend { margin: 0; } +.sentier-diagnostics .sd-dists { display: grid; grid-template-columns: repeat(auto-fill, minmax(14rem, 1fr)); gap: var(--sd-s3); margin-top: var(--sd-s3); } +.sentier-diagnostics .sd-dist { position: relative; display: flex; flex-direction: column; min-width: 0; padding: var(--sd-s3) var(--sd-s4); border: 1px solid var(--sd-line); border-radius: var(--sd-radius); background: var(--sd-surface); } +.sentier-diagnostics .sd-dist:hover { border-color: var(--sd-accent); } +.sentier-diagnostics .sd-dist.sd-chosen { border-color: var(--sd-accent); box-shadow: inset 0 3px 0 var(--sd-accent); } +.sentier-diagnostics .sd-dist.sd-still { opacity: 0.6; } +.sentier-diagnostics .sd-dist > h4 { margin: 0 0 var(--sd-s2); min-height: 2.6em; font-size: var(--sd-t-sm); line-height: 1.3; } +/* the title is the button; it covers the card, so that the whole card is the target */ +.sentier-diagnostics .sd-dist-pick { padding: 0; border: 0; background: none; color: inherit; font: inherit; font-weight: 700; text-align: left; cursor: pointer; } +.sentier-diagnostics .sd-dist-pick::after { content: ""; position: absolute; inset: 0; border-radius: var(--sd-radius); } +.sentier-diagnostics .sd-dist-pick:focus-visible { outline: none; } +.sentier-diagnostics .sd-dist-pick:focus-visible::after { outline: 2px solid var(--sd-accent); outline-offset: 2px; } +.sentier-diagnostics .sd-top { margin: 0 0 0.15rem; font-size: 0.7rem; line-height: 1.2; color: var(--sd-muted); text-align: right; font-variant-numeric: tabular-nums; } +.sentier-diagnostics .sd-hist { display: flex; align-items: flex-end; gap: 2px; height: 4.5rem; border-bottom: 1px solid var(--sd-line-strong); } +.sentier-diagnostics .sd-bin { flex: 1 1 0; min-width: 0; border-radius: 1px 1px 0 0; } +.sentier-diagnostics .sd-bin.sd-gap { margin-left: 5px; } +.sentier-diagnostics .sd-bin.down, .sentier-diagnostics .sd-bin-key.down { background: color-mix(in srgb, var(--sd-down) 70%, var(--sd-surface)); } +.sentier-diagnostics .sd-bin.up, .sentier-diagnostics .sd-bin-key.up { background: color-mix(in srgb, var(--sd-up) 80%, var(--sd-surface)); } +.sentier-diagnostics .sd-bin.mid, .sentier-diagnostics .sd-bin-key.mid { background: color-mix(in srgb, var(--sd-ink) 72%, var(--sd-surface)); } +.sentier-diagnostics .sd-bin.aside, .sentier-diagnostics .sd-bin-key.aside { background: var(--sd-noise); background-image: repeating-linear-gradient(135deg, transparent 0 3px, rgba(255, 255, 255, 0.35) 3px 5px); } +.sentier-diagnostics .sd-bin-key { display: inline-block; width: 0.8em; height: 0.8em; margin-right: 0.35em; border-radius: 2px; vertical-align: -0.05em; } +.sentier-diagnostics .sd-axis { display: flex; gap: 2px; margin-top: 0.2rem; font-size: 0.7rem; line-height: 1.2; color: var(--sd-muted); white-space: nowrap; } +.sentier-diagnostics .sd-axis > span { flex: 1 1 0; min-width: 0; overflow: hidden; text-overflow: clip; } +.sentier-diagnostics .sd-axis-mid { text-align: center; } +.sentier-diagnostics .sd-axis-up { text-align: right; } +.sentier-diagnostics .sd-axis-aside + .sd-axis-down, .sentier-diagnostics .sd-axis-up + .sd-axis-aside { margin-left: 5px; } +.sentier-diagnostics .sd-axis-up + .sd-axis-aside { text-align: right; } +.sentier-diagnostics .sd-dist-meta { margin: var(--sd-s2) 0 0; font-size: var(--sd-t-xs); line-height: 1.45; color: var(--sd-muted); font-variant-numeric: tabular-nums; } +.sentier-diagnostics .sd-dist-meta strong { color: var(--sd-ink); } + +/* who wrote what */ +.sentier-diagnostics .badge { display: inline-block; margin-right: 0.4em; padding: 0 0.5em; border: 1px solid var(--sd-line); border-radius: 999px; font-size: var(--sd-t-xs); } +.sentier-diagnostics .badge.model { border-color: var(--sd-down); } +.sentier-diagnostics .badge.person { border-color: var(--sd-up); } +.sentier-diagnostics .sd-scope { display: inline-block; margin-right: 0.3em; padding: 0 0.4em; border: 1px solid var(--sd-line-strong); border-radius: 3px; background: var(--sd-paper-deep); font-size: var(--sd-t-xs); font-weight: 700; } + +/* the item: a drawer on the right, the whole window on a phone */ +.sentier-diagnostics .sd-drawer { position: fixed; top: 0; right: 0; bottom: 0; z-index: 20; display: flex; flex-direction: column; width: var(--sd-drawer); max-width: 100%; + border-left: 1px solid var(--sd-line-strong); background: var(--sd-surface); box-shadow: -0.5rem 0 2rem color-mix(in srgb, var(--sd-ink) 16%, transparent); } +.sentier-diagnostics .sd-drawer-head { flex: none; display: grid; grid-template-columns: minmax(0, 1fr) auto; column-gap: var(--sd-s4); align-items: start; + padding: var(--sd-s4) var(--sd-s5); border-bottom: 1px solid var(--sd-line); background: var(--sd-paper-deep); } +.sentier-diagnostics .sd-drawer-head > .sd-label { grid-column: 1; margin: 0; font-size: var(--sd-t-xs); font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--sd-muted); } +.sentier-diagnostics .sd-drawer-head > h3 { grid-column: 1; margin: 0.2rem 0 0; font-size: var(--sd-t-lg); line-height: 1.3; overflow-wrap: anywhere; } +.sentier-diagnostics .sd-drawer-head > h3:focus { outline: none; } +.sentier-diagnostics .sd-drawer-head > .sd-button { grid-column: 2; grid-row: 1 / span 2; align-self: center; } +.sentier-diagnostics .sd-drawer-body { flex: 1 1 auto; min-height: 0; overflow: auto; overscroll-behavior: contain; padding: var(--sd-s4) var(--sd-s5) var(--sd-s6); } +.sentier-diagnostics .sd-drawer-body > h4:first-child { margin-top: 0; } +.sentier-diagnostics .sd-why { padding: 0; list-style: none; } +.sentier-diagnostics .sd-why > li { margin: var(--sd-s2) 0; padding: var(--sd-s2) var(--sd-s3); border-left: 3px solid var(--sd-line-strong); border-radius: 0 4px 4px 0; background: var(--sd-paper-deep); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-why > li.sd-none { border-left-color: var(--sd-line); background: none; color: var(--sd-muted); } + +@media (max-width: 47.99rem) { + .sentier-diagnostics { --sd-drawer: 100vw; } + .sentier-diagnostics .sd-drawer { border-left: 0; box-shadow: none; } + .sentier-diagnostics .sd-drawer-head { padding: var(--sd-s3) var(--sd-s4); } + .sentier-diagnostics .sd-drawer-body { padding: var(--sd-s3) var(--sd-s4) var(--sd-s6); } + .sentier-diagnostics .sd-grid tbody th { max-width: 11rem; } + .sentier-diagnostics .sd-grid .sd-group { display: none; } +} diff --git a/src/sentier_diagnostics/page/diagnostics.js b/src/sentier_diagnostics/page/diagnostics.js new file mode 100644 index 0000000..0c7bdb9 --- /dev/null +++ b/src/sentier_diagnostics/page/diagnostics.js @@ -0,0 +1,1618 @@ +/* sentier-diagnostics page: shows a diagnostics file (sentier-diagnostics/1) as it is. + * + * Plain JavaScript, no library, no network but the files the page names. The script mounts + * into every element marked `data-sentier-diagnostics` (the body `page.body` writes), on a + * page of its own or inside a host's page; it looks nothing up outside that element but the + * address and the globals named below. A host that draws the element later calls + * `SentierDiagnostics.mount(element, {data})`, and marks it `data-sentier-diagnostics="manual"` + * so that it is not mounted before. + * + * The page works out no class, direction, verdict or count: they are in the file, checked by + * the validator. It only orders and filters, formats numbers, divides the result after by the + * result before to show their ratio, and subtracts consecutive results to show each step's + * share. Every text of a file is set as text, never as markup. + */ +(function (global) { + 'use strict'; + + var FORMAT = 'sentier-diagnostics/1'; + var MATRIX_FORMAT = 'sentier-diagnostics-matrix/1'; + var MATRIX_MAX_ITEMS = 200000; // matrix.MAX_ITEMS + var MATRIX_MAX_BYTES = 64 * 1024 * 1024; // matrix.MAX_BYTES + var PAGE = 300; // rows of the grid drawn at a time + var MOVES = ['differs', 'beyond 10x', 'zero reference', 'zero result', 'sign change']; + var QUIET = ['agrees', 'within noise', 'near zero']; + // the order of the classes from the worst: the same as the Python rule (rule.RANK) + var RANK = { + 'sign change': 0, 'zero reference': 1, 'zero result': 2, 'beyond 10x': 3, 'differs': 4, + 'missing': 5, 'within noise': 6, 'agrees': 7, 'near zero': 8 + }; + var STRENGTH = { '1-1.1': 0.25, '1.1-1.25': 0.45, '1.25-2': 0.65, '2-10': 0.85, 'over 10': 1 }; + // the ends of the bands, as rule.SIZE_STEPS: the distance (|log10| of the factor) at which each ends + var SIZE_STEPS = [[Math.log10(1.1), '1-1.1'], [Math.log10(1.25), '1.1-1.25'], [Math.log10(2), '1.25-2'], [1, '2-10']]; + var LOOK = ['off claim', 'not claimed', 'claimed, not moved']; + var LOOK_MARK = { 'off claim': '◐', 'not claimed': '■', 'claimed, not moved': '○' }; + var SAID = { + 'agrees': 'the two results differ by no more than the tolerance', + 'within noise': 'they differ by no more than two builds of one state do', + 'differs': 'the results differ', + 'beyond 10x': 'one result is more than ten times the size of the other', + 'zero reference': 'the result before is zero or within the floor; the result after is not', + 'zero result': 'the result after is zero or within the floor; the result before is not', + 'sign change': 'the two results have different signs', + 'near zero': 'both results are within their floor: their ratio says nothing', + 'missing': 'a side has no result; it is not read as zero' + }; + var SHORT = { + 'agrees': '=', 'within noise': '~', 'near zero': '≈ 0', 'missing': '∅', + 'zero reference': '0 →', 'zero result': '→ 0', 'sign change': '±' + }; + // the codes of a matrix (matrix.CODES, matrix.VERDICT_CODES) + var CODES = { + '=': ['agrees', null], '~': ['within noise', null], '0': ['near zero', null], '-': ['missing', null], + 'D': ['differs', 'up'], 'd': ['differs', 'down'], 'B': ['beyond 10x', 'up'], 'b': ['beyond 10x', 'down'], + 'R': ['zero reference', 'up'], 'r': ['zero reference', 'down'], 'T': ['zero result', 'up'], 't': ['zero result', 'down'], + 'S': ['sign change', 'up'], 's': ['sign change', 'down'] + }; + var VERDICT_CODES = { + 'a': 'as claimed', 'o': 'off claim', 'n': 'not claimed', 'c': 'claimed, not moved', + '~': 'within noise', '=': 'unchanged', '?': 'not judged' + }; + var NO_CLAIM = '.'; + + // ------------------------------------------------------------------ pure helpers + + function groupThousands(n) { return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ' '); } + + // a result as a reader reads it: four significant digits; never "0" for a value that is not 0 + function formatNumber(v) { + if (v === null || v === undefined) return 'missing'; + if (v === 0) return '0'; + var a = Math.abs(v); + // one style from 1e4 on: toPrecision(4) would write 12345 as "1.235e+4" itself + if (a >= 1e4 || a < 1e-3) { + return v.toExponential(3).replace(/\.?0+e/, 'e').replace('e+', 'e').replace(/-/g, '−'); + } + var s = v.toPrecision(4); + if (s.indexOf('.') >= 0) s = s.replace(/0+$/, '').replace(/\.$/, ''); + return s.replace('-', '−'); + } + + // the change relative to the size of the result before, signed as the direction is + function percentText(b, a) { + var c = (a - b) / Math.abs(b) * 100; + var size = Math.abs(c); + var text; + if (size === 0) text = '0'; + else if (size < 0.01) text = size.toExponential(1); + else if (size >= 1000) text = groupThousands(Math.round(size)); + else text = size.toFixed(size >= 100 ? 0 : size >= 1 ? 1 : 2); + return (c > 0 ? '+' : c < 0 ? '−' : '') + text + ' %'; + } + + // a step's share of the whole difference, signed; never "0.0 %" for a share that is not 0 + function shareText(part) { + var v = part * 100, size = Math.abs(v); + var text = size === 0 ? '0' : size < 0.1 ? size.toExponential(1) : size.toFixed(1); + return (v > 0 ? '' : v < 0 ? '−' : '') + text + ' %'; + } + + // the result after divided by the result before, of two results of one sign; never "1" for + // a ratio that is not 1, and never "Infinity" where the quotient leaves the floats + function ratioText(before, after) { + var r = after / before; + if (!isFinite(r) || r === 0) { + var orders = Math.log10(Math.abs(after)) - Math.log10(Math.abs(before)); + return '1e' + String(Math.round(orders)).replace('-', '−'); + } + var d = r - 1; + if (d !== 0 && Math.abs(d) < 1e-3) return '1 ' + (d > 0 ? '+ ' : '− ') + formatNumber(Math.abs(d)); + return formatNumber(r); + } + + // a ratio as the grid shows it, compact enough for a column: three significant digits + // ("1.00", "0.984", "2.27"), a power of ten below 0.01 and from 1000 on, and "1e600" where + // the quotient leaves the floats. "1.00" is a rounding, not a claim of agreement: the + // class says whether the two agree, and the cell's title holds both results exactly. + function ratioShort(before, after) { + var r = after / before; + if (!isFinite(r) || r === 0) return ratioText(before, after); + var s = r.toPrecision(3); + if (r < 0.01 || s.indexOf('e') >= 0) s = r.toExponential(2).replace(/\.?0+e/, 'e').replace('e+', 'e'); + return s.replace(/-/g, '−'); + } + + // the 10th, 50th and 90th percentile in one precision: the smallest (three digits at + // least, six at most) at which every two that differ read differently. "1.00, 1.00, 1.00" + // over values that moved would hide the spread a reader looks for, and one line in mixed + // precisions reads a rounded number as exact. Where six digits still read a percentile + // as the median, the line says how it lies, and the numbers left take the digits they need. + function quantileTexts(qs) { + var values = [qs.p10, qs.median, qs.p90]; + var one = function (v, digits) { + if (v !== 0 && (Math.abs(v) < 0.01 || Math.abs(v) >= 1e4)) return v.toExponential(digits - 1).replace(/\.?0+e/, 'e').replace('e+', 'e').replace(/-/g, '−'); + return v.toPrecision(digits).replace(/-/g, '−'); + }; + var pairs = [[0, 1], [1, 2], [0, 2]]; + var digitsFor = function (kept) { + for (var d = 3; d <= 6; d++) { + var apart = pairs.every(function (p) { + return kept.indexOf(p[0]) < 0 || kept.indexOf(p[1]) < 0 || values[p[0]] === values[p[1]] || one(values[p[0]], d) !== one(values[p[1]], d); + }); + if (apart) return d; + } + return null; + }; + var words = {}; + var d = digitsFor([0, 1, 2]); + if (d === null) { + if (values[0] !== values[1] && one(values[0], 6) === one(values[1], 6)) words[0] = 'just under the median'; + if (values[2] !== values[1] && one(values[2], 6) === one(values[1], 6)) words[2] = 'just over the median'; + d = digitsFor([0, 1, 2].filter(function (i) { return !(i in words); })) || 6; + } + return [0, 1, 2].map(function (i) { return i in words ? words[i] : one(values[i], d); }); + } + + // Steps whose differences add up to more than this many times the whole difference + // cancel: their shares would be large and of both signs, and in floating point the + // whole may be mostly rounding. The page says so instead of printing them. + var CANCEL = 10; + + // how an item's whole difference splits among the steps (SPEC 7): `values` are its + // results per state, `explained` says per step whether a cause explains it, `totalClass` + // is the class of the whole way. The split is exact up to rounding, so the page prints + // the sum of the shares it shows, never a fixed 100 %. + function splitOf(values, explained, totalClass) { + if (totalClass === 'missing') return { state: 'missing' }; + if (MOVES.indexOf(totalClass) < 0) return { state: 'still' }; + for (var i = 0; i < values.length; i++) if (values[i] === null) return { state: 'gap' }; + var whole = values[values.length - 1] - values[0]; + var diffs = [], gross = 0; + for (var k = 0; k + 1 < values.length; k++) { + diffs.push(values[k + 1] - values[k]); + gross += Math.abs(values[k + 1] - values[k]); + } + if (!whole || gross > CANCEL * Math.abs(whole)) return { state: 'cancel', factor: whole ? gross / Math.abs(whole) : Infinity }; + var shares = diffs.map(function (d) { return d / whole; }); + var sum = 0, unexplained = 0; + shares.forEach(function (part, j) { sum += part; if (!explained[j]) unexplained += part; }); + return { state: 'split', shares: shares, sum: sum, unexplained: unexplained }; + } + + // what the whole way's row says of a split + function splitText(split) { + switch (split.state) { + case 'split': return shareText(split.sum) + (split.unexplained ? ', of which nothing explains ' + shareText(split.unexplained) : ''); + case 'still': return 'not split: it did not move'; + case 'gap': return 'not split: missing in a state between'; + case 'missing': return 'not split: missing in the first or the last state'; + default: return 'not split: the steps cancel, their differences add up to ' + formatNumber(split.factor) + ' times the whole difference'; + } + } + + // Noise above this is large: it puts every smaller difference of its indicator "within + // noise". Same bound as chain.LARGE_NOISE. + var LARGE_NOISE = 1e-2; + + // what a step's noise says: measured and identical, measured, or not measured. Two + // identical builds are a finding, and never read as "not measured"; an indicator the noise + // holds no size for was not measured, and is never read as identical. `ids` are the + // file's indicators, and `many` what the file calls them. + function noiseText(noise, ids, many) { + if (!noise || !noise.measured) return 'not measured'; + var by = noise.by || {}; + var between = ' between ' + noise.between.join(' and '); + var keys = Object.keys(by); + var all = ids || keys; + var absent = all.filter(function (k) { return !(k in by); }).length; + if (!keys.length) return 'not measured: no result beyond the floor in both builds' + between; + var part = absent ? ', for ' + keys.length + ' of ' + all.length + ' ' + (many || 'indicators') : ''; + var largest = Math.max.apply(null, keys.map(function (k) { return by[k]; })); + if (largest === 0) return 'identical' + between + part; + return 'measured' + between + part + (largest > LARGE_NOISE ? '; large: up to ' + formatNumber(largest * 100) + ' %' : ''); + } + + // a link a file may hold: an https address, or nothing. A page never makes any other + // text a link, whatever the file says. + function safeLink(value) { + return typeof value === 'string' && /^https:\/\/[^\s]+$/.test(value) ? value : null; + } + + // a file a page may load beside itself: a plain relative name, never an address or a way up + function safeName(value) { + return typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/.test(value) && value.indexOf('..') < 0; + } + + function bandStep(band) { + if (!band) return null; + var at = band.indexOf('factor '); + return at < 0 ? null : band.slice(at + 7); + } + + // how far the sizes of two results of one sign lie apart, as rule.distance: the |log10| + // of their factor, or, where the factor is too large or too small for a float, the + // difference of their logarithms, which cannot overflow + var SMALLEST_NORMAL = 2.2250738585072014e-308; + function distance(before, after) { + var factor = Math.abs(after) / Math.abs(before); + if (isFinite(factor) && factor >= SMALLEST_NORMAL) return Math.abs(Math.log10(factor)); + return Math.abs(Math.log10(Math.abs(after)) - Math.log10(Math.abs(before))); + } + + // the band of a pair, as rule.band, for a matrix, which holds no bands: the class and + // direction are the file's, the band only names how far apart the two lie + function bandOf(before, after, cls, dir) { + if (cls === 'beyond 10x') return dir + ', factor over 10'; + if (cls !== 'differs') return null; + var found = distance(before, after); + for (var i = 0; i < SIZE_STEPS.length; i++) if (found <= SIZE_STEPS[i][0]) return dir + ', factor ' + SIZE_STEPS[i][1]; + return dir + ', factor ' + SIZE_STEPS[SIZE_STEPS.length - 1][1]; + } + + // the reader's own size filter: it hides values, it changes no class and no count. + // A value without a ratio (a zero or a sign change) and a missing one always pass. + function sizeAtLeast(cls, band, threshold) { + if (!threshold) return true; + if (['zero reference', 'zero result', 'sign change', 'missing', 'beyond 10x'].indexOf(cls) >= 0) return true; + if (cls !== 'differs') return false; + var step = bandStep(band); + return step !== null && parseFloat(step) >= threshold; + } + + // how one value is drawn: its text, its tone (direction or kind), the strength of the tone + function cellView(before, after, cls, dir, band) { + var view = { text: '', tone: 'none', strength: 0, noRatio: false, mark: '' }; + view.title = cls + (SAID[cls] ? ': ' + SAID[cls] : '') + '. Before ' + + (before === null ? 'missing' : String(before)) + ', after ' + + (after === null ? 'missing' : String(after)) + (dir ? '. Went ' + dir + '.' : '.'); + switch (cls) { + case 'missing': + view.text = before === null && after === null ? 'missing' : before === null ? 'missing before' : 'missing after'; + view.tone = 'missing'; + break; + case 'near zero': + view.text = '≈ 0'; view.tone = 'near'; break; + case 'agrees': + view.text = '='; view.tone = 'none'; break; + case 'within noise': + view.text = '~ ' + percentText(before, after); view.tone = 'noise'; break; + case 'zero reference': + view.text = before === 0 ? 'from 0' : 'from ≈ 0'; view.tone = dir; view.strength = 1; view.noRatio = true; break; + case 'zero result': + view.text = after === 0 ? 'to 0' : 'to ≈ 0'; view.tone = dir; view.strength = 1; view.noRatio = true; break; + case 'sign change': + view.text = (before < 0 ? '−' : '+') + ' → ' + (after < 0 ? '−' : '+'); + view.tone = dir; view.strength = 1; view.noRatio = true; break; + case 'beyond 10x': + var up = Math.abs(after) >= Math.abs(before); + var r = up ? Math.abs(after) / Math.abs(before) : Math.abs(before) / Math.abs(after); + var orders = Math.abs(Math.log10(Math.abs(after)) - Math.log10(Math.abs(before))); + view.text = 'size ' + (up ? '×' : '÷') + (isFinite(r) ? formatNumber(r) : '1e' + Math.round(orders)); + view.tone = dir; view.strength = 1; view.mark = '»'; break; + case 'differs': + view.text = percentText(before, after); view.tone = dir; + view.strength = STRENGTH[bandStep(band)] || 0.5; break; + default: + view.text = String(cls); + } + return view; + } + + // a value of the matrix grid: the ratio after ÷ before in every cell that has one, agreeing + // pairs included ("1.00"), so that a column reads as numbers; else what cellView says + // ("≈ 0", "from 0", "− → +", "missing"). The class shows as the tint and a mark, never in + // place of the number: the colour is the direction's, its strength the band's. The ratio is + // a display; the class stays the file's. + var RATIO_CLASSES = ['agrees', 'within noise', 'differs', 'beyond 10x']; + function ratioView(before, after, cls, dir, band) { + var view = cellView(before, after, cls, dir, band); + var ratio = before !== null && after !== null && before !== 0 && after !== 0 && (before > 0) === (after > 0); + if (ratio && RATIO_CLASSES.indexOf(cls) >= 0) { + view.text = ratioShort(before, after); + if (cls === 'within noise') view.mark = '~'; + view.title += ' After ÷ before: ' + String(after / before) + '.'; + } + return view; + } + + // the key that orders values from the worst: the same as the Python rule (rule.severity) + function severityKey(before, after, cls) { + var rank = RANK.hasOwnProperty(cls) ? RANK[cls] : 9; + if (before === null || after === null) return [rank, 0]; + var magnitude; + if (before !== 0 && after !== 0 && (before > 0) === (after > 0)) { + magnitude = distance(before, after); + } else { + var size = Math.max(Math.abs(before), Math.abs(after)); + magnitude = size ? Math.abs(after - before) / size : 0; + } + return [rank, -magnitude]; + } + + function compareKeys(a, b) { + for (var i = 0; i < Math.min(a.length, b.length); i++) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + return a.length - b.length; + } + + // what a page needs of a file before it shows anything + function checkShape(d) { + if (!d || typeof d !== 'object') return 'not a diagnostics file'; + if (d.format !== FORMAT) { + return typeof d.format === 'string' && d.format.indexOf('sentier-diagnostics/') === 0 + ? 'format ' + d.format + ': this page shows ' + FORMAT + ' only' + : 'not a diagnostics file of format ' + FORMAT; + } + var needed = ['tier', 'audience', 'title', 'item', 'indicators', 'rule', 'states', 'steps', 'counts', + 'attribution', 'items', 'items_count', 'explanations', 'notes']; + for (var i = 0; i < needed.length; i++) { + if (!(needed[i] in d)) return 'the file has no ' + needed[i]; + } + var lists = ['indicators', 'states', 'steps', 'items', 'explanations', 'notes']; + for (var j = 0; j < lists.length; j++) { + if (!Array.isArray(d[lists[j]])) return 'the ' + lists[j] + ' of the file are not a list'; + } + var tables = ['item', 'rule', 'counts', 'attribution', 'items_count']; + for (var k = 0; k < tables.length; k++) { + if (!d[tables[k]] || typeof d[tables[k]] !== 'object' || Array.isArray(d[tables[k]])) return 'the ' + tables[k] + ' of the file is not a table'; + } + if (!d.indicators.length) return 'the file has no indicator'; + if (d.indicator !== undefined && d.indicator !== null && !(d.indicator && typeof d.indicator.one === 'string' && + typeof d.indicator.many === 'string')) return 'the indicator word of the file is not {one, many}'; + if (d.steps.length !== d.states.length - 1) return 'the file has not one step fewer than states'; + if (!d.counts.steps || !d.counts.total) return 'the counts of the file are not {steps, total}'; + return null; + } + + function sameList(a, b) { + if (!Array.isArray(a) || a.length !== b.length) return false; + for (var i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; + } + + // what a page needs of a matrix before it draws a row of it: that it is of the file the + // page shows, and that every row has the shape the format gives. Its codes are not worked + // out again here: `sentier-diagnostics check` does that. + function checkMatrix(m, d) { + if (!m || typeof m !== 'object') return 'not a matrix'; + if (m.format !== MATRIX_FORMAT) return 'not a matrix of format ' + MATRIX_FORMAT; + var of = m.of || {}; + var ids = d.indicators.map(function (i) { return i.id; }); + if (of.title !== d.title || !sameList(of.states, d.states.map(function (s) { return s.id; })) || + !sameList(of.indicators, ids) || !sameList(of.steps, d.steps.map(function (s) { return s.id; }))) { + return 'it is the matrix of another file: its title, states, indicators or steps differ'; + } + if (!Array.isArray(m.items) || m.items.length > MATRIX_MAX_ITEMS) return 'its items are not a list of at most ' + groupThousands(MATRIX_MAX_ITEMS); + var n = m.items.length; + if (d.matrix && d.matrix.items !== n) return 'the file says ' + d.matrix.items + ' items, the matrix holds ' + n; + var claimed = d.steps.some(function (s) { return !!s.claim; }); + if (!Array.isArray(m.values) || m.values.length !== n || !Array.isArray(m.classes) || m.classes.length !== n) return 'its values or classes are not one row per item'; + if (claimed ? !Array.isArray(m.verdicts) || m.verdicts.length !== n : m.verdicts !== null) return 'its verdicts are not one row per item, or null where no step has a claim'; + var ns = d.states.length, nk = d.steps.length, ni = ids.length; + for (var at = 0; at < n; at++) { + var row = m.items[at]; + if (!Array.isArray(row) || row.length !== 5 || typeof row[0] !== 'string' || typeof row[1] !== 'string') return 'item ' + at + ' is not [id, name, place, group, unit]'; + var values = m.values[at]; + if (!Array.isArray(values) || values.length !== ni * ns) return 'item ' + row[0] + ': not one result per indicator and state'; + for (var v = 0; v < values.length; v++) if (values[v] !== null && (typeof values[v] !== 'number' || !isFinite(values[v]))) return 'item ' + row[0] + ': a result that is not a number'; + var codes = m.classes[at]; + if (typeof codes !== 'string' || codes.length !== ni * (nk + 1)) return 'item ' + row[0] + ': not one class code per indicator and comparison'; + for (var c = 0; c < codes.length; c++) if (!CODES.hasOwnProperty(codes[c])) return 'item ' + row[0] + ': an unknown class code'; + if (claimed) { + var said = m.verdicts[at]; + if (typeof said !== 'string' || said.length !== ni * nk) return 'item ' + row[0] + ': not one verdict code per indicator and step'; + for (var s = 0; s < said.length; s++) if (said[s] !== NO_CLAIM && !VERDICT_CODES.hasOwnProperty(said[s])) return 'item ' + row[0] + ': an unknown verdict code'; + } + } + return listedDiffer(matrixRows(m, d), d); + } + + // A matrix says of every item this file lists what the file says: its name, place, group and + // unit, its results, classes, directions and verdicts. A matrix left over from another run of + // the same chain has the same title, states, steps and count, and other numbers; the page + // would draw them in place of the file's. + function listedDiffer(rows, d) { + var ids = d.indicators.map(function (i) { return i.id; }); + var found = 0; + for (var r = 0; r < rows.length; r++) { + var row = rows[r], item = row.item; + if (!item) continue; + found++; + if (row.name !== item.name || row.place !== item.place || row.group !== item.group || row.unit !== item.unit) { + return 'item ' + row.id + ': its name, place, group or unit differ from what this file lists'; + } + for (var x = 0; x < ids.length; x++) { + var id = ids[x]; + var verdicts = row.verdicts && item.verdicts ? sameList(row.verdicts[id], item.verdicts[id] || []) : !row.verdicts === !item.verdicts; + if (!sameList(row.values[id], item.values[id] || []) || !sameList(row.classes[id], item.classes[id] || []) || + !sameList(row.directions[id], item.directions[id] || []) || !verdicts) { + return 'item ' + row.id + ', ' + id + ': its results or classes differ from what this file lists for it'; + } + } + } + return found === d.items.length ? null : (d.items.length - found) + ' of the items this file lists are not in it'; + } + + // the rows of a checked matrix, in the shape of a listed item: values, classes, directions + // and verdicts per indicator. A listed item keeps its own record beside the row. + function matrixRows(m, d) { + var ids = d.indicators.map(function (i) { return i.id; }); + var ns = d.states.length, nk = d.steps.length; + var listed = {}; + d.items.forEach(function (item) { listed[item.id] = item; }); + return m.items.map(function (row, at) { + var out = { id: row[0], name: row[1], place: row[2], group: row[3], unit: row[4], values: {}, classes: {}, directions: {}, + bands: null, verdicts: m.verdicts ? {} : null, item: listed[row[0]] || null }; + ids.forEach(function (id, x) { + out.values[id] = m.values[at].slice(x * ns, (x + 1) * ns); + var codes = m.classes[at].slice(x * (nk + 1), (x + 1) * (nk + 1)).split(''); + out.classes[id] = codes.map(function (c) { return CODES[c][0]; }); + out.directions[id] = codes.map(function (c) { return CODES[c][1]; }); + if (m.verdicts) { + out.verdicts[id] = m.verdicts[at].slice(x * nk, (x + 1) * nk).split('').map(function (c) { return c === NO_CLAIM ? null : VERDICT_CODES[c]; }); + } + }); + if (out.item) out.bands = out.item.bands; + return out; + }); + } + + function listedRow(item) { + return { id: item.id, name: item.name, place: item.place, group: item.group, unit: item.unit, values: item.values, + classes: item.classes, directions: item.directions, bands: item.bands, verdicts: item.verdicts, item: item }; + } + + // ------------------------------------------------------------------ one mounted page + + function mount(root, given) { + if (!root || root.getAttribute('data-sd-mounted') !== null) return null; + root.setAttribute('data-sd-mounted', ''); + given = given || {}; + var doc = root.ownerDocument || global.document; + var win = global; + var standalone = root.getAttribute('data-sd-standalone') !== null; + var base = root.getAttribute('data-sd-base') || ''; + + var D = null, B = {}, LISTED = [], M = null; + var matrix = { state: 'none', said: '' }; + var view = { comp: null, worstN: 20, worstIndicator: '', find: '', cls: '', verdict: '', group: '', limit: PAGE, size: 0, sort: 'worst', open: null, distScale: 'card' }; + var lastFocus = null; + + function q(name) { return root.querySelector('[data-sd="' + name + '"]'); } + function el(tag, props, children) { + var node = doc.createElement(tag); + if (props) { + Object.keys(props).forEach(function (key) { + var value = props[key]; + if (value === null || value === undefined || value === false) return; + if (key === 'text') node.textContent = value; + else if (key === 'cls') node.className = value; + else if (key === 'style') node.setAttribute('style', value); + else node.setAttribute(key, value === true ? '' : value); + }); + } + (children || []).forEach(function (child) { + if (child === null || child === undefined) return; + node.appendChild(typeof child === 'string' ? doc.createTextNode(child) : child); + }); + return node; + } + function clear(node) { while (node.firstChild) node.removeChild(node.firstChild); return node; } + function n(x) { return x === null || x === undefined ? '—' : groupThousands(x); } + function many(count) { return count === 1 ? D.item.one : D.item.many; } + function capital(text) { return text.charAt(0).toUpperCase() + text.slice(1); } + function article(word) { return (/^[aeiou]/i.test(word) ? 'an ' : 'a ') + word; } + + function indicatorName(id) { + for (var i = 0; i < D.indicators.length; i++) { + if (D.indicators[i].id === id) return D.indicators[i].short || D.indicators[i].name; + } + return id; + } + function indicatorIds() { return D.indicators.map(function (i) { return i.id; }); } + function stepIndex(comp) { // the position of a comparison in an item's lists of classes + if (comp === 'total') return D.steps.length; + for (var i = 0; i < D.steps.length; i++) if (D.steps[i].id === comp) return i; + return D.steps.length; + } + function stateIndex(stateId) { + for (var i = 0; i < D.states.length; i++) if (D.states[i].id === stateId) return i; + return 0; + } + function compLabel(comp) { + if (comp === 'total') return 'The whole way: ' + D.states[0].label + ' → ' + D.states[D.states.length - 1].label; + var k = stepIndex(comp); + return 'Step ' + stepNo(k) + ': ' + D.steps[k].label; + } + // a step's number: from 0 where the chain starts from the published results (step 0 + // checks they are reproduced), from 1 otherwise + function stepNo(k) { return k + (D.states[0].kind === 'published' ? 0 : 1); } + // what the file calls an indicator: its own word, or "indicator" + function indWord(many) { + var word = D.indicator || { one: 'indicator', many: 'indicators' }; + return many ? word.many : word.one; + } + function pairOf(row, indicator, k) { + var values = row.values[indicator]; + return k === D.steps.length ? [values[0], values[values.length - 1]] : [values[k], values[k + 1]]; + } + function verdictOf(row, indicator, k) { + if (!row.verdicts || k >= D.steps.length) return null; + return row.verdicts[indicator][k]; + } + function bandAt(row, indicator, k) { + if (row.bands) return row.bands[indicator][k]; + var pair = pairOf(row, indicator, k); + return bandOf(pair[0], pair[1], row.classes[indicator][k], row.directions[indicator][k]); + } + + // the rows the grid and the worst list draw from: every item once the matrix is loaded, + // the listed items before + function rows() { return M || LISTED; } + function rowById(id) { + var all = rows(); + for (var i = 0; i < all.length; i++) if (all[i].id === id) return all[i]; + return null; + } + function heldSaid() { + var c = D.items_count; + if (M) return 'all ' + groupThousands(M.length) + ' ' + many(M.length) + ' (the matrix)'; + return 'the ' + groupThousands(c.listed) + ' ' + many(c.listed) + ' in this file' + + (c.not_listed ? '; ' + groupThousands(c.not_listed) + ' more worth a look are not in it' : ''); + } + + function valueCell(row, indicator, k, ratio) { + var pair = pairOf(row, indicator, k); + var cls = row.classes[indicator][k]; + var v = (ratio ? ratioView : cellView)(pair[0], pair[1], cls, row.directions[indicator][k], bandAt(row, indicator, k)); + var verdict = verdictOf(row, indicator, k); + var title = v.title + (verdict ? ' Verdict: ' + verdict + '.' : ''); + var classes = 'cell tone-' + v.tone + (v.noRatio ? ' no-ratio' : ''); + var td = el('td', { cls: classes, title: title, style: v.strength ? '--s:' + v.strength : null, 'data-class': cls }, + [v.text, v.mark ? el('span', { cls: 'mark', text: v.mark, 'aria-hidden': 'true' }) : null]); + if (verdict && LOOK.indexOf(verdict) >= 0) td.appendChild(el('span', { cls: 'mark verdict-look', text: LOOK_MARK[verdict], title: verdict })); + return td; + } + + function legend(node, ratio) { + clear(node); + // short words beside each key, the whole sentence in its title + var keys = [ + ['tone-up', ratio ? '1.3' : 'up', 'up', 'went up (as signed numbers)'], + ['tone-down', ratio ? '0.77' : 'down', 'down', 'went down (as signed numbers): two results below zero that go further below zero went down'], + ['tone-up no-ratio', '0 →', 'no ratio', 'from zero, to zero or across zero: no ratio'], + ['tone-none', ratio ? '1.00' : '=', 'agrees', 'agrees within the tolerance' + (ratio ? ': no tint' : '')], + ['tone-noise', ratio ? '1.00~' : '~', 'noise', 'within the measured noise'], + ['tone-near', '≈ 0', 'near zero', 'near zero on both sides'], + ['tone-missing', 'missing', '', 'a side has no result'] + ]; + keys.forEach(function (k) { + node.appendChild(el('span', { title: k[3] }, [el('span', { cls: 'key ' + k[0], style: '--s:0.7', text: k[1] }), k[2]])); + }); + node.appendChild(el('span', { title: 'verdicts worth a look' }, ['■ not claimed ◐ off claim ○ claimed, not moved'])); + } + + // the file's own word for an item, in the page's own sentences + function sayWords() { + var spans = root.querySelectorAll('[data-sd-word]'); + var words = { one: D.item.one, many: D.item.many, indicator: indWord(false), indicators: indWord(true), Indicator: capital(indWord(false)) }; + for (var i = 0; i < spans.length; i++) { + var said = words[spans[i].getAttribute('data-sd-word')]; + if (said !== undefined) spans[i].textContent = said; + } + q('grid-region').setAttribute('aria-label', 'After divided by before, per ' + D.item.one + ' and ' + indWord(false)); + q('item-close').setAttribute('aria-label', 'Close the ' + D.item.one); + } + + function renderHeader() { + sayWords(); + if (standalone) doc.title = D.title; + q('title').textContent = D.title; + q('about').textContent = D.about || ''; + var states = clear(q('states')); + D.states.forEach(function (s) { + var identity = Object.keys(s.identity).map(function (k) { return k + ': ' + s.identity[k]; }).join(', '); + var join = s.join ? n(s.join.compared) + ' compared, ' + n(s.join.unmatched) + ' unmatched, ' + n(s.join.ambiguous) + ' ambiguous' : ''; + states.appendChild(el('li', { title: (s.about || '') + (identity ? ' (' + identity + ')' : '') + (s.join && s.join.note ? ' ' + s.join.note : '') }, + [s.label + (s.kind === 'published' ? ' (published)' : ''), join ? el('span', { cls: 'sd-quiet', text: ' · ' + join }) : null])); + }); + tierSaid(); + q('audience').textContent = D.audience === 'local' + ? 'Local file: its writer has not said that it may be published.' : ''; + q('audience').hidden = D.audience !== 'local'; + // only a chain that starts from the published results has a step 0 that reproduces them + q('steps-zero').hidden = D.states[0].kind !== 'published'; + q('made-by').textContent = 'Written by ' + D.made_by + '. ' + FORMAT + '.'; + renderChips(); + renderStepSaid(); + } + + // the comparison is chosen from a row of chips: the whole way, then each step with its + // number, the values that moved in it and a mark for each verdict worth a look + function renderChips() { + var row = clear(q('comparison')); + var chip = function (value, parts, title) { + row.appendChild(el('button', { type: 'button', cls: 'sd-chip', 'data-comp': value, title: title, + 'aria-pressed': view.comp === value ? 'true' : 'false' }, parts)); + }; + var count = function (comp) { + var m = sumOver(comp, moved); + return el('span', { cls: 'sd-chip-count', text: n(m) + ' moved' }); + }; + chip('total', [el('span', { cls: 'sd-chip-label', text: 'The whole way' }), count('total')], compLabel('total')); + D.steps.forEach(function (s, k) { + var parts = [el('span', { cls: 'sd-chip-no', text: String(stepNo(k)) }), el('span', { cls: 'sd-chip-label', text: s.label }), count(s.id)]; + var said = [compLabel(s.id), n(sumOver(s.id, moved)) + ' values moved']; + if (!s.explained) { + parts.push(el('span', { cls: 'sd-chip-flag', text: '!' })); + said.push('nothing explains it'); + } + if (s.claim) LOOK.forEach(function (name) { + var found = sumOver(s.id, function (b) { return b.verdicts[name]; }); + if (!found) return; + parts.push(el('span', { cls: 'sd-chip-mark', text: LOOK_MARK[name] })); + said.push(n(found) + ' ' + name); + }); + chip(s.id, parts, said.join('. ') + '.'); + }); + } + + // under the chips: what the chosen comparison is, the claim of its step and who made it, + // and its rule, so that a reviewer reads what a step said beside what it did + function renderStepSaid() { + var node = q('step-said'); + if (!node) return; + clear(node); + if (view.comp === 'total') { + node.appendChild(el('p', null, [el('strong', { text: 'The whole way: ' }), D.states[0].label + ' → ' + D.states[D.states.length - 1].label + '. ', + el('span', { cls: 'sd-quiet', text: 'Each cell holds the last result divided by the first. Choose a step to see what one change did; "Steps and the whole way" below splits the whole difference between them.' })])); + return; + } + var k = stepIndex(view.comp); + var s = D.steps[k]; + var line = [el('strong', { text: 'Step ' + stepNo(k) + ': ' + s.label + '. ' }), el('span', { cls: 'sd-quiet', text: D.states[k].label + ' → ' + D.states[k + 1].label + '. ' })]; + if (s.about) line.push(el('span', { cls: 'sd-quiet', text: s.about })); + node.appendChild(el('p', null, line)); + if (k === 0 && D.states[0].kind === 'published') { + node.appendChild(el('p', { cls: 'sd-quiet', text: 'What moves here was not reproduced within the tolerance: the residual the chain starts from, not a regression.' })); + } + if (!s.explained) node.appendChild(el('p', { cls: 'sd-error', text: 'Nothing explains this step.' })); + var claim = [el('span', { cls: 'sd-claim-label', text: 'Claim' })]; + if (s.claim) { + claim.push(s.claim.text, ' ', badge(s.claim.by)); + // every verdict on a move, "as claimed" first, each with its count; the ones worth a + // look carry their mark, as on the chips. No colour: a verdict is not an approval. + var found = ['as claimed'].concat(LOOK).map(function (name) { return [name, sumOver(s.id, function (b) { return b.verdicts[name]; })]; }) + .filter(function (p) { return p[1]; }); + claim.push(el('span', { cls: 'sd-verdicts', text: found.length + ? ' ' + found.map(function (p) { return n(p[1]) + ' ' + (p[1] === 1 ? 'value' : 'values') + ' ' + (p[0] === 'as claimed' ? 'moved as claimed' : p[0]) + (LOOK_MARK[p[0]] ? ' ' + LOOK_MARK[p[0]] : ''); }).join(', ') + '.' + : ' Nothing moved, as claimed.' })); + if (s.claim.scope.some(function (e) { return e.items === null; })) { + claim.push(el('span', { cls: 'sd-quiet', text: ' Its list of ' + D.item.many + ' is not in this file' + (s.claim.reach !== null ? ' (' + n(s.claim.reach) + ')' : '') + ': its verdicts were worked out by its writer.' })); + } + } else { + claim.push(el('span', { cls: 'sd-quiet', text: 'none: what moves here is counted, not judged.' })); + } + node.appendChild(el('p', { cls: 'sd-claim' }, claim)); + node.appendChild(el('p', { cls: 'sd-quiet', text: 'Tolerance ' + s.tolerance.value + ' (' + s.tolerance.basis + '). Noise ' + noiseText(s.noise, indicatorIds(), indWord(true)) + '.' })); + } + + function markChips() { + var chips = q('comparison').querySelectorAll('button[data-comp]'); + for (var i = 0; i < chips.length; i++) chips[i].setAttribute('aria-pressed', chips[i].getAttribute('data-comp') === view.comp ? 'true' : 'false'); + } + + // what the page holds of the items: the listed ones, or every one once the matrix is here + function tierSaid() { + var c = D.items_count; + var head = groupThousands(c.all) + ' ' + D.item.many + ', ' + D.indicators.length + ' ' + indWord(D.indicators.length !== 1) + '. '; + var said; + if (D.tier === 'counts') said = 'This file holds counts only: no single ' + D.item.one + ' is shown.'; + else if (M) said = 'All ' + groupThousands(M.length) + ' are loaded from the matrix; ' + groupThousands(c.listed) + ' of the ' + + groupThousands(c.qualifying) + ' worth a look are listed in this file.'; + else said = groupThousands(c.listed) + ' of the ' + groupThousands(c.qualifying) + ' ' + D.item.many + ' worth a look are in this file' + + (c.not_listed ? '; ' + groupThousands(c.not_listed) + ' are not.' : '.'); + q('tier').textContent = head + said; + } + + function sumOver(comp, fn) { + var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; + return Object.keys(blocks).reduce(function (acc, key) { return acc + fn(blocks[key]); }, 0); + } + function moved(block) { return MOVES.reduce(function (a, k) { return a + block.classes[k]; }, 0); } + + function renderSteps() { + var table = clear(q('steps-table')); + var head = ['#', 'Step', 'Kind', 'Explained', 'Claim', 'Moved', 'Up', 'Down', 'As claimed', 'Off claim', 'Not claimed', 'Claimed, not moved', 'Within noise', 'Missing before', 'Missing after', 'Tolerance', 'Noise']; + table.appendChild(el('thead', null, [el('tr', null, head.map(function (h) { return el('th', { scope: 'col', text: h }); }))])); + var body = el('tbody'); + D.steps.forEach(function (s, k) { + var verdict = function (name) { return s.claim ? n(sumOver(s.id, function (b) { return b.verdicts[name]; })) : '—'; }; + var noise = noiseText(s.noise, indicatorIds(), indWord(true)); + // the step from the published results: what moved there is the residual the chain + // starts from, not a regression, whatever its claim says + var unrepro = sumOver(s.id, moved); + var residual = k === 0 && D.states[0].kind === 'published' + ? (unrepro === 1 ? '1 value' : n(unrepro) + ' values') + ' not reproduced within the tolerance: the residual the chain starts from, not a regression.' : null; + body.appendChild(el('tr', null, [ + el('td', { cls: 'num', text: String(stepNo(k)) }), + el('td', null, [safeLink(s.link) ? el('a', { href: safeLink(s.link), text: s.label }) : s.label, s.about ? el('div', { cls: 'sd-quiet', text: s.about }) : null, + residual ? el('div', { cls: 'sd-quiet', text: residual }) : null]), + el('td', { text: s.kind }), + el('td', { text: s.explained ? 'yes' : 'no: nothing explains it', cls: s.explained ? null : 'sd-error' }), + el('td', null, [s.claim ? s.claim.text : 'no claim', s.claim && s.claim.by ? el('div', null, [badge(s.claim.by)]) : null, + s.claim && s.claim.scope.some(function (e) { return e.items === null; }) + ? el('div', { cls: 'sd-quiet', text: 'Its list of ' + D.item.many + ' is not in this file' + (s.claim.reach !== null ? ' (' + n(s.claim.reach) + ')' : '') + ': the verdicts were worked out by its writer and cannot be checked again here.' }) : null]), + el('td', { cls: 'num', text: n(sumOver(s.id, moved)) }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.directions.up; })) }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.directions.down; })) }), + el('td', { cls: 'num', text: verdict('as claimed') }), + el('td', { cls: 'num', text: verdict('off claim') }), + el('td', { cls: 'num', text: verdict('not claimed') }), + el('td', { cls: 'num', text: verdict('claimed, not moved') }), + el('td', { cls: 'num', text: verdict('within noise') }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.missing.before + b.missing.both; })) }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.missing.after + b.missing.both; })) }), + el('td', { cls: 'num', text: String(s.tolerance.value), title: s.tolerance.basis + (s.tolerance.note ? ': ' + s.tolerance.note : '') }), + el('td', { text: noise }) + ])); + }); + table.appendChild(body); + + var total = clear(q('total-table')); + q('total-heading').textContent = 'The whole way: ' + D.states[0].label + ' → ' + D.states[D.states.length - 1].label; + var classes = Object.keys(D.counts.total[D.indicators[0].id].classes); + var cols = [capital(indWord(false))].concat(classes, ['Up', 'Down', 'Largest step', 'Spread over steps', 'Not attributable', 'Moved by what nothing explains']); + total.appendChild(el('thead', null, [el('tr', null, cols.map(function (h) { return el('th', { scope: 'col', text: h, title: SAID[h] || null }); }))])); + var tbody = el('tbody'); + D.indicators.forEach(function (ind) { + var block = D.counts.total[ind.id]; + var att = D.attribution[ind.id]; + var ranked = Object.keys(att.dominant).filter(function (k) { return att.dominant[k]; }) + .sort(function (a, b) { return att.dominant[b] - att.dominant[a]; }) + .map(function (k) { return D.steps[stepIndex(k)].label + ' (' + att.dominant[k] + ')'; }); + var row = [el('th', { scope: 'row', text: ind.name + (ind.unit ? ' (' + ind.unit + ')' : '') })]; + classes.forEach(function (c) { + var title = c === 'missing' ? 'missing before: ' + block.missing.before + ', after: ' + block.missing.after + ', on both sides: ' + block.missing.both : null; + row.push(el('td', { cls: 'num' + (block.classes[c] ? '' : ' zero'), text: n(block.classes[c]), title: title })); + }); + row.push(el('td', { cls: 'num', text: n(block.directions.up) }), el('td', { cls: 'num', text: n(block.directions.down) })); + row.push(el('td', { cls: 'sd-wide', text: ranked.join(', ') || 'none' })); + row.push(el('td', { cls: 'num', text: n(att.spread), title: 'its total moved, and no single step moved it: small steps that add up' })); + row.push(el('td', { cls: 'num', text: n(att.not_attributable), title: 'its total moved, and it has no result in a state between: its difference cannot be split' })); + row.push(el('td', { cls: 'num', text: n(att.unexplained_moved), title: 'moved in a step that no cause explains' })); + tbody.appendChild(el('tr', null, row)); + }); + total.appendChild(tbody); + } + + // the bars of a distribution, left to right: across zero, down from far to near, the + // band around 1 (agrees, within noise), up from near to far, then the values with no + // ratio. [kind, key, tone] + function distBins(block) { + var bands = Object.keys(block.bands); + return [['class', 'sign change', 'aside']] + .concat(bands.filter(function (b) { return b.indexOf('down') === 0; }).map(function (b) { return ['band', b, 'down']; })) + .concat([['class', 'agrees', 'mid'], ['class', 'within noise', 'mid']]) + .concat(bands.filter(function (b) { return b.indexOf('up') === 0; }).map(function (b) { return ['band', b, 'up']; })) + .concat(['zero reference', 'zero result', 'near zero', 'missing'].map(function (c) { return ['class', c, 'aside']; })); + } + // the words under a histogram, each under its run of bars + function axis(bins) { + var runs = []; + bins.forEach(function (b, i) { + var last = runs[runs.length - 1]; + if (last && last.tone === b[2] && !(b[2] === 'aside' && i > 0 && bins[i - 1][2] !== 'aside')) last.n += 1; + else runs.push({ tone: b[2], n: 1, first: !runs.length }); + }); + var words = { down: '↓ ÷10', mid: '1', up: '×10 ↑' }; + return el('div', { cls: 'sd-axis', 'aria-hidden': 'true' }, runs.map(function (run) { + var word = run.tone === 'aside' ? (run.first ? '±' : '0 ∅') : words[run.tone]; + return el('span', { cls: 'sd-axis-' + run.tone, style: 'flex-grow:' + run.n, text: word }); + })); + } + function distLegend() { + var node = clear(q('dist-legend')); + [['down', 'went down'], ['mid', 'agrees or within noise (the band around 1)'], ['up', 'went up'], ['aside', 'across zero, from or to zero, near zero, missing: no ratio']].forEach(function (k) { + node.appendChild(el('span', null, [el('span', { cls: 'sd-bin-key ' + k[0] }), k[1]])); + }); + } + function binWords(bin) { + if (bin[0] === 'band') return bin[1]; + return bin[1] + (SAID[bin[1]] ? ' (' + SAID[bin[1]] + ')' : ''); + } + + // after ÷ before at its 10th, 50th and 90th percentile, over the values of one indicator + // that have a ratio (the same sign on both sides, neither zero nor near it). Only from + // every item: the listed ones are the furthest apart, and would make a median of them + // say more than the whole holds. + function quantiles(indicator, k) { + var held = M || (D.items_count.listed === D.items_count.all && D.tier !== 'counts' ? LISTED : null); + if (!held) return null; + var found = []; + held.forEach(function (row) { + var cls = row.classes[indicator][k]; + if (RATIO_CLASSES.indexOf(cls) < 0) return; + var pair = pairOf(row, indicator, k); + if (pair[0] === null || pair[1] === null || pair[0] === 0 || pair[1] === 0 || (pair[0] > 0) !== (pair[1] > 0)) return; + var r = pair[1] / pair[0]; + if (isFinite(r)) found.push(r); + }); + if (!found.length) return { n: 0 }; + found.sort(function (a, b) { return a - b; }); + var at = function (p) { // linear between the two nearest ranks + var x = p * (found.length - 1), lo = Math.floor(x), hi = Math.ceil(x); + return found[lo] + (found[hi] - found[lo]) * (x - lo); + }; + return { n: found.length, p10: at(0.1), median: at(0.5), p90: at(0.9) }; + } + + // the card of the indicator the matrix is sorted by is marked, wherever the sort was chosen + function markChosenCard() { + var cards = q('dist-cards').childNodes; + for (var i = 0; i < cards.length; i++) { + var card = cards[i]; + var chosen = card.getAttribute('data-indicator') === view.sort; + card.className = card.className.replace(/ sd-chosen/g, '') + (chosen ? ' sd-chosen' : ''); + var button = card.childNodes[0].childNodes[0]; + button.setAttribute('aria-pressed', chosen ? 'true' : 'false'); + } + } + + // one card per indicator, in the order of the matrix's columns: a histogram of the classes + // and bands of the comparison chosen above, each card on the scale of its own tallest bar + function renderDistributions() { + var comp = view.comp; + var k = stepIndex(comp); + var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; + var held = M || (D.items_count.listed === D.items_count.all && D.tier !== 'counts'); + q('dist-said').textContent = 'How many ' + D.item.many + ' fall in each band of after ÷ before, in the comparison chosen above, one card per ' + indWord(false) + + ' in the order of the matrix: from across zero, through down by more than ten times, the band around 1 (dark), to up by more than ten times; then the values with no ratio. ' + + (view.distScale === 'one' ? 'Every card is drawn on one scale, the share of all values, so that a bar of one height means as many in each; ' : + 'Each card is drawn on the scale of its own tallest bar, so that a few values that moved are seen beside many that agree; ') + 'a bar\'s title holds its count. ' + + (held ? 'The median and percentiles are of after ÷ before over every ' + D.item.one + ' with a ratio, in as many digits as keep apart the ones that differ. ' + : 'The median and percentiles need every ' + D.item.one + ', and this page holds the ' + groupThousands(D.items_count.listed) + ' listed, the furthest apart: it gives none. ') + + 'Choose a card to sort the matrix by that ' + indWord(false) + '.'; + var box = clear(q('dist-cards')); + var countsOf = function (block) { + return distBins(block).map(function (b) { return (b[0] === 'band' ? block.bands[b[1]] : block.classes[b[1]]) || 0; }); + }; + // on one scale: the largest share of any bar of any card is the full height + var topShare = 0; + D.indicators.forEach(function (ind) { + var block = blocks[ind.id]; + if (block.pairs) topShare = Math.max(topShare, Math.max.apply(null, countsOf(block)) / block.pairs); + }); + columns(comp).forEach(function (c) { + var ind = c.ind, block = blocks[ind.id]; + var bins = distBins(block); + var counts = countsOf(block); + var top = view.distScale === 'one' ? Math.max(1e-12, topShare) * block.pairs : Math.max.apply(null, counts.concat([1])); + var said = bins.map(function (b, i) { return b[1] + ': ' + groupThousands(counts[i]); }).join('; '); + var bars = bins.map(function (b, i) { + return el('span', { cls: 'sd-bin ' + b[2] + (i && bins[i - 1][2] !== b[2] && (b[2] === 'aside' || bins[i - 1][2] === 'aside') ? ' sd-gap' : ''), + style: 'height:' + (counts[i] ? Math.max(2, 100 * counts[i] / top).toFixed(1) : 0) + '%', + title: binWords(b) + ': ' + groupThousands(counts[i]) + ' of ' + groupThousands(block.pairs) }); + }); + var qs = quantiles(ind.id, k); + var meta = c.active ? capital(activeSaid(c)) + ', of ' + groupThousands(block.pairs) : 'Nothing moved'; + var said3 = qs && qs.n ? quantileTexts(qs) : null; + var left = qs ? block.pairs - qs.n : 0; + var stats = qs === null ? null : (said3 + ? 'Median of ' + groupThousands(qs.n) + ' ' + (qs.n === 1 ? 'ratio' : 'ratios') + ' ' + said3[1] + ', 10th percentile ' + said3[0] + ', 90th ' + said3[2] + + (left ? '; ' + groupThousands(left) + ' without a ratio' : '') + : 'No value with a ratio'); + var button = el('button', { type: 'button', cls: 'sd-dist-pick', 'aria-pressed': view.sort === ind.id ? 'true' : 'false', + title: 'Sort the matrix by ' + ind.name }, [ind.name]); + button.addEventListener('click', function () { + view.sort = view.sort === ind.id ? 'worst' : ind.id; + view.limit = PAGE; + renderGrid(); + markChosenCard(); + var region = q('grid-region'); + if (region && region.scrollIntoView) region.scrollIntoView({ block: 'start', behavior: 'smooth' }); + }); + box.appendChild(el('div', { cls: 'sd-dist' + (c.active ? '' : ' sd-still') + (view.sort === ind.id ? ' sd-chosen' : ''), 'data-indicator': ind.id }, [ + el('h4', null, [button]), + el('p', { cls: 'sd-top', 'aria-hidden': 'true', text: view.distScale === 'one' + ? 'full height: ' + groupThousands(Math.round(top)) + ' of ' + groupThousands(block.pairs) + : 'tallest bar: ' + groupThousands(Math.max.apply(null, counts)) }), + el('div', { cls: 'sd-hist', role: 'img', 'aria-label': said }, bars), + axis(bins), + el('p', { cls: 'sd-dist-meta' }, [el('strong', { text: meta }), stats ? el('br') : null, stats]) + ])); + }); + } + + function sizeSaid() { + return view.size ? ' Only changes by a factor of at least ' + view.size + ' are shown, with every zero, sign change and missing value: a filter of yours, which changes no count.' : ''; + } + + function renderWorst() { + var k = stepIndex(view.comp); + var found = []; + rows().forEach(function (row) { + D.indicators.forEach(function (ind) { + if (view.worstIndicator && ind.id !== view.worstIndicator) return; + var cls = row.classes[ind.id][k]; + if (QUIET.indexOf(cls) >= 0) return; + if (!sizeAtLeast(cls, bandAt(row, ind.id, k), view.size)) return; + var pair = pairOf(row, ind.id, k); + found.push({ row: row, ind: ind.id, key: severityKey(pair[0], pair[1], cls).concat([row.id, ind.id]) }); + }); + }); + found.sort(function (a, b) { return compareKeys(a.key, b.key); }); + var shown = found.slice(0, view.worstN); + q('worst-said').textContent = D.tier === 'counts' + ? 'This file holds counts only: it names no ' + D.item.one + '.' + : 'Showing ' + shown.length + ' of the ' + groupThousands(found.length) + ' values that moved or are missing in ' + compLabel(view.comp) + + ', among ' + heldSaid() + '. Ordered by class (sign change, zero reference, zero result, beyond 10x, differs, missing), then by how far the two sizes lie apart.' + sizeSaid(); + var table = clear(q('worst-table')); + var head = ['#', capital(D.item.one), capital(indWord(false)), 'Class', 'Before', 'After', 'Change', 'Verdict']; + table.appendChild(el('thead', null, [el('tr', null, head.map(function (h) { return el('th', { scope: 'col', text: h }); }))])); + var body = el('tbody'); + shown.forEach(function (one, i) { + var pair = pairOf(one.row, one.ind, k); + body.appendChild(el('tr', { cls: 'sd-row', tabindex: '0', 'data-item': one.row.id }, [ + el('td', { cls: 'num', text: String(i + 1) }), + el('td', { text: one.row.name + (one.row.place ? ' (' + one.row.place + ')' : '') }), + el('td', { text: indicatorName(one.ind) }), + el('td', { text: one.row.classes[one.ind][k] }), + el('td', { cls: 'num', text: formatNumber(pair[0]), title: String(pair[0]) }), + el('td', { cls: 'num', text: formatNumber(pair[1]), title: String(pair[1]) }), + valueCell(one.row, one.ind, k, false), + el('td', { text: verdictOf(one.row, one.ind, k) || '—' }) + ])); + }); + table.appendChild(body); + } + + function gridRows() { + var k = stepIndex(view.comp); + var find = view.find.toLowerCase(); + var ids = indicatorIds(); + var found = rows().filter(function (row) { + if (find && (row.name + ' ' + (row.place || '') + ' ' + row.id).toLowerCase().indexOf(find) < 0) return false; + if (view.group && row.group !== view.group) return false; + if (view.cls && !ids.some(function (id) { return row.classes[id][k] === view.cls; })) return false; + if (view.verdict && !ids.some(function (id) { return verdictOf(row, id, k) === view.verdict; })) return false; + if (view.size && !ids.some(function (id) { return QUIET.indexOf(row.classes[id][k]) < 0 && sizeAtLeast(row.classes[id][k], bandAt(row, id, k), view.size); })) return false; + return true; + }); + var keyed = found.map(function (row, at) { + if (view.sort === 'name') return { row: row, key: [row.name.toLowerCase(), row.place || '', at] }; + if (view.sort === 'worst') return { row: row, key: worstKey(row, k, ids).concat([at]) }; + var pair = pairOf(row, view.sort, k); + return { row: row, key: severityKey(pair[0], pair[1], row.classes[view.sort][k]).concat([at]) }; + }); + keyed.sort(function (a, b) { return compareKeys(a.key, b.key); }); + return keyed.map(function (one) { return one.row; }); + } + + // where a row stands when the grid opens: the largest factor between its two results over + // every indicator, among the values that moved and have a ratio, the furthest from 1 + // first; a row whose moves have no ratio (from or to zero, a sign change) follows, by its + // worst value as severityKey orders values; a row that did not move comes last. Kept on + // the row per comparison. + function worstKey(row, k, ids) { + if (row.worst && row.worst.k === k) return row.worst.key; + var furthest = 0, worst = null; + ids.forEach(function (id) { + var pair = pairOf(row, id, k), cls = row.classes[id][k]; + if (cls === 'differs' || cls === 'beyond 10x') furthest = Math.max(furthest, distance(pair[0], pair[1])); + var one = severityKey(pair[0], pair[1], cls); + if (worst === null || compareKeys(one, worst) < 0) worst = one; + }); + var key = [-furthest].concat(worst); + row.worst = { k: k, key: key }; + return key; + } + + // a heading sorts by itself; a second click goes back to the furthest apart first + function sortButton(label, key, title) { + var button = el('button', { type: 'button', cls: 'sd-sort', text: label, title: title }); + button.addEventListener('click', function () { + view.sort = view.sort === key ? 'worst' : key; + view.limit = PAGE; + renderGrid(); + markChosenCard(); + }); + return button; + } + + // the columns of the matrix: the indicators in which the comparison moved most first, so + // that what a step did is in view without scrolling; the file's order among equals, and + // an indicator in which nothing moved last, quietened + function columns(comp) { + var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; + // a missing value is a finding, not stillness: it counts with what moved + return D.indicators.map(function (ind, at) { + var block = blocks[ind.id]; + return { ind: ind, moved: moved(block), missing: block.classes.missing || 0, at: at }; + }).map(function (c) { c.active = c.moved + c.missing; return c; }) + .sort(function (a, b) { return b.active - a.active || a.at - b.at; }); + } + + function activeSaid(c) { + if (!c.active) return 'nothing moved and nothing is missing'; + return [c.moved ? groupThousands(c.moved) + ' moved' : null, c.missing ? groupThousands(c.missing) + ' missing' : null].filter(Boolean).join(', '); + } + + function renderGrid() { + var k = stepIndex(view.comp); + var cols = columns(view.comp); + var found = gridRows(); + var shown = found.slice(0, view.limit); + q('grid-said').textContent = D.tier === 'counts' + ? 'This file holds counts only: it names no ' + D.item.one + '.' + : 'Showing ' + groupThousands(shown.length) + ' of ' + groupThousands(found.length) + ' ' + many(found.length) + + (M ? '' : ', among ' + heldSaid()) + '.' + sizeSaid(); + q('grid-region').setAttribute('aria-label', 'After divided by before, per ' + D.item.one + ' and ' + indWord(false) + ': ' + compLabel(view.comp) + + (view.sort === 'worst' ? ', the furthest apart first' : '')); + var table = clear(q('grid-table')); + var sorted = function (key) { return view.sort === key ? (key === 'name' ? 'ascending' : 'descending') : null; }; + var head = [ + el('th', { scope: 'col', cls: 'sd-name', 'aria-sort': sorted('name') }, [sortButton(capital(D.item.one), 'name', 'Sort by name')]), + el('th', { scope: 'col', cls: 'sd-group', text: 'Group' }) + ]; + cols.forEach(function (c) { + var ind = c.ind; + head.push(el('th', { scope: 'col', cls: 'num' + (c.active ? '' : ' sd-still'), 'aria-sort': sorted(ind.id), + title: ind.name + (ind.unit ? ' (' + ind.unit + ')' : '') + ': ' + activeSaid(c) + ' here' }, + [sortButton(indicatorName(ind.id), ind.id, 'Sort by how far the two lie apart, the furthest first')])); + }); + table.appendChild(el('thead', null, [el('tr', null, head)])); + var body = el('tbody'); + shown.forEach(function (row) { + var name = row.name + (row.place ? ' (' + row.place + ')' : ''); + var cells = [ + el('th', { scope: 'row', title: name + ', per ' + (row.unit || 'unknown unit') }, [M && row.item ? el('span', { cls: 'sd-listed', title: 'listed in this file' }) : null, name]), + el('td', { cls: 'sd-group', text: row.group || '' }) + ]; + cols.forEach(function (c) { + var td = valueCell(row, c.ind.id, k, true); + if (!c.active) td.className += ' sd-still'; + cells.push(td); + }); + body.appendChild(el('tr', { cls: 'sd-row', tabindex: '0', 'data-item': row.id, 'aria-current': view.open === row.id ? 'true' : null }, cells)); + }); + table.appendChild(body); + var more = q('grid-more'); + more.hidden = found.length <= shown.length; + more.textContent = 'Show ' + groupThousands(Math.min(PAGE, found.length - shown.length)) + ' more'; + } + + function fillSelect(select, options, chosen) { + clear(select); + options.forEach(function (o) { select.appendChild(el('option', { value: o[0], text: o[1] })); }); + select.value = chosen; + } + + function renderFilters() { + var k = stepIndex(view.comp); + var classes = {}, verdicts = {}, groups = {}; + var ids = indicatorIds(); + rows().forEach(function (row) { + ids.forEach(function (id) { + classes[row.classes[id][k]] = 1; + var v = verdictOf(row, id, k); + if (v) verdicts[v] = 1; + }); + if (row.group) groups[row.group] = 1; + }); + var any = [['', 'Any']]; + fillSelect(q('grid-class'), any.concat(Object.keys(classes).sort().map(function (c) { return [c, c]; })), view.cls); + fillSelect(q('grid-verdict'), any.concat(Object.keys(verdicts).sort().map(function (c) { return [c, c]; })), view.verdict); + fillSelect(q('grid-group'), any.concat(Object.keys(groups).sort().map(function (c) { return [c, c]; })), view.group); + q('grid-verdict').disabled = !Object.keys(verdicts).length; + fillSelect(q('worst-indicator'), [['', 'All']].concat(D.indicators.map(function (i) { return [i.id, indicatorName(i.id)]; })), view.worstIndicator); + } + + function badge(by) { + if (!by) return el('span', { cls: 'badge', text: 'author not said' }); + if (by.kind === 'person') return el('span', { cls: 'badge person', text: 'Person: ' + by.name }); + if (by.kind === 'code') return el('span', { cls: 'badge', text: 'Code: ' + by.name }); + if (by.kind === 'model') return el('span', { cls: 'badge model', text: 'Model: ' + by.model + ', run by ' + by.run_by + '. A claim, not checked by a person' }); + return el('span', { cls: 'badge', text: String(by.kind) }); + } + + function aboutText(about) { + var parts = []; + if (about.step) parts.push(about.step === 'total' ? 'the whole way' : 'step ' + stepNo(stepIndex(about.step)) + ': ' + D.steps[stepIndex(about.step)].label); + if (about.indicator) parts.push(indicatorName(about.indicator)); + if (about.item) { + var row = rowById(about.item); + parts.push(row ? row.name : about.item); + } + return parts.length ? 'About ' + parts.join(', ') + '. ' : 'About the whole file. '; + } + + function explanationNode(one, scope) { + return el('li', null, [ + scope ? el('span', { cls: 'sd-scope', text: scope }) : null, + badge(one.by), el('span', { cls: 'sd-quiet', text: aboutText(one.about) }), + one.finding === 'none found' ? el('strong', { text: 'Looked, found no reason: ' }) : null, one.text, + one.not_checked ? el('div', { cls: 'sd-quiet', text: 'Not checked: ' + one.not_checked }) : null, + one.source ? el('div', { cls: 'sd-quiet' }, [safeLink(one.source.link) ? el('a', { href: safeLink(one.source.link), text: one.source.link }) : null, one.source.doc ? ' ' + one.source.doc : null]) : null + ]); + } + + function renderExplanations() { + var list = clear(q('explanation-list')); + if (!D.explanations.length) list.appendChild(el('li', { cls: 'sd-quiet', text: 'The file holds no explanation.' })); + D.explanations.forEach(function (one) { list.appendChild(explanationNode(one)); }); + } + + function renderRule() { + var body = clear(q('rule-body')); + body.appendChild(el('p', null, ['Every value is put into one class by one rule, in this order: missing, near zero, agrees, within noise (both only where the two share a sign), zero reference, zero result, sign change, beyond 10x, differs. ' + + 'Direction is up where the result after is larger as a signed number, down otherwise; colour follows the direction only. ' + + 'Beyond 10x: one size more than ' + D.rule.beyond + ' times the other. The whole way is counted with a tolerance of ' + D.rule.total_tolerance + ', the largest of the steps. ' + + 'The matrix shows the result after divided by the result before; the ratio and the strength of its colour are a display, and change no class.'])); + var stepsTable = el('table', null, [el('thead', null, [el('tr', null, ['Step', 'Tolerance', 'Basis', 'Note', 'Noise'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var tb = el('tbody'); + D.steps.forEach(function (s) { + var noise = noiseText(s.noise, indicatorIds(), indWord(true)) + (s.noise.measured ? ': ' + Object.keys(s.noise.by).map(function (k) { return indicatorName(k) + ' ' + formatNumber(s.noise.by[k]); }).join(', ') : ''); + tb.appendChild(el('tr', null, [el('td', { text: s.label }), el('td', { cls: 'num', text: String(s.tolerance.value) }), el('td', { text: s.tolerance.basis }), el('td', { text: s.tolerance.note || '' }), el('td', { text: noise })])); + }); + stepsTable.appendChild(tb); + body.appendChild(el('div', { cls: 'sd-wrap' }, [stepsTable])); + body.appendChild(el('h4', { text: 'Floors' })); + body.appendChild(el('p', { cls: 'sd-quiet', text: 'One floor per ' + indWord(false) + ' and reference unit: ' + D.rule.floor_factor + ' times the median size of the results of ' + D.states[stateIndex(D.rule.floors_from)].label + ' in that unit, unless declared. Units are never pooled.' })); + var floors = el('table', null, [el('thead', null, [el('tr', null, [capital(indWord(false)), 'Unit', 'Floor', 'Basis', 'From results'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var fb = el('tbody'); + D.rule.floors.forEach(function (f) { + fb.appendChild(el('tr', null, [el('td', { text: indicatorName(f.indicator) }), el('td', { text: f.unit || 'unknown' }), el('td', { cls: 'num', text: formatNumber(f.value), title: String(f.value) }), el('td', { text: f.basis }), el('td', { cls: 'num', text: n(f.n) })])); + }); + floors.appendChild(fb); + body.appendChild(el('div', { cls: 'sd-wrap' }, [floors])); + var notes = clear(q('notes')); + D.notes.forEach(function (text) { notes.appendChild(el('li', { text: text })); }); + if (!D.notes.length) notes.appendChild(el('li', { cls: 'sd-quiet', text: 'None.' })); + } + + // ---------------------------------------------------------------- the item drawer + + // whether a row moved in a comparison and indicator an explanation is about; an + // explanation that names neither is about every one + function movedUnder(row, about) { + var ids = about.indicator ? [about.indicator] : indicatorIds(); + var ks = about.step ? [stepIndex(about.step)] : D.steps.map(function (s, k) { return k; }).concat([D.steps.length]); + return ids.some(function (id) { + return row.classes[id] && ks.some(function (k) { return MOVES.indexOf(row.classes[id][k]) >= 0; }); + }); + } + + function whyNode(row) { + var list = el('ul', { cls: 'sd-why' }); + var own = D.explanations.filter(function (e) { return e.about.item === row.id; }); + var wider = D.explanations.filter(function (e) { return !e.about.item && movedUnder(row, e.about); }); + own.forEach(function (e) { list.appendChild(explanationNode(e, 'This ' + D.item.one)); }); + wider.forEach(function (e) { list.appendChild(explanationNode(e, e.about.step && e.about.indicator ? 'Its step and ' + indWord(false) : e.about.step ? 'A step it moved in' : e.about.indicator ? capital(article(indWord(false))) + ' it moved in' : 'The whole file')); }); + if (!own.length && !wider.length) { + var any = movedUnder(row, {}); + list.appendChild(el('li', { cls: 'sd-none', text: any + ? 'None given: no explanation in this file is about this ' + D.item.one + ', or about a step or ' + indWord(false) + ' in which it moved.' + : 'It moved in no comparison, and no explanation in this file is about it.' })); + } + return list; + } + + function itemTable(row, ind) { + var values = row.values[ind.id]; + var first = values[0], last = values[values.length - 1]; + var totalCls = row.classes[ind.id][D.steps.length]; + var table = el('table', { 'data-indicator': ind.id }, [el('thead', null, [el('tr', null, ['Step', 'Before', 'After', 'Change', 'Verdict', 'Share of the whole difference'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var tb = el('tbody'); + var split = splitOf(values, D.steps.map(function (s) { return s.explained; }), totalCls); + D.steps.forEach(function (s, k) { + var b = values[k], a = values[k + 1]; + var share = split.state === 'split' ? shareText(split.shares[k]) : '—'; + tb.appendChild(el('tr', null, [ + el('td', null, ['Step ' + stepNo(k) + ': ' + s.label, s.explained ? null : el('div', { cls: 'sd-error', text: 'nothing explains it' })]), + el('td', { cls: 'num', text: formatNumber(b), title: String(b) }), + el('td', { cls: 'num', text: formatNumber(a), title: String(a) }), + valueCell(row, ind.id, k, false), + el('td', { text: verdictOf(row, ind.id, k) || '—' }), + el('td', { cls: 'num', text: share }) + ])); + }); + tb.appendChild(el('tr', null, [ + el('th', { scope: 'row', text: 'The whole way' }), + el('td', { cls: 'num', text: formatNumber(first), title: String(first) }), + el('td', { cls: 'num', text: formatNumber(last), title: String(last) }), + valueCell(row, ind.id, D.steps.length, false), + el('td', { text: '—' }), + el('td', { cls: 'num', text: splitText(split) }) + ])); + table.appendChild(tb); + return el('div', { cls: 'sd-wrap' }, [table]); + } + + function renderItem(id) { + var row = rowById(id); + var drawer = q('item'); + if (!row) { closeItem(); return; } + if (drawer.hidden) lastFocus = doc.activeElement || null; + view.open = row.id; + drawer.hidden = false; + q('item-title').textContent = row.name; + q('item-meta').textContent = [row.id, row.place, row.group, row.unit ? 'per ' + row.unit : 'unit unknown', + row.item ? null : 'from the matrix: not listed in this file'].filter(Boolean).join(' · '); + var body = clear(q('item-body')); + body.appendChild(el('h4', { text: 'Why it differs' })); + body.appendChild(whyNode(row)); + body.appendChild(el('p', { cls: 'sd-quiet', text: 'The explanations about this ' + D.item.one + ' first, then those about a step or ' + indWord(false) + ' in which it moved: those may not be about it. An explanation is a claim; it changes no class.' })); + // the indicators, the one whose whole way lies furthest apart first + var last = D.steps.length; + var order = D.indicators.slice().sort(function (a, b) { + var pa = pairOf(row, a.id, last), pb = pairOf(row, b.id, last); + return compareKeys(severityKey(pa[0], pa[1], row.classes[a.id][last]), severityKey(pb[0], pb[1], row.classes[b.id][last])); + }); + order.forEach(function (ind) { + body.appendChild(el('h4', { text: ind.name + (ind.unit ? ' (' + ind.unit + ' per ' + (row.unit || 'unit') + ')' : '') })); + body.appendChild(itemTable(row, ind)); + }); + var named = row.item ? row.item.breakdowns || {} : {}; + Object.keys(named).forEach(function (comp) { + var name = named[comp]; + var holder = el('div'); + body.appendChild(el('h4', { text: 'What it is made of: ' + compLabel(comp) })); + body.appendChild(holder); + var found = B[name]; + if (found) { + if (breakdownFits(found, row.id, comp)) breakdownInto(holder, found); + else holder.appendChild(el('p', { cls: 'sd-error', text: 'The breakdown ' + name + ' is of another ' + D.item.one + ' or another step: not drawn.' })); + return; + } + holder.appendChild(el('p', { cls: 'sd-quiet', text: 'The breakdown ' + name + ' is not with this page.' })); + if (safeName(name) && /^https?:$/.test(win.location.protocol)) { + win.fetch(base + name).then(function (r) { if (!r.ok) throw new Error(); return r.json(); }) + .then(function (data) { if (!breakdownFits(data, row.id, comp)) throw new Error(); clear(holder); breakdownInto(holder, data); }) + .catch(function () { /* the sentence above stays */ }); + } + }); + markOpen(); + var title = q('item-title'); + if (title.focus) title.focus(); + } + + function markOpen() { + var table = q('grid-table'); + var trs = table.querySelectorAll ? table.querySelectorAll('tr[data-item]') : []; + for (var i = 0; i < trs.length; i++) { + if (trs[i].getAttribute('data-item') === view.open) trs[i].setAttribute('aria-current', 'true'); + else if (trs[i].removeAttribute) trs[i].removeAttribute('aria-current'); + } + } + + function closeItem() { + var drawer = q('item'); + if (!drawer || drawer.hidden) return; + drawer.hidden = true; + view.open = null; + markOpen(); + if (hashParts().item) setHash({ step: view.comp }); + if (lastFocus && lastFocus.focus && root.contains && root.contains(lastFocus)) lastFocus.focus(); + lastFocus = null; + } + + // a breakdown is drawn only under the item and the step that it breaks down + function breakdownFits(data, itemId, comp) { + if (!data || data.format !== 'sentier-diagnostics-breakdown/1' || data.item !== itemId) return false; + var k = stepIndex(comp); + var from = comp === 'total' ? D.states[0].id : D.steps[k].from; + var to = comp === 'total' ? D.states[D.states.length - 1].id : D.steps[k].to; + return data.from === from && data.to === to && !!data.indicators && typeof data.indicators === 'object'; + } + + function breakdownInto(holder, data) { + Object.keys(data.indicators).forEach(function (key) { + var entry = data.indicators[key]; + holder.appendChild(el('h4', { text: indicatorName(key) + (entry.unit ? ' (' + entry.unit + ')' : '') })); + var table = el('table', null, [el('thead', null, [el('tr', null, ['Part', 'Kind', 'Before', 'After', 'Change'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var tb = el('tbody'); + var line = function (label, kind, pair, strong) { + var d = pair[1] - pair[0]; + return el('tr', null, [el(strong ? 'th' : 'td', { text: label }), el('td', { text: kind || '' }), + el('td', { cls: 'num', text: formatNumber(pair[0]), title: String(pair[0]) }), + el('td', { cls: 'num', text: formatNumber(pair[1]), title: String(pair[1]) }), + el('td', { cls: 'num cell ' + (d > 0 ? 'tone-up' : d < 0 ? 'tone-down' : 'tone-none'), style: '--s:0.4', text: d === 0 ? '=' : (d > 0 ? '+' : '−') + formatNumber(Math.abs(d)) })]); + }; + entry.parts.forEach(function (p) { tb.appendChild(line(p.label, p.kind, p.values, false)); }); + tb.appendChild(line(entry.parts_not_listed ? 'The other ' + entry.parts_not_listed + ' parts' : 'No other part', '', entry.rest, false)); + tb.appendChild(line('Total', '', entry.total, true)); + table.appendChild(tb); + holder.appendChild(el('div', { cls: 'sd-wrap' }, [table])); + var cover = function (side) { + var listed = entry.parts.reduce(function (a, p) { return a + p.values[side]; }, 0); + return entry.total[side] ? shareText(listed / entry.total[side]) : 'none (the total is 0)'; + }; + holder.appendChild(el('p', { cls: 'sd-quiet', text: (entry.parts_not_listed + ? 'The first ' + entry.parts.length + ' parts make ' + cover(0) + ' of the total before and ' + cover(1) + ' after; the other ' + entry.parts_not_listed + ' are summed in one line. ' + : 'Every part is listed. ') + + 'Parts are listed by the larger of their two sizes, so that a part large on one side only is listed. The listed parts and the others add up to the total.' })); + }); + } + + // ---------------------------------------------------------------- the matrix + + function matrixSaid() { + var node = q('matrix-said'); + if (!node) return; + var c = D.items_count; + var text = { + none: D.tier === 'counts' ? '' : 'This file lists ' + groupThousands(c.listed) + ' of its ' + groupThousands(c.all) + ' ' + D.item.many + ' and names no matrix of every ' + D.item.one + ': the grid shows the listed ones.', + loading: 'Loading the matrix of every ' + D.item.one + ' (' + matrix.said + '); the grid shows the listed ones until it is here.', + loaded: 'The matrix of all ' + groupThousands(M ? M.length : 0) + ' ' + D.item.many + ' is loaded; a dot marks the ' + groupThousands(c.listed) + ' listed in this file.', + refused: 'The matrix was not loaded: ' + matrix.said + '. The grid shows the ' + groupThousands(c.listed) + ' ' + D.item.many + ' listed in this file.' + }[matrix.state]; + node.textContent = text; + } + + function takeMatrix(m) { + if (!D) return; + var problem = checkMatrix(m, D); + if (problem) { matrix.state = 'refused'; matrix.said = problem; matrixSaid(); return; } + M = matrixRows(m, D); + matrix.state = 'loaded'; + matrix.said = ''; + view.limit = PAGE; + tierSaid(); + renderComparison(); + matrixSaid(); + if (view.open) renderItem(view.open); + else route(); + } + + function refuseMatrix(said) { matrix.state = 'refused'; matrix.said = said; matrixSaid(); } + + // the matrix comes after the page is drawn: from a global, or from the file the root or + // the diagnostics file names, as a script beside the page (which opens from disk too) or + // as JSON over http + function loadMatrix() { + if (given.matrix) { takeMatrix(given.matrix); return; } + if (win.SENTIER_DIAGNOSTICS_MATRIX) { takeMatrix(win.SENTIER_DIAGNOSTICS_MATRIX); return; } + var name = root.getAttribute('data-sd-matrix') || (D.matrix ? D.matrix.file : null); + if (!name || D.tier === 'counts') { matrixSaid(); return; } + if (!safeName(name)) { refuseMatrix('its name is not a plain file name beside the page'); return; } + matrix.state = 'loading'; + matrix.said = name; + matrixSaid(); + if (/\.js$/.test(name)) { + var script = doc.createElement('script'); + script.setAttribute('src', base + name); + script.onload = function () { + if (win.SENTIER_DIAGNOSTICS_MATRIX) takeMatrix(win.SENTIER_DIAGNOSTICS_MATRIX); + else refuseMatrix(name + ' holds no matrix'); + }; + script.onerror = function () { refuseMatrix(name + ' is not beside the page'); }; + (doc.head || doc.body || root).appendChild(script); + return; + } + if (!/^https?:$/.test(win.location.protocol)) { + refuseMatrix(name + ' is read over http only; `sentier-diagnostics page` writes it beside the page as a script, which opens from disk'); + return; + } + win.fetch(base + name).then(function (r) { + if (!r.ok) throw new Error(name + ' could not be read (' + r.status + ')'); + return r.text(); + }).then(function (text) { + if (text.length > MATRIX_MAX_BYTES) throw new Error(name + ' is larger than ' + groupThousands(MATRIX_MAX_BYTES) + ' bytes'); + var m; + try { m = JSON.parse(text); } catch (error) { throw new Error(name + ' is not JSON'); } + takeMatrix(m); + }).catch(function (error) { refuseMatrix(error.message || (name + ' could not be read')); }); + } + + // ---------------------------------------------------------------- drawing a file + + function renderComparison() { + renderFilters(); + renderDistributions(); + renderWorst(); + renderGrid(); + } + + // a file the page cannot draw draws nothing: not even the part it drew before it failed + function refuse(problem) { + D = null; M = null; LISTED = []; + if (standalone) doc.title = 'Diagnostics'; + ['title', 'about', 'tier', 'audience', 'unchecked', 'matrix-said'].forEach(function (name) { + var node = q(name); + if (node) node.textContent = ''; + }); + ['audience', 'unchecked', 'item'].forEach(function (name) { if (q(name)) q(name).hidden = true; }); + if (q('states')) clear(q('states')); + showLoad(); + q('content').hidden = true; + q('load-error').textContent = problem; + } + + // `checked`: the file came with the page, which `sentier-diagnostics page` writes only + // after the validator passed it. A file chosen here or named in the address was not + // checked: the page checks its shape only, and says so above everything it draws. + function show(data, checked) { + var problem = checkShape(data); + if (problem) { refuse(problem); return; } + D = data; + M = null; + LISTED = D.items.map(listedRow); + matrix = { state: 'none', said: '' }; + view.comp = 'total'; + view.limit = PAGE; + q('load').hidden = true; + q('content').hidden = false; + var unchecked = q('unchecked'); + if (unchecked) { + unchecked.hidden = !!checked; + unchecked.textContent = checked ? '' : 'Opened in the browser: this page checked the shape of the file, not its counts. ' + + 'Run sentier-diagnostics check on it before relying on what it shows.'; + } + try { + renderHeader(); + renderSteps(); + renderExplanations(); + renderRule(); + legend(q('grid-legend'), true); + distLegend(); + renderComparison(); + matrixSaid(); + route(); + } catch (error) { + refuse('The file is JSON of the right format, but a part of it is missing or not what the format says (' + error.message + '). Run sentier-diagnostics check on it.'); + return; + } + if (checked) loadMatrix(); + } + + // the address holds the chosen step and the open item, `#step=&item=`, each + // optional, so that a reviewer can send someone to exactly what they look at + function hashParts() { + var out = {}; + (win.location.hash || '').slice(1).split('&').forEach(function (part) { + var at = part.indexOf('='); + if (at > 0) out[part.slice(0, at)] = decodeURIComponent(part.slice(at + 1)); + }); + return out; + } + function setHash(parts) { + var said = []; + if (parts.step && parts.step !== 'total') said.push('step=' + encodeURIComponent(parts.step)); + if (parts.item) said.push('item=' + encodeURIComponent(parts.item)); + var wanted = said.join('&'); + if ((win.location.hash || '').replace(/^#/, '') === wanted) return false; + if (wanted) win.location.hash = wanted; + else if (win.history && win.history.replaceState) win.history.replaceState(null, '', win.location.pathname + win.location.search); + return true; + } + + function choose(comp) { + if (comp === view.comp) return; + view.comp = comp; view.cls = ''; view.verdict = ''; view.limit = PAGE; view.sort = 'worst'; + markChips(); + renderStepSaid(); + renderComparison(); + } + + function route() { + if (!D) return; + var parts = hashParts(); + var step = parts.step && stepIndex(parts.step) < D.steps.length ? parts.step : 'total'; + choose(step); + if (parts.item) renderItem(parts.item); + else closeItem(); + } + + function openFrom(event) { + var row = event.target.closest ? event.target.closest('tr[data-item]') : null; + if (!row) return; + var id = row.getAttribute('data-item'); + if (!setHash({ step: view.comp, item: id })) renderItem(id); + } + + function readScript(name) { + var node = q(name); + if (!node || !node.textContent || !/\S/.test(node.textContent)) return undefined; + return JSON.parse(node.textContent); + } + + function start() { + q('comparison').addEventListener('click', function (e) { + var chip = e.target.closest ? e.target.closest('button[data-comp]') : null; + if (!chip || chip.getAttribute('data-comp') === view.comp) return; + choose(chip.getAttribute('data-comp')); + setHash({ step: view.comp, item: view.open }); + }); + // the arrow keys move between the chips + q('comparison').addEventListener('keydown', function (e) { + if (!D || (e.key !== 'ArrowRight' && e.key !== 'ArrowLeft')) return; + var comps = ['total'].concat(D.steps.map(function (s) { return s.id; })); + var at = comps.indexOf(view.comp) + (e.key === 'ArrowRight' ? 1 : -1); + if (at < 0 || at >= comps.length) return; + choose(comps[at]); + setHash({ step: view.comp, item: view.open }); + var next = q('comparison').querySelector('button[data-comp="' + comps[at] + '"]'); + if (next && next.focus) next.focus(); + }); + q('size').addEventListener('change', function (e) { view.size = +e.target.value; view.limit = PAGE; renderWorst(); renderGrid(); }); + q('worst-n').addEventListener('change', function (e) { view.worstN = +e.target.value; renderWorst(); }); + q('dist-scale').addEventListener('change', function (e) { view.distScale = e.target.value === 'one' ? 'one' : 'card'; renderDistributions(); }); + q('worst-indicator').addEventListener('change', function (e) { view.worstIndicator = e.target.value; renderWorst(); }); + q('grid-find').addEventListener('input', function (e) { view.find = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-class').addEventListener('change', function (e) { view.cls = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-verdict').addEventListener('change', function (e) { view.verdict = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-group').addEventListener('change', function (e) { view.group = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-more').addEventListener('click', function () { view.limit += PAGE; renderGrid(); }); + q('item-close').addEventListener('click', closeItem); + root.addEventListener('keydown', function (e) { if (e.key === 'Escape') closeItem(); }); + ['grid-table', 'worst-table'].forEach(function (name) { + q(name).addEventListener('click', openFrom); + q(name).addEventListener('keydown', function (e) { if (e.key === 'Enter') openFrom(e); }); + }); + win.addEventListener('hashchange', route); + q('file').addEventListener('change', function (e) { + var file = e.target.files[0]; + if (!file) return; + var reader = new win.FileReader(); + reader.onload = function () { + var data; + try { data = JSON.parse(reader.result); } catch (error) { refuse('The file is not JSON: ' + error.message); return; } + show(data, false); + }; + reader.readAsText(file); + }); + try { + var breakdowns = given.breakdowns || readScript('breakdowns'); + B = breakdowns || win.SENTIER_DIAGNOSTICS_BREAKDOWNS || {}; + var data = given.data || readScript('data'); + if (data === undefined) data = win.SENTIER_DIAGNOSTICS; + } catch (error) { + refuse('The data of this page is not JSON: ' + error.message); + return; + } + if (data) { show(data, true); return; } + var wanted = new win.URLSearchParams(win.location.search).get('file'); + if (wanted && safeName(wanted) && /\.json$/.test(wanted) && /^https?:$/.test(win.location.protocol)) { + win.fetch(base + wanted).then(function (r) { return r.json(); }).then(function (got) { show(got, false); }, function () { + refuse('Could not read ' + wanted + ' as JSON.'); + }); + return; + } + showLoad(); + } + + // the place for a file: a page of its own offers to open one; a host's page never does, + // and shows only why nothing is drawn + function showLoad() { + q('load').hidden = false; + q('load-choose').hidden = !standalone; + if (!standalone && !q('load-error').textContent) q('load-error').textContent = 'No diagnostics file came with this page.'; + } + + start(); + return { show: show, close: closeItem, open: renderItem }; + } + + // every root of the document not yet mounted, but those a host mounts itself + function mountAll(doc) { + var roots = (doc || global.document).querySelectorAll('[data-sentier-diagnostics]'); + for (var i = 0; i < roots.length; i++) { + if (roots[i].getAttribute('data-sentier-diagnostics') !== 'manual') mount(roots[i]); + } + } + + var api = { formatNumber: formatNumber, percentText: percentText, shareText: shareText, ratioText: ratioText, ratioShort: ratioShort, noiseText: noiseText, + sizeAtLeast: sizeAtLeast, cellView: cellView, ratioView: ratioView, severityKey: severityKey, compareKeys: compareKeys, + bandStep: bandStep, bandOf: bandOf, checkShape: checkShape, checkMatrix: checkMatrix, matrixRows: matrixRows, + safeLink: safeLink, safeName: safeName, splitOf: splitOf, splitText: splitText, distance: distance, quantileTexts: quantileTexts, RANK: RANK, + mount: mount, mountAll: mountAll }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (typeof global.document !== 'undefined') { + global.SentierDiagnostics = api; + if (global.document.readyState === 'loading') global.document.addEventListener('DOMContentLoaded', function () { mountAll(); }); + else mountAll(); + } +})(typeof window !== 'undefined' ? window : this); diff --git a/src/sentier_diagnostics/page/index.html b/src/sentier_diagnostics/page/index.html new file mode 100644 index 0000000..50960be --- /dev/null +++ b/src/sentier_diagnostics/page/index.html @@ -0,0 +1,16 @@ + + + + + +<!--title--> + + + + + +
+ +
+ + diff --git a/src/sentier_diagnostics/page/standalone.css b/src/sentier_diagnostics/page/standalone.css new file mode 100644 index 0000000..75e0671 --- /dev/null +++ b/src/sentier_diagnostics/page/standalone.css @@ -0,0 +1,37 @@ +/* The frame of a page that stands on its own (`sentier-diagnostics page`). A site that holds + the diagnostics in its own page loads diagnostics.css only: this sheet sets the tokens a + site would set, the font and the ground, which diagnostics.css leaves to its host. */ +:root { + --ink: #1d232a; + --muted: #5b6672; + --surface: #ffffff; + --paper-deep: #f3f5f7; + --line: #d9dee3; + --line-strong: #b6bfc8; + --index: #1d625e; + --fail: #a3262a; + --radius: 6px; + --gutter: 16px; + color-scheme: light dark; +} +@media (prefers-color-scheme: dark) { + :root { + --ink: #e6eaee; + --muted: #9aa5b0; + --surface: #15191d; + --paper-deep: #1f252b; + --line: #36404a; + --line-strong: #56616c; + --index: #739e9b; + --fail: #ff8a8e; + } +} +body { + margin: 0; + font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + line-height: 1.45; + color: var(--ink); + background: var(--surface); +} +.sd-standalone { max-width: 1400px; margin: 0 auto; padding: 0 var(--gutter) 2rem; } +.sd-standalone h1 { font-size: 1.6rem; margin: 1rem 0 0.2rem; } diff --git a/src/sentier_diagnostics/rule.py b/src/sentier_diagnostics/rule.py new file mode 100644 index 0000000..0674c2d --- /dev/null +++ b/src/sentier_diagnostics/rule.py @@ -0,0 +1,324 @@ +"""The counting rule: what one pair of results is, which way it went, and how far. + +Every comparison this package writes counts with these functions, so that two +projects mean the same by the same word. A pair is one item in one indicator, before +and after. It gets exactly one class: + + missing a side has no result; it is never read as zero + near zero both sides lie within the floor of the indicator and the item's + reference unit: their ratio says nothing + agrees the two share a sign and differ by no more than the tolerance, + relative to the larger + within noise they share a sign and differ by more, but by no more than two builds + of one state do + zero reference the side before lies within the floor, the side after does not + zero result the side after lies within the floor, the side before does not + sign change both sides lie beyond the floor and have different signs + beyond 10x one side is more than ten times the size of the other + differs any other difference + +The classes are tried in this order; the first that holds is the class. So a pair on +both sides of the floor that agrees is "agrees": crossing the floor by solver noise is not +a change. No class blanks a value, sets it to zero or calls it agreement: the two results +are always kept. + +**Direction** has one rule: a pair that moved (`MOVES`) went up where the result after +is larger than the result before, as signed numbers, and down otherwise. Two results +below zero that go further below zero went down, although the size of the second is +the larger. Colour on a page follows the direction and nothing else. + +**Bands** split a pair that differs, or is beyond 10x, by its direction and by the +factor between the sizes of its two results. + +Standard library only. Parts of this module are adapted from `before_after.py` of +sentier-bafu (MIT, Copyright (c) 2026 sentier.dev); see NOTICE. +""" + +from __future__ import annotations + +import math +import sys +from collections.abc import Iterable, Mapping + +MISSING = "missing" +NEAR_ZERO = "near zero" +ZERO_REFERENCE = "zero reference" +ZERO_RESULT = "zero result" +SIGN_CHANGE = "sign change" +AGREES = "agrees" +WITHIN_NOISE = "within noise" +BEYOND = "beyond 10x" +DIFFERS = "differs" + +#: The classes, in the order a page lists them. +CLASSES = ( + AGREES, + WITHIN_NOISE, + DIFFERS, + BEYOND, + ZERO_REFERENCE, + ZERO_RESULT, + SIGN_CHANGE, + NEAR_ZERO, + MISSING, +) +#: The classes of a pair that moved: they have a direction. +MOVES = frozenset({DIFFERS, BEYOND, ZERO_REFERENCE, ZERO_RESULT, SIGN_CHANGE}) +#: The classes that alone do not make an item worth listing. +QUIET = frozenset({AGREES, WITHIN_NOISE, NEAR_ZERO}) + +UP, DOWN = "up", "down" +DIRECTIONS = (UP, DOWN) + +#: One side more than this many times the size of the other is beyond 10x. +BEYOND_FACTOR = 10.0 +#: A result below this share of the median size of its group is near zero (`floor_of`). +FLOOR_FACTOR = 0.01 +#: Two builds of one database on two machines differ by about 1e-7 to 1e-6 (measured on +#: sentier-bafu: 8e-12 on one runner, 2.8e-7 across runners). The default tolerance lies +#: one order above that, so that solver noise is never counted as a change. +DEFAULT_TOLERANCE = 1e-5 + +#: The factors between the sizes of two results at which a band ends, and the band's name. +SIZE_STEPS = ( + (math.log10(1.1), "1-1.1"), + (math.log10(1.25), "1.1-1.25"), + (math.log10(2.0), "1.25-2"), + (1.0, "2-10"), +) +OVER = "over 10" +_STEPS = tuple(name for _, name in SIZE_STEPS) + (OVER,) +#: The bands, from far down to far up. +BANDS = tuple(f"down, factor {s}" for s in reversed(_STEPS)) + tuple( + f"up, factor {s}" for s in _STEPS +) + +#: How a class ranks when items are ordered from the worst: a lower rank comes first. +RANK = { + SIGN_CHANGE: 0, + ZERO_REFERENCE: 1, + ZERO_RESULT: 2, + BEYOND: 3, + DIFFERS: 4, + MISSING: 5, + WITHIN_NOISE: 6, + AGREES: 7, + NEAR_ZERO: 8, +} + + +def is_value(value) -> bool: + """A result: a finite number that is not a boolean. + + >>> is_value(1.5), is_value(0), is_value(True), is_value(float("nan")), is_value(None) + (True, True, False, False, False) + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + return False + + +def relative_difference(before: float, after: float) -> float: + """How far two results lie apart, relative to the larger of the two; 0 where both are 0. + + >>> relative_difference(2.0, 1.0), relative_difference(0.0, 0.0), relative_difference(1.0, -1.0) + (0.5, 0.0, 2.0) + """ + size = max(abs(before), abs(after)) + return abs(after - before) / size if size else 0.0 + + +def change(before: float, after: float) -> float | None: + """The change relative to the size of the result before, signed as the direction is: + -0.5 is "50 % lower". None where the result before is zero. + + >>> change(2.0, 1.0), change(-2.0, -4.0), change(4.0, 6.0), change(0.0, 1.0) + (-0.5, -1.0, 0.5, None) + """ + return (after - before) / abs(before) if before else None + + +def size_ratio(before: float, after: float) -> float | None: + """The size of the result after divided by the size of the result before, where both are + of one sign and not zero; else None. + + >>> size_ratio(2.0, 3.0), size_ratio(-2.0, -4.0), size_ratio(2.0, -1.0), size_ratio(0.0, 1.0) + (1.5, 2.0, None, None) + """ + if not before or not after or (before > 0) != (after > 0): + return None + found = abs(after) / abs(before) + return found if math.isfinite(found) and found > 0 else None + + +def distance(before: float, after: float) -> float | None: + """How far the sizes of two results lie apart: the |log10| of their factor; None where + they have no factor (a zero, or two signs). Where the factor is too large or too small + for a float, it is |log10|after| - log10|before||, which cannot overflow; elsewhere the + factor itself, so that a factor of exactly 2 lies exactly on the edge of its band. + + >>> distance(1.0, 10.0), distance(-1.0, -0.1), distance(1.0, -1.0), distance(1e-300, 1e300) + (1.0, 1.0, None, 600.0) + """ + if not before or not after or (before > 0) != (after > 0): + return None + factor = abs(after) / abs(before) + if math.isfinite(factor) and factor >= sys.float_info.min: + return abs(math.log10(factor)) + return abs(math.log10(abs(after)) - math.log10(abs(before))) + + +def classify( + before: float | None, + after: float | None, + *, + floor: float = 0.0, + tolerance: float = DEFAULT_TOLERANCE, + noise: float | None = None, +) -> str: + """The class of one pair of results (see the module's text). + + >>> classify(1.0, None), classify(1e-9, -2e-9, floor=1e-6), classify(0.0, 2.0) + ('missing', 'near zero', 'zero reference') + >>> classify(2.0, 0.0), classify(-1.0, 1.0), classify(1.0, 1.000001) + ('zero result', 'sign change', 'agrees') + >>> classify(1.0, 1.02, noise=0.05), classify(1.0, 20.0), classify(-2.0, -4.0) + ('within noise', 'beyond 10x', 'differs') + >>> classify(0.010000001, 0.009999999, floor=0.01), classify(0.02, 0.005, floor=0.01) + ('agrees', 'zero result') + """ + if before is None or after is None: + return MISSING + small_before, small_after = abs(before) <= floor, abs(after) <= floor + if small_before and small_after: + return NEAR_ZERO + same_sign = (before > 0 and after > 0) or (before < 0 and after < 0) + gap = relative_difference(before, after) + if same_sign and gap <= tolerance: + return AGREES + if same_sign and noise and gap <= noise: + return WITHIN_NOISE + if small_before: + return ZERO_REFERENCE + if small_after: + return ZERO_RESULT + if not same_sign: + return SIGN_CHANGE + larger, smaller = max(abs(before), abs(after)), min(abs(before), abs(after)) + if larger > BEYOND_FACTOR * smaller: + return BEYOND + return DIFFERS + + +def direction(before: float | None, after: float | None, kind: str) -> str | None: + """Which way a pair that moved went, as signed numbers; None for a pair that did not. + + >>> direction(-2.0, -4.0, DIFFERS), direction(1.0, 3.0, DIFFERS), direction(1.0, 1.0, AGREES) + ('down', 'up', None) + """ + if kind not in MOVES: + return None + return UP if after > before else DOWN + + +def band(before: float | None, after: float | None, kind: str) -> str | None: + """The band of a pair that differs or is beyond 10x; None for any other class. + + >>> band(1.0, 1.5, DIFFERS), band(-2.0, -4.0, DIFFERS), band(10.0, 0.5, BEYOND) + ('up, factor 1.25-2', 'down, factor 1.25-2', 'down, factor over 10') + """ + if kind not in (DIFFERS, BEYOND): + return None + way = direction(before, after, kind) + if kind == BEYOND: + return f"{way}, factor {OVER}" + found = distance(before, after) + for edge, name in SIZE_STEPS: + if found <= edge: + return f"{way}, factor {name}" + return f"{way}, factor {SIZE_STEPS[-1][1]}" # exactly ten times: not beyond + + +def judge( + before: float | None, + after: float | None, + *, + floor: float = 0.0, + tolerance: float = DEFAULT_TOLERANCE, + noise: float | None = None, +) -> tuple[str, str | None, str | None]: + """Class, direction and band of one pair, from the one rule.""" + kind = classify(before, after, floor=floor, tolerance=tolerance, noise=noise) + return kind, direction(before, after, kind), band(before, after, kind) + + +def severity(before: float | None, after: float | None, kind: str) -> tuple[int, float]: + """The key that orders pairs from the worst: the rank of the class, then how far apart. + + >>> sorted([(1.0, 1.5), (1.0, -1.0), (1.0, 3.0)], key=lambda p: severity(*p, classify(*p))) + [(1.0, -1.0), (1.0, 3.0), (1.0, 1.5)] + """ + if before is None or after is None: + return RANK[kind], 0.0 + found = distance(before, after) + return RANK[kind], -(found if found is not None else relative_difference(before, after)) + + +def largest_step(series: list[float], kinds: list[str]) -> int | None: + """The step that carries an item's whole difference along a chain: of the steps whose + pair moved, the one with the largest difference, the earlier one where two are equal. + None where no step moved: the whole way moved by small steps that add up ("spread"). + + >>> largest_step([1.0, 2.0, 3.0], [DIFFERS, DIFFERS]), largest_step([1.0, 1.04, 1.08], [WITHIN_NOISE] * 2) + (0, None) + >>> largest_step([1.0, 1.045, 1.075], [WITHIN_NOISE, DIFFERS]) + 1 + """ + moved = [k for k, kind in enumerate(kinds) if kind in MOVES] + if not moved: + return None + return max(moved, key=lambda k: (abs(series[k + 1] - series[k]), -k)) + + +def floor_of(values: Iterable[float | None], factor: float = FLOOR_FACTOR) -> tuple[float, int]: + """The floor of one group of results and how many results it was taken from: `factor` + times the median size of the results that are not zero. No such result: no floor. + + >>> floor_of([0.0, 1.0, -3.0, 2.0]), floor_of([0.0, None]), floor_of([1e308, 1.6e308]) + ((0.02, 3), (0.0, 0), (1.3000000000000002e+306, 2)) + """ + sizes = sorted(abs(v) for v in values if is_value(v) and v != 0) + if not sizes: + return 0.0, 0 + middle = len(sizes) // 2 + # halved before they are added: two results near the largest float would add to infinity + median = sizes[middle] if len(sizes) % 2 else sizes[middle - 1] / 2 + sizes[middle] / 2 + return factor * median, len(sizes) + + +def floors_of( + values: Mapping[str, Mapping[str, float | None]], + units: Mapping[str, str | None], + factor: float = FLOOR_FACTOR, +) -> dict[tuple[str, str | None], tuple[float, int]]: + """One floor per indicator and reference unit, from one set of results. + + `values` is `{indicator: {item: result}}`, `units` the reference unit of each item. Items + of different units are never pooled: a result per tractor and a result per kilogram are + not of one size. + + >>> found = floors_of({"x": {"a": 1.0, "b": 3.0, "t": 5000.0}}, {"a": "kg", "b": "kg", "t": "unit"}) + >>> found[("x", "kg")], found[("x", "unit")] + ((0.02, 2), (50.0, 1)) + """ + found = {} + for indicator, by_item in values.items(): + groups: dict[str | None, list[float | None]] = {} + for item, value in by_item.items(): + groups.setdefault(units.get(item), []).append(value) + for unit, group in groups.items(): + found[(indicator, unit)] = floor_of(group, factor) + return found diff --git a/src/sentier_diagnostics/summary.py b/src/sentier_diagnostics/summary.py new file mode 100644 index 0000000..cbd9e40 --- /dev/null +++ b/src/sentier_diagnostics/summary.py @@ -0,0 +1,153 @@ +"""A short text of a diagnostics file, for a pull request, a log or a reader without a page.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping + +from . import chain, claims, fileformat, rule + +MOVES_ORDER = [k for k in rule.CLASSES if k in rule.MOVES] +#: The characters that make Markdown of a text where it stands in a line (GitHub's flavour +#: included: `|` adds a table column, `#12` links an issue, `$` starts maths). Each is +#: escaped with a backslash, which CommonMark allows before any ASCII punctuation. +_MARKUP = re.compile(r"([\\`*_\[\]<>#!|~&$])") + + +def _n(value) -> str: + return f"{value:,}".replace(",", " ") + + +def md(text) -> str: + r"""A text from a file as Markdown that shows it as it is: no table column, emphasis, + link, markup or heading of its own, and an `@` that mentions nobody (a word joiner + follows it). The summary is pasted into pull requests, where a mention notifies. + + >>> print(md("fix|x [a](b) *now*")) + fix\|x \\[a\](b)\ \*now\* + >>> md("@team") == "@\u2060team" + True + """ + return _MARKUP.sub(r"\\\1", str(text)).replace("@", "@\u2060") + + +def noise_text(noise: Mapping, names: Mapping[str, str]) -> str: + """What a step's noise says, with its size per indicator. An indicator the noise holds no + size for was not measured; a size above `chain.LARGE_NOISE` is called large. + + >>> noise_text({"measured": True, "between": ["a", "b"], "by": {"w": 0.0244, "c": 0.0}}, {"w": "Water", "c": "Climate", "t": "Tox"}) + 'noise measured between a and b: Water 2.4 % (large), Climate identical; Tox not measured' + """ + if not noise["measured"]: + return "noise not measured" + said = [] + for key, name in names.items(): + if key in noise["by"]: + size = noise["by"][key] + said.append(f"{name} identical" if size == 0 else + f"{name} {size * 100:.2g} %" + (" (large)" if size > chain.LARGE_NOISE else "")) # fmt: skip + absent = [name for key, name in names.items() if key not in noise["by"]] + between = f"noise measured between {md(noise['between'][0])} and {md(noise['between'][1])}" + return between + ": " + (", ".join(said) or "no result beyond the floor in both builds") + ( + "; " + ", ".join(absent) + " not measured" if absent else "" + ) + + +def text(data: Mapping) -> str: + """The summary as Markdown: the states, every step with what it moved and how that holds + against its claim, the whole way per indicator with its attribution, and what the file + leaves out.""" + states = {s["id"]: s for s in data["states"]} + # every text of the file goes through `md`: it is shown, never read as Markdown + names = {i["id"]: md(i["short"] or i["name"]) for i in data["indicators"]} + labels = {s["id"]: md(s["label"]) for s in data["steps"]} + count = data["items_count"] + many = md(data["item"]["many"]) + word = {k: md(v) for k, v in fileformat.indicator_word(data).items()} + indicators = word["one"] if len(data["indicators"]) == 1 else word["many"] + out = [f"# {md(data['title'])}", ""] + if data["audience"] == "local": + out += ["**Local file: its writer has not said that it may be published.**", ""] + if data.get("about"): + out += [md(data["about"]), ""] + out.append("States: " + " → ".join(md(s["label"]) for s in data["states"])) + for state in data["states"]: + join = state["join"] + if join: + out.append( + f"Read into {md(state['label'])}: {_n(join['compared'])} compared, {_n(join['unmatched'])} " + f"unmatched, {_n(join['ambiguous'])} ambiguous." + (f" {md(join['note'])}" if join["note"] else "") + ) + out.append( + f"{_n(count['all'])} {many}, {len(data['indicators'])} {indicators}. " + + ( + "This file holds counts only." + if data["tier"] == "counts" + else f"{_n(count['listed'])} of the {_n(count['qualifying'])} {many} worth a look are listed" + + (f"; {_n(count['not_listed'])} are not." if count["not_listed"] else ".") + ) + ) + out += ["", "## Steps", "", + "| Step | Kind | Explained | Values moved | Missing before | Missing after | As claimed | Off claim | Not claimed | Claimed, not moved |", + "|---|---|---|---:|---:|---:|---:|---:|---:|---:|"] # fmt: skip + for number, step in enumerate(data["steps"], fileformat.first_step(data)): + blocks = data["counts"]["steps"][step["id"]].values() + moved = sum(b["classes"][k] for b in blocks for k in rule.MOVES) + # a result missing on both sides is missing before and missing after + gone_before = sum(b["missing"]["before"] + b["missing"]["both"] for b in blocks) + gone_after = sum(b["missing"]["after"] + b["missing"]["both"] for b in blocks) + if step["claim"]: + verdicts = [sum(b["verdicts"][v] for b in blocks) for v in + (claims.AS_CLAIMED, claims.OFF_CLAIM, claims.NOT_CLAIMED, claims.NOT_MOVED)] # fmt: skip + said = [_n(v) for v in verdicts] + else: + said = ["no claim", "", "", ""] + out.append( + f"| {number}. {labels[step['id']]} | {md(step['kind'])} | {'yes' if step['explained'] else '**no: nothing explains it**'} " + f"| {_n(moved)} | {_n(gone_before)} | {_n(gone_after)} | " + " | ".join(said) + " |" + ) + if data["states"][0]["kind"] == "published": + blocks = data["counts"]["steps"][data["steps"][0]["id"]].values() + residual = sum(b["classes"][k] for b in blocks for k in rule.MOVES) + said = "1 value is" if residual == 1 else f"{_n(residual)} values are" + out += ["", f"Step 0: {said} not reproduced within its tolerance. That is the " + "residual the chain starts from, not a regression."] + first, last = md(data["states"][0]["label"]), md(data["states"][-1]["label"]) + out += ["", f"## The whole way: {first} → {last}", ""] + header = [word["one"][:1].upper() + word["one"][1:], *rule.CLASSES, "Largest step", "Spread over steps", "Not attributable", + "Moved by what nothing explains"] # fmt: skip + out.append("| " + " | ".join(header) + " |") + out.append("|---|" + "---:|" * (len(header) - 1)) + for indicator in data["indicators"]: + key = indicator["id"] + classes = data["counts"]["total"][key]["classes"] + said = data["attribution"][key] + ranked = sorted(said["dominant"].items(), key=lambda kv: -kv[1]) + largest = ", ".join(f"{labels[s]} ({_n(n)})" for s, n in ranked if n) or "none" + out.append( + f"| {names[key]} | " + " | ".join(_n(classes[k]) for k in rule.CLASSES) + + f" | {largest} | {_n(said['spread'])} | {_n(said['not_attributable'])} | {_n(said['unexplained_moved'])} |" + ) + out += ["", "## Rule", ""] + for step in data["steps"]: + tol = step["tolerance"] + out.append(f"- {labels[step['id']]}: tolerance {tol['value']:g} ({tol['basis']}); {noise_text(step['noise'], names)}") + floors = data["rule"]["floors"] + out.append( + f"- Floors: {len(floors)}, one per {word['one']} and reference unit, " + f"{data['rule']['floor_factor']:g} of the median size in {md(states[data['rule']['floors_from']]['label'])}" + f" ({sum(1 for f in floors if f['basis'] == 'declared')} declared, " + f"{sum(1 for f in floors if f['basis'] == 'none')} without a result to take one from)." + ) + out.append(f"- Beyond 10x: one size more than {data['rule']['beyond']:g} times the other.") + if data["explanations"]: + kinds = {} + for one in data["explanations"]: + kinds[one["by"]["kind"]] = kinds.get(one["by"]["kind"], 0) + 1 + none_found = sum(1 for one in data["explanations"] if one["finding"] == "none found") + out += ["", "Explanations: " + ", ".join(f"{n} by {k}" for k, n in sorted(kinds.items())) + + (f"; {none_found} say that someone looked and found no reason" if none_found else "") + + ". An explanation is a claim; it changes no count."] # fmt: skip + for note in data["notes"]: + out += ["", md(note)] + return "\n".join(out) + "\n" diff --git a/src/sentier_diagnostics/tables.py b/src/sentier_diagnostics/tables.py new file mode 100644 index 0000000..0e309a6 --- /dev/null +++ b/src/sentier_diagnostics/tables.py @@ -0,0 +1,152 @@ +"""Reading tables of results and of items from CSV, by the names in their header. + +A table of results has one row per item and one column per indicator: + + item,x:climate,x:water + p01,0.52,1.25 + p02,,3.0 + +The header names `item` and every indicator id the run compares, each once, in any +order. A column is found by its name, never by its place: two tables with their columns +in another order are read alike. An unknown column, a missing one or one named twice is +refused, and so is a cell that is not a finite number. An empty cell is a missing +result, never zero. + +A table of items names `item` and `unit` (the item's reference unit, which keeps floors +apart), and may name `name`, `place` and `group`. + +This reads tables a project wrote; it reads no database. Standard library only. +""" + +from __future__ import annotations + +import contextlib +import csv +import math +import re +from collections.abc import Iterable +from pathlib import Path + +ITEM_COLUMNS = ("item", "unit", "name", "place", "group") + + +class TableError(ValueError): + """A table that cannot be read; the message names the file, the row and the column.""" + + +def _header(path: Path, rows) -> list[str]: + try: + header = next(rows) + except StopIteration: + raise TableError(f"{path}: empty") from None + header = [h.strip() for h in header] + if header and header[0].startswith(""): + header[0] = header[0][1:] + twice = sorted({h for h in header if header.count(h) > 1}) + if twice: + raise TableError(f"{path}: columns named twice: {', '.join(twice)}") + return header + + +#: A number as a machine writes it: ASCII digits, an optional sign, point and exponent. +#: Python's `float` also takes `1_5` (as 15) and digits of other scripts; a table holding +#: those was more likely written by mistake than on purpose. +DECIMAL = re.compile(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?") + + +def number(text: str, where: str) -> float | None: + """A cell as a result: empty is missing; anything but a finite decimal number is + refused, and so is a number that is not zero but too small to be held (it would be + read as zero). + + >>> number("1.5", "x"), number(" ", "x"), number("-2e-3", "x"), number("0e-999", "x") + (1.5, None, -0.002, 0.0) + """ + text = text.strip() + if not text: + return None + if not DECIMAL.fullmatch(text): + raise TableError(f"{where}: {text!r} is not a decimal number") + value = float(text) + if not math.isfinite(value): + raise TableError(f"{where}: {text!r} is not a finite number") + if value == 0 and re.search(r"[1-9]", re.split(r"[eE]", text)[0]): + raise TableError(f"{where}: {text!r} is too small to be held: it would be read as zero") + return value + + +@contextlib.contextmanager +def _reading(path: Path): + """A table opened for reading, where a file that cannot be opened, cannot be decoded or + is not CSV is a TableError with its name (and the row, where the reader knows it).""" + try: + with path.open(newline="", encoding="utf-8") as handle: + yield handle + except OSError as error: + raise TableError(f"{path}: cannot be read ({error.strerror or error})") from None + except UnicodeDecodeError: + raise TableError(f"{path}: not a text in UTF-8") from None + except csv.Error as error: + raise TableError(f"{path}: not a table that can be read ({error})") from None + + +def read_results(path: Path, indicators: Iterable[str]) -> dict[str, dict[str, float | None]]: + """`{indicator: {item: result}}` from a table of results, fail-closed.""" + path = Path(path) + wanted = list(indicators) + with _reading(path) as handle: + rows = csv.reader(handle) + header = _header(path, rows) + expected = {"item", *wanted} + unknown, missing = set(header) - expected, expected - set(header) + if unknown or missing: + raise TableError( + f"{path}: the header must name `item` and every indicator once" + + (f"; unknown: {', '.join(sorted(unknown))}" if unknown else "") + + (f"; missing: {', '.join(sorted(missing))}" if missing else "") + ) + at = {name: header.index(name) for name in expected} + found: dict[str, dict[str, float | None]] = {i: {} for i in wanted} + seen = set() + for line, row in enumerate(rows, start=2): + if not any(cell.strip() for cell in row): + continue + if len(row) != len(header): + raise TableError(f"{path}, row {line}: {len(row)} cells, the header has {len(header)}") + item = row[at["item"]].strip() + if not item: + raise TableError(f"{path}, row {line}: no item") + if item in seen: + raise TableError(f"{path}, row {line}: item {item!r} a second time") + seen.add(item) + for indicator in wanted: + found[indicator][item] = number(row[at[indicator]], f"{path}, row {line}, {indicator}") + return found + + +def read_items(path: Path) -> dict[str, dict[str, str | None]]: + """`{item: {"unit", "name", "place", "group"}}` from a table of items.""" + path = Path(path) + with _reading(path) as handle: + rows = csv.reader(handle) + header = _header(path, rows) + unknown = set(header) - set(ITEM_COLUMNS) + if unknown or not {"item", "unit"} <= set(header): + raise TableError( + f"{path}: a table of items names `item` and `unit`, and may name " + f"`name`, `place` and `group`" + (f"; unknown: {', '.join(sorted(unknown))}" if unknown else "") + ) + found = {} + for line, row in enumerate(rows, start=2): + if not any(cell.strip() for cell in row): + continue + if len(row) != len(header): + raise TableError(f"{path}, row {line}: {len(row)} cells, the header has {len(header)}") + record = {name: (row[header.index(name)].strip() or None) for name in header} + item = record.pop("item") + if not item: + raise TableError(f"{path}, row {line}: no item") + if item in found: + raise TableError(f"{path}, row {line}: item {item!r} a second time") + found[item] = {k: record.get(k) for k in ("unit", "name", "place", "group")} + return found diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..ce9f8c6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,14 @@ +"""Tests of sentier-diagnostics. Standard library only: `python3 -m unittest discover -s tests -t .` + +Where the package is not installed, the source folder is put on the path, so that a fresh +clone runs the tests without installing anything. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +try: + import sentier_diagnostics # noqa: F401 +except ImportError: + sys.path.insert(0, str(ROOT / "src")) diff --git a/tests/acceptance/cases.json b/tests/acceptance/cases.json new file mode 100644 index 0000000..9a6b0e5 --- /dev/null +++ b/tests/acceptance/cases.json @@ -0,0 +1,28 @@ +{ + "about": "Acceptance cases shared by the Python rule (tests/test_acceptance.py) and the page script (tests/js/page.test.js). Each case is one pair of results, the class, direction and band the rule must give it, and how the page must draw it. `from` names the defect the case holds; see SPEC.md, Acceptance cases.", + "cases": [ + {"id": "A1-down", "from": "before/after v1: two results below zero going further below were coloured as higher", "before": -2.0, "after": -4.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "differs", "direction": "down", "band": "down, factor 1.25-2", "page": {"text": "−100 %", "tone": "down"}, "severity": [4, -0.3010299956639812]}, + {"id": "A1-up", "from": "before/after v1: the mirror case", "before": -4.0, "after": -2.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "differs", "direction": "up", "band": "up, factor 1.25-2", "page": {"text": "+50.0 %", "tone": "up"}, "severity": [4, -0.3010299956639812]}, + {"id": "A2-floor", "from": "before/after v1: the page recounted moved without the floor", "before": 1e-12, "after": 5e-12, "floor": 1e-09, "tolerance": 1e-05, "noise": null, "class": "near zero", "direction": null, "band": null, "page": {"text": "≈ 0", "tone": "near"}, "severity": [8, -0.6989700043360189]}, + {"id": "A3-solver", "from": "before/after v1: 1e-6 solver noise was counted as a change", "before": 1.0, "after": 1.000001, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "agrees", "direction": null, "band": null, "page": {"text": "=", "tone": "none"}, "severity": [7, -4.3429426472042774e-07]}, + {"id": "A3-noise", "from": "before/after v1: a difference within measured noise is not a change", "before": 1.0, "after": 1.0002, "floor": 0.0, "tolerance": 1e-05, "noise": 0.0003, "class": "within noise", "direction": null, "band": null, "page": {"tone": "noise"}, "severity": [6, -8.685021164894766e-05]}, + {"id": "A3-beyond-noise", "from": "before/after v1: a difference larger than the noise is", "before": 1.0, "after": 1.0005, "floor": 0.0, "tolerance": 1e-05, "noise": 0.0003, "class": "differs", "direction": "up", "band": "up, factor 1-1.1", "page": {"tone": "up"}, "severity": [4, -0.00021709297223018438]}, + {"id": "A4-up", "from": "backtest: differences above 1000 % were blanked", "before": 1.0, "after": 25.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "beyond 10x", "direction": "up", "band": "up, factor over 10", "page": {"text": "size ×25", "tone": "up"}, "severity": [3, -1.3979400086720377]}, + {"id": "A4-down", "from": "backtest: the mirror case", "before": 25.0, "after": 1.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "beyond 10x", "direction": "down", "band": "down, factor over 10", "page": {"text": "size ÷25", "tone": "down"}, "severity": [3, -1.3979400086720375]}, + {"id": "A4-ten", "from": "backtest: exactly ten times is not beyond", "before": 1.0, "after": 10.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "differs", "direction": "up", "band": "up, factor 2-10", "page": {"tone": "up"}, "severity": [4, -1.0]}, + {"id": "A5-zeros", "from": "backtest: near-zero pairs were set to agreement", "before": 0.0, "after": 0.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "near zero", "direction": null, "band": null, "page": {"text": "≈ 0", "tone": "near"}, "severity": [8, -0.0]}, + {"id": "A5-below-floor", "from": "backtest: near-zero pairs were set to agreement", "before": 1e-10, "after": 3e-10, "floor": 1e-09, "tolerance": 1e-05, "noise": null, "class": "near zero", "direction": null, "band": null, "page": {"text": "≈ 0", "tone": "near"}, "severity": [8, -0.47712125471966244]}, + {"id": "A7-before", "from": "backtest page: a missing cell was shown as 0.00 %", "before": null, "after": 0.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "missing", "direction": null, "band": null, "page": {"text": "missing before", "tone": "missing"}, "severity": [5, 0.0]}, + {"id": "A7-after", "from": "backtest dashboard: a missing value was turned into 0 and drawn as a match", "before": 0.5, "after": null, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "missing", "direction": null, "band": null, "page": {"text": "missing after", "tone": "missing"}, "severity": [5, 0.0]}, + {"id": "A10-from-zero", "from": "zero and sign change kept apart", "before": 0.0, "after": 1.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "zero reference", "direction": "up", "band": null, "page": {"text": "from 0", "tone": "up"}, "severity": [1, -1.0]}, + {"id": "A10-from-near-zero", "from": "zero and sign change kept apart", "before": 1e-12, "after": 1.0, "floor": 1e-09, "tolerance": 1e-05, "noise": null, "class": "zero reference", "direction": "up", "band": null, "page": {"text": "from ≈ 0", "tone": "up"}, "severity": [1, -12.0]}, + {"id": "A10-to-zero", "from": "zero and sign change kept apart", "before": 1.0, "after": 0.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "zero result", "direction": "down", "band": null, "page": {"text": "to 0", "tone": "down"}, "severity": [2, -1.0]}, + {"id": "A10-sign", "from": "zero and sign change kept apart", "before": -1.0, "after": 1.0, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "sign change", "direction": "up", "band": null, "page": {"text": "− → +", "tone": "up"}, "severity": [0, -2.0]}, + {"id": "A10-sign-down", "from": "zero and sign change kept apart", "before": 0.5, "after": -0.25, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "sign change", "direction": "down", "band": null, "page": {"text": "+ → −", "tone": "down"}, "severity": [0, -1.5]}, + {"id": "A11-unrounded", "from": "trial dashboard and backtest: results were rounded before the ratio", "before": 0.00104, "after": 0.00096, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "differs", "direction": "down", "band": "down, factor 1-1.1", "page": {"tone": "down"}, "severity": [4, -0.03476210625921192]}, + {"id": "A26-floor-agrees", "from": "review of PR 2: a pair on both sides of the floor that agrees was a zero result", "before": 0.010000001, "after": 0.009999999, "floor": 0.01, "tolerance": 1e-05, "noise": null, "class": "agrees", "direction": null, "band": null, "page": {"text": "=", "tone": "none"}, "severity": [7, -8.685889634187353e-08]}, + {"id": "A26-floor-noise", "from": "review of PR 2: the same within the measured noise", "before": 0.0101, "after": 0.0099, "floor": 0.01, "tolerance": 1e-05, "noise": 0.05, "class": "within noise", "direction": null, "band": null, "page": {"tone": "noise"}, "severity": [6, -0.008686179185092617]}, + {"id": "A26-floor-moves", "from": "review of PR 2: a pair that crosses the floor by more than the noise still moved", "before": 0.02, "after": 0.005, "floor": 0.01, "tolerance": 1e-05, "noise": 0.05, "class": "zero result", "direction": "down", "band": null, "page": {"text": "to ≈ 0", "tone": "down"}, "severity": [2, -0.6020599913279624]}, + {"id": "A27-far-apart", "from": "review of PR 2: a factor too large for a float; Python and the page ordered it differently", "before": 1e-300, "after": 1e+300, "floor": 0.0, "tolerance": 1e-05, "noise": null, "class": "beyond 10x", "direction": "up", "band": "up, factor over 10", "page": {"text": "size ×1e600", "tone": "up"}, "severity": [3, -600.0]} + ] +} diff --git a/tests/fixtures/located-factors/after.csv b/tests/fixtures/located-factors/after.csv new file mode 100644 index 0000000..6f07144 --- /dev/null +++ b/tests/fixtures/located-factors/after.csv @@ -0,0 +1,7 @@ +item,x:water,x:climate,x:ecotox,x:land +d1,0.126,0.52,1.10,1.62 +d2,0.310,0.70,0.90,0.80 +d3,0.004,0.03,0.05,0.40 +d4,2.700,1.90,2.40,3.10 +d5,0.300,2.35,4.00,0.25 +d6,0.0020000002,0.10,0.20,0.06 diff --git a/tests/fixtures/located-factors/before.csv b/tests/fixtures/located-factors/before.csv new file mode 100644 index 0000000..c1eda05 --- /dev/null +++ b/tests/fixtures/located-factors/before.csv @@ -0,0 +1,7 @@ +item,x:water,x:climate,x:ecotox,x:land +d1,0.120,0.52,1.10,1.50 +d2,0.080,0.70,0.90,0.80 +d3,0.004,0.03,0.05,0.40 +d4,0.950,1.90,2.40,3.10 +d5,0.300,2.10,4.00,0.25 +d6,0.0020,0.10,0.20,0.06 diff --git a/tests/fixtures/located-factors/description.json b/tests/fixtures/located-factors/description.json new file mode 100644 index 0000000..2694ad8 --- /dev/null +++ b/tests/fixtures/located-factors/description.json @@ -0,0 +1,25 @@ +{ + "title": "Made-up: located factors, before and after", + "about": "The shape of the first step sentier-bafu will show: a flow that names a country takes that country's factor. Six made-up datasets, four indicators. The claim names the indicators whose factors are taken by country; sentier-bafu builds that list from its build.", + "item": {"one": "dataset", "many": "datasets"}, + "audience": "public", + "indicators": [ + {"id": "x:water", "name": "Water use", "short": "Water", "unit": "m3 world eq"}, + {"id": "x:climate", "name": "Climate change", "short": "Climate", "unit": "kg CO2 eq"}, + {"id": "x:ecotox", "name": "Ecotoxicity, freshwater", "short": "Ecotox", "unit": "CTUe"}, + {"id": "x:land", "name": "Land use", "short": "Land", "unit": "Pt"} + ], + "items": "items.csv", + "states": [ + {"id": "before", "label": "Without located factors", "results": "before.csv", "identity": {"commit": "0000001"}}, + {"id": "after", "label": "With located factors", "results": "after.csv", "identity": {"commit": "0000002"}} + ], + "steps": [ + {"id": "located-factors", "label": "Located factors", "kind": "method", + "about": "A flow that names a country takes that country's factor instead of the world's.", + "claim": {"text": "Water use and land use may move; nothing else moves.", + "by": {"kind": "person", "name": "A. Author"}, + "scope": [{"indicators": ["x:water", "x:land"], "items": "*", "direction": "any", "expect": "may"}]}} + ], + "notes": ["Made-up results: nobody's data."] +} diff --git a/tests/fixtures/located-factors/items.csv b/tests/fixtures/located-factors/items.csv new file mode 100644 index 0000000..4d42b72 --- /dev/null +++ b/tests/fixtures/located-factors/items.csv @@ -0,0 +1,7 @@ +item,unit,name,place,group +d1,kg,Wheat grain,CH,Crops +d2,kg,Tomato,ES,Crops +d3,MJ,Heat from wood,CH,Energy +d4,kg,Rice,IN,Crops +d5,kg,Steel,RER,Materials +d6,kWh,Electricity,CH,Energy diff --git a/tests/fixtures/tables/again.csv b/tests/fixtures/tables/again.csv new file mode 100644 index 0000000..eeb49ad --- /dev/null +++ b/tests/fixtures/tables/again.csv @@ -0,0 +1,5 @@ +item,x:climate,x:water +w1,0.5000002,1.2000004 +w2,0.62,1.1 +h1,0.03,0.002 +c1,0.2, diff --git a/tests/fixtures/tables/description.json b/tests/fixtures/tables/description.json new file mode 100644 index 0000000..ad1fb38 --- /dev/null +++ b/tests/fixtures/tables/description.json @@ -0,0 +1,35 @@ +{ + "title": "Made-up tables: published, recomputed, fixed", + "about": "Four made-up items in two indicators, read from CSV tables.", + "item": {"one": "product", "many": "products"}, + "audience": "public", + "indicators": [ + {"id": "x:climate", "name": "Climate change", "short": "Climate", "unit": "kg CO2 eq"}, + {"id": "x:water", "name": "Water use", "short": "Water", "unit": "m3"} + ], + "items": "items.csv", + "states": [ + {"id": "published", "label": "Published", "kind": "published", "results": "published.csv", + "join": {"compared": 4, "unmatched": 0, "ambiguous": 0, "note": null}}, + {"id": "repro", "label": "Recomputed", "results": "repro.csv", "identity": {"commit": "0000001"}}, + {"id": "fixed", "label": "Fixed", "results": "fixed.csv", "identity": {"commit": "0000002"}} + ], + "steps": [ + {"id": "step-0", "label": "Reproduction", "kind": "reproduction", + "tolerance": {"value": 5e-4, "basis": "published rounding"}, + "claim": {"text": "Nothing moves.", "by": {"kind": "code", "name": "reproduction rule"}, "scope": []}}, + {"id": "fix-1", "label": "Fix of wheat", "kind": "fix", + "noise": {"tables": [{"label": "build a", "results": "repro.csv"}, {"label": "build b", "results": "again.csv"}]}, + "claim": {"text": "Wheat goes down in climate change.", "by": {"kind": "person", "name": "A. Author"}, + "scope": [{"indicators": ["x:climate"], "items": ["w1"], "direction": "down", "largest": 0.5, "expect": "must"}]}} + ], + "explanations": [ + {"text": "Compost's water use was not computed in the reproduction.", "by": {"kind": "person", "name": "B. Reviewer"}, + "step": "step-0", "indicator": "x:water", "item": "c1", "finding": "none found"}, + {"text": "Oats moved in the reproduction because a parameter rounds differently.", + "by": {"kind": "model", "model": "example-model-1", "run_by": "example maintainers (agent-run)"}, + "step": "step-0", "indicator": "x:climate", "item": "w2", + "not_checked": "Nobody compared the parameter with its source."} + ], + "notes": ["Made-up tables for the tests."] +} diff --git a/tests/fixtures/tables/fixed.csv b/tests/fixtures/tables/fixed.csv new file mode 100644 index 0000000..db8abdf --- /dev/null +++ b/tests/fixtures/tables/fixed.csv @@ -0,0 +1,5 @@ +item,x:climate,x:water +w1,0.35,1.2000001 +w2,0.62,1.1 +h1,0.03,0.002 +c1,0.2,0.012 diff --git a/tests/fixtures/tables/items.csv b/tests/fixtures/tables/items.csv new file mode 100644 index 0000000..8356796 --- /dev/null +++ b/tests/fixtures/tables/items.csv @@ -0,0 +1,5 @@ +item,unit,name,place,group +w1,kg,Wheat,CH,Cereals +w2,kg,Oats,CH,Cereals +h1,MJ,Heat from wood,CH,Energy +c1,kg,Compost,CH,Fertiliser diff --git a/tests/fixtures/tables/published.csv b/tests/fixtures/tables/published.csv new file mode 100644 index 0000000..a32e4ce --- /dev/null +++ b/tests/fixtures/tables/published.csv @@ -0,0 +1,5 @@ +item,x:climate,x:water +w1,0.5,1.2 +w2,0.45,1.1 +h1,0.03,0.002 +c1,-0.5,0.01 diff --git a/tests/fixtures/tables/repro.csv b/tests/fixtures/tables/repro.csv new file mode 100644 index 0000000..19d1a1b --- /dev/null +++ b/tests/fixtures/tables/repro.csv @@ -0,0 +1,5 @@ +x:water,item,x:climate +1.2000001,w1,0.5000002 +1.1,w2,0.62 +0.002,h1,0.03 +,c1,0.2 diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..4c4a832 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,54 @@ +"""Small chains built in tests. Every value here is made up.""" + +from __future__ import annotations + +import copy + +from sentier_diagnostics import chain, claims + +A, B = "x:a", "x:b" +INDICATORS = [(A, "Indicator A", "A", "kg CO2 eq"), (B, "Indicator B", "B", "m3")] + + +def two_states(before: dict, after: dict, *, claim=None, units=None, first_kind="published", **more) -> dict: + """A chain of one step: `before` and `after` are `{indicator: {item: value}}`.""" + items = {item for side in (before, after) for column in side.values() for item in column} + return chain.build( + "Two states", + indicators=INDICATORS, + states=[chain.state("s0", "Before", kind=first_kind), chain.state("s1", "After")], + steps=[chain.step("one", "One step", kind="fix", claim=claim)], + values={"s0": before, "s1": after}, + units=units or dict.fromkeys(items, "kg"), + **more, + ) + + +def three_states(first: dict, middle: dict, last: dict, *, explained=(True, True), **more) -> dict: + items = {item for side in (first, middle, last) for column in side.values() for item in column} + return chain.build( + "Three states", + indicators=INDICATORS, + states=[chain.state("s0", "Published", kind="published"), chain.state("s1", "Middle"), + chain.state("s2", "Last")], # fmt: skip + steps=[chain.step("one", "First step", kind="fix", explained=explained[0]), + chain.step("two", "Second step", kind="fix", explained=explained[1])], # fmt: skip + values={"s0": first, "s1": middle, "s2": last}, + units=dict.fromkeys(items, "kg"), + **more, + ) + + +def claim_down(items, largest=None, expect=claims.MUST, indicators=(A,)): + return claims.claim( + "They go down.", + [claims.scope(list(indicators), set(items), "down", largest, expect)], + by=claims.origin("person", "A. Author"), + ) + + +def mutated(data: dict, change) -> dict: + """A deep copy of a file with one change applied, for tests of the validator.""" + copied = copy.deepcopy(data) + change(copied) + return copied diff --git a/tests/js/dom.js b/tests/js/dom.js new file mode 100644 index 0000000..ee1e2d7 --- /dev/null +++ b/tests/js/dom.js @@ -0,0 +1,192 @@ +// A small stand-in for the part of the DOM the page script uses, so that the page can be +// drawn under Node without a browser or a package. Elements hold children, attributes and +// listeners; text nodes hold text. `page(data, breakdowns)` builds the elements of body.html +// that carry `data-sd`, runs diagnostics.js against them and returns what it drew. +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +const PAGE = path.join(__dirname, '..', '..', 'src', 'sentier_diagnostics', 'page'); + +// the selectors the script asks for: [attr], [attr="value"], tag[attr], tag[attr="value"] +function matches(node, selector) { + const one = /^([a-z0-9]*)\[([a-z-]+)(?:="([^"]*)")?\]$/.exec(selector); + if (!one) throw new Error('the stand-in does not know the selector ' + selector); + const [, tag, key, value] = one; + if (node.text !== undefined) return false; + if (tag && node.tagName !== tag.toUpperCase()) return false; + if (!(key in node.attributes)) return false; + return value === undefined || node.attributes[key] === value; +} + +class FakeNode { + constructor(tag, text) { + this.tagName = tag ? tag.toUpperCase() : null; + this.text = text; // a text node holds this; an element has none + this.childNodes = []; + this.parentNode = null; + this.attributes = {}; + this.listeners = {}; + this.hidden = false; + this.className = ''; + this.value = ''; + this.disabled = false; + this.focused = false; + } + + get firstChild() { return this.childNodes[0] || null; } + + appendChild(child) { + if (child.parentNode) child.parentNode.removeChild(child); + child.parentNode = this; + this.childNodes.push(child); + return child; + } + + removeChild(child) { + const at = this.childNodes.indexOf(child); + if (at < 0) throw new Error('not a child of this node'); + this.childNodes.splice(at, 1); + child.parentNode = null; + return child; + } + + get textContent() { + return this.text !== undefined ? this.text : this.childNodes.map((c) => c.textContent).join(''); + } + + set textContent(value) { + if (this.text !== undefined) { this.text = String(value); return; } + this.childNodes.forEach((c) => { c.parentNode = null; }); + this.childNodes = []; + if (value !== '' && value !== null && value !== undefined) this.appendChild(new FakeNode(null, String(value))); + } + + setAttribute(key, value) { + this.attributes[key] = String(value); + if (key === 'value') this.value = String(value); + } + + getAttribute(key) { return key in this.attributes ? this.attributes[key] : null; } + + removeAttribute(key) { delete this.attributes[key]; } + + addEventListener(type, listener) { (this.listeners[type] = this.listeners[type] || []).push(listener); } + + // an event on this node, passed up to its parents as a browser does + dispatch(type, event) { + const full = { target: this, ...event }; + for (let node = this; node; node = node.parentNode) (node.listeners[type] || []).forEach((listener) => listener(full)); + } + + closest(selector) { + for (let node = this; node; node = node.parentNode) if (matches(node, selector)) return node; + return null; + } + + querySelectorAll(selector) { return this.all().slice(1).filter((n) => matches(n, selector)); } + + querySelector(selector) { return this.querySelectorAll(selector)[0] || null; } + + contains(other) { for (let n = other; n; n = n.parentNode) if (n === this) return true; return false; } + + focus() { this.focused = true; } + + scrollIntoView() {} + + // every text a reader sees: text nodes and tooltips, in order; nothing under a hidden node + seen() { + if (this.text !== undefined) return [this.text]; + if (this.hidden) return []; + const own = this.attributes.title ? [this.attributes.title] : []; + return own.concat(...this.childNodes.map((c) => c.seen())); + } + + // every element below this one, this one first + all() { return this.text !== undefined ? [] : [this].concat(...this.childNodes.map((c) => c.all())); } +} + +// the root of body.html and every element in it that carries `data-sd` or `data-sd-word`, each a child of the +// root; the data scripts as the Python writes them where `scripts` holds them +function body(scripts) { + const html = fs.readFileSync(path.join(PAGE, 'body.html'), 'utf8'); + const root = new FakeNode('div'); + root.setAttribute('class', 'sentier-diagnostics'); + root.setAttribute('data-sentier-diagnostics', ''); + for (const [whole, tag] of html.matchAll(/<([a-z0-9]+)\b[^>]*>/g)) { + const word = /\sdata-sd-word="([^"]+)"/.exec(whole); + if (word) { const span = new FakeNode(tag); span.setAttribute('data-sd-word', word[1]); root.appendChild(span); continue; } + const name = /\sdata-sd="([^"]+)"/.exec(whole); + if (!name) continue; + const node = new FakeNode(tag); + node.setAttribute('data-sd', name[1]); + node.hidden = /\shidden[\s>=]/.test(whole); + root.appendChild(node); + } + for (const [name, value] of Object.entries(scripts || {})) { + const node = new FakeNode('script'); + node.setAttribute('type', 'application/json'); + node.setAttribute('data-sd', name); + node.textContent = JSON.stringify(value); + root.appendChild(node); + } + return root; +} + +// `options`: {standalone, attributes, globals, fetch, protocol, scripts} +function page(data, breakdowns, options) { + options = options || {}; + const scripts = options.scripts || {}; + if (data !== undefined && !options.globals) scripts.data = data; + if (breakdowns && !options.globals) scripts.breakdowns = breakdowns; + const root = body(scripts); + if (options.standalone) root.setAttribute('data-sd-standalone', ''); + for (const [key, value] of Object.entries(options.attributes || {})) root.setAttribute(key, value); + const head = new FakeNode('head'); + const listeners = {}; + const document = { + title: '', + readyState: 'complete', + head, + activeElement: null, + createElement: (tag) => new FakeNode(tag), + createTextNode: (text) => new FakeNode(null, String(text)), + querySelectorAll: (selector) => (matches(root, selector) ? [root] : []), + addEventListener: () => {}, + }; + root.ownerDocument = document; + const context = { + document, + location: { hash: '', search: '', pathname: '/index.html', protocol: options.protocol || 'file:' }, + history: { replaceState: () => { context.location.hash = ''; } }, + addEventListener: (type, listener) => { (listeners[type] = listeners[type] || []).push(listener); }, + URLSearchParams, + FileReader: class {}, + fetch: options.fetch || (() => Promise.reject(new Error('no network'))), + ...(options.globals || {}), + }; + if (options.globals && data !== undefined) context.SENTIER_DIAGNOSTICS = data; + if (options.globals && breakdowns) context.SENTIER_DIAGNOSTICS_BREAKDOWNS = breakdowns; + context.window = context; + vm.createContext(context); + vm.runInContext(fs.readFileSync(path.join(PAGE, 'diagnostics.js'), 'utf8'), context, { filename: 'diagnostics.js' }); + const byName = {}; + root.childNodes.forEach((n) => { if (n.getAttribute('data-sd')) byName[n.getAttribute('data-sd')] = n; }); + return { + root, + byId: byName, // by the name in data-sd + head, + document, + context, + // the text of everything drawn and not hidden, tooltips included + text: () => root.childNodes.filter((n) => !n.hidden && n.tagName !== 'SCRIPT').map((n) => n.seen().join(' ')).join('\n'), + open: (hash) => { + context.location.hash = '#' + hash; + (listeners.hashchange || []).forEach((listener) => listener({})); + }, + }; +} + +module.exports = { FakeNode, page, body }; diff --git a/tests/js/page.test.js b/tests/js/page.test.js new file mode 100644 index 0000000..33254fb --- /dev/null +++ b/tests/js/page.test.js @@ -0,0 +1,844 @@ +// The page script against the acceptance cases the Python rule is tested on. +// Run with `node --test tests/js/page.test.js`. Node's own test runner: no package, no install. +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const page = require('../../src/sentier_diagnostics/page/diagnostics.js'); +const cases = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'acceptance', 'cases.json'), 'utf8')).cases; + +test('every case is drawn with the tone of its direction and the text it must have', () => { + for (const c of cases) { + const view = page.cellView(c.before, c.after, c.class, c.direction, c.band); + assert.equal(view.tone, c.page.tone, c.id + ': ' + c.from); + if (c.page.text !== undefined) assert.equal(view.text, c.page.text, c.id + ': ' + c.from); + if (c.class === 'beyond 10x') assert.ok(view.text.length > 0 && view.mark === '»', c.id + ': never blank'); + } +}); + +test('the tone comes from the direction only, never from the ratio (A1)', () => { + // two results below zero going further below zero: the ratio is 2, the direction is down + assert.equal(page.cellView(-2, -4, 'differs', 'down', 'down, factor 1.25-2').tone, 'down'); + assert.equal(page.cellView(-4, -2, 'differs', 'up', 'up, factor 1.25-2').tone, 'up'); +}); + +test('the page takes the class from the file and works out none (A2)', () => { + // the file says agrees: the page draws agreement, whatever the numbers + assert.equal(page.cellView(1, 2, 'agrees', null, null).text, '='); + assert.equal(page.cellView(1e-12, 5e-12, 'near zero', null, null).text, '≈ 0'); +}); + +test('a missing side is never shown as a number (A7)', () => { + for (const [b, a, said] of [[null, 0, 'missing before'], [0.5, null, 'missing after'], [null, null, 'missing']]) { + const view = page.cellView(b, a, 'missing', null, null); + assert.equal(view.text, said); + assert.doesNotMatch(view.text, /\d/); + } +}); + +test('the order worst first is the order of the Python rule', () => { + for (const c of cases) { + const key = page.severityKey(c.before, c.after, c.class); + assert.equal(key[0], c.severity[0], c.id); + assert.ok(Math.abs(key[1] - c.severity[1]) <= 1e-12, c.id + ': ' + key[1] + ' against ' + c.severity[1]); + } +}); + +test('numbers keep their sign and never read 0 when they are not', () => { + assert.equal(page.formatNumber(-2e-6), '−2e−6'); + assert.equal(page.formatNumber(0), '0'); + assert.equal(page.formatNumber(1234.5), '1235'); + assert.equal(page.formatNumber(null), 'missing'); + assert.notEqual(page.percentText(1, 1.0000003), '0 %'); + assert.equal(page.shareText(0), '0 %'); + assert.notEqual(page.shareText(-0.00025), '−0.0 %'); +}); + +test('the size filter hides small changes and keeps every value without a ratio', () => { + assert.equal(page.sizeAtLeast('differs', 'up, factor 1.1-1.25', 1.25), false); + assert.equal(page.sizeAtLeast('differs', 'up, factor 1.25-2', 1.25), true); + assert.equal(page.sizeAtLeast('differs', 'down, factor 1-1.1', 0), true); + for (const kind of ['zero reference', 'zero result', 'sign change', 'missing', 'beyond 10x']) { + assert.equal(page.sizeAtLeast(kind, null, 10), true, kind); + } + assert.equal(page.sizeAtLeast('agrees', null, 1.1), false); +}); + +test('the page refuses a file of another format, and an older one', () => { + assert.match(page.checkShape({ format: 'sentier-diagnostics/2' }), /shows sentier-diagnostics\/1 only/); + assert.match(page.checkShape({ format: 'sentier/before-after/2' }), /not a diagnostics file/); + const example = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'examples', 'diagnostics.json'), 'utf8')); + assert.equal(page.checkShape(example), null); +}); + +test('two identical builds are a finding, not "not measured"', () => { + assert.equal(page.noiseText({ measured: false, between: null, by: null }), 'not measured'); + assert.equal(page.noiseText({ measured: true, between: ['a', 'b'], by: { 'x:a': 0, 'x:b': 0 } }), 'identical between a and b'); + assert.equal(page.noiseText({ measured: true, between: ['a', 'b'], by: { 'x:a': 0, 'x:b': 2e-7 } }), 'measured between a and b'); +}); + +test('noise that was not measured for an indicator is never "identical"', () => { + const ids = ['x:a', 'x:b', 'x:c']; + assert.equal(page.noiseText({ measured: true, between: ['a', 'b'], by: {} }, ids), + 'not measured: no result beyond the floor in both builds between a and b'); + assert.equal(page.noiseText({ measured: true, between: ['a', 'b'], by: { 'x:a': 0 } }, ids), + 'identical between a and b, for 1 of 3 indicators'); + assert.equal(page.noiseText({ measured: true, between: ['a', 'b'], by: { 'x:a': 0, 'x:b': 0, 'x:c': 0.21 } }, ids), + 'measured between a and b; large: up to 21 %'); +}); + +test('only an https address becomes a link', () => { + assert.equal(page.safeLink('https://example.org/a'), 'https://example.org/a'); + for (const bad of ['javascript:alert(1)', 'JAVASCRIPT:x', 'http://example.org', 'data:text/html,x', ' https://x', 'https://a b', null, 7]) { + assert.equal(page.safeLink(bad), null, String(bad)); + } +}); + +test('a file with a part of the wrong shape is refused before anything is drawn', () => { + const example = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'examples', 'diagnostics.json'), 'utf8')); + for (const [key, value] of [['items', {}], ['counts', []], ['steps', null], ['indicators', []], ['counts', { steps: {} }]]) { + assert.notEqual(page.checkShape({ ...example, [key]: value }), null, key + ': ' + JSON.stringify(value)); + } +}); + +test('one number style from 1e4 on, never the "e+" of toPrecision', () => { + assert.equal(page.formatNumber(9999), '9999'); + assert.equal(page.formatNumber(12345), '1.235e4'); + assert.equal(page.formatNumber(1e6), '1e6'); + assert.equal(page.formatNumber(-123456), '−1.235e5'); + assert.equal(page.formatNumber(0.00042), '4.2e−4'); +}); + +test('an item\'s whole difference is split by the steps, the sum of the shares shown (A18)', () => { + const split = page.splitOf([1, 1.5, 1.75], [true, false], 'differs'); + assert.equal(split.state, 'split'); + assert.deepEqual(split.shares, [2 / 3, 1 / 3]); + assert.equal(page.splitText(split), '100.0 %, of which nothing explains 33.3 %'); + // in floating point the shares need not add up to exactly 1: the page prints their sum + const rounded = page.splitOf([0.1, 0.2, 0.7], [true, true], 'differs'); + assert.equal(page.splitText(rounded), page.shareText(rounded.shares[0] + rounded.shares[1])); +}); + +test('steps that cancel are said to cancel, not split into large shares of both signs (A18)', () => { + const split = page.splitOf([1, 1e16, 1, 2], [true, true, true], 'differs'); + assert.equal(split.state, 'cancel'); + assert.match(page.splitText(split), /^not split: the steps cancel, their differences add up to 2e16 times/); +}); + +test('an item missing in any state is not split, not even for the steps before the gap (A18)', () => { + assert.equal(page.splitOf([1, 2, null, 4], [true, true, true], 'differs').state, 'gap'); + assert.equal(page.splitOf([null, 2, 4], [true, true], 'missing').state, 'missing'); + assert.equal(page.splitOf([1, 1, 1], [true, true], 'agrees').state, 'still'); +}); + +const dom = require('./dom.js'); +const EXAMPLE = path.join(__dirname, '..', '..', 'examples'); +const example = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'diagnostics.json'), 'utf8')); +const wheat = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'p01-fix-1.json'), 'utf8')); + +test('the example page is drawn whole', () => { + const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }, { standalone: true }); + assert.equal(drawn.byId.load.hidden, true, drawn.byId['load-error'].textContent); + assert.equal(drawn.byId.content.hidden, false); + assert.equal(drawn.byId.unchecked.hidden, true); + assert.equal(drawn.document.title, example().title); + const text = drawn.text(); + for (const said of ['Reproduction', 'Spread over steps', 'Not attributable', 'Water 2.4 %'.slice(0, 5)]) assert.ok(text.includes(said), said); + assert.equal(drawn.byId['steps-table'].all().filter((n) => n.tagName === 'TR').length, 1 + example().steps.length); + assert.ok(drawn.byId['grid-table'].all().some((n) => n.getAttribute('data-item') === 'p01')); +}); + +test('the steps table prints the file\'s counts of what moved, never a recount from the items (A2)', () => { + const data = example(); + const moves = ['differs', 'beyond 10x', 'zero reference', 'zero result', 'sign change']; + const drawn = dom.page(data); + const rows = drawn.byId['steps-table'].all().filter((n) => n.tagName === 'TR').slice(1); + data.steps.forEach((s, k) => { + const said = Object.values(data.counts.steps[s.id]).reduce((a, b) => a + moves.reduce((m, c) => m + b.classes[c], 0), 0); + assert.equal(rows[k].childNodes[5].textContent, String(said), s.id); + const side = (name) => Object.values(data.counts.steps[s.id]).reduce((a, b) => a + b.missing[name] + b.missing.both, 0); + assert.equal(rows[k].childNodes[13].textContent, String(side('before')), s.id + ': missing before'); + assert.equal(rows[k].childNodes[14].textContent, String(side('after')), s.id + ': missing after'); + }); + assert.notEqual(rows.map((r) => r.childNodes[13].textContent).join(), rows.map((r) => r.childNodes[14].textContent).join(), + 'the example has a step with missing values on one side only'); +}); + +test('a share is signed against the whole difference, whichever way it goes (A18)', () => { + // the whole way goes down: a step that goes down carries a positive share of it + assert.deepEqual(page.splitOf([2, 1], [true], 'differs').shares, [1]); + const split = page.splitOf([4, 5, 2], [true, true], 'differs'); + assert.deepEqual(split.shares, [-0.5, 1.5]); + assert.equal(split.sum, 1); +}); + +test('step 0 is said to reproduce the published results only where the chain starts from them', () => { + assert.equal(dom.page(example()).byId['steps-zero'].hidden, false); + const built = example(); + built.states[0].kind = 'build'; + assert.equal(dom.page(built).byId['steps-zero'].hidden, true); +}); + +test('a local file says so at its top, a public one does not (A25)', () => { + const local = example(); + local.audience = 'local'; + const drawn = dom.page(local); + assert.equal(drawn.byId.audience.hidden, false); + assert.ok(drawn.byId.audience.textContent.includes('Local file')); + assert.equal(dom.page(example()).byId.audience.hidden, true); +}); + +test('a text by a model is drawn as a claim nobody checked (A9)', () => { + const drawn = dom.page(example()); + const list = drawn.byId['explanation-list']; + const model = list.childNodes.find((li) => li.seen().join(' ').includes('model')); + assert.ok(model, 'the example has an explanation by a model'); + assert.ok(model.seen().join(' ').includes('A claim, not checked by a person')); +}); + +test('nothing drawn calls a move better or worse (A22)', () => { + const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }); + drawn.open('item=p01'); + const html = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'sentier_diagnostics', 'page', 'body.html'), 'utf8') + .replace(/<[^>]*>/g, ' '); + const text = (drawn.text() + '\n' + html).toLowerCase(); + for (const word of ['better', 'worse', 'improve', 'closer']) assert.ok(!text.includes(word), word); +}); + +test('the item view splits only what can be split and draws the breakdown of its own step', () => { + const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }); + drawn.open('item=p01'); + const body = drawn.byId['item-body'].seen().join(' '); + assert.ok(body.includes('What it is made of')); + assert.ok(body.includes('Fertiliser') || body.includes('Total'), 'the breakdown is drawn'); + assert.ok(/100\.0 %/.test(body), 'the sum of the shares'); + drawn.open('item=p03'); // barley: no water use in the rebuilt state + const barley = drawn.byId['item-body']; + const water = barley.querySelector('table[data-indicator="x:water"]'); + const shares = water.all().filter((n) => n.tagName === 'TR').slice(1, -1).map((tr) => tr.childNodes[5].textContent); + assert.deepEqual(shares, shares.map(() => '—')); + assert.ok(water.seen().join(' ').includes('not split: missing in a state between')); +}); + +test('a breakdown named by another item is not drawn under it (issue 5)', () => { + const data = example(); + const rice = data.items.find((i) => i.id === 'p05'); + rice.breakdowns = { total: 'p01-fix-1.json' }; + const drawn = dom.page(data, { 'p01-fix-1.json': wheat() }); + drawn.open('item=p05'); + const body = drawn.byId['item-body'].seen().join(' '); + assert.ok(body.includes('is of another product or another step: not drawn'), body.slice(-300)); +}); + +test('a breakdown of the right step and another item is not drawn under it (issue 5)', () => { + const data = example(); + const step = Object.keys(data.items.find((i) => i.id === 'p01').breakdowns)[0]; + const rice = data.items.find((i) => i.id === 'p05'); + rice.breakdowns = { [step]: 'p01-fix-1.json' }; + const drawn = dom.page(data, { 'p01-fix-1.json': wheat() }); + drawn.open('item=p05'); + const body = drawn.byId['item-body'].seen().join(' '); + assert.ok(body.includes('is of another product or another step: not drawn'), body.slice(-300)); +}); + +test('a file the page cannot draw draws nothing', () => { + const data = example(); + data.counts.total = null; + const drawn = dom.page(data); + assert.equal(drawn.byId.content.hidden, true); + assert.equal(drawn.byId.load.hidden, false); + assert.equal(drawn.byId.title.textContent, ''); +}); + +test('a host page never offers to open a file; a page of its own does', () => { + const host = dom.page(undefined); + assert.equal(host.byId.load.hidden, false); + assert.equal(host.byId['load-choose'].hidden, true); + assert.match(host.byId['load-error'].textContent, /No diagnostics file came with this page/); + const broken = example(); broken.counts.total = null; + assert.equal(dom.page(broken).byId['load-choose'].hidden, true); + const own = dom.page(undefined, undefined, { standalone: true }); + assert.equal(own.byId['load-choose'].hidden, false); + assert.equal(own.byId['load-error'].textContent, ''); +}); + +test('chips choose the comparison: the whole way first, then each step with what moved in it', () => { + const data = example(); + const drawn = dom.page(data); + const chips = () => drawn.byId.comparison.querySelectorAll('button[data-comp]'); + assert.deepEqual(chips().map((c) => c.getAttribute('data-comp')), ['total'].concat(data.steps.map((s) => s.id))); + assert.deepEqual(chips().map((c) => c.getAttribute('aria-pressed')), ['true'].concat(data.steps.map(() => 'false'))); + const moved = (comp) => Object.values(comp === 'total' ? data.counts.total : data.counts.steps[comp]) + .reduce((sum, b) => sum + ['differs', 'beyond 10x', 'zero reference', 'zero result', 'sign change'].reduce((a, k) => a + b.classes[k], 0), 0); + chips().forEach((chip) => assert.ok(chip.textContent.includes(moved(chip.getAttribute('data-comp')) + ' moved'), chip.textContent)); + // the unexplained step is flagged, the verdicts worth a look are marked + const rebuilt = data.steps.findIndex((s) => !s.explained); + assert.ok(rebuilt >= 0); + assert.ok(chips()[rebuilt + 1].textContent.includes('!')); + assert.match(chips()[rebuilt + 1].getAttribute('title'), /nothing explains it/); + const claimed = data.steps.findIndex((s) => s.claim && Object.values(data.counts.steps[s.id]).some((b) => b.verdicts['not claimed'])); + assert.ok(chips()[claimed + 1].textContent.includes('■'), chips()[claimed + 1].textContent); + // a click on a step's label chooses it, and the grid follows + const label = chips()[2].childNodes[1]; // the label inside the chip: the click reaches the chip + label.dispatch('click', {}); + assert.deepEqual(chips().map((c) => c.getAttribute('aria-pressed')), ['false', 'false', 'true'].concat(data.steps.slice(2).map(() => 'false'))); + assert.ok(drawn.byId['grid-region'].getAttribute('aria-label').endsWith(': Step 1: ' + data.steps[1].label + ', the furthest apart first'), + drawn.byId['grid-region'].getAttribute('aria-label')); +}); + +// ------------------------------------------------------------------ the matrix and the drawer + +const local = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'local', 'diagnostics.json'), 'utf8')); +const full = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'local', 'matrix.json'), 'utf8')); +const gridRows = (drawn) => drawn.byId['grid-table'].querySelectorAll('tr[data-item]'); + +test('the band the page names for a matrix row is the band of the Python rule', () => { + for (const c of cases) { + if (c.class !== 'differs' && c.class !== 'beyond 10x') continue; + assert.equal(page.bandOf(c.before, c.after, c.class, c.direction), c.band, c.id); + } +}); + +test('a ratio never reads 1 when it is not, nor Infinity', () => { + assert.equal(page.ratioText(2, 3), '1.5'); + assert.equal(page.ratioText(-2, -4), '2'); + assert.equal(page.ratioText(1, 1.0000003), '1 + 3e−7'); + assert.equal(page.ratioText(1, 0.9999), '1 − 1e−4'); + assert.equal(page.ratioText(1e-300, 1e300), '1e600'); + assert.equal(page.ratioText(1e300, 1e-300), '1e−600'); +}); + +test('the matrix the Python writes is one the page takes, and the rows are the file\'s', () => { + const data = local(); + assert.equal(page.checkMatrix(full(), data), null); + const rows = page.matrixRows(full(), data); + assert.equal(rows.length, data.items_count.all); + for (const item of data.items) { + const row = rows.find((r) => r.id === item.id); + assert.deepEqual(row.values, item.values, item.id); + assert.deepEqual(row.classes, item.classes, item.id); + assert.deepEqual(row.directions, item.directions, item.id); + assert.deepEqual(row.verdicts, item.verdicts, item.id); + } +}); + +test('a matrix of another file, or of another shape, is refused', () => { + const data = local(); + const other = full(); other.of.title = 'Another'; + assert.match(page.checkMatrix(other, data), /another file/); + const short = full(); short.classes[0] = short.classes[0].slice(1); + assert.match(page.checkMatrix(short, data), /class code/); + const wrong = full(); wrong.values[2][0] = 'x'; + assert.match(page.checkMatrix(wrong, data), /not a number/); + const fewer = full(); fewer.items.pop(); fewer.values.pop(); fewer.classes.pop(); fewer.verdicts.pop(); + assert.match(page.checkMatrix(fewer, data), /says 10 items/); +}); + +test('a matrix that says other numbers of a listed item than its file is refused', () => { + // a matrix left over from another run of the same chain: same title, states, steps and count + const data = local(); + const listed = data.items[0].id; + const at = full().items.findIndex((row) => row[0] === listed); + const tripled = full(); tripled.values[at] = tripled.values[at].map((v) => (typeof v === 'number' ? v * 3 : v)); + assert.match(page.checkMatrix(tripled, data), new RegExp('item ' + listed + ', .*: its results or classes differ')); + const reclassed = full(); + const code = reclassed.classes[at][0]; + const other = [...reclassed.classes.join('')].find((c) => c !== code); // another class the matrix itself uses + reclassed.classes[at] = other + reclassed.classes[at].slice(1); + assert.match(page.checkMatrix(reclassed, data), /its results or classes differ/); + const renamed = full(); renamed.items[at][1] = 'Another name'; + assert.match(page.checkMatrix(renamed, data), /its name, place, group or unit differ/); + const swapped = full(); swapped.items[at][0] = 'not-listed'; + assert.match(page.checkMatrix(swapped, data), /1 of the items this file lists are not in it/); + // an item the file does not list has nothing to be held to + const unlisted = full(); + const free = unlisted.items.findIndex((row) => !data.items.some((item) => item.id === row[0])); + unlisted.values[free] = unlisted.values[free].map((v) => (typeof v === 'number' ? v * 3 : v)); + assert.equal(page.checkMatrix(unlisted, data), null); +}); + +test('the grid shows the listed items, says so, and every item once the matrix is here', () => { + const data = local(); + const before = dom.page(data); // over file: the matrix.json the file names cannot be read + assert.equal(gridRows(before).length, data.items.length); + assert.match(before.byId['matrix-said'].textContent, /not loaded: matrix\.json is read over http only/); + assert.match(before.byId['grid-said'].textContent, new RegExp('of ' + data.items.length + ' products, among the ' + data.items.length + ' products in this file; ' + data.items_count.not_listed + ' more')); + const after = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: full() } }); + assert.equal(gridRows(after).length, data.items_count.all); + assert.match(after.byId['matrix-said'].textContent, /matrix of all 10 products is loaded/); + assert.match(before.byId.tier.textContent, /6 of the \d+ products worth a look are in this file; \d+ are not\./); + assert.match(after.byId.tier.textContent, /All 10 are loaded from the matrix; 6 of the \d+ worth a look are listed in this file\./); + assert.equal(after.byId['grid-table'].all().filter((n) => n.className === 'sd-listed').length, data.items.length); +}); + +test('a matrix beside the page is loaded as a script after the page is drawn', () => { + const drawn = dom.page(local(), undefined, { attributes: { 'data-sd-matrix': 'matrix.js', 'data-sd-base': 'runs/1/' } }); + const script = drawn.head.querySelector('script[src]'); + assert.equal(script.getAttribute('src'), 'runs/1/matrix.js'); + assert.match(drawn.byId['matrix-said'].textContent, /^Loading/); + drawn.context.SENTIER_DIAGNOSTICS_MATRIX = full(); + script.onload(); + assert.equal(gridRows(drawn).length, 10); +}); + +test('a matrix is fetched as JSON over http, and one of another file is not drawn', async () => { + const served = (m) => () => Promise.resolve({ ok: true, status: 200, text: () => Promise.resolve(JSON.stringify(m)) }); + const drawn = dom.page(local(), undefined, { protocol: 'https:', fetch: served(full()) }); + await new Promise((done) => setImmediate(done)); + assert.equal(gridRows(drawn).length, 10); + const other = full(); other.of.steps = other.of.steps.slice().reverse(); + const refused = dom.page(local(), undefined, { protocol: 'https:', fetch: served(other) }); + await new Promise((done) => setImmediate(done)); + assert.equal(gridRows(refused).length, 6); + assert.match(refused.byId['matrix-said'].textContent, /not loaded: it is the matrix of another file/); +}); + +test('a grid cell is the ratio after ÷ before, coloured by its direction', () => { + const drawn = dom.page(example()); + const wheat = gridRows(drawn).find((tr) => tr.getAttribute('data-item') === 'p01'); + const item = example().items.find((i) => i.id === 'p01'); + const values = item.values['x:climate']; + const cell = wheat.childNodes[2]; // name, group, then the first indicator: no unit column + assert.equal(item.classes['x:climate'][4], 'differs'); + assert.equal(cell.textContent, page.ratioShort(values[0], values[values.length - 1])); + assert.match(cell.className, new RegExp('tone-' + item.directions['x:climate'][4])); + const heads = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').map((th) => th.textContent); + assert.equal(heads.length, 2 + example().indicators.length); + assert.ok(!heads.includes('Unit')); +}); + +test('a ratio in the grid has three significant digits, and never "Infinity"', () => { + assert.equal(page.ratioShort(1, 1), '1.00'); + assert.equal(page.ratioShort(1, 1.0000003), '1.00'); + assert.equal(page.ratioShort(1, 0.984), '0.984'); + assert.equal(page.ratioShort(-2, -4.54), '2.27'); + assert.equal(page.ratioShort(1, 25), '25.0'); + assert.equal(page.ratioShort(1, 999.7), '1e3'); + assert.equal(page.ratioShort(1, 12345), '1.23e4'); + assert.equal(page.ratioShort(1, 0.004), '4e−3'); + assert.equal(page.ratioShort(1e-300, 1e300), '1e600'); + const noise = page.ratioView(1, 1.001, 'within noise', 'up', null); + assert.deepEqual([noise.text, noise.mark, noise.tone], ['1.00', '~', 'noise']); + const agrees = page.ratioView(2, 2, 'agrees', null, null); + assert.deepEqual([agrees.text, agrees.tone, agrees.strength], ['1.00', 'none', 0]); + assert.equal(page.ratioView(0, 0, 'agrees', null, null).text, '='); + assert.equal(page.ratioView(-1, 2, 'sign change', 'up', null).text, '− → +'); +}); + +test('every grid cell with a ratio shows it, agreeing ones too; the class is the tint and a mark', () => { + const data = example(); + const drawn = dom.page(data); + let agrees = 0, noise = 0; + for (const tr of gridRows(drawn)) { + const item = data.items.find((i) => i.id === tr.getAttribute('data-item')); + data.indicators.forEach((ind, at) => { + const cell = tr.childNodes[2 + at]; + const v = item.values[ind.id], before = v[0], after = v[v.length - 1], cls = item.classes[ind.id][4]; + const shared = before !== null && after !== null && before !== 0 && after !== 0 && (before > 0) === (after > 0); + if (shared && ['agrees', 'within noise', 'differs', 'beyond 10x'].includes(cls)) { + assert.ok(cell.textContent.startsWith(page.ratioShort(before, after)), item.id + ' ' + ind.id + ': ' + cell.textContent); + if (cls === 'agrees') { agrees += 1; assert.match(cell.className, /tone-none/); } + if (cls === 'within noise') { noise += 1; assert.match(cell.textContent, /~$/); } + } else { + assert.doesNotMatch(cell.textContent, /^\d/, item.id + ' ' + ind.id + ' (' + cls + '): ' + cell.textContent); + } + assert.notEqual(cell.textContent, '='); + }); + } + assert.ok(agrees > 5, 'agreeing cells were checked'); +}); + +test('the grid opens with the items that lie furthest apart, and a second click goes back to that', () => { + const data = example(); + data.items.reverse(); // the file lists them worst first already: the page must order them itself + const drawn = dom.page(data); + // the largest factor among the values that moved with a ratio first, then the worst value + const worst = (id) => { + const i = data.items.find((x) => x.id === id); + let furthest = 0; + const keys = data.indicators.map((ind) => { + const v = i.values[ind.id], cls = i.classes[ind.id][4]; + if (cls === 'differs' || cls === 'beyond 10x') furthest = Math.max(furthest, Math.abs(Math.log10(Math.abs(v[v.length - 1] / v[0])))); + return page.severityKey(v[0], v[v.length - 1], cls); + }); + return [-furthest].concat(keys.sort(page.compareKeys)[0]); + }; + const order = () => gridRows(drawn).map((tr) => tr.getAttribute('data-item')); + const first = order(); + for (let k = 1; k < first.length; k++) assert.ok(page.compareKeys(worst(first[k - 1]), worst(first[k])) <= 0, first.join()); + assert.notDeepEqual(first, data.items.map((i) => i.id), 'the example is not already in that order'); + assert.match(drawn.byId['grid-region'].getAttribute('aria-label'), /, the furthest apart first$/); + const button = () => drawn.byId['grid-table'].querySelectorAll('th[scope="col"]')[0].querySelector('button[type="button"]'); + button().dispatch('click', {}); + assert.doesNotMatch(drawn.byId['grid-region'].getAttribute('aria-label'), /furthest apart first/); + button().dispatch('click', {}); + assert.deepEqual(order(), first); +}); + +test('a heading sorts the grid by how far the two lie apart, the furthest first, and says so', () => { + const data = example(); + const drawn = dom.page(data); + const head = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]'); + const water = head.find((th) => th.getAttribute('title') && th.getAttribute('title').indexOf('Water') === 0); + water.querySelector('button[type="button"]').dispatch('click', {}); + const after = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').find((th) => th.getAttribute('aria-sort')); + assert.equal(after.getAttribute('aria-sort'), 'descending'); + const order = gridRows(drawn).map((tr) => tr.getAttribute('data-item')); + const key = (id) => { const i = data.items.find((x) => x.id === id); const v = i.values['x:water']; return page.severityKey(v[0], v[v.length - 1], i.classes['x:water'][4]); }; + for (let k = 1; k < order.length; k++) assert.ok(page.compareKeys(key(order[k - 1]), key(order[k])) <= 0, order.join()); +}); + +test('the grid draws 300 rows at a time', () => { + const data = local(); + const m = full(); + const n = 650; + const grown = { ...m, items: [], values: [], classes: [], verdicts: [] }; + for (let i = 0; i < n; i++) { + const at = i % m.items.length; + grown.items.push(i < m.items.length ? m.items[at] : [m.items[at][0] + '-' + i].concat(m.items[at].slice(1))); + grown.values.push(m.values[at]); grown.classes.push(m.classes[at]); grown.verdicts.push(m.verdicts[at]); + } + data.matrix.items = n; + const drawn = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: grown } }); + assert.equal(gridRows(drawn).length, 300); + assert.equal(drawn.byId['grid-more'].textContent, 'Show 300 more'); + drawn.byId['grid-more'].dispatch('click', {}); + drawn.byId['grid-more'].dispatch('click', {}); + assert.equal(gridRows(drawn).length, n); + assert.equal(drawn.byId['grid-more'].hidden, true); +}); + +test('the drawer opens on an item of the matrix, says why first, and closes with Escape', () => { + const data = local(); + const drawn = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: full() } }); + const unlisted = full().items.map((r) => r[0]).find((id) => !data.items.some((i) => i.id === id)); + gridRows(drawn).find((tr) => tr.getAttribute('data-item') === unlisted).childNodes[0].dispatch('click', {}); + assert.equal(drawn.context.location.hash, 'item=' + unlisted); + drawn.open('item=' + unlisted); + assert.equal(drawn.byId.item.hidden, false); + assert.match(drawn.byId['item-meta'].textContent, /from the matrix: not listed in this file/); + assert.equal(drawn.byId['item-body'].childNodes[0].textContent, 'Why it differs'); + assert.equal(drawn.byId['item-title'].focused, true); + drawn.root.dispatch('keydown', { key: 'Escape' }); + assert.equal(drawn.byId.item.hidden, true); +}); + +test('why it differs: the item\'s own explanations, then those of a step or indicator it moved in, never a "none" over a part', () => { + const data = example(); + const drawn = dom.page(data); + for (const item of data.items) { + drawn.open('item=' + item.id); + const why = drawn.byId['item-body'].childNodes[1]; + const own = data.explanations.filter((e) => e.about.item === item.id).length; + const scopes = why.childNodes.map((li) => (li.childNodes[0].className === 'sd-scope' ? li.childNodes[0].textContent : 'none')); + assert.equal(scopes.filter((s) => s === 'This product').length, own, item.id); + assert.ok(scopes.indexOf('This product') <= 0 || scopes.slice(0, own).every((s) => s === 'This product'), item.id + ': its own first'); + if (scopes.includes('none')) assert.equal(why.childNodes.length, 1, item.id + ': "none" only where nothing is given'); + } +}); + +test('the page sets the title of a page of its own only, never a host\'s', () => { + assert.equal(dom.page(example()).document.title, ''); + assert.equal(dom.page(example(), undefined, { standalone: true }).document.title, example().title); +}); + +test('the data can come from a global, as a site\'s own data.js sets it', () => { + const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }, { globals: {} }); + assert.equal(drawn.byId.content.hidden, false); + assert.equal(drawn.byId.unchecked.hidden, true); +}); + +test('the page\'s own sentences say the file\'s word for an item', () => { + const data = example(); + data.item = { one: 'dataset', many: 'datasets' }; + const drawn = dom.page(data); + const words = drawn.root.querySelectorAll('[data-sd-word]'); + assert.ok(words.length >= 4); + const items = words.filter((span) => ['one', 'many'].includes(span.getAttribute('data-sd-word'))); + assert.ok(items.length >= 4); + for (const span of items) assert.equal(span.textContent, span.getAttribute('data-sd-word') === 'many' ? 'datasets' : 'dataset'); + assert.equal(drawn.byId['item-close'].getAttribute('aria-label'), 'Close the dataset'); + assert.ok(drawn.byId['grid-region'].getAttribute('aria-label').startsWith('After divided by before, per dataset and indicator: The whole way')); +}); + +test('the page says the file\'s word for an indicator, and "indicator" where it gives none', () => { + const plain = dom.page(example()); + const said = (drawn, value) => drawn.root.querySelectorAll('[data-sd-word]') + .filter((span) => span.getAttribute('data-sd-word') === value).map((span) => span.textContent); + assert.deepEqual([...new Set(said(plain, 'indicator'))], ['indicator']); + const data = example(); + data.indicator = { one: 'impact category', many: 'impact categories' }; + const drawn = dom.page(data); + assert.equal(drawn.byId.content.hidden, false); + assert.ok(said(drawn, 'indicator').length >= 3); + assert.deepEqual([...new Set(said(drawn, 'indicator'))], ['impact category']); + assert.deepEqual(said(drawn, 'Indicator'), ['Impact category']); + assert.ok(drawn.byId['grid-region'].getAttribute('aria-label').startsWith('After divided by before, per ' + data.item.one + ' and impact category: ')); + const text = drawn.text(); + assert.match(text, new RegExp(data.indicators.length + ' impact categories\\. ')); + assert.match(text, /One floor per impact category and reference unit/); + assert.doesNotMatch(text.replace(data.about, ''), /\bindicators?\b/i); +}); + +test('a page refuses an indicator word that is not {one, many}', () => { + const data = example(); + data.indicator = { one: 'impact category' }; + const drawn = dom.page(data); + assert.equal(drawn.byId.content.hidden, true); + assert.match(drawn.text(), /indicator word of the file is not \{one, many\}/); +}); + +test('steps count from 1 where the chain does not start from the published results', () => { + const published = example(); + assert.equal(published.states[0].kind, 'published'); + const first = dom.page(published); + assert.match(first.text(), /Step 0: /); + assert.equal(first.byId['steps-zero'].hidden, false); + const data = example(); + data.states[0].kind = 'build'; + const drawn = dom.page(data); + assert.equal(drawn.byId.content.hidden, false); + assert.equal(drawn.byId['steps-zero'].hidden, true); + const text = drawn.text(); + assert.doesNotMatch(text, /Step 0\b/); + data.steps.forEach((s, k) => assert.ok(text.includes('Step ' + (k + 1) + ': ' + s.label), s.label)); + const rows = drawn.byId['steps-table'].all().filter((n) => n.tagName === 'TR' && n.firstChild && n.firstChild.tagName === 'TD'); + assert.deepEqual(rows.map((row) => row.firstChild.textContent), data.steps.map((s, k) => String(k + 1))); + drawn.open('item=' + encodeURIComponent(data.items[0].id)); + const drawer = drawn.byId['item-body'].seen().join(' '); + assert.ok(drawer.includes('Step 1: ' + data.steps[0].label), drawer.slice(0, 200)); + assert.doesNotMatch(drawer, /Step 0\b/); +}); + + +// ------------------------------------------------------------------ reviewing one step + +const chips = (drawn) => drawn.byId.comparison.querySelectorAll('button[data-comp]'); + +test('choosing a step says its claim and what the claim found, under the chips, and the address keeps it', () => { + const data = example(); + const drawn = dom.page(data); + assert.match(drawn.byId['step-said'].textContent, /^The whole way: /); + const k = data.steps.findIndex((s) => s.claim && s.claim.scope.length); + const step = data.steps[k]; + chips(drawn)[k + 1].dispatch('click', {}); + const said = drawn.byId['step-said'].textContent; + assert.ok(said.includes(step.claim.text), said); + // what the claim found, from the file's own counts + const look = ['off claim', 'not claimed', 'claimed, not moved']; + const found = look.filter((v) => Object.values(data.counts.steps[step.id]).some((b) => b.verdicts[v])); + if (found.length) found.forEach((v) => assert.ok(said.includes(v), v)); + else assert.match(said, /as claimed/); + assert.equal(drawn.context.location.hash, 'step=' + encodeURIComponent(step.id)); + assert.equal(chips(drawn)[k + 1].getAttribute('aria-pressed'), 'true'); +}); + +test('the matrix shows first the indicators in which the chosen comparison moved most', () => { + const data = example(); + const drawn = dom.page(data); + const moves = ['differs', 'beyond 10x', 'zero reference', 'zero result', 'sign change']; + // what moved, and the missing values with it: missing is a finding, not stillness + const moved = (comp, id) => moves.concat(['missing']).reduce((m, c) => m + (comp === 'total' ? data.counts.total : data.counts.steps[comp])[id].classes[c], 0); + for (const comp of ['total'].concat(data.steps.map((s) => s.id))) { + drawn.byId.comparison.querySelectorAll('button[data-comp="' + comp + '"]')[0].dispatch('click', {}); + const heads = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').slice(2); + const counts = heads.map((th) => moved(comp, data.indicators.find((i) => th.getAttribute('title').indexOf(i.name) === 0).id)); + for (let k = 1; k < counts.length; k++) assert.ok(counts[k - 1] >= counts[k], comp + ': ' + counts.join()); + heads.forEach((th, k) => assert.equal(/sd-still/.test(th.className), counts[k] === 0, comp)); + // the cells of a quiet column are quiet too + const row = gridRows(drawn)[0]; + const cells = row.childNodes.filter((n) => n.tagName === 'TD').slice(1); // after the group + cells.forEach((td, k) => assert.equal(/sd-still/.test(td.className), counts[k] === 0, comp + ' cell ' + k)); + } +}); + +test('an address with a step opens the page on that step, and with an item on that item', () => { + const data = example(); + const step = data.steps[2]; + const drawn = dom.page(data); + drawn.open('step=' + encodeURIComponent(step.id) + '&item=' + encodeURIComponent(data.items[0].id)); + assert.equal(chips(drawn).find((c) => c.getAttribute('aria-pressed') === 'true').getAttribute('data-comp'), step.id); + assert.equal(drawn.byId.item.hidden, false); + // closing the item keeps the step + drawn.byId['item-close'].dispatch('click', {}); + assert.equal(drawn.byId.item.hidden, true); + assert.equal(drawn.context.location.hash, 'step=' + encodeURIComponent(step.id)); + // an unknown step falls back to the whole way + drawn.open('step=nothing-of-the-kind'); + assert.equal(chips(drawn)[0].getAttribute('aria-pressed'), 'true'); +}); + +test('the arrow keys move between the chips', () => { + const data = example(); + const drawn = dom.page(data); + drawn.byId.comparison.dispatch('keydown', { key: 'ArrowRight' }); + assert.equal(chips(drawn)[1].getAttribute('aria-pressed'), 'true'); + drawn.byId.comparison.dispatch('keydown', { key: 'ArrowLeft' }); + assert.equal(chips(drawn)[0].getAttribute('aria-pressed'), 'true'); + drawn.byId.comparison.dispatch('keydown', { key: 'ArrowLeft' }); // nothing before the whole way + assert.equal(chips(drawn)[0].getAttribute('aria-pressed'), 'true'); +}); + +const cards = (drawn) => drawn.byId['dist-cards'].childNodes; + +test('a distribution card per indicator, in the order of the matrix, its bars the file\'s counts', () => { + const data = example(); + const drawn = dom.page(data); + for (const comp of ['total'].concat(data.steps.map((s) => s.id))) { + drawn.byId.comparison.querySelectorAll('button[data-comp="' + comp + '"]')[0].dispatch('click', {}); + const heads = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').slice(2).map((th) => th.getAttribute('title')); + const ids = cards(drawn).map((card) => card.getAttribute('data-indicator')); + assert.equal(ids.length, data.indicators.length); + ids.forEach((id, k) => assert.ok(heads[k].indexOf(data.indicators.find((i) => i.id === id).name) === 0, comp + ' ' + k)); + for (const card of cards(drawn)) { + const block = (comp === 'total' ? data.counts.total : data.counts.steps[comp])[card.getAttribute('data-indicator')]; + const bars = card.childNodes[2].childNodes; + const sum = bars.reduce((s, bar) => s + Number(bar.getAttribute('title').match(/: ([\d,]+) of/)[1].replace(/,/g, '')), 0); + // every pair is in one bar: the bands split what moved and what agrees, the rest is a class + assert.equal(sum, block.pairs, comp + ' ' + card.getAttribute('data-indicator')); + } + } +}); + +test('a median of after ÷ before only from every item, never from the listed ones', () => { + const data = local(); + const listed = dom.page(data); + for (const card of cards(listed)) assert.doesNotMatch(card.textContent, /Median/); + assert.match(listed.byId['dist-said'].textContent, /need every product/); + const loaded = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: full() } }); + const rows = page.matrixRows(full(), data); + const k = data.steps.length; // the whole way + const card = cards(loaded)[0]; + const id = card.getAttribute('data-indicator'); + const ratios = rows.filter((r) => ['agrees', 'within noise', 'differs', 'beyond 10x'].includes(r.classes[id][k])) + .map((r) => [r.values[id][0], r.values[id][r.values[id].length - 1]]) + .filter(([a, b]) => a && b && (a > 0) === (b > 0)).map(([a, b]) => b / a).sort((a, b) => a - b); + const x = 0.5 * (ratios.length - 1), lo = Math.floor(x), hi = Math.ceil(x); + const median = ratios[lo] + (ratios[hi] - ratios[lo]) * (x - lo); + assert.ok(ratios.length > 0); + const shown = Number(card.textContent.match(/Median of [\d ]+ ratios? ([\d.e−]+)/)[1].replace('−', '-')); + assert.ok(Math.abs(shown - median) <= 5e-3 * Math.abs(median), shown + ' vs ' + median); +}); + +test('percentiles share one precision: the fewest digits that keep apart the ones that differ', () => { + // a spread only the fifth digit shows + const text = page.quantileTexts({ p10: 0.99981, median: 1.0, p90: 1.00042 }); + assert.notEqual(text[0], text[2]); + assert.deepEqual(text, ['0.99981', '1.0000', '1.0004']); + // never mixed: a 10th percentile that reads as exactly 1 beside a six-digit median + assert.deepEqual(page.quantileTexts({ p10: 1.00004, median: 1.00005, p90: 1.00006 }), ['1.00004', '1.00005', '1.00006']); + assert.deepEqual(page.quantileTexts({ p10: 0.999, median: 1, p90: 1.0001 }), ['0.99900', '1.0000', '1.0001']); + assert.deepEqual(page.quantileTexts({ p10: 0.5, median: 1, p90: 2 }), ['0.500', '1.00', '2.00']); + assert.deepEqual(page.quantileTexts({ p10: 1, median: 1, p90: 1 }), ['1.00', '1.00', '1.00']); +}); + +test('a card sorts the matrix by its indicator, and a heading marks its card', () => { + const data = example(); + const drawn = dom.page(data); + const card = cards(drawn)[1]; + const id = card.getAttribute('data-indicator'); + card.childNodes[0].childNodes[0].dispatch('click', {}); + const sorted = drawn.byId['grid-table'].querySelectorAll('th[aria-sort]'); + assert.equal(sorted.length, 1); + assert.ok(sorted[0].getAttribute('title').indexOf(data.indicators.find((i) => i.id === id).name) === 0); + assert.equal(cards(drawn)[1].childNodes[0].childNodes[0].getAttribute('aria-pressed'), 'true'); + assert.match(cards(drawn)[1].className, /sd-chosen/); + // the heading of another indicator moves the mark to its card + const other = cards(drawn)[0].getAttribute('data-indicator'); + const head = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').slice(2) + .find((th) => th.getAttribute('title').indexOf(data.indicators.find((i) => i.id === other).name) === 0); + head.childNodes[0].dispatch('click', {}); + assert.equal(cards(drawn)[0].childNodes[0].childNodes[0].getAttribute('aria-pressed'), 'true'); + assert.equal(cards(drawn)[1].childNodes[0].childNodes[0].getAttribute('aria-pressed'), 'false'); +}); + +test('the bars of every card can be put on one scale, the share of all values (A21)', () => { + const data = example(); + const drawn = dom.page(data); + const heights = () => cards(drawn).map((card) => card.childNodes[2].childNodes.map((bar) => parseFloat(/height:([\d.]+)%/.exec(bar.getAttribute('style'))[1]))); + // each card to its own tallest bar: every card that has values reaches the top + heights().forEach((h) => assert.equal(Math.max(...h), 100)); + const select = drawn.byId['dist-scale']; + select.value = 'one'; + select.dispatch('change', { target: select }); + const one = heights(); + assert.equal(Math.max(...one.flat()), 100); + // on one scale, a bar's height is its share of the values of its card, the same factor in every card + const shares = cards(drawn).map((card, i) => { + const block = data.counts.total[card.getAttribute('data-indicator')]; + // a bar is at least 2% high, so that one value is seen: those are left out + return card.childNodes[2].childNodes.map((bar, j) => { + const n = Number(bar.getAttribute('title').match(/: ([\d,]+) of/)[1].replace(/,/g, '')); + return n && one[i][j] > 2 ? one[i][j] / (n / block.pairs) : null; + }).filter((x) => x !== null); + }).flat(); + for (const f of shares) assert.ok(Math.abs(f - shares[0]) < 0.06 * shares[0], f + ' vs ' + shares[0]); + assert.match(drawn.byId['dist-said'].textContent, /one scale/); +}); + +test('the claim line names every verdict on a move, "as claimed" first, in words and without colour', () => { + const data = example(); + const drawn = dom.page(data); + const k = data.steps.findIndex((s) => s.claim && s.claim.scope.length); + const step = data.steps[k]; + chips(drawn)[k + 1].dispatch('click', {}); + const said = drawn.byId['step-said'].textContent; + const total = (v) => Object.values(data.counts.steps[step.id]).reduce((a, b) => a + b.verdicts[v], 0); + for (const v of ['as claimed', 'off claim', 'not claimed', 'claimed, not moved']) { + const word = v === 'as claimed' ? 'moved as claimed' : v; + if (total(v)) assert.ok(said.includes(total(v) + ' value' + (total(v) === 1 ? '' : 's') + ' ' + word), v + ': ' + said); + else assert.ok(!said.includes(word), v); + } + if (total('as claimed')) assert.ok(said.indexOf('moved as claimed') < Math.max(said.indexOf('off claim'), said.indexOf('not claimed')) || !/off claim|not claimed/.test(said)); + assert.doesNotMatch(said, /moves as claimed/); + const all = drawn.byId['step-said'].all(); + assert.ok(!all.some((node) => /sd-good|sd-look/.test(node.className || ''))); +}); + +test('a column whose only change is a missing value is not quietened as if nothing moved', () => { + const data = example(); + const step = data.steps[0].id; + const id = data.indicators[data.indicators.length - 1].id; + const block = data.counts.steps[step][id]; + // nothing moved in this indicator, two values went missing + for (const c of ['differs', 'beyond 10x', 'zero reference', 'zero result', 'sign change']) block.classes[c] = 0; + block.classes.missing = 2; + const drawn = dom.page(data); + drawn.byId.comparison.querySelectorAll('button[data-comp="' + step + '"]')[0].dispatch('click', {}); + const name = data.indicators.find((i) => i.id === id).name; + const head = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').slice(2).find((th) => th.getAttribute('title').indexOf(name) === 0); + assert.doesNotMatch(head.className, /sd-still/); + assert.match(head.getAttribute('title'), /2 missing/); + const card = cards(drawn).find((c) => c.getAttribute('data-indicator') === id); + assert.doesNotMatch(card.className, /sd-still/); +}); + +test('two percentiles that six digits cannot part are said to lie just under or over the median', () => { + const t = page.quantileTexts({ p10: 1 - 1e-9, median: 1, p90: 5.94545 }); + assert.equal(t[0], 'just under the median'); + assert.notEqual(t[2], t[1]); + assert.equal(t[2], '5.95'); // the pair (median, 90th) never blurred + assert.equal(t[1], '1.00'); // the words took the place of the six digits: the rest go back to three + // only the pair that blurs is widened: 5.94545 needs no six digits beside 1 + assert.equal(page.quantileTexts({ p10: 0.5, median: 1, p90: 1 + 1e-9 })[0], '0.500'); + assert.equal(page.quantileTexts({ p10: 0.5, median: 1, p90: 1 + 1e-9 })[2], 'just over the median'); +}); + +test('a card says over how many ratios its median is, how many have none, and what full height is', () => { + const data = local(); + const loaded = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: full() } }); + for (const card of cards(loaded)) { + const text = card.textContent; + const block = data.counts.total[card.getAttribute('data-indicator')]; + const m = text.match(/Median of ([\d ]+) ratios?/); + if (!m) { assert.match(text, /No value with a ratio/); continue; } + const n = Number(m[1].replace(/ /g, '')); + const left = block.pairs - n; + if (left) assert.ok(text.includes(left + ' without a ratio'), text); + else assert.doesNotMatch(text, /without a ratio/); + assert.match(card.childNodes[1].textContent, /^tallest bar: \d/); + } +}); diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py new file mode 100644 index 0000000..a7cc39b --- /dev/null +++ b/tests/test_acceptance.py @@ -0,0 +1,302 @@ +"""The acceptance cases of SPEC.md: one test (or one shared case) for each defect found in +the tools this one replaces, and for each rule the sessions asked for. The pair cases in +`acceptance/cases.json` are run here against the Python rule and in `js/page.test.js` +against the page script.""" + +from __future__ import annotations + +import json +import math +import tempfile +import unittest +from pathlib import Path + +from sentier_diagnostics import breakdown, chain, claims, example, fileformat, rule, summary, tables + +from . import helpers +from .helpers import A, B + +HERE = Path(__file__).parent +CASES = json.loads((HERE / "acceptance" / "cases.json").read_text(encoding="utf-8"))["cases"] + + +class PairCases(unittest.TestCase): + """A1, A2, A3, A4, A5, A7, A10, A11: the shared cases, against the rule.""" + + def test_every_case(self): + self.assertGreaterEqual(len(CASES), 15) + for case in CASES: + with self.subTest(case["id"]): + found = rule.judge(case["before"], case["after"], floor=case["floor"], + tolerance=case["tolerance"], noise=case["noise"]) # fmt: skip + self.assertEqual(found, (case["class"], case["direction"], case["band"]), case["from"]) + rank, far = rule.severity(case["before"], case["after"], case["class"]) + self.assertEqual(rank, case["severity"][0]) + # log10 may differ in its last digit between C libraries (Linux, macOS) + self.assertTrue(math.isclose(far, case["severity"][1], rel_tol=1e-12, abs_tol=1e-300)) + + def test_every_class_has_a_case(self): + self.assertEqual({c["class"] for c in CASES}, set(rule.CLASSES)) + + +class Acceptance(unittest.TestCase): + def test_a01_direction_is_the_sign_of_after_minus_before(self): + """A1: going further below zero is down, whatever the ratio says.""" + for before, after, way in ((-2.0, -4.0, "down"), (-4.0, -2.0, "up"), (2.0, 4.0, "up"), (4.0, 2.0, "down")): + kind = rule.classify(before, after) + self.assertEqual(rule.direction(before, after, kind), way) + + def test_a02_the_file_carries_the_class_and_the_floor(self): + """A2: what is listed as moved is what the rule counted with its floor; a page never + recounts. A value below its floor in a file is near zero, and the validator refuses + a file that says otherwise.""" + data = helpers.two_states({A: {"p": 1e-12, "q": 1.0}}, {A: {"p": 5e-12, "q": 2.0}}, + floors={(A, "kg"): 1e-9}) # fmt: skip + listed = {i["id"]: i for i in data["items"]} + self.assertEqual(listed["p"]["classes"][A], [rule.NEAR_ZERO, rule.NEAR_ZERO]) + self.assertEqual(data["counts"]["steps"]["one"][A]["classes"]["near zero"], 1) + floors = {(f["indicator"], f["unit"]): f for f in data["rule"]["floors"]} + self.assertEqual(floors[(A, "kg")]["value"], 1e-9) + self.assertEqual(floors[(A, "kg")]["basis"], "declared") + + def test_a03_solver_noise_agrees(self): + """A3: 1e-6 is within the default tolerance; measured noise has a class of its own.""" + self.assertEqual(rule.classify(1.0, 1.000001), rule.AGREES) + self.assertEqual(rule.classify(1.0, 1.0 + 2.8e-7), rule.AGREES) # two runners, measured + self.assertEqual(rule.classify(1.0, 1.0002, noise=3e-4), rule.WITHIN_NOISE) + self.assertEqual(rule.classify(1.0, 1.0002), rule.DIFFERS) # without the noise, a change + + def test_a04_beyond_ten_is_never_blanked(self): + """A4: a factor of 25 is a class and a band, and its item is listed.""" + data = helpers.two_states({A: {"p": 1.0}}, {A: {"p": 25.0}}) + self.assertEqual(data["counts"]["total"][A]["classes"][rule.BEYOND], 1) + self.assertEqual(data["items"][0]["classes"][A], [rule.BEYOND, rule.BEYOND]) + self.assertEqual(data["items"][0]["values"][A], [1.0, 25.0]) + + def test_a05_near_zero_is_not_agreement(self): + """A5: two results below the floor are near zero, a class of their own.""" + self.assertEqual(rule.classify(1e-10, 3e-10, floor=1e-9), rule.NEAR_ZERO) + data = helpers.two_states({A: {"p": 0.0, "q": 1.0}}, {A: {"p": 0.0, "q": 1.0}}) + classes = data["counts"]["total"][A]["classes"] + self.assertEqual((classes["near zero"], classes["agrees"]), (1, 1)) + + def test_a06_floors_are_kept_per_reference_unit(self): + """A6: a result per machine hour and a result per kilogram are never pooled.""" + before = {A: {"kg1": 100.0, "kg2": 200.0, "mj1": 0.001, "mj2": 0.002}} + after = {A: {"kg1": 100.0, "kg2": 200.0, "mj1": 0.002, "mj2": 0.002}} + units = {"kg1": "kg", "kg2": "kg", "mj1": "MJ", "mj2": "MJ"} + data = helpers.two_states(before, after, units=units) + floors = {(f["indicator"], f["unit"]): f["value"] for f in data["rule"]["floors"]} + self.assertAlmostEqual(floors[(A, "kg")], 1.5) + self.assertAlmostEqual(floors[(A, "MJ")], 1.5e-5) + mj1 = next(i for i in data["items"] if i["id"] == "mj1") + self.assertEqual(mj1["classes"][A][0], rule.DIFFERS) # pooled, it would be near zero + + def test_a07_missing_is_never_zero(self): + """A7: a result missing on one side is missing, in the counts and in the list.""" + data = helpers.two_states({A: {"p": 0.5, "q": 1.0}}, {A: {"p": None, "q": 1.0}}) + block = data["counts"]["steps"]["one"][A] + self.assertEqual(block["classes"]["missing"], 1) + self.assertEqual(block["missing"], {"before": 0, "after": 1, "both": 0}) + self.assertEqual(data["items"][0]["values"][A], [0.5, None]) + self.assertEqual(rule.classify(None, 0.0), rule.MISSING) + + def test_a08_columns_are_read_by_name(self): + """A8: two tables with their columns in another order are read alike.""" + with tempfile.TemporaryDirectory() as folder: + one, two = Path(folder) / "one.csv", Path(folder) / "two.csv" + one.write_text("item,x:a,x:b\np,1.5,2\nq,,3\n", encoding="utf-8") + two.write_text("x:b,item,x:a\n2,p,1.5\n3,q,\n", encoding="utf-8") + self.assertEqual(tables.read_results(one, [A, B]), tables.read_results(two, [A, B])) + + def test_a09_every_explanation_says_who_wrote_it(self): + """A9: an explanation without its origin is refused; a model's names the model and + who ran it, and the page says that it is a claim nobody checked.""" + data = example.build() + self.assertEqual(fileformat.errors(data), []) + unsigned = helpers.mutated(data, lambda d: d["explanations"][0].update(by=None)) + self.assertTrue(any("says who wrote it" in e for e in fileformat.errors(unsigned))) + model = next(k for k, e in enumerate(data["explanations"]) if e["by"]["kind"] == "model") + half = helpers.mutated(data, lambda d: d["explanations"][model]["by"].update(run_by=None)) + self.assertTrue(any("who ran it" in e for e in fileformat.errors(half))) + # the page draws it as a claim nobody checked: node "a text by a model is drawn as a claim" + + def test_a10_zero_and_sign_change_are_apart(self): + self.assertEqual(rule.classify(0.0, 1.0), rule.ZERO_REFERENCE) + self.assertEqual(rule.classify(1.0, 0.0), rule.ZERO_RESULT) + self.assertEqual(rule.classify(-1.0, 1.0), rule.SIGN_CHANGE) + self.assertIsNone(rule.band(-1.0, 1.0, rule.SIGN_CHANGE)) + + def test_a11_results_are_not_rounded_before_the_ratio(self): + """A11: 0.00104 and 0.00096 are both 0.001 at three decimals, and 8 % apart.""" + self.assertEqual(rule.classify(0.00104, 0.00096), rule.DIFFERS) + data = helpers.two_states({A: {"p": 0.00104}}, {A: {"p": 0.00096}}) + self.assertEqual(data["items"][0]["values"][A], [0.00104, 0.00096]) + text = chain.dumps(data) + self.assertIn("0.00104", text) + + def test_a12_not_a_number_is_refused(self): + with self.assertRaises(ValueError): + helpers.two_states({A: {"p": math.nan}}, {A: {"p": 1.0}}) + with self.assertRaises(ValueError): + helpers.two_states({A: {"p": 1.0}}, {A: {"p": math.inf}}) + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "d.json" + path.write_text('{"format": "sentier-diagnostics/1", "x": NaN}', encoding="utf-8") + with self.assertRaises(ValueError): + chain.read(path) + table = Path(folder) / "t.csv" + table.write_text("item,x:a,x:b\np,nan,1\n", encoding="utf-8") + with self.assertRaises(tables.TableError): + tables.read_results(table, [A, B]) + + def test_a13_a_missing_indicator_column_is_refused(self): + with tempfile.TemporaryDirectory() as folder: + table = Path(folder) / "t.csv" + table.write_text("item,x:a\np,1\n", encoding="utf-8") + with self.assertRaises(tables.TableError): + tables.read_results(table, [A, B]) + + def test_a14_a_breakdown_never_falls_back_silently(self): + """A14: a breakdown whose totals are not the item's results is refused, and one + whose parts do not add up is refused.""" + data = example.build() + good = example.wheat_breakdown() + self.assertEqual(breakdown.against(good, data), []) + moved = helpers.mutated(good, lambda d: d["indicators"]["x:climate"]["total"].__setitem__(1, 0.5)) + self.assertTrue(breakdown.against(moved, data)) + self.assertTrue(breakdown.errors(moved)) # the parts no longer add up to it either + other = helpers.mutated(good, lambda d: d.update(item="p02")) + self.assertTrue(breakdown.against(other, data)) + + def test_a15_breakdown_parts_keep_their_sign(self): + """A15: a part below zero stays below zero; bars are drawn from the signed value.""" + parts = [("credit", "flow", -0.3, -0.1), ("burden", "flow", 1.0, 1.0)] + found = breakdown.build("p", "a", "b", {A: ("kg", parts)}, top=5) + credit = next(p for p in found["indicators"][A]["parts"] if p["label"] == "credit") + self.assertEqual(credit["values"], [-0.3, -0.1]) + + def test_a16_nothing_is_cut_short_silently(self): + """A16: a list cut at `listed` says how many it left out, and a file that hides + the cut is refused. A breakdown cut at `top` says how many parts it left out.""" + before = {A: {f"p{k}": 1.0 for k in range(6)}} + after = {A: {f"p{k}": 2.0 + k for k in range(6)}} + data = helpers.two_states(before, after, listed=2) + self.assertEqual(data["items_count"], {"all": 6, "qualifying": 6, "listed": 2, "not_listed": 4}) + hidden = helpers.mutated(data, lambda d: d["items_count"].update(not_listed=0)) + self.assertTrue(fileformat.errors(hidden)) + cut = example.wheat_breakdown()["indicators"]["x:climate"] + self.assertEqual((len(cut["parts"]), cut["parts_not_listed"]), (5, 3)) + + def test_a17_a_file_of_counts_names_no_item(self): + data = example.build(tier="counts") + self.assertEqual(data["items"], []) + self.assertEqual(data["items_count"]["listed"], 0) + for step in data["steps"]: + for entry in (step["claim"] or {"scope": []})["scope"]: + self.assertIn(entry["items"], ("*", None)) + # an item slipped into a file of counts, with every count made to agree with it: + # only the rule that a file of counts holds no item can refuse it + listed = example.build()["items"][:1] + + def leak(d): + d["items"] = listed + d["items_count"].update(listed=1, not_listed=d["items_count"]["qualifying"] - 1) + + found = fileformat.errors(helpers.mutated(data, leak)) + self.assertEqual(found, ["a file of counts holds no item"]) + + def test_a18_attribution_and_what_it_cannot_split(self): + """A18: an item is attributed to the step that moved it most; an item missing in a + state between is counted as not attributable, never split. The page's split of one + item into shares, and the sum it prints, are tested under node (A18 there).""" + data = helpers.three_states({A: {"p": 1.0, "q": 1.0}}, {A: {"p": 1.5, "q": None}}, + {A: {"p": 1.75, "q": 3.0}}, explained=(True, False)) # fmt: skip + attribution = data["attribution"][A] + self.assertEqual(attribution["dominant"], {"one": 1, "two": 0}) + self.assertEqual(attribution["not_attributable"], 1) + self.assertEqual(attribution["unexplained_moved"], 1) # p moved in the step nothing explains + self.assertEqual(attribution["spread"], 0) + # the validator holds the file to it: q said to be attributed to a step is refused + def claim_q(d): + d["attribution"][A].update(not_attributable=0, dominant={"one": 2, "two": 0}) + + found = fileformat.errors(helpers.mutated(data, claim_q)) + self.assertTrue(any("2 dominant one, 1 in the listed items" in e for e in found), found) + + def test_a19_claims_must_and_may(self): + """A19: only a `must` entry can be unmet; an empty scope says nothing moves; a + result before of zero cannot be within a relative `largest`.""" + must = claims.scope([A], {"p"}, "down", None, claims.MUST) + may = claims.scope([A], {"p"}, "down") + self.assertEqual(may["expect"], claims.MAY) # the default + self.assertEqual(claims.verdict(rule.AGREES, 1.0, 1.0, None, [must]), claims.NOT_MOVED) + self.assertEqual(claims.verdict(rule.AGREES, 1.0, 1.0, None, [may]), claims.UNCHANGED) + self.assertEqual(claims.verdict(rule.DIFFERS, 1.0, 2.0, "up", []), claims.NOT_CLAIMED) + capped = claims.scope([A], "*", "up", 0.5) + self.assertEqual(claims.verdict(rule.ZERO_REFERENCE, 0.0, 1.0, "up", [capped]), claims.OFF_CLAIM) + uncapped = claims.scope([A], "*", "up") + self.assertEqual(claims.verdict(rule.ZERO_REFERENCE, 0.0, 1.0, "up", [uncapped]), claims.AS_CLAIMED) + nothing = claims.claim("Nothing moves.", [], by=claims.origin("code", "rule")) + data = helpers.two_states({A: {"p": 1.0}}, {A: {"p": 2.0}}, claim=nothing) + self.assertEqual(data["counts"]["steps"]["one"][A]["verdicts"]["not claimed"], 1) + + def test_a19b_a_claim_list_can_stay_with_its_writer(self): + """A19: with `claim_lists` false, the file lists the items that moved but not the + claim's list; the verdicts are kept and cannot be checked again.""" + data = example.build(claim_lists=False) + self.assertEqual(fileformat.errors(data), []) + fix = next(s for s in data["steps"] if s["id"] == "fix-1") + self.assertEqual([e["items"] for e in fix["claim"]["scope"]], [None]) + self.assertEqual(fix["claim"]["reach"], 2) + self.assertEqual(data["counts"], example.build()["counts"]) + + def test_a20_no_field_of_acceptance(self): + """A20 (sentier-bafu-agribalyse ADR 0010): a file carries no status and no + `accepted_by`; the validator refuses fields it does not know, anywhere.""" + data = example.build() + for change in ( + lambda d: d.update(status="accepted"), + lambda d: d["explanations"][0].update(accepted_by="someone"), + lambda d: d["steps"][1].update(status="ok"), + lambda d: d["items"][0].update(status="ok"), + ): + self.assertTrue(fileformat.errors(helpers.mutated(data, change))) + + def test_a22_the_reference_is_neutral_after_step_0(self): + """A22: coming closer to the published results is counted, never called better. + Step 0 and the whole way have no such count, and the page shows none.""" + data = example.build() + self.assertIsNone(data["counts"]["steps"]["step-0"]["x:climate"]["closer"]) + self.assertIsNone(data["counts"]["total"]["x:climate"]["closer"]) + self.assertIsNotNone(data["counts"]["steps"]["fix-1"]["x:climate"]["closer"]) + said = summary.text(data).lower() + for word in ("better", "worse", "improve", "closer"): + self.assertNotIn(word, said) + # what the page draws is read under node: "nothing drawn calls a move better or worse" + + def test_a23_indicators_are_matched_by_their_exact_id(self): + """A23: a column `x:a:biogenic` is not the indicator `x:a`.""" + with tempfile.TemporaryDirectory() as folder: + table = Path(folder) / "t.csv" + table.write_text("item,x:a:biogenic,x:b\np,1,2\n", encoding="utf-8") + with self.assertRaises(tables.TableError): + tables.read_results(table, [A, B]) + + def test_a24_a_result_on_one_side_only_is_counted_by_side(self): + """A24 (sentier-bafu): items on one side only are counted, before and after apart.""" + data = helpers.two_states({A: {"p": 1.0, "q": None}}, {A: {"p": None, "q": 2.0, "r": None}}) + self.assertEqual(data["counts"]["steps"]["one"][A]["missing"], {"before": 1, "after": 1, "both": 1}) + wrong = helpers.mutated(data, lambda d: d["counts"]["steps"]["one"][A]["missing"].update(both=0)) + self.assertTrue(fileformat.errors(wrong)) + + def test_a25_a_file_is_local_unless_its_writer_says_otherwise(self): + data = helpers.two_states({A: {"p": 1.0}}, {A: {"p": 2.0}}) + self.assertEqual(data["audience"], "local") + self.assertEqual(helpers.two_states({A: {"p": 1.0}}, {A: {"p": 2.0}}, audience="public")["audience"], "public") + with self.assertRaises(ValueError): + helpers.two_states({A: {"p": 1.0}}, {A: {"p": 2.0}}, audience="published") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_matrix_page.py b/tests/test_matrix_page.py new file mode 100644 index 0000000..bc59985 --- /dev/null +++ b/tests/test_matrix_page.py @@ -0,0 +1,357 @@ +"""The full matrix of a local file, and the page as a frameless body a site holds.""" + +from __future__ import annotations + +import contextlib +import io +import json +import re +import tempfile +import unittest +from html.parser import HTMLParser +from pathlib import Path + +from sentier_diagnostics import chain, cli, example, fileformat, from_tables, matrix, page + +from . import helpers + + +def local(): + return example.build_local() + + +class Matrix(unittest.TestCase): + def test_the_local_example_and_its_matrix_agree(self): + data, full = local() + self.assertEqual(fileformat.errors(data), []) + self.assertEqual(matrix.errors(full), []) + self.assertEqual(matrix.against(full, data), []) + self.assertEqual(data["matrix"], {"file": "matrix.json", "items": 10}) + self.assertEqual(len(data["items"]), 6) + + def test_the_matrix_is_ordered_as_the_listed_items_are(self): + data, full = local() + self.assertEqual( + [r[0] for r in full["items"]][: len(data["items"])], [i["id"] for i in data["items"]] + ) + + def test_a_matrix_survives_its_own_text(self): + _, full = local() + self.assertEqual(json.loads(matrix.dumps(full)), full) + + def test_every_changed_code_is_found(self): + data, full = local() + cases = { + "a class code": lambda m: m["classes"].__setitem__(0, ("=" if m["classes"][0][0] != "=" else "D") + m["classes"][0][1:]), + "a direction": lambda m: m["classes"].__setitem__(0, m["classes"][0].swapcase()), + "a result": lambda m: m["values"][7].__setitem__(0, (m["values"][7][0] or 1.0) * 3), + "a verdict": lambda m: m["verdicts"].__setitem__( + 6, "".join("o" if c == "a" else "a" if c == "o" else c for c in m["verdicts"][6])), + "a name of a listed item": lambda m: m["items"][0].__setitem__(1, "Another name"), + "one item fewer": lambda m: [m[k].pop() for k in ("items", "values", "classes", "verdicts")], + "the title": lambda m: m["of"].__setitem__("title", "Another"), + } # fmt: skip + for name, change in cases.items(): + with self.subTest(name): + changed = helpers.mutated(full, change) + self.assertNotEqual(changed, full) + found = matrix.errors(changed) or matrix.against(changed, data) + self.assertNotEqual(found, [], name) + + def test_counts_that_are_not_the_matrix_are_found(self): + data, full = local() + cases = { + "a count": lambda d: d["counts"]["total"]["x:climate"]["classes"].__setitem__( + "agrees", d["counts"]["total"]["x:climate"]["classes"]["agrees"] + 1), + "the attribution": lambda d: d["attribution"]["x:climate"].__setitem__( + "spread", d["attribution"]["x:climate"]["spread"] + 1), + "the items worth a look": lambda d: d["items_count"].__setitem__("qualifying", d["items_count"]["qualifying"] - 1), + "the items of the matrix": lambda d: d["matrix"].__setitem__("items", 11), + } # fmt: skip + for name, change in cases.items(): + with self.subTest(name): + self.assertNotEqual(matrix.against(full, helpers.mutated(data, change)), [], name) + + def test_shape_faults_never_raise(self): + _, full = local() + for bad in (None, [], {"format": "sentier-diagnostics-matrix/2"}, {**full, "extra": 1}, + {**full, "items": [[1, 2, 3, 4, 5]]}, {**full, "classes": ["?" * 15] * 10}): # fmt: skip + with self.subTest(repr(bad)[:60]): + self.assertNotEqual(matrix.errors(bad), []) + + def test_a_matrix_is_written_and_read_back_checked(self): + data, full = local() + with tempfile.TemporaryDirectory() as folder: + where = matrix.write(full, Path(folder) / "matrix.json") + self.assertEqual(matrix.read(where, data), full) + other = helpers.mutated(data, lambda d: d.update(title="Another")) + with self.assertRaises(ValueError): + matrix.read(where, other) + + +class NamesAMatrix(unittest.TestCase): + """The key `matrix` of a diagnostics file (SPEC 4.2).""" + + def test_only_a_local_file_of_items_names_a_matrix(self): + data, _ = local() + public = helpers.mutated(data, lambda d: d.update(audience="public")) + self.assertTrue(any("only a local file" in e for e in fileformat.errors(public))) + + def test_the_matrix_holds_every_item(self): + data, _ = local() + wrong = helpers.mutated(data, lambda d: d["matrix"].update(items=9)) + self.assertNotEqual(fileformat.errors(wrong), []) + + def test_a_matrix_is_named_by_a_plain_file_name(self): + data, _ = local() + for name in ("../matrix.json", "matrix.js", "/abs.json", "a b.json"): + with self.subTest(name): + self.assertNotEqual( + fileformat.errors(helpers.mutated(data, lambda d: d["matrix"].update(file=name))), [] + ) + + def test_no_matrix_is_no_key_or_null(self): + data, _ = local() + self.assertEqual(fileformat.errors(helpers.mutated(data, lambda d: d.update(matrix=None))), []) + self.assertEqual(fileformat.errors(helpers.mutated(data, lambda d: d.pop("matrix"))), []) + + def test_a_public_file_is_never_built_with_a_matrix(self): + with self.assertRaises(ValueError): + example._made(chain.build_with_matrix, audience="public") + + +class Elements(HTMLParser): + """Every start tag of a fragment, with its attributes.""" + + def __init__(self): + super().__init__() + self.tags = [] + + def handle_starttag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + + +def tags_of(text: str) -> list[tuple[str, dict]]: + parser = Elements() + parser.feed(text) + return parser.tags + + +class Body(unittest.TestCase): + def test_one_root_and_no_frame_no_id(self): + text = page.body(example.build(), breakdowns={"p01-fix-1.json": example.wheat_breakdown()}) + tags = tags_of(text) + self.assertEqual(tags[0][0], "div") + self.assertEqual(tags[0][1].get("class"), "sentier-diagnostics") + self.assertIn("data-sentier-diagnostics", tags[0][1]) + self.assertEqual([t for t, a in tags if "data-sentier-diagnostics" in a], ["div"]) + self.assertFalse({"html", "head", "body", "main", "link", "style", "h1"} & {t for t, _ in tags}) + self.assertFalse([t for t, a in tags if "id" in a], "no id a host's page could hold twice") + self.assertTrue(text.rstrip().endswith("
")) + + def test_the_data_inside_is_the_file_and_cannot_end_its_element(self): + data = helpers.mutated(example.build(), lambda d: d.update(about="