From b2c4fc4f1c216d491c40c296189c7352669448d0 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 22 Oct 2026 10:34:03 +0200 Subject: [PATCH] test(site): FTW's scenario pack on device-simulator's house, run on a machine of its own The end-to-end check from the 21 October simulator test becomes a maintained part of the stack, split as Fredrik asked: the house lives in srcfl/device-simulator (its headless run builds it from a spec, applies a timeline and faults, and records what flows through each fuse), and FTW keeps what is FTW's. go/test/site (test files only) holds the scenario pack: the house spec, FTW's configuration and requests, and criteria read from the house's record and FTW's API, for every case of the report plus s11 (pause, then restart), s9r, s5t, s12 (the one-phase night), s13 (socket reconnect) and s14 (a one-phase car). A criterion that fails on master is marked known; when a fix makes it pass the run fails until the mark goes. Each case names the fault-catalog entries it covers and its sources. FTW_SITE picks short, full, names or random::; lanes run several cases at once on ports 100 apart. A random seed draws FTW's side and the house draws its own timeline from the same seed; eight invariants are checked every second, and a failing seed is cut after its first violation and kept for the full set. make siterig, siterig-full, siterig-random and siterig-coverage run it. go/test/site/runner is siterig-runner: on its own machine it runs the short set on pull requests that touch control, loadpoint, OCPP or telemetry code and the full set on each master commit, and posts siterig/short, siterig/full and siterig/random commit statuses. Nothing runs in GitHub Actions. Core is unchanged. Co-Authored-By: Claude Opus 5.5 --- Makefile | 28 +- go/test/site/README.md | 100 ++ go/test/site/catalog_test.go | 171 +++ go/test/site/devsim.version | 1 + go/test/site/evaluate_test.go | 682 +++++++++++ go/test/site/procattr_linux_test.go | 9 + go/test/site/procattr_other_test.go | 11 + go/test/site/random_test.go | 549 +++++++++ go/test/site/rig_test.go | 413 +++++++ go/test/site/rig_unit_test.go | 141 +++ go/test/site/runner/README.md | 78 ++ .../runner/energy.ftw.siterig-runner.plist | 24 + go/test/site/runner/main.go | 232 ++++ go/test/site/runner/runner.go | 173 +++ go/test/site/runner/runner_test.go | 94 ++ go/test/site/runner/siterig-runner.service | 18 + go/test/site/scenarios_test.go | 1026 +++++++++++++++++ go/test/site/stack_test.go | 537 +++++++++ go/test/site/testdata/found/.gitkeep | 0 go/test/site/verdicts_test.go | 553 +++++++++ 20 files changed, 4839 insertions(+), 1 deletion(-) create mode 100644 go/test/site/README.md create mode 100644 go/test/site/catalog_test.go create mode 100644 go/test/site/devsim.version create mode 100644 go/test/site/evaluate_test.go create mode 100644 go/test/site/procattr_linux_test.go create mode 100644 go/test/site/procattr_other_test.go create mode 100644 go/test/site/random_test.go create mode 100644 go/test/site/rig_test.go create mode 100644 go/test/site/rig_unit_test.go create mode 100644 go/test/site/runner/README.md create mode 100644 go/test/site/runner/energy.ftw.siterig-runner.plist create mode 100644 go/test/site/runner/main.go create mode 100644 go/test/site/runner/runner.go create mode 100644 go/test/site/runner/runner_test.go create mode 100644 go/test/site/runner/siterig-runner.service create mode 100644 go/test/site/scenarios_test.go create mode 100644 go/test/site/stack_test.go create mode 100644 go/test/site/testdata/found/.gitkeep create mode 100644 go/test/site/verdicts_test.go diff --git a/Makefile b/Makefile index afe4082ae..c24d24bb0 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ # make clean — remove all build artifacts .PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 release release-linux release-linux-arm64 release-linux-amd64 \ - run-sim sim-ocpp dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ + run-sim sim-ocpp dev fmt vet clean e2e siterig siterig-full siterig-random siterig-coverage siterig-judge ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions protocol-boundary cloudlink-boundary \ drivers drivers-present driver-versions-across-pin @@ -121,6 +121,32 @@ release-workflow-test: e2e: drivers-present cd go && FTW_E2E=1 go test ./test/e2e -v -timeout 180s +# FTW against device-simulator's house with a real fuse +# (go/test/site/README.md). SITERIG_SET is short, full or scenario names; +# SITERIG_LANES runs that many at once on ports 100 apart. +SITERIG_SET ?= short +SITERIG_LANES ?= 2 +siterig: drivers-present + cd go && FTW_SITE=$(SITERIG_SET) FTW_SITE_LANES=$(SITERIG_LANES) \ + go test -count=1 -timeout 300m -run 'TestSite$$' -v ./test/site/ + +siterig-full: + $(MAKE) siterig SITERIG_SET=full + +# Random cases: SEED=n replays one, COUNT=m runs m seeds from SEED. +SEED ?= 1 +COUNT ?= 1 +siterig-random: + $(MAKE) siterig SITERIG_SET=random:$(SEED):$(COUNT) + +# Which fault-catalog entries have a scenario: CATALOG=path/to/catalog.yaml +siterig-coverage: + cd go && FTW_SITE_CATALOG=$(CATALOG) go test -count=1 -run TestCatalogCoverage -v ./test/site/ + +# Judge a recorded run again with the current criteria: DIR=results directory +siterig-judge: + cd go && FTW_SITE_JUDGE=$(DIR) go test -count=1 -run 'TestJudge$$' -v ./test/site/ + driver-repository-validate: drivers-present cd go && go run ./cmd/ftw-driver-repository publish -unsigned -drivers ../drivers -output ../dist/driver-repository -base-url https://example.invalid/releases/download/drivers-local -repository https://github.com/srcfl/ftw diff --git a/go/test/site/README.md b/go/test/site/README.md new file mode 100644 index 000000000..3289356fb --- /dev/null +++ b/go/test/site/README.md @@ -0,0 +1,100 @@ +# Site test: FTW against a simulated house + +`TestSite` runs the real `ftw` binary against a house from +[srcfl/device-simulator](https://github.com/srcfl/device-simulator): its +headless run (`simulator run`, docs/HEADLESS_RUN.md there) builds a Sungrow +hybrid whose battery follows FTW's EMS writes after 2 s, an SDM630 site +meter behind an IEC gG fuse on each phase, and OCPP 1.6 chargers with the +car on them. FTW talks to it over Modbus and OCPP as on a real site. Each +scenario starts a fresh house and a fresh `state.db`, drives the house and +FTW, and judges the run against fixed criteria from what flowed through the +fuse (the house's record) and what FTW said (its API, once a second). FTW +runs `planner_self` with the Core planner and SE3 prices. + +The house lives in device-simulator; this directory keeps only what is +FTW's: the scenarios, FTW's configuration, the criteria and the pinned +simulator commit in [`devsim.version`](devsim.version). + +## Run it + +```bash +make siterig # the short set, two lanes +make siterig-full # every scenario +make siterig SITERIG_SET=s1-25,s9-25 # named scenarios +make siterig-coverage CATALOG=../reports/felkatalog/catalog.yaml +make siterig-judge DIR=/tmp/ftw-site-123 # judge a recorded run again +``` + +| Variable | Meaning | +|---|---| +| `FTW_SITE` | `short`, `full`, or names. Unset skips the test. | +| `FTW_SITE_LANES` | scenarios at once (`SITERIG_LANES`, default 2), each lane on its own 100 ports | +| `FTW_SITE_PORT_BASE` | first port on 127.0.0.1 (default 18700): the house on base+0..59, FTW on base+90 and +91 | +| `FTW_SITE_SHARD` | `k/n`: shard k of n, for several machines | +| `FTW_SITE_OUT` | run directories, `summary.md` and `summary.json` (default: a new temporary directory) | +| `FTW_SITE_DEVSIM_SRC` | a device-simulator checkout at the pinned commit; without it the test clones the pin into the user cache | +| `FTW_SITE_FTW_BIN`, `FTW_SITE_DEVSIM_BIN` | prebuilt binaries | +| `FTW_SITE_CATALOG` | the fault catalog, for `TestCatalogCoverage` | +| `FTW_SITE_JUDGE` | a results directory: `TestJudge` judges its runs again with the criteria as they are now, and rewrites their `result.json` and the summary | + +Each run directory keeps `house/plant.jsonl` (the house once a second, with +every event and mark), `house/ocpp-.jsonl` (every OCPP frame the +charger read or wrote), `ftw.jsonl` (FTW's `/api/status` and +`/api/loadpoints` once a second), `ftw.log`, `house.log`, `config.yaml`, +`house.json` and `result.json`. + +A run whose record has a hole over 15 s fails as "host paused": a sleeping +laptop stretches every wait and proves nothing. + +## Where it runs + +Not in GitHub Actions. `siterig-runner` ([runner/](runner/README.md)) runs +on a machine of its own, watches `master` and open pull requests, and posts +the result on the exact commit as the status `siterig/short` (pull requests +that touch control, loadpoint, OCPP or telemetry code), `siterig/full` +(each new `master` commit) or `siterig/random`. A beta needs +`siterig/full` on its commit. + +## Known failures + +A criterion named in a scenario's `known` map fails on master for the +finding it names. The run passes while it keeps failing, and fails once it +passes: a fix deletes its line in the same pull request. + +## The fault catalog + +Each scenario names the entries of FTW's field fault catalog it covers and +its sources (`provenance` in [catalog_test.go](catalog_test.go)). +`make siterig-coverage` lists, per area, which faults have a scenario. + +## Scenarios + +"Over the limit" is the worst phase above the fuse − 1 A, measured through +the fuse. A device's response time counts as 2 s, the meter and control +tick as 2 s more. + +| Scenario | House and steps | Criteria | Short | +|---|---|---|---| +| `s0-25` Idle charger | 25 A, 3 A per phase, a charger with no car, heartbeat 60 s, 300 s | Home use `measured` in 95 % of samples; no OCPP disconnect | yes | +| `s0b-25` | as s0, Core's heartbeat | as s0 | | +| `s1-16`, `s1-25` Charge now | 3 A per phase, Charge now 16 A for 180 s | the room the fuse gives (25 A: 11 040 W) within 60 s; held 95 %; fuse holds; on 25 A no `fuse_limit` | yes | +| `s2-16` Charge now during export | 16 A, 1 A house, 7 kW PV, full battery: 9.1 A export per phase | car starts within 60 s (**known**: export counted as import); fuse holds | | +| `s3-16`, `s3-25` Load step | car and a 3 kW battery hold charging, then 13 A (16 A) or 21 A (25 A) per phase for 180 s | fuse holds; over the limit at most 7 s after the step | yes | +| `s4-25`, `s4b-25` Saved unlock | `surplus_unlock_bat_soc` 0.9 saved, not armed; Charge now 3p, and bare (s4b) | no stops; no offer drops to 0 W; full power over the last minute | | +| `s5-25`, `s5s-25` Surplus only | 25 A, 9.3 kW surplus, `auto`; s5 without steps, s5s with Easee's | no offer above 16 A; fuse holds; only 6–16 A; ≥ 4 140 W 90 % of the time (**known** in both) | | +| `s5t-25` Surplus with a goal | as s5s with a goal, 3.8 kW of export, a charger that switches phases | no offer above 16 A; fuse holds; ≥ 6 A on one phase 90 % of the time | | +| `s6-25` Inverter offline | 1.5 kW PV, the inverter silent for 300 s | Home use shown and marked estimated throughout; measured before and after; never estimated while every driver reads | | +| `s7-25` Restart | Charge now, 15 min, Core down 20 s, 420 s after | the day's Home use is kept; Home use back within 30 s; Charge now never below 4 140 W; full power within 420 s | | +| `s7s-25` Short restart | as s7, restarted after 60 s | as s7 without the day total, which needs the long run (10 % rule) | yes | +| `s8-60-25`, `s8-120-25` Slow charger | 15 A per phase, battery at its 5 % floor, profiles applied 60 or 120 s late | fuse holds; over the limit at most the delay + 7 s | 60 s | +| `s9-25` 3p to 1p | 10 A per phase, a 32 A charger that switches phases; Charge now 7 360 W on 3p, after 90 s on 1p | fuse holds; over the limit at most 3 s before and after the switch | | +| `s9r-25` 1p to 3p beside a loaded L2 | house 2/20/2 A, the 32 A charger; Charge now 7 360 W on 1p, after 90 s on 3p | as s9 | | +| `s10-25` Two chargers | `carport` 60 s late; cut 6 A and raise 16 A together | fuse holds; over the limit at most 3 s after the step | | +| `s11-25` Pause, then restart | a charger applying 5 s late; Charge now 16 A, 0 W for 25 s, then 6 A | an offer within 60 s of the restart; over the limit at most 3 s; fuse holds | | +| `s12-20` One-phase night | 3 × 20 A, house 1.6/1.7/2.3 A, a charger applying 40 s late, the field's step table on `auto`, a plan for 6.6 kWh by the first quarter hour 40 min out | fuse holds; planned charging never orders one phase; no order cut while every phase has room, unless the plan paused; 90 % of the plan by the ready time | | +| `s13-25` Socket reconnect | Charge now 16 A, the charger's connection gone for 10 s | Charge now goes on after the charger returns (**known**: #1788); no unplug seen | | +| `s15-25` One-phase charger, Charge now | 25 A, 3 A per phase, an Easee wired on L1 (6-16 A, steps to 3 680 W, no phase mode); Charge now on the phases `/api/loadpoints` reports, as the app sends it, 180 s | 16 A on L1 within 60 s; held 95 %; fuse holds; no `fuse_limit` | yes | +| `s15b-25`, `s15c-25` | as s15, phase mode `1p` and `auto` | as s15 | | +| `s15bat-16` One-phase charger, battery rising | 16 A, 3 A per phase, the s15 charger, battery at 50 % following its EMS 8 s late; PV 2.1 kW rising every 30 s (+2.5 kW, then 1.5 kW) so Core charges the battery in steps; Charge now 6 s after the second step, while the raise is on its way | the battery order rose over 1 150 W in the 20 s before (setup check); car charging within 10 s; fuse holds; over the limit at most 3 s | | +| `s15p-25` One-phase charger, planned | as s15, a plan for 600 Wh by the first quarter hour 15 min out | fuse holds; 90 % of the plan by the ready time; never called not following | | +| `s14-25` One-phase car | an 11 kW charger with a one-phase car, a plan it cannot meet, 20 min | 3.4–3.8 kW over the last 10 min; never called not following | | diff --git a/go/test/site/catalog_test.go b/go/test/site/catalog_test.go new file mode 100644 index 000000000..ab7a55199 --- /dev/null +++ b/go/test/site/catalog_test.go @@ -0,0 +1,171 @@ +//go:build unix + +package site + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// provenance says where each scenario comes from: the entries of FTW's +// field fault catalog it covers (reports/felkatalog/catalog.yaml in the FTW +// stack workspace) and its sources. +var provenance = map[string]struct { + catalog []string + sources []string +}{ + "s0-25": {[]string{"home-070", "ocpp-011", "ocpp-013", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1779", "https://github.com/srcfl/ftw/pull/1790"}}, + "s0b-25": {[]string{"home-070", "ocpp-011", "ocpp-013", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1790"}}, + "s1-16": {nil, []string{"simulator test before beta 0.155.0, 21 Oct 2026"}}, + "s1-25": {nil, []string{"simulator test before beta 0.155.0, 21 Oct 2026"}}, + "s2-16": {[]string{"ev-078"}, []string{"https://github.com/srcfl/ftw/pull/1782"}}, + "s3-16": {[]string{"ev-100"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s3-25": {[]string{"ev-100"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s4-25": {[]string{"ev-093"}, []string{"https://github.com/srcfl/ftw/pull/1774"}}, + "s4b-25": {[]string{"ev-093"}, []string{"https://github.com/srcfl/ftw/pull/1774"}}, + "s5-25": {[]string{"ev-089", "ev-110"}, []string{"https://github.com/srcfl/ftw/pull/1774"}}, + "s5s-25": {[]string{"ev-089", "ev-102"}, []string{"https://github.com/srcfl/ftw/pull/1791"}}, + "s6-25": {[]string{"home-061", "ocpp-010", "tel-035", "tel-037", "tel-038", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1780"}}, + "s7-25": {[]string{"dev-100", "ev-085", "ev-091", "home-061", "home-074", "hw-061"}, []string{"https://github.com/srcfl/ftw/pull/1778", "https://github.com/srcfl/ftw/pull/1787"}}, + "s7s-25": {[]string{"ev-085", "ev-091"}, []string{"https://github.com/srcfl/ftw/pull/1778"}}, + "s8-60-25": {[]string{"dev-076", "ev-099", "ev-107"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s8-120-25": {[]string{"dev-076", "ev-099", "ev-107"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s9-25": {[]string{"ev-094"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s9r-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (probe: a held 1p order raised to 3p beside L2 at 20 A)"}}, + "s10-25": {[]string{"dev-076", "ev-105"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s11-25": {[]string{"ev-104"}, []string{"https://github.com/srcfl/ftw/pull/1763", "https://github.com/srcfl/ftw/pull/1793"}}, + "s5t-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (probe: surplus-only with a goal)", "https://github.com/srcfl/ftw/pull/1799"}}, + "s12-20": {[]string{"ev-097"}, []string{"https://github.com/srcfl/ftw/pull/1799", "field report, 22 Oct 2026, v0.155.0-beta.1"}}, + "s13-25": {[]string{"ocpp-009", "dev-119"}, []string{"https://github.com/srcfl/ftw/issues/1788"}}, + "s14-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (help1022 car1p, fix D)"}}, + "s15-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now"}}, + "s15b-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now (phase mode 1p)"}}, + "s15c-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now (phase mode auto)"}}, + "s15bat-16": {nil, []string{"https://github.com/srcfl/ftw/pull/1805", "release blocker 22 Oct 2026: a one-phase charger never starts while Core raises its battery"}}, + "s15p-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger, planned charging"}}, +} + +func TestEveryScenarioSaysWhereItComesFrom(t *testing.T) { + for _, sc := range scenarios { + p, ok := provenance[sc.name] + if !ok || len(p.catalog)+len(p.sources) == 0 { + t.Errorf("%s has no catalog entry or source", sc.name) + } + } + for name := range provenance { + found := false + for _, sc := range scenarios { + found = found || sc.name == name + } + if !found { + t.Errorf("provenance for %s, which is no scenario", name) + } + } +} + +type catalogEntry struct { + ID string `yaml:"id"` + Kind string `yaml:"kind"` + Area string `yaml:"area"` + Title string `yaml:"title"` + Status string `yaml:"status"` + Sources []string `yaml:"sources"` +} + +// TestCatalogCoverage reads the fault catalog at FTW_SITE_CATALOG and writes +// coverage.md: per area, which faults have a scenario here and which do not. +// It fails when a scenario names an entry the catalog does not have. +func TestCatalogCoverage(t *testing.T) { + path := os.Getenv("FTW_SITE_CATALOG") + if path == "" { + t.Skip("set FTW_SITE_CATALOG to the fault catalog (catalog.yaml)") + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var cat struct { + Entries []catalogEntry `yaml:"entries"` + } + if err := yaml.Unmarshal(b, &cat); err != nil { + t.Fatal(err) + } + byID := map[string]catalogEntry{} + for _, e := range cat.Entries { + byID[e.ID] = e + } + covered := map[string][]string{} + for name, p := range provenance { + for _, id := range p.catalog { + if _, ok := byID[id]; !ok { + t.Errorf("%s names catalog entry %s, which the catalog does not have", name, id) + } + covered[id] = append(covered[id], name) + } + } + type tally struct{ faults, open, covered, openCovered int } + areas := map[string]*tally{} + var md strings.Builder + md.WriteString("# Site-test coverage of the fault catalog\n\n") + md.WriteString("| Entry | Status | Title | Scenarios |\n|---|---|---|---|\n") + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + e := byID[id] + if e.Kind != "fault" { + if len(covered[id]) > 0 { + sort.Strings(covered[id]) + fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", id, e.Kind, e.Title, strings.Join(covered[id], ", ")) + } + continue + } + a := areas[e.Area] + if a == nil { + a = &tally{} + areas[e.Area] = a + } + open := e.Status == "open" || e.Status == "partly_fixed" || e.Status == "fix_unmerged" + a.faults++ + if open { + a.open++ + } + if len(covered[id]) > 0 { + a.covered++ + if open { + a.openCovered++ + } + sort.Strings(covered[id]) + fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", id, e.Status, e.Title, strings.Join(covered[id], ", ")) + } + } + md.WriteString("\n| Area | Faults | With a scenario | Open | Open with a scenario |\n|---|---:|---:|---:|---:|\n") + names := make([]string, 0, len(areas)) + for a := range areas { + names = append(names, a) + } + sort.Strings(names) + for _, a := range names { + x := areas[a] + fmt.Fprintf(&md, "| %s | %d | %d | %d | %d |\n", a, x.faults, x.covered, x.open, x.openCovered) + } + out := os.Getenv("FTW_SITE_OUT") + if out == "" { + // No place to keep it: the log is the report. + t.Logf("%d catalog entries, %d with a scenario\n\n%s", len(byID), len(covered), md.String()) + return + } + dst := filepath.Join(out, "coverage.md") + if err := os.WriteFile(dst, []byte(md.String()), 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%d catalog entries, %d with a scenario; wrote %s", len(byID), len(covered), dst) +} diff --git a/go/test/site/devsim.version b/go/test/site/devsim.version new file mode 100644 index 000000000..c0ca38fa8 --- /dev/null +++ b/go/test/site/devsim.version @@ -0,0 +1 @@ +a3030d3ff2d4b547d0aa596022ca85daa69904eb diff --git a/go/test/site/evaluate_test.go b/go/test/site/evaluate_test.go new file mode 100644 index 000000000..83b34ff20 --- /dev/null +++ b/go/test/site/evaluate_test.go @@ -0,0 +1,682 @@ +//go:build unix + +package site + +import ( + "bufio" + "encoding/json" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + volt = 230.0 + maxW = 11040.0 // the rig charger, 16 A on three phases + minW = 4140.0 // the loadpoints' minimum, 6 A on three phases +) + +// ---- what the house and the recorder wrote ---- + +// plantLine is a line of the house's plant.jsonl: a sample a second, an +// event or a mark. +type plantLine struct { + T float64 `json:"t"` + Mark string `json:"mark"` + Event json.RawMessage `json:"event"` + PhaseA [3]float64 `json:"phase_a"` // through each fuse, + into the site + Fuse *plantFuse `json:"fuse"` + Devices map[string]plantDevice `json:"devices"` +} + +type plantFuse struct { + Damage [3]float64 `json:"damage"` + Blown [3]bool `json:"blown"` +} + +type plantDevice struct { + W float64 `json:"w"` // a charger's draw + BatteryW float64 `json:"battery_w"` // an inverter's battery, + charging + SoCPct float64 `json:"soc_pct"` + EMSCmd float64 `json:"ems_cmd"` // 170 charge, 187 discharge, 204 stop + EMSSetW float64 `json:"ems_setpoint_w"` + Connected bool `json:"connected"` +} + +type ftwStatus struct { + Drivers map[string]struct { + Status string `json:"status"` + } `json:"drivers"` + LoadQuality *string `json:"load_quality"` + LoadReason *string `json:"load_reason"` + LoadW *float64 `json:"load_w"` + Energy struct { + Today struct { + LoadWh *float64 `json:"load_wh"` + HistoryQuality *string `json:"history_quality"` + } `json:"today"` + } `json:"energy"` +} + +type ftwLoadpoint struct { + ID string `json:"id"` + Plugged bool `json:"plugged_in"` + Phases int `json:"phases"` + Charger struct { + Known bool `json:"known"` + } `json:"charger"` + CommandedReason string `json:"commanded_reason"` + ControlFeedback struct { + Status string `json:"status"` // following, not_following, limited... + Severity string `json:"severity"` + Reason string `json:"reason"` + } `json:"control_feedback"` +} + +type ftwLine struct { + T float64 `json:"t"` + Status *ftwStatus `json:"status"` + LPs *struct { + Loadpoints []ftwLoadpoint `json:"loadpoints"` + } `json:"lps"` +} + +// sample is one second: the house's truth and the last FTW reading at or +// before it. +type sample struct { + T float64 + PhaseA [3]float64 + Damage [3]float64 + Blown [3]bool + CarW []float64 // by loadpoint, in lpIDs order + BatW float64 + Bat plantDevice // the inverter + Status *ftwStatus + LPs []ftwLoadpoint +} + +type frame struct { + T float64 `json:"t"` + Dir string `json:"dir"` + Frame json.RawMessage `json:"frame"` +} + +// houseEventLine is a change the house made, as its record has it. +type houseEventLine struct { + T float64 + Event houseEvent +} + +// runData is one finished scenario's record. +type runData struct { + dir string + samples []*sample + marks map[string]float64 + allMarks []markLine // every mark, in order, also repeated ones + events []houseEventLine + frames []frame // garage's charger + meta map[string]any +} + +type markLine struct { + T float64 + Label string +} + +func loadRun(dir string, meta map[string]any) (*runData, error) { + d := &runData{dir: dir, marks: map[string]float64{}, meta: meta} + var ftw []ftwLine + if err := readLines(filepath.Join(dir, "ftw.jsonl"), func(b []byte) { + var l ftwLine + if json.Unmarshal(b, &l) == nil { + ftw = append(ftw, l) + } + }); err != nil { + return nil, err + } + next := 0 + var last *ftwLine + err := readLines(filepath.Join(dir, "house", "plant.jsonl"), func(b []byte) { + var l plantLine + if json.Unmarshal(b, &l) != nil { + return + } + switch { + case l.Mark != "": + if _, ok := d.marks[l.Mark]; !ok { + d.marks[l.Mark] = l.T + } + d.allMarks = append(d.allMarks, markLine{l.T, l.Mark}) + return + case l.Event != nil: + var ev houseEvent + if json.Unmarshal(l.Event, &ev) == nil { + d.events = append(d.events, houseEventLine{l.T, ev}) + } + return + case l.Fuse == nil: + return + } + for next < len(ftw) && ftw[next].T <= l.T { + last = &ftw[next] + next++ + } + s := &sample{T: l.T, PhaseA: l.PhaseA, Damage: l.Fuse.Damage, Blown: l.Fuse.Blown, + BatW: l.Devices["inverter"].BatteryW, Bat: l.Devices["inverter"]} + for _, id := range lpIDs { + if dev, ok := l.Devices[id]; ok { + s.CarW = append(s.CarW, dev.W) + } + } + // A reading older than 3 s is no reading: FTW was down or slow. + if last != nil && l.T-last.T <= 3 { + s.Status = last.Status + if last.LPs != nil { + s.LPs = last.LPs.Loadpoints + } + } + d.samples = append(d.samples, s) + }) + if err != nil { + return nil, err + } + d.frames = d.loadFrames(0) + return d, nil +} + +// loadFrames is the OCPP frames charger i read and wrote, from its own log. +func (d *runData) loadFrames(i int) []frame { + var out []frame + readLines(filepath.Join(d.dir, "house", "ocpp-"+lpIDs[i]+".jsonl"), func(b []byte) { + var f frame + if json.Unmarshal(b, &f) == nil { + out = append(out, f) + } + }) + return out +} + +func readLines(path string, fn func([]byte)) error { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 64<<20) + for sc.Scan() { + fn(sc.Bytes()) + } + return sc.Err() +} + +func (d *runData) metaF(k string) float64 { + v, _ := d.meta[k].(float64) + if i, ok := d.meta[k].(int); ok { + v = float64(i) + } + return v +} + +// longestGap is the longest time between two samples, and when it began. +func (d *runData) longestGap() (gap, at float64) { + for i := 1; i < len(d.samples); i++ { + if g := d.samples[i].T - d.samples[i-1].T; g > gap { + gap, at = g, d.samples[i-1].T-d.samples[0].T + } + } + return gap, at +} + +// ---- per-sample readings ---- + +func carW(r *sample, i int) float64 { + if i < len(r.CarW) { + return r.CarW[i] + } + return 0 +} + +// phaseI is the current through each phase's fuse, + into the site. +func phaseI(r *sample) [3]float64 { return r.PhaseA } + +func maxAbsI(r *sample) float64 { + m := 0.0 + for _, i := range r.PhaseA { + m = math.Max(m, math.Abs(i)) + } + return m +} + +func damage(r *sample) float64 { return math.Max(r.Damage[0], math.Max(r.Damage[1], r.Damage[2])) } + +func blown(r *sample) bool { return r.Blown[0] || r.Blown[1] || r.Blown[2] } + +func firstLP(r *sample) ftwLoadpoint { + if len(r.LPs) > 0 { + return r.LPs[0] + } + return ftwLoadpoint{} +} + +func reasons(r *sample) []string { + lp := firstLP(r) + set := map[string]bool{} + for _, k := range []string{lp.CommandedReason, lp.ControlFeedback.Reason} { + if k != "" { + set[k] = true + } + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + return out +} + +func status(r *sample) ftwStatus { + if r.Status != nil { + return *r.Status + } + return ftwStatus{} +} + +func str(p *string) string { + if p == nil { + return "null" + } + return *p +} + +func allRead(r *sample) bool { + st := status(r) + if len(st.Drivers) == 0 { + return false + } + for _, d := range st.Drivers { + if d.Status != "ok" { + return false + } + } + return true +} + +func batSigned(r *sample) float64 { return r.BatW } + +// ---- windows ---- + +func (d *runData) window(t0, t1 float64) []*sample { + var out []*sample + for _, r := range d.samples { + if t0 <= r.T && r.T < t1 { + out = append(out, r) + } + } + return out +} + +// firstT is the seconds from t0 to the first sample from t0 on that +// satisfies pred, or nil. +func (d *runData) firstT(t0 float64, pred func(*sample) bool) *float64 { + for _, r := range d.samples { + if r.T >= t0 && pred(r) { + v := r.T - t0 + return &v + } + } + return nil +} + +// runsOver is the longest contiguous stretch and the total seconds with the +// worst phase above limit. +func (d *runData) runsOver(t0, t1, limit float64) (longest, total float64) { + var curStart, prevT *float64 + for _, r := range d.window(t0, t1) { + t := r.T + if maxAbsI(r) > limit { + if curStart == nil { + curStart = &t + } + if prevT != nil { + total += t - *prevT + } + } else { + if curStart != nil { + longest = math.Max(longest, t-*curStart) + } + curStart = nil + } + prevT = &t + } + if curStart != nil && prevT != nil { + longest = math.Max(longest, *prevT-*curStart) + } + return round(longest, 1), round(total, 1) +} + +// i2Over is the seconds the worst phase's I² filter (tau 60 s) is above +// limit, and the filter's peak. +func (d *runData) i2Over(t0, t1, limit float64) (over, peak float64) { + const tau = 60.0 + var y, prev float64 + started := false + for _, r := range d.window(t0, t1) { + x := math.Pow(maxAbsI(r), 2) + if !started { + y, started = x, true + } else { + dt := r.T - prev + y += (1 - math.Exp(-dt/tau)) * (x - y) + if math.Sqrt(y) > limit { + over += dt + } + } + prev = r.T + peak = math.Max(peak, math.Sqrt(y)) + } + return round(over, 1), round(peak, 2) +} + +func mean(xs []float64) float64 { + if len(xs) == 0 { + return 0 + } + s := 0.0 + for _, x := range xs { + s += x + } + return s / float64(len(xs)) +} + +func meanOf(ws []*sample, f func(*sample) float64) float64 { + xs := make([]float64, len(ws)) + for i, r := range ws { + xs[i] = f(r) + } + return mean(xs) +} + +func maxOf(ws []*sample, f func(*sample) float64) float64 { + m := 0.0 + for _, r := range ws { + m = math.Max(m, f(r)) + } + return m +} + +func round(x float64, n int) float64 { + p := math.Pow(10, float64(n)) + return math.Round(x*p) / p +} + +func roundp(p *float64, n int) *float64 { + if p == nil { + return nil + } + v := round(*p, n) + return &v +} + +// counter counts strings and lists the commonest first. +type counter map[string]int + +func (c counter) top(n int) map[string]int { + type kv struct { + k string + v int + } + var all []kv + for k, v := range c { + all = append(all, kv{k, v}) + } + sort.Slice(all, func(i, j int) bool { + if all[i].v != all[j].v { + return all[i].v > all[j].v + } + return all[i].k < all[j].k + }) + out := map[string]int{} + for i, x := range all { + if i >= n { + break + } + out[x.k] = x.v + } + return out +} + +// ---- OCPP profiles FTW sent ---- + +type profile struct { + T float64 `json:"t"` + Conn int `json:"conn"` + Purpose string `json:"purpose"` + Unit string `json:"unit"` + Limit float64 `json:"limit"` + Phases *float64 `json:"phases"` +} + +func (p profile) phasesOr3() float64 { + if p.Phases == nil || *p.Phases == 0 { + return 3 + } + return *p.Phases +} + +func (p profile) amps() bool { return strings.HasPrefix(strings.ToUpper(p.Unit), "A") } + +func (p profile) offerW() float64 { + if p.amps() { + return p.Limit * volt * p.phasesOr3() + } + return p.Limit +} + +// perPhaseA is the offer in amperes per phase. +func (p profile) perPhaseA() float64 { + if p.amps() { + return p.Limit + } + return p.Limit / (volt * p.phasesOr3()) +} + +// profiles lists the SetChargingProfile CALLs FTW sent in [t0, t1), timed +// from t0. +func profiles(frames []frame, t0, t1 float64) []profile { + var out []profile + for _, f := range frames { + if f.T < t0 || f.T >= t1 || f.Dir != "cs->cp" { + continue + } + var arr []json.RawMessage + if json.Unmarshal(f.Frame, &arr) != nil || len(arr) < 4 { + continue + } + var typ int + var action string + json.Unmarshal(arr[0], &typ) + json.Unmarshal(arr[2], &action) + if typ != 2 || action != "SetChargingProfile" { + continue + } + var req struct { + ConnectorID int `json:"connectorId"` + Profiles struct { + Purpose string `json:"chargingProfilePurpose"` + Schedule struct { + Unit string `json:"chargingRateUnit"` + Periods []struct { + Limit float64 `json:"limit"` + Phases *float64 `json:"numberPhases"` + } `json:"chargingSchedulePeriod"` + } `json:"chargingSchedule"` + } `json:"csChargingProfiles"` + } + json.Unmarshal(arr[3], &req) + p := profile{T: round(f.T-t0, 1), Conn: req.ConnectorID, Purpose: req.Profiles.Purpose, Unit: req.Profiles.Schedule.Unit} + if per := req.Profiles.Schedule.Periods; len(per) > 0 { + p.Limit, p.Phases = per[0].Limit, per[0].Phases + } + out = append(out, p) + } + return out +} + +func connector1(ps []profile) []profile { + var out []profile + for _, p := range ps { + if p.Conn == 1 { + out = append(out, p) + } + } + return out +} + +// ---- verdicts ---- + +// metrics are a run's numbers, written to result.json. +type metrics map[string]any + +// criterion is one pass condition. A scenario's known map names the +// criteria that fail on master and why. +type criterion struct { + Name string `json:"name"` + OK bool `json:"ok"` +} + +func crit(name string, ok bool) criterion { return criterion{Name: name, OK: ok} } + +// ---- shared summaries ---- + +func (d *runData) carSummary(t0, t1, targetW float64) metrics { + w := d.window(t0, t1) + ts := d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.95*minW }) + tm := d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.98*targetW }) + var hold *float64 + if tm != nil { + var after []*sample + for _, r := range w { + if r.T >= t0+*tm { + after = append(after, r) + } + } + n := 0 + for _, r := range after { + if carW(r, 0) >= 0.95*targetW { + n++ + } + } + h := round(float64(n)/math.Max(1, float64(len(after))), 3) + hold = &h + } + drops, on := 0, false + for _, r := range w { + c := carW(r, 0) + if c >= 0.9*minW { + on = true + } else if c < 500 && on { + drops++ + on = false + } + } + rs := counter{} + for _, r := range w { + for _, x := range reasons(r) { + rs[x]++ + } + } + fuseLimit := 0 + for k, v := range rs { + if strings.Contains(k, "fuse") { + fuseLimit += v + } + } + var last60 []float64 + for _, r := range w { + if r.T >= t1-60 { + last60 = append(last60, carW(r, 0)) + } + } + var minAfter *float64 + if ts != nil { + m := math.Inf(1) + for _, r := range w { + if r.T >= t0+*ts { + m = math.Min(m, carW(r, 0)) + } + } + m = math.Round(m) + minAfter = &m + } + return metrics{ + "time_to_start_s": roundp(ts, 1), + "time_to_target_s": roundp(tm, 1), + "hold_fraction": hold, + "stops_after_start": drops, + "car_w_last60_mean": math.Round(mean(last60)), + "car_w_min_after_start": minAfter, + "fuse_limit_samples": fuseLimit, + "reasons": rs.top(6), + "max_abs_i": round(maxOf(w, maxAbsI), 2), + "fuse_damage_max": round(maxOf(w, damage), 4), + "fuse_blown": anyBlown(w), + } +} + +func anyBlown(ws []*sample) bool { + for _, r := range ws { + if blown(r) { + return true + } + } + return false +} + +func le(p *float64, limit float64) bool { return p != nil && *p <= limit } + +func (d *runData) overage(t0, t1, fuse float64) metrics { + w := d.window(t0, t1) + m := metrics{"max_i": round(maxOf(w, maxAbsI), 2)} + m["over_limit_longest_s"], m["over_limit_total_s"] = d.runsOver(t0, t1, fuse-1) + m["over_fuse_longest_s"], _ = d.runsOver(t0, t1, fuse) + m["fuse_damage_max"] = round(maxOf(w, damage), 4) + m["fuse_blown"] = anyBlown(w) + return m +} + +func (d *runData) phaseMean(t0, t1 float64) [3]float64 { + w := d.window(t0, t1) + var out [3]float64 + for k := 0; k < 3; k++ { + out[k] = round(meanOf(w, func(r *sample) float64 { return math.Abs(phaseI(r)[k]) }), 1) + } + return out +} + +// disconnects lists "OCPP charger disconnected" lines in FTW's log between +// t0 and t1, in seconds from t0. +func (d *runData) disconnects(t0, t1 float64) []float64 { + out := []float64{} + readLines(filepath.Join(d.dir, "ftw.log"), func(b []byte) { + line := string(b) + if !strings.Contains(line, "OCPP charger disconnected") || !strings.HasPrefix(line, "time=") { + return + } + ts, _, _ := strings.Cut(strings.TrimPrefix(line, "time="), " ") + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return + } + at := float64(t.UnixMicro()) / 1e6 + if t0 <= at && at <= t1 { + out = append(out, round(at-t0, 1)) + } + }) + return out +} diff --git a/go/test/site/procattr_linux_test.go b/go/test/site/procattr_linux_test.go new file mode 100644 index 000000000..7fa4fff10 --- /dev/null +++ b/go/test/site/procattr_linux_test.go @@ -0,0 +1,9 @@ +package site + +import "syscall" + +// childProcAttr puts a child in its own process group and kills it if the +// test binary dies, as when go test's -timeout fires. +func childProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL} +} diff --git a/go/test/site/procattr_other_test.go b/go/test/site/procattr_other_test.go new file mode 100644 index 000000000..c38c4e548 --- /dev/null +++ b/go/test/site/procattr_other_test.go @@ -0,0 +1,11 @@ +//go:build unix && !linux + +package site + +import "syscall" + +// childProcAttr puts a child in its own process group. Without Pdeathsig a +// child outlives a test binary that dies; stop it by its run directory. +func childProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setpgid: true} +} diff --git a/go/test/site/random_test.go b/go/test/site/random_test.go new file mode 100644 index 000000000..aeaa2b10d --- /dev/null +++ b/go/test/site/random_test.go @@ -0,0 +1,549 @@ +//go:build unix + +package site + +import ( + "encoding/json" + "fmt" + "math" + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// A random case draws FTW's side from a seed: the fuse, the house, the +// loadpoint's settings and FTW's requests (Charge now, a pause, a plan, a +// Core restart, an import ceiling) at random times. The house draws its own +// timeline from the same seed (device-simulator's random block): load steps +// and spikes, PV clouds, a battery at its floor, a car taking less, slow +// chargers, silent devices, reconnects, a late or glitched meter, signed +// currents. FTW's invariants are checked every second. The house writes +// the timeline it drew, so a failing seed can be replayed and kept. + +// randomDurationS is how long a random case runs. +const randomDurationS = 1200.0 + +// randomTolerated are invariants a seed may break for a finding still open +// on master. A fix deletes its line here. +var randomTolerated = map[string]string{ + "no discharge asked at the battery's floor": "ev-099: the fuse guard drives the battery below its floor; fix unmerged", +} + +// ftwAction is one of FTW's requests in a random case. +type ftwAction struct { + AtS float64 `json:"at_s"` + Do string `json:"do"` // charge_now, pause, clear, plan, restart, ceiling + W float64 `json:"w,omitempty"` + PhaseMode string `json:"phase_mode,omitempty"` + DownS float64 `json:"down_s,omitempty"` + CeilingW float64 `json:"ceiling_w,omitempty"` + SoCPct float64 `json:"soc_pct,omitempty"` + GoalPct float64 `json:"goal_pct,omitempty"` + ReadyMin float64 `json:"ready_min,omitempty"` +} + +// randomPlan is everything a random case does, so it can be saved and run +// again: House is set once the house has drawn its timeline. +type randomPlan struct { + Seed int64 `json:"seed"` + FuseA float64 `json:"fuse_a"` + HouseA [3]float64 `json:"house_a"` + PVW float64 `json:"pv_w"` + SoCPct float64 `json:"soc_pct"` + LPExtra map[string]any `json:"lp_extra,omitempty"` + Actions []ftwAction `json:"actions"` + DurationS float64 `json:"duration_s"` + House *houseSpec `json:"house,omitempty"` + Found *foundNote `json:"found,omitempty"` +} + +// foundNote says what a saved case failed and when it was found. +type foundNote struct { + Invariant string `json:"invariant"` + On string `json:"on"` // the date + Commit string `json:"commit,omitempty"` +} + +func drawPlan(seed int64) randomPlan { + rng := rand.New(rand.NewSource(seed)) + pick := func(xs ...float64) float64 { return xs[rng.Intn(len(xs))] } + between := func(lo, hi float64) float64 { return math.Round((lo+rng.Float64()*(hi-lo))*10) / 10 } + p := randomPlan{Seed: seed, DurationS: randomDurationS, + FuseA: pick(16, 20, 25), PVW: pick(0, 0, 2000, 5000, 8000), SoCPct: pick(6, 15, 40, 70, 95)} + for k := range p.HouseA { + p.HouseA[k] = between(0.5, p.FuseA/2) + } + p.LPExtra = map[string]any{} + switch rng.Intn(3) { + case 1: + p.LPExtra["phase_mode"] = "auto" + case 2: + p.LPExtra["phase_mode"] = "3p" + } + if rng.Intn(2) == 0 { + p.LPExtra["allowed_steps_w"] = easeeSteps + } + for t := 40 + between(0, 60); t < p.DurationS-60; t += between(60, 300) { + a := ftwAction{AtS: t} + switch rng.Intn(10) { + case 0, 1, 2: + a.Do, a.W, a.PhaseMode = "charge_now", pick(4140, 7360, 11040), "3p" + case 3: + a.Do, a.W, a.PhaseMode = "charge_now", pick(1380, 3680), "1p" + case 4: + a.Do = "pause" + case 5: + a.Do = "clear" + case 6: + a.Do, a.SoCPct, a.GoalPct, a.ReadyMin = "plan", between(20, 50), between(60, 90), pick(30, 60, 240) + case 7: + a.Do, a.DownS = "restart", between(5, 30) + case 8: + a.Do, a.CeilingW = "ceiling", pick(0, 3000, 5000, 8000) + default: + continue + } + p.Actions = append(p.Actions, a) + } + return p +} + +// randomScenario runs a plan drawn from seed, or a saved one. +func randomScenario(p randomPlan) scenario { + name, dir := fmt.Sprintf("r%d", p.Seed), fmt.Sprintf("random-%d", p.Seed) + sc := scenario{name: name, dir: dir, minutes: p.DurationS/60 + 1, + run: func(c *runCtx) (map[string]any, error) { return runPlan(c, p) }, + judge: judgeInvariants, + tolerate: randomTolerated} + if p.Found != nil { + sc.name, sc.dir = "found-"+fmt.Sprint(p.Seed), "found-"+fmt.Sprint(p.Seed) + sc.known = map[string]string{p.Found.Invariant: fmt.Sprintf("found by random seed %d on %s; triage it", p.Seed, p.Found.On)} + } + return sc +} + +func runPlan(c *runCtx, p randomPlan) (map[string]any, error) { + loads := [3]float64{p.HouseA[0] * volt, p.HouseA[1] * volt, p.HouseA[2] * volt} + spec := stackSpec{fuseA: p.FuseA, loadsW: loads, pvW: p.PVW, socPct: p.SoCPct, lpExtra: p.LPExtra} + if p.House != nil { + spec.house = p.House + } else { + spec.random = &houseRandom{Seed: p.Seed, DurationS: p.DurationS, GapS: 45} + } + plan, _ := json.MarshalIndent(p, "", " ") + err := with(c, spec, func(st *stack) error { + os.WriteFile(filepath.Join(st.rundir, "plan.json"), plan, 0o644) + if err := st.plug(0); err != nil { + return err + } + t0 := time.Now() + st.mark("start") + for _, a := range p.Actions { + sleepS(time.Until(t0.Add(time.Duration(a.AtS * float64(time.Second)))).Seconds()) + st.mark("ftw:" + a.Do) + var err error + switch a.Do { + case "charge_now": + err = st.chargeNow("garage", 0, a.PhaseMode, a.W) + case "pause": + err = st.chargeNow("garage", 0, "3p", -1) + case "clear": + _, err = call("DELETE", st.ftw+"/api/loadpoints/garage/manual_hold", nil, nil) + case "plan": + _, err = st.planTo("garage", a.SoCPct, a.GoalPct, time.Duration(a.ReadyMin*float64(time.Minute))) + case "restart": + st.stopFTW() + sleepS(a.DownS) + err = st.startFTW() + st.mark("ftw:up") + case "ceiling": + err = st.post("/api/peak_import_ceiling", map[string]any{"peak_import_ceiling_w": a.CeilingW}, nil) + } + if err != nil { + st.mark("ftw:error") + } + } + sleepS(time.Until(t0.Add(time.Duration(p.DurationS * float64(time.Second)))).Seconds()) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "random", "seed": p.Seed, "fuse": p.FuseA}, err +} + +// ---- invariants, every second ---- + +// houseTiming is what the house picked for the charger, from the spec it +// ran (timeline.json for a random house). +type houseTiming struct { + commandS, startupS float64 + floorPct float64 +} + +func (d *runData) timing() houseTiming { + h := houseTiming{floorPct: 5} + for _, name := range []string{"timeline.json", "../house.json"} { + b, err := os.ReadFile(filepath.Join(d.dir, "house", name)) + if err != nil { + continue + } + var spec houseSpec + if json.Unmarshal(b, &spec) != nil { + continue + } + for _, dev := range spec.Devices { + if dev.ID == "garage" { + if dev.CommandResponseS != nil { + h.commandS = *dev.CommandResponseS + } + if dev.StartupDelayS != nil { + h.startupS = *dev.StartupDelayS + } + } + if dev.ID == "inverter" && dev.MinSoCPct != nil { + h.floorPct = *dev.MinSoCPct + } + } + break + } + return h +} + +// eventWindow is when the house changed on its own, and how long FTW may +// take to answer it. +type eventWindow struct{ from, to float64 } + +func judgeInvariants(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + limit := fuse - 1 + h := d.timing() + t0, t1 := d.marks["start"], d.marks["end"] + run := d.window(t0, t1) + m := metrics{"charger_command_s": h.commandS, "charger_startup_s": h.startupS, "events": len(d.events)} + + // Windows in which an over-limit second is the house's doing. + var windows []eventWindow + faultUntil := map[float64]float64{} // a fault's start and end + for _, ev := range d.events { + batteryUsable := true + if r := d.at(ev.T); r != nil && r.Bat.SoCPct <= h.floorPct+1 { + batteryUsable = false + } + // The fastest lever: the battery answers in 2 s, the charger in its + // own time, never under 2 s; then the meter and tick, 2 s, and 3 s. + lever := math.Max(2, h.commandS) + if batteryUsable { + lever = 2 + } + allow := lever + 2 + 3 + switch ev.Event.Do { + case "meter_late", "silence", "offline", "reconnect": + allow += ev.Event.ForS + faultUntil[ev.T] = ev.T + ev.Event.ForS + case "meter_glitch": + faultUntil[ev.T] = ev.T + 5 + } + windows = append(windows, eventWindow{ev.T, ev.T + allow}) + } + for _, mk := range d.allMarks { + if mk.Label == "ftw:restart" { + windows = append(windows, eventWindow{mk.T, mk.T + 60}) + } + } + explained := func(t float64) (float64, bool) { + best, ok := 0.0, false + for _, w := range windows { + if t >= w.from && t <= w.to { + best, ok = math.Max(best, w.to), true + } + } + return best, ok + } + + // 1, 2: the fuse holds; over the limit only as long as an outside step + // allows, and at most 3 s after FTW's own raise. + blownAny := anyBlown(run) + unexplained, longestOwn := 0.0, 0.0 + var firstBad float64 + var start *sample + for i, r := range run { + over := maxAbsI(r) > limit + if over && start == nil { + start = r + } + if (!over || i == len(run)-1) && start != nil { + end := r.T + if to, ok := explained(start.T); ok { + if end > to { + unexplained = math.Max(unexplained, end-to) + if firstBad == 0 { + firstBad = to + } + } + } else if end-start.T > 3 { + longestOwn = math.Max(longestOwn, end-start.T) + if firstBad == 0 { + firstBad = start.T + 3 + } + } + start = nil + } + } + m["over_limit_after_steps_s"], m["over_limit_own_s"] = round(unexplained, 1), round(longestOwn, 1) + + // 3: with every device reading and no fault, Home use is measured. + inFault := func(t float64) bool { + for from, to := range faultUntil { + if t >= from && t <= to+30 { + return true + } + } + return false + } + restartAt := []float64{t0} + for _, mk := range d.allMarks { + if mk.Label == "ftw:up" { + restartAt = append(restartAt, mk.T) + } + } + nearRestart := func(t float64) bool { + for _, r := range restartAt { + if t >= r && t <= r+60 { + return true + } + } + return false + } + first := map[string]float64{} + note := func(name string, t float64) { + if _, ok := first[name]; !ok { + first[name] = t + } + } + if firstBad > 0 { + note("limit", firstBad) + } + for _, r := range run { + if blown(r) { + note("fuse", r.T) + break + } + } + dishonest, run3 := 0.0, 0.0 + for i, r := range run { + bad := r.Status != nil && allRead(r) && !inFault(r.T) && !nearRestart(r.T) && str(status(r).LoadQuality) != "measured" + if bad && i > 0 { + run3 += r.T - run[i-1].T + dishonest = math.Max(dishonest, run3) + if dishonest > 10 { + note("home use", r.T) + } + } else { + run3 = 0 + } + } + m["home_use_estimated_while_reading_s"] = round(dishonest, 1) + + // 4: during Charge now the car charges when there is room for it. + var holds []eventWindow + var holdFrom float64 + for _, mk := range d.allMarks { + switch mk.Label { + case "ftw:charge_now": + if holdFrom == 0 { + holdFrom = mk.T + } + case "ftw:pause", "ftw:clear", "ftw:plan", "end": + if holdFrom > 0 { + holds = append(holds, eventWindow{holdFrom, mk.T}) + holdFrom = 0 + } + } + } + grace := 60 + h.commandS + h.startupS + 10 + idle, run4 := 0.0, 0.0 + for _, hw := range holds { + run4 = 0 + for i, r := range d.window(hw.from+grace, hw.to) { + roomA := limit - (maxAbsI(r) - carW(r, 0)/(3*volt)) + bad := carW(r, 0) < 0.5*1380 && roomA >= 7 && !inFault(r.T) && !nearRestart(r.T) + if bad && i > 0 { + run4++ + idle = math.Max(idle, run4) + if idle > 30 { + note("charge now", r.T) + } + } else { + run4 = 0 + } + } + } + m["charge_now_idle_with_room_s"] = idle + + // 5: no order above the charger's 16 A per phase. + over := 0 + for _, p := range connector1(profiles(d.frames, t0, t1)) { + if p.Limit > 0 && p.perPhaseA() > 16+1e-6 { + over++ + note("rating", t0+p.T) + } + } + m["offers_over_rating"] = over + + // 6: FTW does not ask the battery to discharge at its floor. + floorS := 0.0 + for _, r := range run { + if r.Bat.EMSCmd == 187 && r.Bat.EMSSetW > 50 && r.Bat.SoCPct <= h.floorPct+0.5 { + floorS++ + if floorS > 10 { + note("floor", r.T) + } + } + } + m["discharge_at_floor_s"] = floorS + + // 7: a stale meter means no raise. + raises := 0 + prs := connector1(profiles(d.frames, t0, t1)) + for _, ev := range d.events { + if ev.Event.Do != "meter_late" { + continue + } + for i := 1; i < len(prs); i++ { + at := t0 + prs[i].T + if at > ev.T+2 && at <= ev.T+ev.Event.ForS && prs[i].offerW() > prs[i-1].offerW()+1 { + raises++ + note("stale meter", at) + } + } + } + m["raises_on_a_stale_meter"] = raises + // The first second an invariant broke, from the start, per invariant: + // a kept seed is cut a minute after the one it failed on. + firsts := map[string]float64{} + for k, t := range first { + firsts[k] = round(t-t0, 1) + } + m["first_violation_s"] = firsts + return m, []criterion{ + crit("fuse holds", !blownAny), + crit("over the limit no longer than an outside step allows", unexplained == 0), + crit("over the limit at most 3 s after FTW's own raise", longestOwn == 0), + crit("Home use measured while every device reads", dishonest <= 10), + crit("Charge now charges when there is room", idle <= 30), + crit("no offer above the charger's rating", over == 0), + crit("no discharge asked at the battery's floor", floorS <= 10), + crit("no raise on a stale meter", raises == 0), + } +} + +// ---- seeds, replays and saved cases ---- + +// randomSet parses FTW_SITE=random:: (or random:). +func randomSet(which string) ([]scenario, bool) { + if !strings.HasPrefix(which, "random:") { + return nil, false + } + var first, count int64 = 0, 1 + parts := strings.Split(strings.TrimPrefix(which, "random:"), ":") + fmt.Sscan(parts[0], &first) + if len(parts) > 1 { + fmt.Sscan(parts[1], &count) + } + var out []scenario + for s := first; s < first+count; s++ { + out = append(out, randomScenario(drawPlan(s))) + } + return out, true +} + +// foundScenarios are saved failing seeds (testdata/found/*.json): fixed +// cases in the full set, known until someone triages them. +func foundScenarios(root string) []scenario { + files, _ := filepath.Glob(filepath.Join(root, "go", "test", "site", "testdata", "found", "*.json")) + sort.Strings(files) + var out []scenario + for _, f := range files { + b, err := os.ReadFile(f) + if err != nil { + continue + } + var p randomPlan + if json.Unmarshal(b, &p) == nil && p.House != nil && p.Found != nil { + out = append(out, randomScenario(p)) + } + } + return out +} + +// invariantKey ties each criterion to its first_violation_s entry. +var invariantKey = map[string]string{ + "fuse holds": "fuse", + "over the limit no longer than an outside step allows": "limit", + "over the limit at most 3 s after FTW's own raise": "limit", + "Home use measured while every device reads": "home use", + "Charge now charges when there is room": "charge now", + "no offer above the charger's rating": "rating", + "no discharge asked at the battery's floor": "floor", + "no raise on a stale meter": "stale meter", +} + +// keepFailure writes a failing random run as a case to keep: its plan with +// the house's timeline, both cut a minute after the first violation. +func keepFailure(rundir, out string, o outcome) string { + b, err := os.ReadFile(filepath.Join(rundir, "plan.json")) + if err != nil { + return "" + } + var p randomPlan + if json.Unmarshal(b, &p) != nil { + return "" + } + var h houseSpec + if hb, err := os.ReadFile(filepath.Join(rundir, "house", "timeline.json")); err == nil && json.Unmarshal(hb, &h) == nil { + p.House = &h + } else if p.House == nil { + return "" + } + failed := "" + for _, c := range o.Criteria { + if _, tolerated := randomTolerated[c.Name]; !c.OK && !tolerated { + failed = c.Name + break + } + } + var res struct { + Metrics map[string]any `json:"metrics"` + } + if rb, err := os.ReadFile(filepath.Join(rundir, "result.json")); err == nil && json.Unmarshal(rb, &res) == nil { + firsts, _ := res.Metrics["first_violation_s"].(map[string]any) + at, ok := firsts[invariantKey[failed]].(float64) + if ok && at+60 < p.DurationS { + p.DurationS = math.Ceil(at + 60) + var evs []houseEvent + for _, ev := range p.House.Timeline { + if ev.AtS <= p.DurationS { + evs = append(evs, ev) + } + } + p.House.Timeline = evs + var acts []ftwAction + for _, a := range p.Actions { + if a.AtS <= p.DurationS { + acts = append(acts, a) + } + } + p.Actions = acts + } + } + p.Found = &foundNote{Invariant: failed, On: time.Now().Format("2006-01-02")} + dst := filepath.Join(out, "found", fmt.Sprintf("%d.json", p.Seed)) + os.MkdirAll(filepath.Dir(dst), 0o755) + nb, _ := json.MarshalIndent(p, "", " ") + if os.WriteFile(dst, nb, 0o644) != nil { + return "" + } + return dst +} diff --git a/go/test/site/rig_test.go b/go/test/site/rig_test.go new file mode 100644 index 000000000..35184ffb2 --- /dev/null +++ b/go/test/site/rig_test.go @@ -0,0 +1,413 @@ +//go:build unix + +// Package site runs the real FTW binary against srcfl/device-simulator: a +// Sungrow hybrid over Modbus, an SDM630 site meter behind an IEC gG fuse on +// each phase, and OCPP 1.6 chargers. Each scenario builds a fresh house and +// a fresh state.db, drives it, records the simulator and FTW once a second +// and judges the run. See README.md for the scenarios and how to run them. +package site + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// rigEnv is what every scenario shares. +type rigEnv struct { + root string // repository root: drivers/ and web/ + ftwBin string + devsimBin string + out string +} + +func repoRoot(t *testing.T) string { + _, file, _, _ := runtime.Caller(0) + dir := filepath.Dir(file) + for i := 0; i < 6; i++ { + if _, err := os.Stat(filepath.Join(dir, "Makefile")); err == nil { + return dir + } + dir = filepath.Dir(dir) + } + t.Fatal("could not locate the repository root") + return "" +} + +// devsimCommit is the device-simulator commit the rig runs, from +// devsim.version. +func devsimCommit(t *testing.T, root string) string { + b, err := os.ReadFile(filepath.Join(root, "go", "test", "site", "devsim.version")) + if err != nil { + t.Fatal(err) + } + c := strings.TrimSpace(string(b)) + if len(c) != 40 { + t.Fatalf("devsim.version: want a full commit hash, got %q", c) + } + return c +} + +func mustRun(t *testing.T, dir string, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } +} + +// buildDevsim builds the pinned device-simulator, from FTW_SITE_DEVSIM_SRC +// when set (it must be at the pinned commit) or from a clone in the cache. +func buildDevsim(t *testing.T, root, bin string) { + commit := devsimCommit(t, root) + src := os.Getenv("FTW_SITE_DEVSIM_SRC") + if src == "" { + cache, err := os.UserCacheDir() + if err != nil { + t.Fatal(err) + } + src = filepath.Join(cache, "ftw-site", "device-simulator-"+commit[:12]) + if _, err := os.Stat(filepath.Join(src, "go.mod")); err != nil { + os.RemoveAll(src) + mustRun(t, "", "git", "init", "-q", src) + mustRun(t, src, "git", "fetch", "-q", "--depth=1", "https://github.com/srcfl/device-simulator", commit) + mustRun(t, src, "git", "checkout", "-q", "FETCH_HEAD") + } + } + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = src + head, err := cmd.Output() + if err != nil || strings.TrimSpace(string(head)) != commit { + t.Fatalf("device-simulator in %s is at %s, not the pinned %s", src, strings.TrimSpace(string(head)), commit) + } + cmd = exec.Command("go", "build", "-o", bin, "./cmd/simulator/") + cmd.Dir = src + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build device-simulator: %v\n%s", err, out) + } +} + +func setup(t *testing.T) *rigEnv { + root := repoRoot(t) + for _, f := range []string{"sdm630.lua", "sungrow.lua"} { + if _, err := os.Stat(filepath.Join(root, "drivers", f)); err != nil { + t.Fatalf("drivers/%s missing: run make drivers first", f) + } + } + out := os.Getenv("FTW_SITE_OUT") + if out == "" { + var err error + if out, err = os.MkdirTemp("", "ftw-site-"); err != nil { + t.Fatal(err) + } + } + env := &rigEnv{root: root, out: out, + ftwBin: os.Getenv("FTW_SITE_FTW_BIN"), devsimBin: os.Getenv("FTW_SITE_DEVSIM_BIN")} + bin := filepath.Join(out, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + if env.ftwBin == "" { + env.ftwBin = filepath.Join(bin, "ftw") + mustRun(t, filepath.Join(root, "go"), "go", "build", "-o", env.ftwBin, "./cmd/ftw") + } + if env.devsimBin == "" { + env.devsimBin = filepath.Join(bin, "devsim") + buildDevsim(t, root, env.devsimBin) + } + t.Logf("results in %s", out) + return env +} + +// selected resolves FTW_SITE: "short" (the pull-request set), "full" (every +// scenario and every saved random case), "random::", or +// a comma-separated list of names. +func selected(t *testing.T, which string) []scenario { + if rs, ok := randomSet(which); ok { + return rs + } + var out []scenario + switch which { + case "full", "all": + for _, sc := range scenarios { + if sc.name != "s7s-25" { // the full set runs the long s7 + out = append(out, sc) + } + } + out = append(out, foundScenarios(repoRoot(t))...) + case "short": + for _, sc := range scenarios { + if sc.short { + out = append(out, sc) + } + } + default: + byName := map[string]scenario{} + for _, sc := range scenarios { + byName[sc.name] = sc + } + for _, n := range strings.Split(which, ",") { + sc, ok := byName[strings.TrimSpace(n)] + if !ok { + t.Fatalf("unknown scenario %q", n) + } + out = append(out, sc) + } + } + return out +} + +// shard picks shard k of n (FTW_SITE_SHARD "k/n"). +func shard(t *testing.T, scs []scenario, spec string) []scenario { + if spec == "" { + return scs + } + k, n := 0, 0 + if _, err := fmt.Sscanf(spec, "%d/%d", &k, &n); err != nil || n < 1 || k < 1 || k > n { + t.Fatalf("FTW_SITE_SHARD %q: want k/n", spec) + } + return split(scs, n)[k-1] +} + +// split deals scenarios into n groups of about equal time: longest first, +// each to the group with the least time so far. +func split(scs []scenario, n int) [][]scenario { + sorted := append([]scenario(nil), scs...) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].minutes > sorted[j].minutes }) + load := make([]float64, n) + out := make([][]scenario, n) + for _, sc := range sorted { + best := 0 + for i := range load { + if load[i] < load[best] { + best = i + } + } + load[best] += sc.minutes + out[best] = append(out[best], sc) + } + return out +} + +// outcome is one scenario's line in summary.json. +type outcome struct { + Name string `json:"name"` + Dir string `json:"dir"` + Verdict string `json:"verdict"` // pass, known, fail + DurationS float64 `json:"duration_s"` + Criteria []criterion `json:"criteria"` + Known []string `json:"known,omitempty"` + Failures []string `json:"failures,omitempty"` + Error string `json:"error,omitempty"` +} + +// TestSite runs the scenarios FTW_SITE names. FTW_SITE_LANES runs that many +// at once on ports 100 apart from FTW_SITE_PORT_BASE (default 18700); +// FTW_SITE_SHARD splits a set over machines; FTW_SITE_OUT keeps the run +// directories. +func TestSite(t *testing.T) { + which := os.Getenv("FTW_SITE") + if which == "" { + t.Skip("set FTW_SITE=short, full or a list of scenarios; see go/test/site/README.md") + } + scs := shard(t, selected(t, which), os.Getenv("FTW_SITE_SHARD")) + base := 18700 + if s := os.Getenv("FTW_SITE_PORT_BASE"); s != "" { + var err error + if base, err = strconv.Atoi(s); err != nil { + t.Fatalf("FTW_SITE_PORT_BASE: %v", err) + } + } + lanes := 1 + if s := os.Getenv("FTW_SITE_LANES"); s != "" { + var err error + if lanes, err = strconv.Atoi(s); err != nil || lanes < 1 { + t.Fatalf("FTW_SITE_LANES %q", s) + } + } + env := setup(t) + var mu sync.Mutex + var outcomes []outcome + t.Run("lanes", func(t *testing.T) { + for k, group := range split(scs, lanes) { + k, group := k, group + t.Run(fmt.Sprintf("lane%d", k+1), func(t *testing.T) { + if lanes > 1 { + t.Parallel() + } + for _, sc := range group { + sc := sc + t.Run(sc.name, func(t *testing.T) { + o := runScenario(env, lane(base+100*k), sc) + mu.Lock() + outcomes = append(outcomes, o) + mu.Unlock() + if strings.HasPrefix(sc.dir, "random-") && o.Verdict == "fail" && o.Error == "" { + if kept := keepFailure(filepath.Join(env.out, sc.dir), env.out, o); kept != "" { + t.Logf("kept as %s; copy it to go/test/site/testdata/found/ to run it in the full set", kept) + } + } + t.Logf("%s in %.0f s: %v", o.Verdict, o.DurationS, o.Criteria) + for _, k := range o.Known { + t.Logf("fails as known: %s", k) + } + if o.Error != "" { + t.Fatal(o.Error) + } + for _, f := range o.Failures { + t.Error(f) + } + }) + } + }) + } + }) + sort.Slice(outcomes, func(i, j int) bool { return outcomes[i].Name < outcomes[j].Name }) + writeSummary(t, env.out, outcomes) +} + +func runScenario(env *rigEnv, l lane, sc scenario) outcome { + o := outcome{Name: sc.name, Dir: sc.dir, Verdict: "fail"} + rundir := filepath.Join(env.out, sc.dir) + t0 := time.Now() + meta, err := sc.run(&runCtx{env: env, lane: l, rundir: rundir}) + o.DurationS = time.Since(t0).Round(time.Second).Seconds() + if meta != nil { + b, _ := json.MarshalIndent(meta, "", " ") + os.WriteFile(filepath.Join(rundir, "meta.json"), b, 0o644) + } + if err != nil { + o.Error = err.Error() + return o + } + return judgeRun(sc, rundir, meta, o) +} + +// judgeRun judges a recorded run against the scenario's criteria and its +// known and tolerated lines, and writes result.json. +func judgeRun(sc scenario, rundir string, meta map[string]any, o outcome) outcome { + // Numbers go through JSON, as a rerun from the run directory sees them. + var m map[string]any + b, _ := json.Marshal(meta) + json.Unmarshal(b, &m) + d, err := loadRun(rundir, m) + if err != nil { + o.Error = err.Error() + return o + } + // The criteria count seconds. A host that sleeps or stalls (a laptop + // lid) stretches the run and leaves holes in the record. A Core restart + // costs a sample or two at most: FTW refuses at once or times out in 5 s. + if gap, at := d.longestGap(); gap > 15 { + o.Error = fmt.Sprintf("the recorder missed %.0f s at %.0f s into the run: the host paused, so the timing proves nothing", gap, at) + return o + } + numbers, criteria := sc.judge(d) + o.Criteria = criteria + for _, c := range criteria { + if finding, ok := sc.tolerate[c.Name]; ok { + if !c.OK { + o.Known = append(o.Known, fmt.Sprintf("%s: %s (tolerated)", c.Name, finding)) + } + continue + } + finding, known := sc.known[c.Name] + switch { + case known && c.OK: + o.Failures = append(o.Failures, fmt.Sprintf("%q passes now: delete its known line (%s)", c.Name, finding)) + case known: + o.Known = append(o.Known, fmt.Sprintf("%s: %s", c.Name, finding)) + case !c.OK: + o.Failures = append(o.Failures, fmt.Sprintf("%q fails", c.Name)) + } + } + for name := range sc.known { + found := false + for _, c := range criteria { + found = found || c.Name == name + } + if !found { + o.Failures = append(o.Failures, fmt.Sprintf("known line names no criterion: %q", name)) + } + } + switch { + case len(o.Failures) > 0: + o.Verdict = "fail" + case len(o.Known) > 0: + o.Verdict = "known" + default: + o.Verdict = "pass" + } + res, _ := json.MarshalIndent(map[string]any{"verdict": o.Verdict, "criteria": criteria, "known": o.Known, + "failures": o.Failures, "duration_s": o.DurationS, "metrics": numbers, "meta": meta, "samples": len(d.samples)}, "", " ") + os.WriteFile(filepath.Join(rundir, "result.json"), res, 0o644) + return o +} + +// TestJudge judges the recorded runs in FTW_SITE_JUDGE again with the +// criteria as they are now, without running anything: a changed criterion +// or known line is checked against a long run in seconds. +func TestJudge(t *testing.T) { + dir := os.Getenv("FTW_SITE_JUDGE") + if dir == "" { + t.Skip("set FTW_SITE_JUDGE to a results directory of an earlier run") + } + var outs []outcome + for _, sc := range selected(t, "full") { + rundir := filepath.Join(dir, sc.dir) + b, err := os.ReadFile(filepath.Join(rundir, "meta.json")) + if err != nil { + continue + } + var meta map[string]any + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatalf("%s: %v", rundir, err) + } + o := outcome{Name: sc.name, Dir: sc.dir, Verdict: "fail"} + if b, err := os.ReadFile(filepath.Join(rundir, "result.json")); err == nil { + var prev struct { + DurationS float64 `json:"duration_s"` + } + json.Unmarshal(b, &prev) + o.DurationS = prev.DurationS + } + o = judgeRun(sc, rundir, meta, o) + t.Logf("%s: %s %v", sc.name, o.Verdict, append(o.Failures, o.Known...)) + outs = append(outs, o) + } + if len(outs) == 0 { + t.Fatalf("no recorded run of a scenario in %s", dir) + } + writeSummary(t, dir, outs) +} + +func writeSummary(t *testing.T, out string, outs []outcome) { + if len(outs) == 0 { + return + } + b, _ := json.MarshalIndent(outs, "", " ") + os.WriteFile(filepath.Join(out, "summary.json"), b, 0o644) + var md strings.Builder + md.WriteString("| Scenario | Verdict | Time | Failed or known |\n|---|---|---|---|\n") + for _, o := range outs { + notes := append(append([]string{}, o.Failures...), o.Known...) + if o.Error != "" { + notes = append([]string{o.Error}, notes...) + } + fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", o.Name, o.Verdict, + (time.Duration(o.DurationS) * time.Second).String(), strings.ReplaceAll(strings.Join(notes, "; "), "|", "/")) + } + os.WriteFile(filepath.Join(out, "summary.md"), []byte(md.String()), 0o644) + t.Logf("summary:\n%s", md.String()) +} diff --git a/go/test/site/rig_unit_test.go b/go/test/site/rig_unit_test.go new file mode 100644 index 000000000..7e921c2ad --- /dev/null +++ b/go/test/site/rig_unit_test.go @@ -0,0 +1,141 @@ +//go:build unix + +package site + +import ( + "encoding/json" + "testing" +) + +// These run in every go test, without a simulator. + +func TestShardsCoverEveryScenarioOnce(t *testing.T) { + for _, set := range []string{"short", "full"} { + all := selected(t, set) + for n := 1; n <= 7; n++ { + seen := map[string]int{} + for k := 1; k <= n; k++ { + for _, sc := range shard(t, all, fmtShard(k, n)) { + seen[sc.name]++ + } + } + for _, sc := range all { + if seen[sc.name] != 1 { + t.Fatalf("%s in %d shards: %s runs %d times", set, n, sc.name, seen[sc.name]) + } + } + } + } +} + +func fmtShard(k, n int) string { return string(rune('0'+k)) + "/" + string(rune('0'+n)) } + +func TestFullSetRunsTheLongRestart(t *testing.T) { + names := map[string]bool{} + for _, sc := range selected(t, "full") { + names[sc.name] = true + } + if !names["s7-25"] || names["s7s-25"] || len(names) != len(scenarios)-1 { + t.Fatalf("full set %v", names) + } +} + +// A known line must name a criterion its judge reports. +func TestKnownLinesNameCriteria(t *testing.T) { + for _, sc := range scenarios { + _, criteria := sc.judge(&runData{marks: map[string]float64{}, meta: map[string]any{"fuse": 25.0}}) + have := map[string]bool{} + for _, c := range criteria { + have[c.Name] = true + } + for name := range sc.known { + if !have[name] { + t.Errorf("%s: known line names no criterion: %q", sc.name, name) + } + } + } + _, criteria := judgeInvariants(&runData{marks: map[string]float64{}, meta: map[string]any{"fuse": 25.0}}) + have := map[string]bool{} + for _, c := range criteria { + have[c.Name] = true + if invariantKey[c.Name] == "" { + t.Errorf("invariant %q has no first_violation_s key", c.Name) + } + } + for name := range randomTolerated { + if !have[name] { + t.Errorf("tolerated line names no invariant: %q", name) + } + } +} + +func TestRunsOver(t *testing.T) { + d := &runData{} + for i, a := range []float64{20, 25, 26, 26, 20, 25, 20} { + d.samples = append(d.samples, &sample{T: float64(i), PhaseA: [3]float64{a, 0, 0}}) + } + // As the Python rig counted: a run lasts until the first sample under + // the limit, and each sample over it adds the time since the last one. + longest, total := d.runsOver(0, 10, 24) + if longest != 3 || total != 4 { + t.Fatalf("longest %v total %v, want 3 and 4", longest, total) + } +} + +func TestRandomPlansAreSeeded(t *testing.T) { + a, b, c := drawPlan(3), drawPlan(3), drawPlan(4) + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + jc, _ := json.Marshal(c) + if string(ja) != string(jb) || string(ja) == string(jc) { + t.Fatal("seed 3 twice differs, or seed 4 is the same") + } + for seed := int64(1); seed <= 30; seed++ { + p := drawPlan(seed) + for _, a := range p.Actions { + if a.AtS < 40 || a.AtS > p.DurationS-60 { + t.Fatalf("seed %d: %s at %.0f s", seed, a.Do, a.AtS) + } + } + for _, x := range p.HouseA { + if x > p.FuseA/2 { + t.Fatalf("seed %d: house %.1f A on %.0f A", seed, x, p.FuseA) + } + } + } +} + +// An over-limit stretch right after an outside step is the house's; the +// same stretch with no step before it is FTW's own and fails after 3 s. +func TestInvariantsTellStepsFromOwnRaises(t *testing.T) { + mk := func(withStep bool) *runData { + d := &runData{marks: map[string]float64{"start": 0, "end": 100}, meta: map[string]any{"fuse": 25.0}} + for i := 0; i <= 100; i++ { + a := 10.0 + if i >= 50 && i < 56 { + a = 26 // 6 s over the 24 A limit + } + d.samples = append(d.samples, &sample{T: float64(i), PhaseA: [3]float64{a, a, a}, Bat: plantDevice{SoCPct: 50}}) + } + if withStep { + d.events = append(d.events, houseEventLine{T: 49.5, Event: houseEvent{Do: "loads"}}) + } + return d + } + failed := func(cs []criterion) []string { + var out []string + for _, c := range cs { + if !c.OK { + out = append(out, c.Name) + } + } + return out + } + if _, cs := judgeInvariants(mk(true)); len(failed(cs)) != 0 { + t.Fatalf("after a step: %v", failed(cs)) + } + _, cs := judgeInvariants(mk(false)) + if f := failed(cs); len(f) != 1 || f[0] != "over the limit at most 3 s after FTW's own raise" { + t.Fatalf("without a step: %v", f) + } +} diff --git a/go/test/site/runner/README.md b/go/test/site/runner/README.md new file mode 100644 index 000000000..346d7747c --- /dev/null +++ b/go/test/site/runner/README.md @@ -0,0 +1,78 @@ +# siterig-runner + +Runs FTW's site test (`go/test/site`) on a machine of its own and publishes +the result as a GitHub commit status on the exact commit it tested: + +| Status | When | What runs | +|---|---|---| +| `siterig/full` | each new commit on `master` | every scenario | +| `siterig/short` | the head of each open pull request from this repository that touches `go/internal/{control,loadpoint,ocpp,telemetry}` or `go/test/site` | the short set | +| `siterig/random` | once a night on `master` | that night's random seeds | + +A commit that already has the status is not run again. To run one again, +post a new `pending` status or remove the result and wait a day. No GitHub +Actions workflow runs the rig; `native-release.yml` only reads +`siterig/full`. + +The description says how it went, for example `16 pass, 3 known, 0 fail of +19 in 24m`. The run directories stay in `~/siterig/results` for 14 days. + +## The machine + +Any Mac or Linux host with Go, git, make and the `gh` CLI that can stay +awake. A laptop works if it does not sleep: a sleeping host stretches the +rig's timing, and the rig then fails the run as "host paused". Not the home +energy box: the rig needs the CPU, and the box is production. + +The rig uses 100 ports per lane from `-port-base` (default 18700) on +127.0.0.1. Leave them free. + +Pull requests from forks are skipped unless `-allow-forks` is set: the rig +builds and runs the pull request's code on this machine. + +## Install + +```bash +gh auth login # an account that may write commit statuses on srcfl/ftw +mkdir -p ~/siterig/bin +git clone https://github.com/srcfl/ftw ~/siterig/src +cd ~/siterig/src/go && go build -o ~/siterig/bin/siterig-runner ./test/site/runner +~/siterig/bin/siterig-runner -once # one pass in the foreground +``` + +macOS (launchd): + +```bash +sed "s/USER/$USER/g" ~/siterig/src/go/test/site/runner/energy.ftw.siterig-runner.plist \ + > ~/Library/LaunchAgents/energy.ftw.siterig-runner.plist +launchctl load ~/Library/LaunchAgents/energy.ftw.siterig-runner.plist +``` + +Linux (systemd): edit `siterig-runner.service`, copy it to +`/etc/systemd/system/` and run `sudo systemctl enable --now siterig-runner`. + +The log is `~/siterig/runner.log` (launchd) or `journalctl -u +siterig-runner` (systemd). Rebuild the binary after a change to this +directory; the scenarios themselves always come from the commit under test. + +## How long it takes + +On an M5 MacBook (10 cores), 22 October 2026: the 23 scenarios of that +morning took 72 minutes in three lanes. With the one-phase charger cases the +set adds up to about 270 minutes, so three lanes need about 90 and four +about 70; `s12-20` alone takes up to 56, so more than four do not help. +Each lane is one `ftw` and one `simulator run`, both mostly idle. + +## Flags + +| Flag | Default | | +|---|---|---| +| `-repo` | `srcfl/ftw` | | +| `-work` | `~/siterig` | clone, worktrees, results, `last-random` | +| `-every` | `5m` | how often to look for new commits | +| `-lanes` | `3` | scenarios at once | +| `-port-base` | `18700` | first port | +| `-random-at`, `-random-seeds` | `2`, `10` | local hour and count of the nightly random seeds; `-random-at -1` turns them off | +| `-drafts` | `true` | also run draft pull requests | +| `-allow-forks` | `false` | also run forks' pull requests | +| `-once` | | one pass, then exit | diff --git a/go/test/site/runner/energy.ftw.siterig-runner.plist b/go/test/site/runner/energy.ftw.siterig-runner.plist new file mode 100644 index 000000000..10f4604c6 --- /dev/null +++ b/go/test/site/runner/energy.ftw.siterig-runner.plist @@ -0,0 +1,24 @@ + + + + + + Labelenergy.ftw.siterig-runner + ProgramArguments + + /Users/USER/siterig/bin/siterig-runner + -work/Users/USER/siterig + -lanes3 + + EnvironmentVariables + + PATH/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + HOME/Users/USER + + RunAtLoad + KeepAlive + StandardOutPath/Users/USER/siterig/runner.log + StandardErrorPath/Users/USER/siterig/runner.log + + diff --git a/go/test/site/runner/main.go b/go/test/site/runner/main.go new file mode 100644 index 000000000..6878257c9 --- /dev/null +++ b/go/test/site/runner/main.go @@ -0,0 +1,232 @@ +// siterig-runner runs FTW's site test (go/test/site) on a machine of its own +// and publishes the result as a GitHub commit status on the exact commit: +// siterig/full on each new master commit, siterig/short on the head of each +// open pull request that touches control, loadpoint, OCPP or telemetry code, +// and siterig/random for a night's random seeds. native-release.yml reads +// siterig/full before it publishes a beta. See README.md. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" +) + +func main() { + var cfg config + flag.StringVar(&cfg.Repo, "repo", "srcfl/ftw", "GitHub repository") + work := flag.String("work", filepath.Join(os.Getenv("HOME"), "siterig"), "clone, worktrees and results") + every := flag.Duration("every", 5*time.Minute, "how often to look for new commits") + lanes := flag.Int("lanes", 3, "scenarios run at once") + portBase := flag.Int("port-base", 18700, "first port; lanes use 100 each") + keep := flag.Duration("keep", 14*24*time.Hour, "how long to keep results") + flag.BoolVar(&cfg.AllowForks, "allow-forks", false, "also run pull requests from forks (their code runs here)") + flag.BoolVar(&cfg.Drafts, "drafts", true, "also run draft pull requests") + flag.IntVar(&cfg.RandomAt, "random-at", 2, "local hour of the nightly random seeds; -1 never") + flag.IntVar(&cfg.RandomSeeds, "random-seeds", 10, "random seeds a night") + once := flag.Bool("once", false, "run what is due and exit") + flag.Parse() + + r := &runner{cfg: cfg, gh: ghCLI{repo: cfg.Repo}, work: *work, lanes: *lanes, portBase: *portBase, keep: *keep} + if err := r.prepare(); err != nil { + log.Fatal(err) + } + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + for { + r.tick() + if *once { + return + } + select { + case <-stop: + return + case <-time.After(*every): + } + } +} + +type runner struct { + cfg config + gh github + work string + lanes int + portBase int + keep time.Duration +} + +func (r *runner) clone() string { return filepath.Join(r.work, "ftw") } +func (r *runner) results() string { return filepath.Join(r.work, "results") } + +// prepare clones the repository once. +func (r *runner) prepare() error { + if err := os.MkdirAll(r.results(), 0o755); err != nil { + return err + } + if _, err := os.Stat(filepath.Join(r.clone(), ".git")); err == nil { + return nil + } + return run(r.work, "git", "clone", "--quiet", "https://github.com/"+r.cfg.Repo, r.clone()) +} + +// tick runs everything that is due, one job at a time. +func (r *runner) tick() { + stateFile := filepath.Join(r.work, "last-random") + last, _ := os.ReadFile(stateFile) + jobs, err := plan(r.gh, r.cfg, time.Now(), strings.TrimSpace(string(last))) + if err != nil { + log.Printf("plan: %v", err) + return + } + for _, j := range jobs { + log.Printf("start %s", j) + state, desc := r.runJob(j) + log.Printf("done %s: %s, %s", j, state, desc) + if err := r.gh.SetStatus(j.SHA, j.Context, state, desc); err != nil { + log.Printf("status: %v", err) + } + if j.Context == ctxRandom { + os.WriteFile(stateFile, []byte(time.Now().Format("2006-01-02")), 0o644) + } + } + r.prune() +} + +// runJob checks out the commit in a worktree of its own and runs the rig. +func (r *runner) runJob(j job) (state, desc string) { + if err := r.gh.SetStatus(j.SHA, j.Context, "pending", "running on "+hostname()); err != nil { + log.Printf("status: %v", err) + } + stamp := time.Now().Format("20060102-150405") + out := filepath.Join(r.results(), fmt.Sprintf("%s-%s-%s", stamp, short(j.SHA), strings.TrimPrefix(j.Context, "siterig/"))) + wt := filepath.Join(r.work, "wt-"+short(j.SHA)+"-"+stamp) + if err := os.MkdirAll(out, 0o755); err != nil { + return "error", err.Error() + } + defer func() { + run(r.clone(), "git", "worktree", "remove", "--force", wt) + run(r.clone(), "git", "worktree", "prune") + }() + fetch := []string{"fetch", "--quiet", "origin", "master"} + if j.PR > 0 { + fetch = append(fetch, fmt.Sprintf("refs/pull/%d/head", j.PR)) + } + if err := run(r.clone(), "git", fetch...); err != nil { + return "error", "git fetch failed" + } + if err := run(r.clone(), "git", "worktree", "add", "--detach", wt, j.SHA); err != nil { + return "error", "commit not found" + } + log1, _ := os.Create(filepath.Join(out, "rig.log")) + defer log1.Close() + mk := exec.Command("make", "drivers") + mk.Dir, mk.Stdout, mk.Stderr = wt, log1, log1 + if err := mk.Run(); err != nil { + return "error", "make drivers failed" + } + t0 := time.Now() + test := exec.Command("go", "test", "-count=1", "-timeout", "300m", "-run", "TestSite$", "-v", "./test/site/") + test.Dir, test.Stdout, test.Stderr = filepath.Join(wt, "go"), log1, log1 + test.Env = append(os.Environ(), "FTW_SITE="+j.Set, fmt.Sprintf("FTW_SITE_LANES=%d", r.lanes), + fmt.Sprintf("FTW_SITE_PORT_BASE=%d", r.portBase), "FTW_SITE_OUT="+out) + test.Run() // the summary, not the exit code, says what passed + summary, _ := os.ReadFile(filepath.Join(out, "summary.json")) + state, desc = describe(summary, time.Since(t0)) + if state == "success" && test.ProcessState != nil && !test.ProcessState.Success() { + state, desc = "failure", "the rig failed outside the scenarios: "+desc + } + return state, desc +} + +// prune drops results older than keep. +func (r *runner) prune() { + entries, _ := os.ReadDir(r.results()) + for _, e := range entries { + if info, err := e.Info(); err == nil && time.Since(info.ModTime()) > r.keep { + os.RemoveAll(filepath.Join(r.results(), e.Name())) + } + } +} + +func run(dir, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("%s %s: %v: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return nil +} + +func hostname() string { + h, _ := os.Hostname() + return h +} + +// ghCLI asks GitHub through the gh CLI, as the machine's logged-in user. +type ghCLI struct{ repo string } + +func (g ghCLI) api(out any, args ...string) error { + b, err := exec.Command("gh", append([]string{"api"}, args...)...).Output() + if err != nil { + return fmt.Errorf("gh api %s: %w", strings.Join(args, " "), err) + } + if out == nil { + return nil + } + return json.Unmarshal(b, out) +} + +func (g ghCLI) MasterSHA() (string, error) { + var c struct { + SHA string `json:"sha"` + } + err := g.api(&c, "repos/"+g.repo+"/commits/master") + return c.SHA, err +} + +func (g ghCLI) Pulls() ([]pull, error) { + var ps []pull + err := g.api(&ps, "repos/"+g.repo+"/pulls?state=open&per_page=100") + return ps, err +} + +func (g ghCLI) Files(pr int) ([]string, error) { + var fs []struct { + Filename string `json:"filename"` + } + if err := g.api(&fs, fmt.Sprintf("repos/%s/pulls/%d/files?per_page=100", g.repo, pr)); err != nil { + return nil, err + } + out := make([]string, len(fs)) + for i, f := range fs { + out[i] = f.Filename + } + return out, nil +} + +func (g ghCLI) Contexts(sha string) (map[string]bool, error) { + var ss []struct { + Context string `json:"context"` + } + if err := g.api(&ss, "repos/"+g.repo+"/commits/"+sha+"/statuses?per_page=100"); err != nil { + return nil, err + } + have := map[string]bool{} + for _, s := range ss { + have[s.Context] = true + } + return have, nil +} + +func (g ghCLI) SetStatus(sha, context, state, desc string) error { + return g.api(nil, "-X", "POST", "repos/"+g.repo+"/statuses/"+sha, + "-f", "state="+state, "-f", "context="+context, "-f", "description="+desc) +} diff --git a/go/test/site/runner/runner.go b/go/test/site/runner/runner.go new file mode 100644 index 000000000..1a0def0c0 --- /dev/null +++ b/go/test/site/runner/runner.go @@ -0,0 +1,173 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// Status contexts the runner publishes on a commit. +const ( + ctxShort = "siterig/short" + ctxFull = "siterig/full" + ctxRandom = "siterig/random" +) + +// watchedPaths are the parts of the tree whose changes get the short set on +// a pull request. +var watchedPaths = []string{ + "go/internal/control/", "go/internal/loadpoint/", "go/internal/ocpp/", "go/internal/telemetry/", + "go/test/site/", +} + +// job is one run of the rig on one commit. +type job struct { + Context string // which status it publishes + SHA string + Set string // FTW_SITE: short, full or random:: + PR int // 0 for master +} + +func (j job) String() string { + if j.PR > 0 { + return fmt.Sprintf("%s on #%d %s", j.Context, j.PR, short(j.SHA)) + } + return fmt.Sprintf("%s on master %s", j.Context, short(j.SHA)) +} + +func short(sha string) string { + if len(sha) > 8 { + return sha[:8] + } + return sha +} + +// pull is the part of a pull request the runner needs. +type pull struct { + Number int `json:"number"` + Draft bool `json:"draft"` + Head struct { + SHA string `json:"sha"` + Repo struct { + FullName string `json:"full_name"` + } `json:"repo"` + } `json:"head"` +} + +// github is what the runner asks GitHub; gh api serves it in production. +type github interface { + MasterSHA() (string, error) + Pulls() ([]pull, error) + Files(pr int) ([]string, error) + Contexts(sha string) (map[string]bool, error) // status contexts already on a commit + SetStatus(sha, context, state, description string) error +} + +// config is the runner's flags. +type config struct { + Repo string + AllowForks bool // a fork's code runs on this machine; off by default + Drafts bool + RandomAt int // local hour of the nightly random run; -1 never + RandomSeeds int +} + +// plan lists what to run now, master first. A commit that already carries +// the context, in any state, is not run again. Random seeds run once a day +// (lastRandom is the day they last ran) on whatever master is then. +func plan(gh github, cfg config, now time.Time, lastRandom string) ([]job, error) { + var jobs []job + master, err := gh.MasterSHA() + if err != nil { + return nil, err + } + have, err := gh.Contexts(master) + if err != nil { + return nil, err + } + if !have[ctxFull] { + jobs = append(jobs, job{Context: ctxFull, SHA: master, Set: "full"}) + } + pulls, err := gh.Pulls() + if err != nil { + return nil, err + } + for _, p := range pulls { + if p.Draft && !cfg.Drafts { + continue + } + if p.Head.Repo.FullName != cfg.Repo && !cfg.AllowForks { + continue + } + have, err := gh.Contexts(p.Head.SHA) + if err != nil { + return nil, err + } + if have[ctxShort] { + continue + } + files, err := gh.Files(p.Number) + if err != nil { + return nil, err + } + if touchesWatched(files) { + jobs = append(jobs, job{Context: ctxShort, SHA: p.Head.SHA, Set: "short", PR: p.Number}) + } + } + day := now.Format("2006-01-02") + if cfg.RandomAt >= 0 && cfg.RandomSeeds > 0 && now.Hour() >= cfg.RandomAt && lastRandom != day { + // One seed range per day, so each night tries new timelines. + first := int(now.Unix()/86400) * cfg.RandomSeeds + jobs = append(jobs, job{Context: ctxRandom, SHA: master, Set: fmt.Sprintf("random:%d:%d", first, cfg.RandomSeeds)}) + } + return jobs, nil +} + +func touchesWatched(files []string) bool { + for _, f := range files { + for _, p := range watchedPaths { + if strings.HasPrefix(f, p) { + return true + } + } + } + return false +} + +// outcome is one line of the rig's summary.json. +type outcome struct { + Name string `json:"name"` + Verdict string `json:"verdict"` // pass, known, fail + DurationS float64 `json:"duration_s"` + Error string `json:"error"` +} + +// describe turns summary.json into a status: success when nothing failed, +// with counts that fit GitHub's 140 characters. +func describe(summary []byte, took time.Duration) (state, desc string) { + var outs []outcome + if err := json.Unmarshal(summary, &outs); err != nil || len(outs) == 0 { + return "error", "no summary: the rig did not finish (see the runner's log)" + } + n := map[string]int{} + var failed []string + for _, o := range outs { + n[o.Verdict]++ + if o.Verdict == "fail" { + failed = append(failed, o.Name) + } + } + state = "success" + if len(failed) > 0 { + state = "failure" + } + desc = fmt.Sprintf("%d pass, %d known, %d fail of %d in %s", n["pass"], n["known"], n["fail"], len(outs), took.Round(time.Minute)) + if len(failed) > 0 { + desc += ": " + strings.Join(failed, " ") + } + if len(desc) > 140 { + desc = desc[:137] + "..." + } + return state, desc +} diff --git a/go/test/site/runner/runner_test.go b/go/test/site/runner/runner_test.go new file mode 100644 index 000000000..d05a1dd36 --- /dev/null +++ b/go/test/site/runner/runner_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +type fakeGitHub struct { + master string + pulls []pull + files map[int][]string + contexts map[string]map[string]bool +} + +func (f fakeGitHub) MasterSHA() (string, error) { return f.master, nil } +func (f fakeGitHub) Pulls() ([]pull, error) { return f.pulls, nil } +func (f fakeGitHub) Files(pr int) ([]string, error) { return f.files[pr], nil } +func (f fakeGitHub) Contexts(sha string) (map[string]bool, error) { + return f.contexts[sha], nil +} +func (f fakeGitHub) SetStatus(sha, context, state, description string) error { return nil } + +func pr(n int, sha, repo string, draft bool) pull { + p := pull{Number: n, Draft: draft} + p.Head.SHA, p.Head.Repo.FullName = sha, repo + return p +} + +func TestPlanRunsWhatIsDue(t *testing.T) { + gh := fakeGitHub{ + master: "m1", + pulls: []pull{ + pr(1, "a1", "srcfl/ftw", false), // control code: short set + pr(2, "b1", "srcfl/ftw", false), // docs only: nothing + pr(3, "c1", "someone/ftw", false), // a fork: never by default + pr(4, "d1", "srcfl/ftw", false), // already has its status + }, + files: map[int][]string{ + 1: {"go/internal/control/dispatch.go"}, 2: {"docs/x.md"}, + 3: {"go/internal/ocpp/handlers.go"}, 4: {"go/internal/loadpoint/x.go"}, + }, + contexts: map[string]map[string]bool{"d1": {ctxShort: true}}, + } + cfg := config{Repo: "srcfl/ftw", Drafts: true, RandomAt: 2, RandomSeeds: 10} + night := time.Date(2026, 10, 23, 3, 0, 0, 0, time.Local) + jobs, err := plan(gh, cfg, night, "") + if err != nil { + t.Fatal(err) + } + var got []string + for _, j := range jobs { + got = append(got, j.String()+" "+j.Set) + } + want := []string{"siterig/full on master m1 full", "siterig/short on #1 a1 short", "siterig/random on master m1 random:"} + if len(got) != len(want) { + t.Fatalf("jobs %v, want %v", got, want) + } + for i := range want { + if !strings.HasPrefix(got[i], want[i]) { + t.Fatalf("job %d: %q, want %q", i, got[i], want[i]) + } + } + + // Master already tested, random already run today, before the hour. + gh.contexts["m1"] = map[string]bool{ctxFull: true} + jobs, _ = plan(gh, cfg, night, "2026-10-23") + if len(jobs) != 1 || jobs[0].PR != 1 { + t.Fatalf("second look: %v", jobs) + } + cfg.AllowForks = true + jobs, _ = plan(gh, cfg, night.Add(-2*time.Hour), "") + if len(jobs) != 2 || jobs[1].PR != 3 { + t.Fatalf("with forks allowed, before the random hour: %v", jobs) + } +} + +func TestDescribe(t *testing.T) { + state, desc := describe([]byte(`[{"name":"s1-25","verdict":"pass"},{"name":"s2-16","verdict":"known"},{"name":"s9-25","verdict":"fail"}]`), 24*time.Minute) + if state != "failure" || desc != "1 pass, 1 known, 1 fail of 3 in 24m0s: s9-25" { + t.Fatalf("%s %q", state, desc) + } + state, _ = describe([]byte(`[{"name":"s1-25","verdict":"pass"},{"name":"s2-16","verdict":"known"}]`), time.Minute) + if state != "success" { + t.Fatalf("known only: %s", state) + } + if state, _ := describe(nil, time.Minute); state != "error" { + t.Fatalf("no summary: %s", state) + } + long := `[` + strings.Repeat(`{"name":"a-very-long-scenario-name","verdict":"fail"},`, 9) + `{"name":"x","verdict":"fail"}]` + if _, desc := describe([]byte(long), time.Minute); len(desc) > 140 { + t.Fatalf("%d characters", len(desc)) + } +} diff --git a/go/test/site/runner/siterig-runner.service b/go/test/site/runner/siterig-runner.service new file mode 100644 index 000000000..f287a597b --- /dev/null +++ b/go/test/site/runner/siterig-runner.service @@ -0,0 +1,18 @@ +# systemd unit for siterig-runner (README.md). Replace USER and the paths, +# then: sudo cp siterig-runner.service /etc/systemd/system/ && +# sudo systemctl enable --now siterig-runner +[Unit] +Description=FTW site test runner (siterig/short, siterig/full commit statuses) +After=network-online.target +Wants=network-online.target + +[Service] +User=USER +Environment=HOME=/home/USER +Environment=PATH=/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin +ExecStart=/home/USER/siterig/bin/siterig-runner -work /home/USER/siterig -lanes 3 +Restart=always +RestartSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/go/test/site/scenarios_test.go b/go/test/site/scenarios_test.go new file mode 100644 index 000000000..4c42c9c68 --- /dev/null +++ b/go/test/site/scenarios_test.go @@ -0,0 +1,1026 @@ +//go:build unix + +package site + +import ( + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "time" +) + +// scenario is one house and what happens in it. known names the criteria +// that fail on master and the finding behind each: such a criterion must +// keep failing, and once a fix makes it pass the run fails until the fix +// deletes its line. Criteria come from reports/simulatorprov-20261021. +type scenario struct { + name string // what FTW_SITE names it by + dir string // its run directory + minutes float64 // its run time on a GitHub runner, for sharding + short bool // in the pull-request set + run func(c *runCtx) (map[string]any, error) + judge func(d *runData) (metrics, []criterion) + known map[string]string + // tolerate names criteria that may fail for a known finding but need + // not: random cases, where a seed may or may not reach it. + tolerate map[string]string +} + +var scenarios = []scenario{ + {name: "s0-25", dir: "s0-idle-charger-25A", minutes: 5.1, short: true, + run: func(c *runCtx) (map[string]any, error) { return s0(c, 25, 60) }, judge: judgeS0}, + {name: "s0b-25", dir: "s0b-idle-charger-default-hb-25A", minutes: 5.1, + run: func(c *runCtx) (map[string]any, error) { return s0(c, 25, 0) }, judge: judgeS0}, + {name: "s1-16", dir: "s1-chargenow-16A", minutes: 3.6, short: true, + run: func(c *runCtx) (map[string]any, error) { return s1(c, 16) }, judge: judgeS1}, + {name: "s1-25", dir: "s1-chargenow-25A", minutes: 3.6, short: true, + run: func(c *runCtx) (map[string]any, error) { return s1(c, 25) }, judge: judgeS1}, + {name: "s2-16", dir: "s2-export-16A", minutes: 3.1, + run: func(c *runCtx) (map[string]any, error) { return s2(c, 16) }, judge: judgeS2, + known: map[string]string{"car starts within 60 s": "export counted as import (review fuse finding 1), next beta"}}, + {name: "s3-16", dir: "s3-step-16A", minutes: 6.7, short: true, + run: func(c *runCtx) (map[string]any, error) { return s3(c, 16) }, judge: judgeS3}, + {name: "s3-25", dir: "s3-step-25A", minutes: 6.7, short: true, + run: func(c *runCtx) (map[string]any, error) { return s3(c, 25) }, judge: judgeS3}, + {name: "s4-25", dir: "s4-unlock-25A", minutes: 3.6, + run: func(c *runCtx) (map[string]any, error) { return s4(c, 25, "3p") }, judge: judgeS4}, + {name: "s4b-25", dir: "s4b-unlock-nophase-25A", minutes: 3.6, + run: func(c *runCtx) (map[string]any, error) { return s4(c, 25, "") }, judge: judgeS4}, + {name: "s5-25", dir: "s5-surplus-auto-25A", minutes: 5.2, + run: func(c *runCtx) (map[string]any, error) { return s5(c, 25, nil) }, judge: judgeS5, + known: map[string]string{"charges at least the minimum 90 % of the time": "surplus without allowed_steps_w never starts, loadpoint/controller.go surplus3PhaseSteps; next beta"}}, + {name: "s5s-25", dir: "s5s-surplus-auto-steps-25A", minutes: 5.2, + run: func(c *runCtx) (map[string]any, error) { return s5(c, 25, easeeSteps) }, judge: judgeS5, + known: map[string]string{"charges at least the minimum 90 % of the time": "phase mode auto stays on 1p under 9 kW surplus (R1); next beta"}}, + {name: "s6-25", dir: "s6-inverter-offline-25A", minutes: 7.7, + run: func(c *runCtx) (map[string]any, error) { return s6(c, 25) }, judge: judgeS6}, + {name: "s7-25", dir: "s7-restart-25A", minutes: 23, + run: func(c *runCtx) (map[string]any, error) { return s7(c, 25, false) }, judge: judgeS7}, + // The full s7 runs 15 min before the restart, because the day's Home + // use may have at most 10 % unknown time. The short one restarts after + // a minute and leaves the day total out. + {name: "s7s-25", dir: "s7s-restart-short-25A", minutes: 9, short: true, + run: func(c *runCtx) (map[string]any, error) { return s7(c, 25, true) }, judge: judgeS7}, + {name: "s8-60-25", dir: "s8-slow60-25A", minutes: 7.5, short: true, + run: func(c *runCtx) (map[string]any, error) { return s8(c, 25, 60) }, judge: judgeS8}, + {name: "s8-120-25", dir: "s8-slow120-25A", minutes: 12.5, + run: func(c *runCtx) (map[string]any, error) { return s8(c, 25, 120) }, judge: judgeS8}, + {name: "s9-25", dir: "s9-phase-switch-25A", minutes: 4.7, + run: func(c *runCtx) (map[string]any, error) { return s9(c, 25, 7360) }, judge: judgeS9}, + {name: "s10-25", dir: "s10-two-chargers-25A", minutes: 7.9, + run: func(c *runCtx) (map[string]any, error) { return s10(c, 25, 60) }, judge: judgeS10}, + {name: "s11-25", dir: "s11-pause-restart-25A", minutes: 5.1, + run: func(c *runCtx) (map[string]any, error) { return s11(c, 25) }, judge: judgeS11}, + {name: "s9r-25", dir: "s9r-house-on-l2-25A", minutes: 5, + run: func(c *runCtx) (map[string]any, error) { return s9r(c, 25, 7360) }, judge: judgeS9}, + {name: "s5t-25", dir: "s5t-surplus-goal-25A", minutes: 6, + run: func(c *runCtx) (map[string]any, error) { return s5t(c, 25) }, judge: judgeS5t}, + {name: "s12-20", dir: "s12-one-phase-night-20A", minutes: 57, + run: func(c *runCtx) (map[string]any, error) { return s12(c, 20) }, judge: judgeS12}, + {name: "s13-25", dir: "s13-socket-reconnect-25A", minutes: 6, + run: func(c *runCtx) (map[string]any, error) { return s13(c, 25) }, judge: judgeS13, + known: map[string]string{"Charge now goes on after the charger returns": "a socket reconnect turns Charge now into a 0 W pause; srcfl/ftw#1788"}}, + {name: "s15-25", dir: "s15-one-phase-charger-chargenow-25A", minutes: 4.6, short: true, + run: func(c *runCtx) (map[string]any, error) { return s15(c, 25) }, judge: judgeS15}, + {name: "s15b-25", dir: "s15b-one-phase-charger-1p-25A", minutes: 4.6, + run: func(c *runCtx) (map[string]any, error) { return s15mode(c, 25, "1p") }, judge: judgeS15}, + {name: "s15c-25", dir: "s15c-one-phase-charger-auto-25A", minutes: 4.6, + run: func(c *runCtx) (map[string]any, error) { return s15mode(c, 25, "auto") }, judge: judgeS15}, + {name: "s15bat-16", dir: "s15bat-one-phase-charger-battery-rising-16A", minutes: 5.5, + run: func(c *runCtx) (map[string]any, error) { return s15bat(c, 16) }, judge: judgeS15bat}, + {name: "s15p-25", dir: "s15p-one-phase-charger-plan-25A", minutes: 32, + run: func(c *runCtx) (map[string]any, error) { return s15p(c, 25) }, judge: judgeS15p}, + {name: "s14-25", dir: "s14-one-phase-car-25A", minutes: 22, + run: func(c *runCtx) (map[string]any, error) { return s14(c, 25) }, judge: judgeS14}, +} + +// runCtx is where one scenario runs. +type runCtx struct { + env *rigEnv + lane lane + rundir string +} + +func (c *runCtx) stack(spec stackSpec) *stack { return newStack(c.env, c.lane, c.rundir, spec) } + +func houseA(a float64) [3]float64 { return [3]float64{a * volt, a * volt, a * volt} } + +// ---- what a client does ---- + +type loadpointView struct { + ID string `json:"id"` + Plugged bool `json:"plugged_in"` + Charger struct { + Known bool `json:"known"` + } `json:"charger"` +} + +func (st *stack) loadpoint(id string) (loadpointView, error) { + var j struct { + Loadpoints []loadpointView `json:"loadpoints"` + } + if err := st.get("/api/loadpoints", &j); err != nil { + return loadpointView{}, err + } + for _, x := range j.Loadpoints { + if x.ID == id { + return x, nil + } + } + if len(j.Loadpoints) > 0 { + return j.Loadpoints[0], nil + } + return loadpointView{}, fmt.Errorf("no loadpoints") +} + +// chargeNow is what the web, PWA and iOS clients send. An empty phaseMode +// is a bare API call. powerW 0 means amps on three phases, below 0 a pause. +func (st *stack) chargeNow(lpID string, amps float64, phaseMode string, powerW float64) error { + switch { + case powerW < 0: // a pause + powerW = 0 + case powerW == 0: + powerW = amps * volt * 3 + } + body := map[string]any{"power_w": powerW, "hold_s": 0} + if phaseMode != "" { + body["phase_mode"] = phaseMode + } + return st.post("/api/loadpoints/"+lpID+"/manual_hold", body, nil) +} + +// chargerOnline waits until charger i talks to FTW. +func (st *stack) chargerOnline(i int) error { + serial := st.serial(i) + ok := waitUntil(90*time.Second, time.Second, func() bool { + var s struct { + Drivers map[string]json.RawMessage `json:"drivers"` + } + if st.get("/api/status", &s) != nil { + return false + } + _, ok := s.Drivers[serial] + return ok + }) + if !ok { + return fmt.Errorf("charger %s never connected to FTW", serial) + } + return nil +} + +// plug plugs a car into charger i once the charger is online, and waits +// for FTW to see it. +func (st *stack) plug(i int) error { + if err := st.chargerOnline(i); err != nil { + return err + } + time.Sleep(2 * time.Second) + label := "plug" + if i > 0 { + label = fmt.Sprintf("plug%d", i) + } + st.mark(label) + on := true + if err := st.event(houseEvent{Do: "plug", Device: lpIDs[i], On: &on}); err != nil { + return err + } + ok := waitUntil(90*time.Second, time.Second, func() bool { + lp, err := st.loadpoint(lpIDs[i]) + return err == nil && lp.Plugged && lp.Charger.Known + }) + if !ok { + return fmt.Errorf("loadpoint %s never saw the car plugged in", lpIDs[i]) + } + return nil +} + +func sleepS(s float64) { time.Sleep(time.Duration(s * float64(time.Second))) } + +// with brings a stack up, runs body and always takes the stack down. +func with(c *runCtx, spec stackSpec, body func(st *stack) error) error { + st := c.stack(spec) + defer st.down() + if err := st.up(); err != nil { + return err + } + if err := body(st); err != nil { + return err + } + return st.markErr +} + +// ---- 0. Adopted OCPP charger, no car: is Home use measured? ---- + +func s0(c *runCtx, fuse float64, heartbeatS int) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, heartbeatS: heartbeatS}, func(st *stack) error { + if err := st.chargerOnline(0); err != nil { + return err + } + st.mark("connected") + // Several heartbeats: #1779 keeps the idle 0 W fresh with them. + sleepS(300) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s0", "fuse": fuse, "house_a": 3, "heartbeat_s": heartbeatS}, err +} + +// ---- 1. Charge now, 3 A house load per phase ---- + +func s1(c *runCtx, fuse float64) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s1", "fuse": fuse, "house_a": 3}, err +} + +// ---- 2. Charge now during PV export of 9 A per phase, 16 A fuse ---- + +func s2(c *runCtx, fuse float64) (map[string]any, error) { + // Full battery so it cannot soak up the export. PV 7 kW, house 1 A: + // 6.3 kW export, 9.1 A per phase out of the site. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(1), pvW: 7000, socPct: 100}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(150) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s2", "fuse": fuse, "house_a": 1, "pv_w": 7000, "soc": 100}, err +} + +// ---- 3. House load steps to near the fuse while car and battery charge ---- + +func s3(c *runCtx, fuse float64) (map[string]any, error) { + stepA := map[float64]float64{16: 13, 25: 21}[fuse] + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 40}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("battery_hold") + if err := st.post("/api/battery/manual_hold", map[string]any{"direction": "charge", "power_w": 3000, "hold_s": 1800}, nil); err != nil { + return err + } + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(120) + st.mark("step") + if err := st.setLoadsA(stepA); err != nil { + return err + } + sleepS(180) + st.mark("step_back") + if err := st.setLoadsA(3); err != nil { + return err + } + sleepS(60) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s3", "fuse": fuse, "house_a": 3, "step_a": stepA, "battery_hold_w": 3000, "charger_delay_s": 0}, err +} + +// ---- 4. surplus_unlock_bat_soc saved but not armed, Charge now ---- + +func s4(c *runCtx, fuse float64, phaseMode string) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + st.mark("schedule") + // Only the unlock, as in #1774's R2 test: a SoC goal would make the + // plan charge at once and hide the fault. + var resp json.RawMessage + if err := st.post("/api/loadpoints/garage/target", map[string]any{"schedule": map[string]any{ + "soc": 0, "time_of_day_min": 7 * 60, "time_of_day_min_utc": 5 * 60, + "recurring": false, "days": 0, "surplus_unlock_bat_soc": 0.9}}, &resp); err != nil { + return err + } + os.WriteFile(filepath.Join(st.rundir, "target-response.json"), resp, 0o644) + sleepS(10) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, phaseMode, 0); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s4", "fuse": fuse, "house_a": 3, "unlock": 0.9, "soc": 50, "phase_mode": phaseMode}, err +} + +// ---- 5. Surplus only, phase mode auto, ~9 kW surplus ---- + +// easeeSteps are Easee's whole-amp offers at 230 V, as in #1774's tests: +// 6-16 A on one phase, then 6-16 A on three. +var easeeSteps = []float64{0, 1380, 1610, 1840, 2070, 2300, 2530, 2760, 2990, 3220, 3450, 3680, + 4140, 4830, 5520, 6210, 6900, 7590, 8280, 8970, 9660, 10350, 11040} + +func s5(c *runCtx, fuse float64, steps []float64) (map[string]any, error) { + extra := map[string]any{"surplus_only": true, "phase_mode": "auto"} + if steps != nil { + extra["allowed_steps_w"] = steps + } + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(1), pvW: 10000, socPct: 100, lpExtra: extra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + st.mark("plugged") + sleepS(300) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s5", "fuse": fuse, "house_a": 1, "pv_w": 10000, "soc": 100, "phase_mode": "auto", "steps": steps != nil}, err +} + +// ---- 6. PV inverter offline for 5 minutes ---- + +func s6(c *runCtx, fuse float64) (map[string]any, error) { + // A car plugged in, so that case 0 does not interfere. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), pvW: 1500, socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(60) + st.mark("offline") + if err := st.event(houseEvent{Do: "offline", Device: "inverter"}); err != nil { + return err + } + sleepS(300) + st.mark("online") + if err := st.event(houseEvent{Do: "online", Device: "inverter"}); err != nil { + return err + } + sleepS(90) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s6", "fuse": fuse, "house_a": 3, "pv_w": 1500, "offline_s": 300}, err +} + +// ---- 7. Core restart of ~20 s while charging ---- + +func s7(c *runCtx, fuse float64, short bool) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + if short { + sleepS(60) + } else { + // A new box has minutes of history; unknown time may be at most + // 10 % of it (history_dashboard.go). Wait until the day total is + // known, then long enough that the restart's ~110 s of unknown + // time stays below the rule, as on a box that has run since morning. + if waitUntil(900*time.Second, 5*time.Second, func() bool { + var s ftwStatus + return st.get("/api/status", &s) == nil && s.Energy.Today.LoadWh != nil + }) { + st.mark("today_known") + } + sleepS(900) + } + st.mark("ftw_stop") + st.stopFTW() + sleepS(20) + st.mark("ftw_start") + if err := st.startFTW(); err != nil { + return err + } + st.mark("ftw_up") + // Long enough to see whether the car gets back to full power. + sleepS(420) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s7", "fuse": fuse, "house_a": 3, "down_s": 20, "short": short}, err +} + +// ---- 8. Slow charger: profiles apply 60 s or 120 s late ---- + +func s8(c *runCtx, fuse, delayS float64) (map[string]any, error) { + // Battery at the rig's 5 % floor: it cannot cover the house. At 8 % FTW + // forced it to discharge for the fuse and the meter showed room that + // vanished when it ran empty, which hid the stacking (see the report). + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(15), socPct: 5, chargers: []charger{{delayS: delayS}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + // The charger draws its maximum until FTW's first profile lands, + // delayS after plug-in. Start from there, at 0 W. + st.mark("plug_wait") + sleepS(delayS + 20) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + // Long enough for the last rise and the cut after it to land. + sleepS(max(300, 4*delayS+120)) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s8", "fuse": fuse, "house_a": 15, "charger_delay_s": delayS, "soc_pct": 5}, err +} + +// ---- 9. Charge now switched from 3p to 1p at the same watts ---- + +func s9(c *runCtx, fuse, watts float64) (map[string]any, error) { + // A 32 A charger that can switch phases. 7 360 W is 10.7 A on three + // phases and 32 A on one. House 10 A per phase, empty battery: one phase + // has 14 A of room below FTW's 24 A limit. + spec := stackSpec{fuseA: fuse, loadsW: houseA(10), socPct: 5, chargers: []charger{{kw: 22.08, phaseSwitch: true}}, + lpExtra: map[string]any{"min_charge_w": 1380, "max_charge_w": 22080, "phase_mode": "auto"}} + err := with(c, spec, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 0, "3p", watts); err != nil { + return err + } + sleepS(90) + st.mark("switch") + if err := st.chargeNow("garage", 0, "1p", watts); err != nil { + return err + } + sleepS(150) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s9", "fuse": fuse, "house_a": 10, "watts": watts, "charger_a": 32}, err +} + +// ---- 10. Two chargers: one raised while the other has just been cut ---- + +func s10(c *runCtx, fuse, delayS float64) (map[string]any, error) { + // garage answers at once; carport is a cloud charger that applies a + // profile delayS late. House 3 A per phase, empty battery. Both charge + // at 10 A (23 A per phase). Then carport is cut to 6 A and garage raised + // to 16 A in the same second: the room carport frees is not there until + // its cut lands. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 5, chargers: []charger{{}, {delayS: delayS}}}, func(st *stack) error { + if err := st.plug(1); err != nil { + return err + } + // The slow charger draws its maximum until FTW's first profile lands. + st.mark("plug_wait") + sleepS(delayS + 20) + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + st.mark("charge_now") + if err := st.chargeNow("garage", 10, "3p", 0); err != nil { + return err + } + if err := st.chargeNow("carport", 10, "3p", 0); err != nil { + return err + } + sleepS(delayS + 60) + st.mark("step") + if err := st.chargeNow("carport", 6, "3p", 0); err != nil { + return err + } + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(2*delayS + 120) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s10", "fuse": fuse, "house_a": 3, "charger_delay_s": []float64{0, delayS}}, err +} + +// ---- 11. Pause and restart under a long declared response time (#1793) ---- + +func s11(c *runCtx, fuse float64) (map[string]any, error) { + // OCPP declares no response time, so Core reserves 180 s for each order, + // as for a charger that declares 180 s. Charge now 16 A, pause (0 W) for + // 25 s, then start at 6 A: the cut must end the old order's reservation. + // The charger applies a profile 5 s late (ev-104's plant). + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, chargers: []charger{{delayS: 5}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(60) + st.mark("pause") + if err := st.chargeNow("garage", 0, "3p", -1); err != nil { + return err + } + sleepS(25) + st.mark("restart") + if err := st.chargeNow("garage", 6, "3p", 0); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s11", "fuse": fuse, "house_a": 3, "pause_s": 25, "charger_delay_s": 5}, err +} + +// ---- planned charging ---- + +// planTo tells FTW the car is at socPct and must reach goalPct by the first +// quarter hour at least readyIn from now. It returns the ready time. +func (st *stack) planTo(lp string, socPct, goalPct float64, readyIn time.Duration) (time.Time, error) { + if err := st.post("/api/loadpoints/"+lp+"/soc", map[string]any{"soc_pct": socPct}, nil); err != nil { + return time.Time{}, err + } + ready := time.Now().Add(readyIn).Truncate(15 * time.Minute).Add(15 * time.Minute) + if err := st.post("/api/loadpoints/"+lp+"/target", map[string]any{"schedule": map[string]any{ + "soc": goalPct / 100, "time_of_day_min": ready.Hour()*60 + ready.Minute(), "recurring": false, "days": 0}}, nil); err != nil { + return time.Time{}, err + } + return ready, nil +} + +// ---- 9r. s9 reversed: a 1p order raised to 3p beside a loaded L2 ---- + +func s9r(c *runCtx, fuse, watts float64) (map[string]any, error) { + // House 2/20/2 A on 25 A: L2 has 4 A left under FTW's 24 A limit. A + // 32 A charger that switches phases gets Charge now 7 360 W on 1p + // (L1 has room), then the same watts on 3p, which would put 10.7 A on + // L2 too. + spec := stackSpec{fuseA: fuse, loadsW: [3]float64{2 * volt, 20 * volt, 2 * volt}, socPct: 5, + chargers: []charger{{kw: 22.08, phaseSwitch: true}}, + lpExtra: map[string]any{"min_charge_w": 1380, "max_charge_w": 22080, "phase_mode": "auto"}} + err := with(c, spec, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 0, "1p", watts); err != nil { + return err + } + sleepS(90) + st.mark("switch") + if err := st.chargeNow("garage", 0, "3p", watts); err != nil { + return err + } + sleepS(150) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s9", "fuse": fuse, "house_a": []float64{2, 20, 2}, "watts": watts, "charger_a": 32, "order": "1p then 3p"}, err +} + +// ---- 5t. s5s with a goal: surplus below the three-phase minimum ---- + +func s5t(c *runCtx, fuse float64) (map[string]any, error) { + // PV 4.5 kW, house 1 A per phase, full battery: about 3.8 kW of surplus, + // under the 4 140 W three-phase minimum. With a goal saved, a + // surplus-only loadpoint must still charge, on one phase: the charger + // switches phases, as an Easee does. + extra := map[string]any{"surplus_only": true, "phase_mode": "auto", "allowed_steps_w": easeeSteps} + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(1), pvW: 4500, socPct: 100, lpExtra: extra, + chargers: []charger{{phaseSwitch: true}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + if _, err := st.planTo("garage", 40, 80, 8*time.Hour); err != nil { + return err + } + st.mark("plugged") + sleepS(300) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s5t", "fuse": fuse, "house_a": 1, "pv_w": 4500, "soc": 100}, err +} + +func judgeS5t(d *runData) (metrics, []criterion) { + m, cs := judgeS5(d) + t0, t1 := d.marks["plugged"], d.marks["end"] + w := d.window(t0+60, t1) + n := 0 + for _, r := range w { + if carW(r, 0) >= 0.95*1380 { + n++ + } + } + frac := round(float64(n)/math.Max(1, float64(len(w))), 3) + ones := 0 + prs := connector1(profiles(d.frames, t0, t1)) + for _, p := range prs { + if p.Limit > 0 && p.phasesOr3() == 1 { + ones++ + } + } + m["charging_fraction_ge_1p_min"], m["one_phase_offers"] = frac, ones + out := []criterion{} + for _, c := range cs { + if c.Name == "no offer above the charger's rating" || c.Name == "fuse holds" { + out = append(out, c) + } + } + return m, append(out, crit("charges at least 6 A on one phase 90 % of the time", frac >= 0.9)) +} + +// ---- 12. One-phase night: planned charging on auto with Easee's steps ---- + +// fieldSteps is the field site's step table (22 Oct): one-phase levels +// 1 600-3 800 W and three-phase levels 4 800-11 100 W. +var fieldSteps = []float64{0, 1600, 1840, 2070, 2300, 2530, 2760, 2990, 3220, 3450, 3680, 3800, + 4800, 5520, 6210, 6900, 7590, 8280, 8970, 9660, 10350, 11100} + +func s12(c *runCtx, fuse float64) (map[string]any, error) { + // 3 × 20 A, house 1.6/1.7/2.3 A, an Easee-like charger applying in + // 40 s, a three-phase car, and a plan for 6.6 kWh by the first quarter + // hour 40 min out: 7-10 kW on three phases. One phase cannot deliver + // that, so this tests the cuts and the shortfall of the field night, + // not its one-phase choice (#1799 cause 1, which needs a lighter plan). + extra := map[string]any{"phase_mode": "auto", "allowed_steps_w": fieldSteps, "min_charge_w": 1600, "max_charge_w": 11100} + var ready time.Time + err := with(c, stackSpec{fuseA: fuse, loadsW: [3]float64{1.6 * volt, 1.7 * volt, 2.3 * volt}, socPct: 5, + chargers: []charger{{delayS: 40}}, lpExtra: extra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + var err error + if ready, err = st.planTo("garage", 40, 51, 40*time.Minute); err != nil { + return err + } + st.mark("plan") + sleepS(time.Until(ready).Seconds()) + st.mark("ready") + sleepS(30) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s12", "fuse": fuse, "plan_wh": 0.11 * 60000, "charger_delay_s": 40}, err +} + +func judgeS12(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + t0, tReady := d.marks["plan"], d.marks["ready"] + m := metrics{} + // Energy the car took between the plan and the ready time. + wh := 0.0 + w := d.window(t0, tReady) + for i := 1; i < len(w); i++ { + wh += carW(w[i], 0) * (w[i].T - w[i-1].T) / 3600 + } + plan := d.metaF("plan_wh") + m["delivered_wh"], m["plan_wh"] = math.Round(wh), plan + prs := connector1(profiles(d.frames, t0, tReady)) + ones, cuts := 0, 0 + for i, p := range prs { + if p.Limit > 0 && p.phasesOr3() == 1 { + ones++ + } + // A lower order while every phase has room is a cut on room alone, + // unless the plan itself asked for less (a pause between slots). + if i > 0 && p.offerW() < prs[i-1].offerW()-1 { + if r := d.at(t0 + p.T); r != nil && maxAbsI(r) < fuse-1-0.5 { + if after := d.at(t0 + p.T + 2); after == nil || firstLP(after).CommandedReason != "plan" { + cuts++ + } + } + } + } + // Seconds FTW gave fuse_limit while the worst phase had 6 A of room: + // recorded, not judged, since a slow charger's pending order may hold it. + limited := 0 + for _, r := range w { + if r.LPs != nil && firstLP(r).CommandedReason == "fuse_limit" && maxAbsI(r) < fuse-1-6 { + limited++ + } + } + m["one_phase_orders"], m["cuts_with_room"], m["profiles"], m["fuse_limit_with_room_s"] = ones, cuts, len(prs), limited + isBlown := anyBlown(d.window(t0, d.marks["end"])) + return m, []criterion{ + crit("fuse holds", !isBlown), + crit("planned charging never orders one phase", ones == 0), + crit("no order cut while every phase has room", cuts == 0), + crit("at least 90 % of the plan by the ready time", wh >= 0.9*plan), + } +} + +// at is the last sample at or before t. +func (d *runData) at(t float64) *sample { + var last *sample + for _, r := range d.samples { + if r.T > t { + break + } + last = r + } + return last +} + +// ---- 13. Charge now across a charger socket reconnect ---- + +func s13(c *runCtx, fuse float64) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(60) + st.mark("drop") + if err := st.event(houseEvent{Do: "reconnect", Device: "garage", ForS: 10}); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s13", "fuse": fuse, "house_a": 3, "drop_s": 10}, err +} + +func judgeS13(d *runData) (metrics, []criterion) { + tDrop, t1 := d.marks["drop"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + minAfter := minOf(d.window(tDrop, t1), car) + plugged := 0 + for _, r := range d.window(tDrop, t1) { + if lp := firstLP(r); r.LPs != nil && !lp.Plugged { + plugged++ + } + } + m := metrics{"car_min_after_drop": math.Round(minAfter), "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), car)), + "unplugged_samples": plugged} + return m, []criterion{ + crit("Charge now goes on after the charger returns", minAfter >= 0.95*minW), + crit("no unplug seen", plugged == 0), + } +} + +// ---- 14. A one-phase car under a three-phase order ---- + +func s14(c *runCtx, fuse float64) (map[string]any, error) { + // A one-phase car on an 11 kW charger draws on L1 whatever FTW offers. + // A plan asks for more than it can take, so FTW should plan its 16 A + // on one phase (3.5-3.7 kW) and not call the charger not following. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 5, chargers: []charger{{carPhases: 1}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + // 24 kWh in about 45 min: more than even 11 kW gives, so FTW must + // charge from the start. + if _, err := st.planTo("garage", 40, 80, 30*time.Minute); err != nil { + return err + } + st.mark("plan") + sleepS(20 * 60) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s14", "fuse": fuse, "house_a": 3, "car_phases": 1}, err +} + +func judgeS14(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["plan"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + last := meanOf(d.window(t1-600, t1), car) + notFollowing := 0 + for _, r := range d.window(t0+180, t1) { + if firstLP(r).ControlFeedback.Status == "not_following" { + notFollowing++ + } + } + m := metrics{"car_w_last10min": math.Round(last), "not_following_samples": notFollowing} + return m, []criterion{ + crit("delivers 3.4-3.8 kW over the last 10 min", last >= 3400 && last <= 3800), + crit("never called not following", notFollowing == 0), + } +} + +// ---- 15. A charger wired on one phase: Charge now and planned charging ---- + +// onePhaseCharger is an Easee wired on L1: 6-16 A there, nothing on three +// phases. Its loadpoint names no phase mode, as most do. +var onePhaseCharger = charger{kw: 3.68, wiredPhases: 1} + +var onePhaseExtra = map[string]any{"min_charge_w": 1380, "max_charge_w": 3680, + "allowed_steps_w": easeeSteps[:12]} // 0, 1 380 ... 3 680 W + +// chargeNowAsTheApp sends Charge now at 16 A on the phases /api/loadpoints +// reports, as the app does. +func (st *stack) chargeNowAsTheApp(lpID string) (int, error) { + var lps struct { + Loadpoints []ftwLoadpoint `json:"loadpoints"` + } + if err := st.get("/api/loadpoints", &lps); err != nil { + return 0, err + } + phases := 3 + for _, lp := range lps.Loadpoints { + if lp.ID == lpID && lp.Phases > 0 { + phases = lp.Phases + } + } + mode := "3p" + if phases == 1 { + mode = "1p" + } + return phases, st.chargeNow(lpID, 0, mode, 16*volt*float64(phases)) +} + +func s15(c *runCtx, fuse float64) (map[string]any, error) { return s15mode(c, fuse, "") } + +// s15mode is s15 with the loadpoint's phase mode set ("" leaves it out). +func s15mode(c *runCtx, fuse float64, mode string) (map[string]any, error) { + extra := map[string]any{} + for k, v := range onePhaseExtra { + extra[k] = v + } + if mode != "" { + extra["phase_mode"] = mode + } + phases := 0 + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, chargers: []charger{onePhaseCharger}, + lpExtra: extra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + var err error + if phases, err = st.chargeNowAsTheApp("garage"); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s15", "fuse": fuse, "house_a": 3, "api_phases": phases, "phase_mode": mode}, err +} + +func judgeS15(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + const target = 16 * volt // all a one-phase charger gives + m := d.carSummary(t0, t1, target) + m["target_w"], m["api_phases"] = target, d.meta["api_phases"] + hold, _ := m["hold_fraction"].(*float64) + return m, []criterion{ + crit("16 A on L1 within 60 s", le(m["time_to_target_s"].(*float64), 60)), + crit("holds 95 % of the time", hold != nil && *hold >= 0.95), + crit("fuse holds", !m["fuse_blown"].(bool)), + crit("no fuse_limit with room to spare", m["fuse_limit_samples"].(int) == 0), + } +} + +func s15p(c *runCtx, fuse float64) (map[string]any, error) { + // 600 Wh by the first quarter hour 15 min out: about 10 min at 3.68 kW. + var ready time.Time + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 5, chargers: []charger{onePhaseCharger}, + lpExtra: onePhaseExtra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + var err error + if ready, err = st.planTo("garage", 40, 41, 15*time.Minute); err != nil { + return err + } + st.mark("plan") + sleepS(time.Until(ready).Seconds()) + st.mark("ready") + sleepS(30) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s15p", "fuse": fuse, "house_a": 3, "plan_wh": 0.01 * 60000}, err +} + +func judgeS15p(d *runData) (metrics, []criterion) { + t0, tReady := d.marks["plan"], d.marks["ready"] + wh := 0.0 + w := d.window(t0, tReady) + for i := 1; i < len(w); i++ { + wh += carW(w[i], 0) * (w[i].T - w[i-1].T) / 3600 + } + plan := d.metaF("plan_wh") + notFollowing := 0 + for _, r := range w { + if r.LPs != nil && firstLP(r).ControlFeedback.Status == "not_following" { + notFollowing++ + } + } + m := metrics{"delivered_wh": math.Round(wh), "plan_wh": plan, "not_following_samples": notFollowing, + "car_w_max": math.Round(maxOf(w, func(r *sample) float64 { return carW(r, 0) }))} + return m, []criterion{ + crit("fuse holds", !anyBlown(d.window(t0, d.marks["end"]))), + crit("at least 90 % of the plan by the ready time", wh >= 0.9*plan), + crit("never called not following", notFollowing == 0), + } +} + +// s15bat is Charge now on a one-phase charger while Core is raising its own +// battery order: PV climbs every 30 s (+2.5 kW for the second step, 1.5 kW +// for the others) and self-consumption charges the battery with it. The +// battery follows 8 s after each write, and Charge now goes 6 s after the +// second step, while Core's raise is on its way. On +// 16 A with 3 A of house load a raise of more than +// (15 - 3) x 230 - (1 380 + 230) = 1 150 W on its way once kept the car from +// starting at all (#1805). With the battery's 2 s the car started 25 s late +// when Charge now caught a raise in flight, and at once when it did not. +func s15bat(c *runCtx, fuse float64) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, chargers: []charger{onePhaseCharger}, + lpExtra: onePhaseExtra, batteryDelayS: 8}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + for i, pv := range []float64{2100, 4600, 6100, 7600, 9100} { + if err := st.event(houseEvent{Do: "pv", Device: "inverter", PVW: f64(pv)}); err != nil { + return err + } + if i == 1 { + sleepS(6) + st.mark("charge_now") + if _, err := st.chargeNowAsTheApp("garage"); err != nil { + return err + } + sleepS(24) + continue + } + sleepS(30) + } + sleepS(90) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s15bat", "fuse": fuse, "house_a": 3, "battery_delay_s": 8}, err +} + +func judgeS15bat(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + fuse := d.metaF("fuse") + // Core's battery order: the EMS command and setpoint it wrote. + order := func(r *sample) float64 { + switch r.Bat.EMSCmd { + case 170: + return r.Bat.EMSSetW + case 187: + return -r.Bat.EMSSetW + } + return 0 + } + rise, low := 0.0, math.Inf(1) + for _, r := range d.window(t0-20, t0) { + low = math.Min(low, order(r)) + rise = math.Max(rise, order(r)-low) + } + start := d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.95*1380 }) + over, longest := 0.0, 0.0 + for _, r := range d.window(t0, t1) { + if maxAbsI(r) > fuse-1 { + over++ + longest = math.Max(longest, over) + } else { + over = 0 + } + } + m := metrics{"battery_rise_w_20s_before": math.Round(rise), "car_start_s": roundp(start, 1), + "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), func(r *sample) float64 { return carW(r, 0) })), + "over_fuse_minus1_longest_s": longest, "max_abs_i": round(maxOf(d.window(t0, t1), maxAbsI), 1)} + return m, []criterion{ + crit("the battery order rose over 1 150 W in the 20 s before Charge now", rise > 1150), // the setup + crit("car charging within 10 s of Charge now", le(start, 10)), + crit("fuse holds", !anyBlown(d.window(t0, t1))), + crit("over fuse - 1 A at most 3 s", longest <= 3), + } +} diff --git a/go/test/site/stack_test.go b/go/test/site/stack_test.go new file mode 100644 index 000000000..aa4b156bc --- /dev/null +++ b/go/test/site/stack_test.go @@ -0,0 +1,537 @@ +//go:build unix + +package site + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sync" + "syscall" + "time" +) + +// ocppPass is the shared OCPP credential the rig gives FTW and the chargers. +const ocppPass = "sim-ocpp-rig-20261021" + +// lane is the first of the 100 ports one rig uses on 127.0.0.1. The house +// (device-simulator's headless run) takes base+0 to base+59; FTW takes +// base+90 (API) and base+91 (OCPP). +type lane int + +func (l lane) simHTTP() int { return int(l) } +func (l lane) ftwAPI() int { return int(l) + 90 } +func (l lane) ftwOCPP() int { return int(l) + 91 } + +// ---- processes ---- + +type proc struct { + cmd *exec.Cmd + done chan struct{} +} + +// procs runs each child in its own process group, so a stop reaches +// everything it started. +type procs struct { + mu sync.Mutex + m map[string]*proc +} + +func (ps *procs) start(name string, argv []string, dir string, env []string, logPath string) error { + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return err + } + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = append(append(os.Environ(), "GOMAXPROCS=2"), env...) + cmd.Stdout, cmd.Stderr = f, f + cmd.SysProcAttr = childProcAttr() + if err := cmd.Start(); err != nil { + f.Close() + return fmt.Errorf("start %s: %w", name, err) + } + p := &proc{cmd: cmd, done: make(chan struct{})} + go func() { cmd.Wait(); f.Close(); close(p.done) }() + ps.mu.Lock() + if ps.m == nil { + ps.m = map[string]*proc{} + } + ps.m[name] = p + ps.mu.Unlock() + return nil +} + +func (ps *procs) stop(name string, sig syscall.Signal, wait time.Duration) { + ps.mu.Lock() + p := ps.m[name] + delete(ps.m, name) + ps.mu.Unlock() + if p == nil { + return + } + syscall.Kill(-p.cmd.Process.Pid, sig) + select { + case <-p.done: + case <-time.After(wait): + syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL) + <-p.done + } +} + +func (ps *procs) stopAll() { + ps.mu.Lock() + names := make([]string, 0, len(ps.m)) + for n := range ps.m { + names = append(names, n) + } + ps.mu.Unlock() + for _, n := range names { + ps.stop(n, syscall.SIGTERM, 10*time.Second) + } +} + +// ---- HTTP ---- + +var client = &http.Client{Timeout: 5 * time.Second} + +// call sends body as JSON and decodes the answer into out when out is not +// nil. It returns the raw answer. +func call(method, url string, body, out any) ([]byte, error) { + var rd io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return nil, err + } + rd = bytes.NewReader(b) + } + req, err := http.NewRequest(method, url, rd) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 { + return raw, fmt.Errorf("%s %s: %s: %s", method, url, resp.Status, bytes.TrimSpace(raw)) + } + if out != nil && len(raw) > 0 { + if err := json.Unmarshal(raw, out); err != nil { + return raw, fmt.Errorf("%s %s: %w", method, url, err) + } + } + return raw, nil +} + +func waitHTTP(url string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + if _, last = call("GET", url, nil, nil); last == nil { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("%s did not answer: %v", url, last) +} + +// waitUntil polls pred every step until it holds or the timeout passes. +func waitUntil(timeout, step time.Duration, pred func() bool) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if pred() { + return true + } + time.Sleep(step) + } + return false +} + +func now() float64 { return float64(time.Now().UnixMicro()) / 1e6 } + +// ---- the house: device-simulator's headless run ---- + +// houseSpec is device-simulator's run spec (docs/HEADLESS_RUN.md there). +type houseSpec struct { + Name string `json:"name"` + Site houseSite `json:"site"` + Devices []houseDevice `json:"devices"` + Timeline []houseEvent `json:"timeline,omitempty"` + Random *houseRandom `json:"random,omitempty"` +} + +// houseRandom asks the house to draw its own timeline from a seed. +type houseRandom struct { + Seed int64 `json:"seed"` + DurationS float64 `json:"duration_s"` + GapS float64 `json:"gap_s,omitempty"` +} + +type houseSite struct { + FuseA int `json:"fuse_a"` + FuseProfile string `json:"fuse_profile,omitempty"` + HouseW [3]float64 `json:"house_w"` + SignedPhaseCurrents *bool `json:"signed_phase_currents,omitempty"` +} + +type houseDevice struct { + ID string `json:"id"` + Driver string `json:"driver"` + CommandResponseS *float64 `json:"command_response_s,omitempty"` + StartupDelayS *float64 `json:"startup_delay_s,omitempty"` + LoadBalancerA *float64 `json:"load_balancer_a,omitempty"` + BatteryKWh float64 `json:"battery_kwh,omitempty"` + SoCPct *float64 `json:"soc_pct,omitempty"` + MinSoCPct *float64 `json:"min_soc_pct,omitempty"` + PVW float64 `json:"pv_w,omitempty"` + OCPPURL string `json:"ocpp_url,omitempty"` + MaxKW float64 `json:"max_kw,omitempty"` + Vendor string `json:"vendor,omitempty"` + Model string `json:"model,omitempty"` + PhaseSwitch bool `json:"phase_switch,omitempty"` + OCPPConfig map[string]string `json:"ocpp_config,omitempty"` + CarMaxA float64 `json:"car_max_a,omitempty"` + CarPhases int `json:"car_phases,omitempty"` + WiredPhases int `json:"wired_phases,omitempty"` +} + +// houseEvent is one change to the house (device-simulator's runEvent). +type houseEvent struct { + AtS float64 `json:"at_s,omitempty"` + Do string `json:"do"` + Device string `json:"device,omitempty"` + Label string `json:"label,omitempty"` + W *[3]float64 `json:"w,omitempty"` + A *[3]float64 `json:"a,omitempty"` + PVW *float64 `json:"pv_w,omitempty"` + Pct *float64 `json:"pct,omitempty"` + On *bool `json:"on,omitempty"` + MaxA *float64 `json:"max_a,omitempty"` + Phases *int `json:"phases,omitempty"` + ForS float64 `json:"for_s,omitempty"` +} + +type housePorts struct { + HTTP int `json:"http"` + Devices map[string]struct { + Driver string `json:"driver"` + Modbus int `json:"modbus_port"` + UnitID int `json:"unit_id"` + Serial string `json:"serial"` + } `json:"devices"` +} + +// ---- one run ---- + +// charger is one OCPP charger and the car on it. FTW's loadpoint has the +// charger's id. +type charger struct { + kw float64 // three-phase rating; default 11.04 + phaseSwitch bool + delayS float64 // applies a profile this late, like a cloud charger + carPhases int // 0: as many as offered + wiredPhases int // 1: wired on L1 only, kw its rating there + carMaxA float64 +} + +type stackSpec struct { + // house replays a spec the house wrote (timeline.json); random asks it + // to draw a timeline. Without either the house has no timeline. + house *houseSpec + random *houseRandom + + fuseA float64 + loadsW [3]float64 + pvW float64 + socPct float64 + chargers []charger // nil: one 11 kW charger + lpExtra map[string]any + heartbeatS int // FTW's OCPP heartbeat; 0 leaves Core's default + // batteryDelayS is how long the battery takes to follow an EMS write; + // 0 leaves the model's (2 s for sungrow). + batteryDelayS float64 +} + +type stack struct { + env *rigEnv + lane lane + rundir string + spec stackSpec + ps procs + ports housePorts + ftw string // FTW's API base URL + sim string // the house's API base URL + rec *recorder + cfg string + // markErr is the first mark the house did not take; the run cannot be + // judged without it. + markErr error +} + +func newStack(env *rigEnv, l lane, rundir string, spec stackSpec) *stack { + if spec.chargers == nil { + spec.chargers = []charger{{}} + } + return &stack{env: env, lane: l, rundir: rundir, spec: spec, + ftw: fmt.Sprintf("http://127.0.0.1:%d", l.ftwAPI()), + sim: fmt.Sprintf("http://127.0.0.1:%d", l.simHTTP())} +} + +func f64(v float64) *float64 { return &v } + +func (st *stack) houseSpec() houseSpec { + if st.spec.house != nil { + // A replay: the same house on this lane's ports. + h := *st.spec.house + h.Name = filepath.Base(st.rundir) + h.Random = nil + h.Devices = append([]houseDevice(nil), h.Devices...) + for i := range h.Devices { + if h.Devices[i].Driver == "ocpp" { + h.Devices[i].OCPPURL = fmt.Sprintf("ws://ftw:%s@127.0.0.1:%d/", ocppPass, st.lane.ftwOCPP()) + } + } + return h + } + h := houseSpec{Name: filepath.Base(st.rundir), Random: st.spec.random, + Site: houseSite{FuseA: int(st.spec.fuseA), FuseProfile: "iec_gg", HouseW: st.spec.loadsW}, + Devices: []houseDevice{{ID: "inverter", Driver: "sungrow", BatteryKWh: 10, + SoCPct: f64(st.spec.socPct), MinSoCPct: f64(5), PVW: st.spec.pvW}}} + if st.spec.batteryDelayS > 0 { + h.Devices[0].CommandResponseS = f64(st.spec.batteryDelayS) + } + for i, c := range st.spec.chargers { + var delay *float64 + if st.spec.random == nil { + delay = f64(c.delayS) // a random house draws its own + } + kw := c.kw + if kw == 0 { + kw = 11.04 + } + h.Devices = append(h.Devices, houseDevice{ID: lpIDs[i], Driver: "ocpp", + OCPPURL: fmt.Sprintf("ws://ftw:%s@127.0.0.1:%d/", ocppPass, st.lane.ftwOCPP()), + MaxKW: kw, Vendor: "RigCo", Model: fmt.Sprintf("Rig%.0f", kw), PhaseSwitch: c.phaseSwitch, CommandResponseS: delay, + CarPhases: c.carPhases, CarMaxA: c.carMaxA, WiredPhases: c.wiredPhases, + OCPPConfig: map[string]string{"HeartbeatInterval": "60", "MeterValueSampleInterval": "10"}}) + } + return h +} + +func (st *stack) up() error { + if err := os.RemoveAll(st.rundir); err != nil { + return err + } + if err := os.MkdirAll(st.rundir, 0o755); err != nil { + return err + } + spec, _ := json.MarshalIndent(st.houseSpec(), "", " ") + specPath := filepath.Join(st.rundir, "house.json") + if err := os.WriteFile(specPath, spec, 0o644); err != nil { + return err + } + houseDir := filepath.Join(st.rundir, "house") + if err := st.ps.start("house", []string{st.env.devsimBin, "run", "-spec", specPath, + "-port-base", fmt.Sprint(int(st.lane)), "-out", houseDir}, st.rundir, nil, + filepath.Join(st.rundir, "house.log")); err != nil { + return err + } + if !waitUntil(30*time.Second, 200*time.Millisecond, func() bool { + b, err := os.ReadFile(filepath.Join(houseDir, "ports.json")) + return err == nil && json.Unmarshal(b, &st.ports) == nil && len(st.ports.Devices) > 0 + }) { + return fmt.Errorf("the house never wrote ports.json (see house.log)") + } + if err := waitHTTP(st.sim+"/api/run", 10*time.Second); err != nil { + return err + } + var err error + if st.cfg, err = st.ftwConfig(); err != nil { + return err + } + if err := st.startFTW(); err != nil { + return err + } + if _, err := call("POST", st.ftw+"/api/mode", map[string]any{"mode": "planner_self"}, nil); err != nil { + return err + } + st.rec = &recorder{ftw: st.ftw, path: filepath.Join(st.rundir, "ftw.jsonl")} + st.rec.start() + return nil +} + +func (st *stack) startFTW() error { + if err := st.ps.start("ftw", []string{st.env.ftwBin, "-config", st.cfg, + "-web", filepath.Join(st.env.root, "web"), "-drivers", filepath.Join(st.env.root, "drivers")}, + st.rundir, []string{"GOMAXPROCS=4"}, filepath.Join(st.rundir, "ftw.log")); err != nil { + return err + } + return waitHTTP(st.ftw+"/api/status", 90*time.Second) +} + +func (st *stack) stopFTW() { st.ps.stop("ftw", syscall.SIGTERM, 30*time.Second) } + +func (st *stack) down() { + if st.rec != nil { + st.rec.close() + } + st.ps.stop("ftw", syscall.SIGTERM, 30*time.Second) + st.ps.stop("house", syscall.SIGTERM, 15*time.Second) + st.ps.stopAll() +} + +// event changes the house now. +func (st *stack) event(ev houseEvent) error { + _, err := call("POST", st.sim+"/api/run/event", ev, nil) + return err +} + +// mark names this moment in the house's record. +func (st *stack) mark(label string) { + if _, err := call("POST", st.sim+"/api/run/mark", map[string]string{"label": label}, nil); err != nil && st.markErr == nil { + st.markErr = fmt.Errorf("mark %s: %w", label, err) + } +} + +func (st *stack) get(path string, out any) error { + _, err := call("GET", st.ftw+path, nil, out) + return err +} + +func (st *stack) post(path string, body, out any) error { + _, err := call("POST", st.ftw+path, body, out) + return err +} + +func (st *stack) setLoadsA(a float64) error { + return st.event(houseEvent{Do: "loads", A: &[3]float64{a, a, a}}) +} + +func (st *stack) serial(i int) string { return st.ports.Devices[lpIDs[i]].Serial } + +// ---- FTW's API, once a second ---- + +type recorder struct { + ftw string + path string + mu sync.Mutex + stop chan struct{} + done chan struct{} +} + +func (r *recorder) start() { + r.stop, r.done = make(chan struct{}), make(chan struct{}) + go r.run() +} + +func (r *recorder) close() { + if r.stop == nil { + return + } + close(r.stop) + <-r.done +} + +func (r *recorder) append(rec map[string]any) { + b, _ := json.Marshal(rec) + r.mu.Lock() + defer r.mu.Unlock() + f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return + } + f.Write(append(b, '\n')) + f.Close() +} + +func (r *recorder) run() { + defer close(r.done) + for { + t0 := time.Now() + rec := map[string]any{"t": now()} + for key, path := range map[string]string{"status": "/api/status", "lps": "/api/loadpoints"} { + if b, err := call("GET", r.ftw+path, nil, nil); err != nil { + rec[key+"_err"] = err.Error() + } else { + rec[key] = json.RawMessage(b) + } + } + r.append(rec) + select { + case <-r.stop: + return + case <-time.After(time.Second - time.Since(t0)): + } + } +} + +// ---- FTW's configuration ---- + +var lpIDs = []string{"garage", "carport"} + +func (st *stack) ftwConfig() (string, error) { + var lps []map[string]any + for i := range st.spec.chargers { + lp := map[string]any{ + "id": lpIDs[i], "driver_name": st.serial(i), + "min_charge_w": 4140, "max_charge_w": 11040, "vehicle_capacity_wh": 60000, + } + for k, v := range st.spec.lpExtra { + lp[k] = v + } + lps = append(lps, lp) + } + ocpp := map[string]any{"enabled": true, "bind": "127.0.0.1", "port": st.lane.ftwOCPP(), "path": "/", + "username": "ftw", "password": ocppPass} + // 60 s is what every box configured from the old example has; 0 leaves + // it unset so Core's default applies. + if st.spec.heartbeatS > 0 { + ocpp["heartbeat_interval_s"] = st.spec.heartbeatS + } + modbus := func(id string) map[string]any { + d := st.ports.Devices[id] + return map[string]any{"modbus": map[string]any{"host": "127.0.0.1", "port": d.Modbus, "unit_id": max(d.UnitID, 1)}} + } + cfg := map[string]any{ + "site": map[string]any{"name": "rig", "control_interval_s": 2, "grid_target_w": 0, + "grid_tolerance_w": 42, "watchdog_timeout_s": 60}, + "fuse": map[string]any{"max_amps": st.spec.fuseA, "phases": 3, "voltage": 230}, + "drivers": []map[string]any{ + {"name": "meter", "lua": "drivers/sdm630.lua", "is_site_meter": true, "capabilities": modbus("meter")}, + {"name": "sungrow", "lua": "drivers/sungrow.lua", "battery_capacity_wh": 10000, "capabilities": modbus("inverter")}, + }, + "api": map[string]any{"port": st.lane.ftwAPI()}, + "ocpp": ocpp, + "loadpoints": lps, + "state": map[string]any{"path": filepath.Join(st.rundir, "state.db")}, + "app_link": map[string]any{"enabled": false}, + "fleet_ping": map[string]any{"enabled": false}, + "device_repository": map[string]any{"enabled": false}, + // A real box plans. A box without a planner runs Self, where the + // battery chases the raw meter and covers the car (dispatch.go), + // which hides the stress on the fuse. + "planner": map[string]any{"enabled": true, "engine": "core", "mode": "self_consumption"}, + "price": map[string]any{"provider": "elprisetjustnu", "zone": "SE3"}, + } + path := filepath.Join(st.rundir, "config.yaml") + b, err := json.MarshalIndent(cfg, "", " ") // JSON is valid YAML + if err != nil { + return "", err + } + return path, os.WriteFile(path, b, 0o644) +} diff --git a/go/test/site/testdata/found/.gitkeep b/go/test/site/testdata/found/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/go/test/site/verdicts_test.go b/go/test/site/verdicts_test.go new file mode 100644 index 000000000..9cb180db0 --- /dev/null +++ b/go/test/site/verdicts_test.go @@ -0,0 +1,553 @@ +//go:build unix + +package site + +import ( + "math" + "sort" +) + +// Each judge turns a finished run into numbers and pass criteria. The +// numbers and limits are the ones reports/simulatorprov-20261021 used. + +func judgeS0(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["connected"], d.marks["end"] + w := d.window(t0+20, t1) + lq, lr, today := counter{}, counter{}, counter{} + nAllRead := 0 + for _, r := range w { + st := status(r) + lq[str(st.LoadQuality)]++ + lr[str(st.LoadReason)]++ + today[str(st.Energy.Today.HistoryQuality)]++ + if allRead(r) { + nAllRead++ + } + } + var last *float64 + if len(w) > 0 { + last = status(w[len(w)-1]).Energy.Today.LoadWh + } + disc := d.disconnects(t0, t1-2) // the rig stops FTW at the end + m := metrics{"load_quality": lq, "load_reason": lr, "today_history_quality": today, + "all_read_samples": nAllRead, "samples_window": len(w), "today_load_wh_last": last, + "disconnects_s": disc} + return m, []criterion{ + crit("Home use measured in 95 % of samples", float64(lq["measured"]) >= 0.95*float64(len(w))), + crit("no OCPP disconnects", len(disc) == 0), + } +} + +func judgeS1(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + fuse := d.metaF("fuse") + target := maxW + if fuse < 25 { + // Room on a 16 A fuse: fuse - 1 A margin - house, in 1 A steps. + target = math.Min(maxW, math.Floor(fuse-1-d.metaF("house_a"))*volt*3) + } + m := d.carSummary(t0, t1, target) + m["target_w"] = target + hold, _ := m["hold_fraction"].(*float64) + cs := []criterion{ + crit("target within 60 s", le(m["time_to_target_s"].(*float64), 60)), + crit("holds 95 % of the time", hold != nil && *hold >= 0.95), + crit("fuse holds", !m["fuse_blown"].(bool)), + } + if fuse >= 25 { + cs = append(cs, crit("no fuse_limit with room to spare", m["fuse_limit_samples"].(int) == 0)) + } + return m, cs +} + +func judgeS2(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + pre := d.window(t0-15, t0) + var exp [3]float64 + for k := 0; k < 3; k++ { + exp[k] = round(meanOf(pre, func(r *sample) float64 { return phaseI(r)[k] }), 2) + } + m := d.carSummary(t0, t1, maxW) + m["export_a_before"] = exp + return m, []criterion{ + crit("car starts within 60 s", le(m["time_to_start_s"].(*float64), 60)), + crit("fuse holds", !m["fuse_blown"].(bool)), + } +} + +func judgeS3(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tStep, tBack, t1 := d.marks["charge_now"], d.marks["step"], d.marks["step_back"], d.marks["end"] + pre := d.window(tStep-30, tStep) + m := metrics{ + "pre_step_car_w": math.Round(meanOf(pre, func(r *sample) float64 { return carW(r, 0) })), + "pre_step_bat_w": math.Round(meanOf(pre, batSigned)), + "pre_step_max_i": round(maxOf(pre, maxAbsI), 2), + } + m["post_step_max_i"] = round(maxOf(d.window(tStep, tBack), maxAbsI), 2) + m["over_fuse_longest_s"], m["over_fuse_total_s"] = d.runsOver(tStep, tBack, fuse) + overLongest, overTotal := d.runsOver(tStep, tBack, fuse-1) + m["over_fuse_minus1_longest_s"], m["over_fuse_minus1_total_s"] = overLongest, overTotal + m["i2_over_fuse_minus1_s"], m["i2_peak_a"] = d.i2Over(tCN, t1, fuse-1) + m["back_under_fuse_minus1_s"] = roundp(d.firstT(tStep, func(r *sample) bool { return maxAbsI(r) <= fuse-1 && r.T > tStep+1 }), 1) + last := d.window(tBack-60, tBack) + m["post_step_car_w_last60"] = math.Round(meanOf(last, func(r *sample) float64 { return carW(r, 0) })) + m["post_step_bat_w_last60"] = math.Round(meanOf(last, batSigned)) + var after []*sample + for _, r := range d.samples { + if r.T >= tCN { + after = append(after, r) + } + } + m["fuse_damage_max"] = round(maxOf(after, damage), 4) + isBlown := anyBlown(d.samples) + m["fuse_blown"] = isBlown + // Device response + meter and tick (2 + 2 s) + 3 s. + allowed := d.metaF("charger_delay_s") + 2 + 2 + 3 + m["allowed_over_s"] = allowed + m["run_over_fuse_minus1_longest_s"], _ = d.runsOver(tCN, t1, fuse-1) + m["run_max_i"] = round(maxOf(after, maxAbsI), 2) + // Before the step, layers with different margins may hold a phase + // between fuse - 1 A and fuse - 0.5 A; reported, not judged. + m["pre_step_over_fuse_minus1_longest_s"], _ = d.runsOver(tCN, tStep, fuse-1) + m["pre_step_over_fuse_minus_half_longest_s"], _ = d.runsOver(tCN, tStep, fuse-0.5) + return m, []criterion{ + crit("fuse holds", !isBlown), + crit("over fuse - 1 A no longer than the response time + 3 s after the step", overLongest <= allowed), + } +} + +func judgeS4(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + m := d.carSummary(t0, t1, maxW) + tx := connector1(profiles(d.frames, t0, t1)) + m["profiles_sent"] = len(tx) + offers := map[float64]bool{} + for _, p := range tx { + offers[math.Round(p.offerW())] = true + } + m["distinct_offers_w"] = sortedKeys(offers) + zeros := 0 + for i := 1; i < len(tx); i++ { + if tx[i-1].offerW() > 0 && tx[i].offerW() == 0 { + zeros++ + } + } + m["offer_drops_to_zero"] = zeros + return m, []criterion{ + crit("no stops", m["stops_after_start"].(int) == 0), + crit("no offer drops to 0 W", zeros == 0), + crit("car starts", m["time_to_start_s"].(*float64) != nil), + crit("full power over the last minute", m["car_w_last60_mean"].(float64) >= 0.95*maxW), + } +} + +func sortedKeys(set map[float64]bool) []float64 { + out := make([]float64, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Float64s(out) + return out +} + +func judgeS5(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["plugged"], d.marks["end"] + prs := connector1(profiles(d.frames, t0, t1)) + ratingA := maxW / (3 * volt) + var bad []profile + over, maxOffer := 0, 0.0 + offers := map[float64]bool{} + type ap struct { + A float64 + Phases float64 + } + offersAP := map[ap]bool{} + for _, p := range prs { + offers[math.Round(p.offerW())] = true + offersAP[ap{round(p.Limit, 2), p.phasesOr3()}] = true + maxOffer = math.Max(maxOffer, p.perPhaseA()) + if p.Limit <= 0 { + continue + } + a := p.perPhaseA() + // A valid step is 6-16 A per phase: the charger is rated 16 A. + if a < 6-1e-6 || a > ratingA+1e-6 { + bad = append(bad, p) + } + if a > ratingA+1e-6 { + over++ + } + } + w := d.window(t0+60, t1) + pre := d.window(t0-10, t0+5) + surplus := meanOf(pre, func(r *sample) float64 { + i := phaseI(r) + return -(i[0] + i[1] + i[2]) * volt + }) + carMean := math.Round(meanOf(w, func(r *sample) float64 { return carW(r, 0) })) + charging := 0 + for _, r := range w { + if carW(r, 0) >= 0.95*minW { + charging++ + } + } + frac := round(float64(charging)/math.Max(1, float64(len(w))), 3) + rs := counter{} + for _, r := range w { + for _, x := range reasons(r) { + rs[x]++ + } + } + all := d.window(t0, t1) + isBlown := anyBlown(all) + if len(bad) > 5 { + bad = bad[:5] + } + m := metrics{ + "surplus_w_at_plug": math.Round(surplus), + "profiles_sent": len(prs), + "invalid_examples": bad, + "offers_w": firstN(sortedKeys(offers), 12), + "car_w_mean_after_60s": carMean, + "charging_fraction_ge_min": frac, + "time_to_start_s": roundp(d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.95*minW }), 1), + "reasons": rs.top(6), + "offers_over_rating": over, + "max_offer_a": round(maxOffer, 2), + "fuse_blown": isBlown, + "fuse_damage_max": round(maxOf(all, damage), 4), + "surplus_used_fraction": round(carMean/math.Max(1, math.Round(surplus)), 3), + } + nInvalid := 0 + for _, p := range prs { + if a := p.perPhaseA(); p.Limit > 0 && (a < 6-1e-6 || a > ratingA+1e-6) { + nInvalid++ + } + } + m["invalid_steps"] = nInvalid + return m, []criterion{ + // The beta criteria (Fredrik, 21 Oct): no offer above the charger's + // rating per phase, and the fuse never trips. + crit("no offer above the charger's rating", over == 0), + crit("fuse holds", !isBlown), + crit("only valid steps (6-16 A)", nInvalid == 0), + crit("charges at least the minimum 90 % of the time", frac >= 0.9), + } +} + +func firstN(xs []float64, n int) []float64 { + if len(xs) > n { + return xs[:n] + } + return xs +} + +type qualitySummary struct { + Samples int `json:"samples"` + LoadWShown int `json:"load_w_shown"` + LoadQuality map[string]int `json:"load_quality"` + TodayFirst *float64 `json:"today_load_wh_first"` + TodayLast *float64 `json:"today_load_wh_last"` +} + +func summarize(ws []*sample) qualitySummary { + q := qualitySummary{Samples: len(ws), LoadQuality: counter{}} + for i, r := range ws { + st := status(r) + q.LoadQuality[str(st.LoadQuality)]++ + if st.LoadW != nil { + q.LoadWShown++ + } + if i == 0 { + q.TodayFirst = st.Energy.Today.LoadWh + } + q.TodayLast = st.Energy.Today.LoadWh + } + return q +} + +func judgeS6(d *runData) (metrics, []criterion) { + tOff, tOn, t1 := d.marks["offline"], d.marks["online"], d.marks["end"] + before := summarize(d.window(tOff-40, tOff)) + during := summarize(d.window(tOff+15, tOn)) + after := summarize(d.window(tOn+30, t1)) + // From 20 s after plug-in: FTW's start and the charger's first + // MeterValues after plug-in are reported apart (plug_in_approx_s). + plugT, ok := d.marks["plug"] + if !ok && len(d.samples) > 0 { + plugT = d.samples[0].T + } + var viol []*sample + for _, r := range d.samples { + if r.T >= plugT+20 && allRead(r) && str(status(r).LoadQuality) != "measured" && !(tOff <= r.T && r.T < tOn+30) { + viol = append(viol, r) + } + } + plugIn := 0 + for _, r := range d.window(plugT, plugT+20) { + if allRead(r) && str(status(r).LoadQuality) != "measured" { + plugIn++ + } + } + var firstViol *float64 + if len(viol) > 0 { + v := round(viol[0].T-tOff, 1) + firstViol = &v + } + marked := 0 + for k, v := range during.LoadQuality { + if k != "null" && k != "" && k != "measured" { + marked += v + } + } + m := metrics{"before": before, "during": during, "after": after, + "approx_while_all_read": len(viol), "approx_while_all_read_first": firstViol, + "load_w_missing_during": during.Samples - during.LoadWShown, "plug_in_approx_s": plugIn} + return m, []criterion{ + crit("Home use shown throughout the outage", during.LoadWShown == during.Samples), + crit("Home use marked estimated throughout the outage", marked == during.Samples), + crit("measured before the outage", before.LoadQuality["measured"] == before.Samples), + crit("measured after the outage", after.LoadQuality["measured"] == after.Samples), + crit("never estimated while every driver reads", len(viol) == 0), + } +} + +func judgeS7(d *runData) (metrics, []criterion) { + tCN, tStop, tUp, t1 := d.marks["charge_now"], d.marks["ftw_stop"], d.marks["ftw_up"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + pre := d.window(tStop-30, tStop) + preCar := meanOf(pre, car) + post := d.window(tUp, t1) + m := metrics{"pre_car_w": math.Round(preCar), "downtime_s": round(tUp-tStop, 1)} + var before *float64 + if len(pre) > 0 { + m["pre_load_w"] = status(pre[len(pre)-1]).LoadW + before = status(pre[len(pre)-1]).Energy.Today.LoadWh + } + m["today_load_wh_before"] = before + var afterFirst, afterLast *float64 + nullAfter := 0 + hq, lq := counter{}, counter{} + for _, r := range post { + // A sample before the rig's first status read after the restart + // has no reading; only a status without the day total counts. + if r.Status == nil { + continue + } + st := status(r) + v := st.Energy.Today.LoadWh + if v == nil { + nullAfter++ + } else if afterFirst == nil { + afterFirst = v + } + afterLast = v + hq[str(st.Energy.Today.HistoryQuality)]++ + lq[str(st.LoadQuality)]++ + } + m["today_load_wh_after_first"], m["today_load_wh_null_after"], m["today_load_wh_after_last"] = afterFirst, nullAfter, afterLast + m["history_quality_after"], m["load_quality_after"] = hq, lq + loadBack := d.firstT(tUp, func(r *sample) bool { return status(r).LoadW != nil }) + m["load_w_back_s"] = roundp(loadBack, 1) + m["car_back_s"] = roundp(d.firstT(tUp, func(r *sample) bool { return preCar > 0 && car(r) >= 0.95*preCar }), 1) + m["car_resumed_s"] = roundp(d.firstT(tUp, func(r *sample) bool { return car(r) >= 0.95*minW }), 1) + m["car_w_150s_after_up"] = math.Round(meanOf(d.window(tUp+90, tUp+150), car)) + rs := counter{} + for _, r := range d.window(tUp+10, t1) { + for _, x := range reasons(r) { + rs[x]++ + } + } + m["reasons_after_up"] = rs.top(4) + m["car_min_during_restart"] = math.Round(minOf(d.window(tStop, tUp+60), car)) + m["car_w_last60"] = math.Round(meanOf(d.window(t1-60, t1), car)) + if tk, ok := d.marks["today_known"]; ok { + m["today_known_after_s"] = math.Round(tk - tCN) + } + minAfterStop := minOf(d.window(tStop, t1), car) + m["car_min_after_stop"] = math.Round(minAfterStop) + var fullFrom *float64 + dip, dipMin := 0, preCar + for _, r := range post { + if car(r) >= 0.95*preCar { + if fullFrom == nil { + v := r.T - tUp + fullFrom = &v + } + } else { + fullFrom = nil + dip++ + dipMin = math.Min(dipMin, car(r)) + } + } + m["full_power_from_s"], m["dip_s"], m["dip_min_w"] = roundp(fullFrom, 1), dip, math.Round(dipMin) + cs := []criterion{ + crit("Home use back within 30 s", le(loadBack, 30)), + // Charge now continues without a stop and is back at full power + // within 420 s of Core being up, for at least the last 30 s. The + // dip is recorded. + crit("Charge now never stops", minAfterStop >= 0.95*minW), + crit("full power within 420 s", fullFrom != nil && *fullFrom <= 420 && t1-tUp-*fullFrom >= 30), + } + if d.meta["short"] != true { + cs = append([]criterion{crit("the day's Home use is kept", before != nil && nullAfter == 0 && + afterLast != nil && *afterLast >= *before*0.99)}, cs...) + } + return m, cs +} + +func minOf(ws []*sample, f func(*sample) float64) float64 { + if len(ws) == 0 { + return 0 + } + m := math.Inf(1) + for _, r := range ws { + m = math.Min(m, f(r)) + } + return m +} + +func judgeS8(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + t0, t1 := d.marks["charge_now"], d.marks["end"] + m := d.carSummary(t0, t1, maxW) + m["over_fuse_longest_s"], m["over_fuse_total_s"] = d.runsOver(t0, t1, fuse) + overLongest, overTotal := d.runsOver(t0, t1, fuse-1) + m["over_fuse_minus1_longest_s"], m["over_fuse_minus1_total_s"] = overLongest, overTotal + m["i2_over_fuse_minus1_s"], m["i2_peak_a"] = d.i2Over(t0, t1, fuse-1) + var prs [][2]float64 + for i, p := range connector1(profiles(d.frames, t0, t1)) { + if i >= 20 { + break + } + prs = append(prs, [2]float64{p.T, math.Round(p.offerW())}) + } + m["profiles"] = prs + allowed := d.metaF("charger_delay_s") + 2 + 2 + 3 + m["allowed_over_s"] = allowed + if plug, ok := d.marks["plug"]; ok { + // Before FTW's first profile reaches the charger it draws its maximum. + m["plugin_max_i"] = round(maxOf(d.window(plug, t0), maxAbsI), 2) + m["plugin_over_fuse_longest_s"], _ = d.runsOver(plug, t0, fuse) + } + return m, []criterion{ + crit("fuse holds", !m["fuse_blown"].(bool)), + crit("over fuse - 1 A no longer than the charger's delay + 7 s", overLongest <= allowed), + } +} + +func judgeS9(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tSw, t1 := d.marks["charge_now"], d.marks["switch"], d.marks["end"] + before, after := d.overage(tCN, tSw, fuse), d.overage(tSw, t1, fuse) + car := func(r *sample) float64 { return carW(r, 0) } + m := metrics{"before": before, "after": after, + "before_phase_a": d.phaseMean(tSw-30, tSw), "after_phase_a_last60": d.phaseMean(t1-60, t1), + "car_w_before": math.Round(meanOf(d.window(tSw-30, tSw), car)), + "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), car))} + var seq [][3]any + for _, p := range connector1(profiles(d.frames, tSw-5, t1)) { + a := round(p.Limit, 2) + if len(seq) == 0 || seq[len(seq)-1][1] != a || !samePhases(seq[len(seq)-1][2], p.Phases) { + seq = append(seq, [3]any{p.T, a, p.Phases}) + } + } + if len(seq) > 12 { + seq = seq[:12] + } + m["profiles_after_switch"] = seq + return m, []criterion{ + crit("fuse holds", !before["fuse_blown"].(bool) && !after["fuse_blown"].(bool)), + crit("over fuse - 1 A at most 3 s on 3p", before["over_limit_longest_s"].(float64) <= 3), + crit("over fuse - 1 A at most 3 s after the switch to 1p", after["over_limit_longest_s"].(float64) <= 3), + } +} + +func samePhases(a any, b *float64) bool { + pa, _ := a.(*float64) + if pa == nil || b == nil { + return pa == nil && b == nil + } + return *pa == *b +} + +func judgeS10(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tSt, t1 := d.marks["charge_now"], d.marks["step"], d.marks["end"] + before, after := d.overage(tCN, tSt, fuse), d.overage(tSt, t1, fuse) + m := metrics{"before": before, "after": after} + pre, last := d.window(tSt-30, tSt), d.window(t1-60, t1) + var preW, lastW [2]float64 + for i := 0; i < 2; i++ { + i := i + preW[i] = math.Round(meanOf(pre, func(r *sample) float64 { return carW(r, i) })) + lastW[i] = math.Round(meanOf(last, func(r *sample) float64 { return carW(r, i) })) + } + m["pre_step_w"], m["last60_w"] = preW, lastW + for i, name := range lpIDs { + var seq [][2]float64 + for _, p := range connector1(profiles(d.loadFrames(i), tSt-5, t1)) { + a := round(p.Limit, 1) + if len(seq) == 0 || seq[len(seq)-1][1] != a { + seq = append(seq, [2]float64{p.T, a}) + } + } + if len(seq) > 10 { + seq = seq[:10] + } + m[name+"_profiles_after_step"] = seq + } + return m, []criterion{ + crit("fuse holds", !before["fuse_blown"].(bool) && !after["fuse_blown"].(bool)), + crit("over fuse - 1 A at most 3 s after the step", after["over_limit_longest_s"].(float64) <= 3), + } +} + +func judgeS11(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tP, tR, t1 := d.marks["charge_now"], d.marks["pause"], d.marks["restart"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + prs := connector1(profiles(d.frames, tCN-1, t1)) + var firstPos, paused *profile + for i := range prs { + p := &prs[i] + if firstPos == nil && p.T >= tR-tCN && p.Limit > 0 { + firstPos = p + } + if paused == nil && p.T >= tP-tCN && p.Limit == 0 { + paused = p + } + } + m := metrics{ + "car_w_before_pause": math.Round(meanOf(d.window(tP-20, tP), car)), + "car_started_s": roundp(d.firstT(tR, func(r *sample) bool { return car(r) >= 0.95*minW }), 1), + "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), car)), + } + var firstPosS *float64 + if paused != nil { + m["pause_profile_at_s"] = round(paused.T-(tP-tCN), 1) + } + if firstPos != nil { + v := round(firstPos.T-(tR-tCN), 1) + firstPosS = &v + m["first_positive_offer_a"] = firstPos.Limit + } + m["first_positive_offer_s"] = firstPosS + rs := counter{} + for _, r := range d.window(tR+2, t1) { + for _, x := range reasons(r) { + rs[x]++ + } + } + m["reasons_after_restart"] = rs.top(4) + run := d.overage(tCN, t1, fuse) + for k, v := range run { + m["run_"+k] = v + } + return m, []criterion{ + crit("an offer within 60 s of the restart", le(firstPosS, 60)), + crit("over fuse - 1 A at most 3 s", run["over_limit_longest_s"].(float64) <= 3), + crit("fuse holds", !run["fuse_blown"].(bool)), + } +}