From 1ac43da0f17a6c6d25b3206963f03eadb8fd714f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 20 Oct 2026 06:53:33 +0200 Subject: [PATCH] fix(history): keep unseen optional PV out of load training History leaves an optional PV source out of Home use when it has never reported, as a hybrid may have no panels. The forecast requires it, and both load models now train on history intervals, so 2 kW of unseen PV could become a 1 kW label for a 3 kW house (Codex review of #1754). History options now carry the optional flows. While one has no live reading, the snapshot is marked EstimatedLoad: Home use is unchanged and training skips it. Past civil dusk the PV is 0 W by physics, so the night still trains. Forecast revisions are unchanged. Co-Authored-By: Claude Opus 5.5 --- .changeset/training-requires-optional-pv.md | 5 ++ go/cmd/ftw/forecast_measurements.go | 4 +- go/cmd/ftw/history_dark_pv.go | 27 +++++++ go/cmd/ftw/history_optional_pv_test.go | 82 +++++++++++++++++++++ go/cmd/ftw/main.go | 1 + go/internal/telemetry/forecast.go | 10 ++- 6 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 .changeset/training-requires-optional-pv.md create mode 100644 go/cmd/ftw/history_optional_pv_test.go diff --git a/.changeset/training-requires-optional-pv.md b/.changeset/training-requires-optional-pv.md new file mode 100644 index 00000000..cfdefa96 --- /dev/null +++ b/.changeset/training-requires-optional-pv.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Keep load-model training strict on hybrid inverters whose PV reading is optional. While such a PV source has not reported, Home use still shows the house, but the interval does not train the forecast; after dark the PV counts as 0 W and the night trains as before. diff --git a/go/cmd/ftw/forecast_measurements.go b/go/cmd/ftw/forecast_measurements.go index 861548dd..96768b99 100644 --- a/go/cmd/ftw/forecast_measurements.go +++ b/go/cmd/ftw/forecast_measurements.go @@ -26,7 +26,9 @@ func historyMeasurementOptions(cfg *config.Config, catalog []drivers.CatalogEntr opts, _, optional := forecastMeasurementTopology(cfg, catalog) flows := opts.ExpectedFlows[:0:0] for _, flow := range opts.ExpectedFlows { - if !optional[flow] { + if optional[flow] { + opts.OptionalFlows = append(opts.OptionalFlows, flow) + } else { flows = append(flows, flow) } } diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go index 909b2476..40e0e82e 100644 --- a/go/cmd/ftw/history_dark_pv.go +++ b/go/cmd/ftw/history_dark_pv.go @@ -58,6 +58,33 @@ func darkPV(p *state.HistorySources, sun historySun) bool { return true } +// optionalPVUnread marks a snapshot as an estimate while an optional PV +// source has no live reading. History leaves that flow out of Home use, but +// unseen PV would make a training label under-count the house. Past civil +// dusk the PV is 0 W by physics, so the night stays measured. +func optionalPVUnread(p *state.HistorySources, tel *telemetry.Store, flows []telemetry.ForecastFlow, sun historySun) bool { + if p == nil || tel == nil { + return false + } + unread := false + for _, f := range flows { + if f.DerType != telemetry.DerPV { + continue + } + if h := tel.DriverHealth(f.Driver); tel.Get(f.Driver, telemetry.DerPV) == nil || h == nil || !h.TelemetryLive() { + unread = true + } + } + if !unread { + return false + } + if z, placed := sun.zenith(p.AtMS); placed && z > civilDuskZenithDeg { + return false + } + p.Missing |= state.EstimatedLoad + return true +} + // historyDevices names the hardware behind a history snapshot, as the // topology's devices line. The watchdog drops an offline driver from the // energy record. After sunset a silent inverter is asleep, not replaced, so it diff --git a/go/cmd/ftw/history_optional_pv_test.go b/go/cmd/ftw/history_optional_pv_test.go new file mode 100644 index 00000000..d30ac86c --- /dev/null +++ b/go/cmd/ftw/history_optional_pv_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// A hybrid without a declared PV reading: History leaves its PV out of Home +// use, the forecast requires it, and the forecast revision must not change. +func optionalPVConfig() *config.Config { + return &config.Config{Drivers: []config.Driver{ + {Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}, + {Name: "hybrid", Lua: "drivers/hybrid.lua", BatteryCapacityWh: 10000}, + }} +} + +func TestHistoryOptionsKeepOptionalPVForTraining(t *testing.T) { + cfg := optionalPVConfig() + history := historyMeasurementOptions(cfg, measurementCatalog()) + pv := telemetry.ForecastFlow{Driver: "hybrid", DerType: telemetry.DerPV} + for _, f := range history.ExpectedFlows { + if f == pv { + t.Fatal("History requires optional PV") + } + } + if len(history.OptionalFlows) != 1 || history.OptionalFlows[0] != pv { + t.Fatalf("optional flows = %+v", history.OptionalFlows) + } + raw, err := json.Marshal(forecastMeasurementOptions(cfg, measurementCatalog())) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "OptionalFlows") { + t.Fatalf("forecast options changed their stored identity: %s", raw) + } +} + +func TestOptionalPVUnreadKeepsDaytimeOutOfTraining(t *testing.T) { + stockholm, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatal(err) + } + pinned := historySun{Lat: 59.6, Lon: 16.5, Placed: true} + flows := historyMeasurementOptions(optionalPVConfig(), measurementCatalog()).OptionalFlows + for _, c := range []struct { + name string + at time.Time + sun historySun + pvLive bool + estimated bool + }{ + {"day, PV never reported", time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), pinned, false, true}, + {"twilight, PV never reported", time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), pinned, false, true}, + {"past civil dusk, PV never reported", time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), pinned, false, false}, + {"night without a pin", time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), historySun{}, false, true}, + {"day, PV reporting", time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), pinned, true, false}, + } { + t.Run(c.name, func(t *testing.T) { + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("site") + if c.pvLive { + tel.Update("hybrid", telemetry.DerPV, -2000, nil, nil) + tel.RecordDriverSuccess("hybrid") + } + p := &state.HistorySources{AtMS: c.at.UnixMilli(), GridW: 1000} + got := optionalPVUnread(p, tel, flows, c.sun) + if got != c.estimated || (p.Missing&state.EstimatedLoad != 0) != c.estimated { + t.Fatalf("estimated=%v missing=%b, want estimated=%v", got, p.Missing, c.estimated) + } + if p.Missing&state.MissingPower != 0 { + t.Fatal("optional PV changed what History shows") + } + }) + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index dedbb874..e62a6745 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -4008,6 +4008,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersis } hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options) dark := darkPV(hp.Sources, sun) + optionalPVUnread(hp.Sources, tel, options.OptionalFlows, sun) if hp.Sources != nil { // Live Load is not an energy observation. The source interval writer // owns historical Load; the chart placeholder stays explicitly unknown. diff --git a/go/internal/telemetry/forecast.go b/go/internal/telemetry/forecast.go index d5e3743e..b5a31152 100644 --- a/go/internal/telemetry/forecast.go +++ b/go/internal/telemetry/forecast.go @@ -35,9 +35,13 @@ type ForecastOptions struct { HouseholdInvalidReason string PVInvalidReason string ExpectedFlows []ForecastFlow - MaxAge time.Duration - MaxSkew time.Duration - UseWatchdogTimeout bool `json:",omitempty"` + // OptionalFlows are flows History does not require but a training label + // does, such as PV on a hybrid inverter that may have no panels. Omitted + // when empty, so forecast revisions keep their stored identity. + OptionalFlows []ForecastFlow `json:",omitempty"` + MaxAge time.Duration + MaxSkew time.Duration + UseWatchdogTimeout bool `json:",omitempty"` // Live display may use the driver's measured site-side control power // (for example Pixii AC instead of battery DC). Raw history stays separate. UseControlPower bool `json:",omitempty"`