From 7d2c4ae4fa7e69df99da685d56e1b40d81216a80 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 19 Oct 2026 19:40:32 +0200 Subject: [PATCH 1/5] fix(history): count unknown PV as 0 W after sunset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PV inverter that sleeps at night is the most common remaining reason for a day without Home use. Göran's SMA was silent for 145 s at 02:22 on 19 October with 0 W on both sides, longer than the 2-minute bridge. While the sun is below the horizon at the site's map pin, Core now records an unknown PV source as 0 W, marked EstimatedLoad, never as a measurement. An inverter the watchdog took offline keeps its device in the history topology, so the night is not cut out. The source writer carries the marker into every interval an estimated snapshot bounds. Without a pin (unset, 0,0 or the untouched default) nothing changes. By day the gap stays unknown. Grid, battery and EV are not affected. Relates to #1750. Co-Authored-By: Claude Opus 5.5 --- .changeset/unknown-pv-after-sunset.md | 5 + docs/site-convention.md | 6 + go/cmd/ftw/forecast_site.go | 2 +- go/cmd/ftw/forecast_tracking.go | 1 + go/cmd/ftw/history_dark_pv.go | 67 +++++++++ go/cmd/ftw/history_dark_pv_test.go | 166 ++++++++++++++++++++++ go/cmd/ftw/main.go | 19 ++- go/internal/state/history_sources.go | 2 + go/internal/state/history_sources_test.go | 25 ++++ 9 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 .changeset/unknown-pv-after-sunset.md create mode 100644 go/cmd/ftw/history_dark_pv.go create mode 100644 go/cmd/ftw/history_dark_pv_test.go diff --git a/.changeset/unknown-pv-after-sunset.md b/.changeset/unknown-pv-after-sunset.md new file mode 100644 index 00000000..a93efb61 --- /dev/null +++ b/.changeset/unknown-pv-after-sunset.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Keep the day's Home use when a PV inverter goes silent at night. While the sun is below the horizon at the site's map pin, an unknown PV source counts as 0 W, marked as an estimate. Without a pin, or by day, the gap stays unknown as before. diff --git a/docs/site-convention.md b/docs/site-convention.md index 72872e5f..9b65d6e6 100644 --- a/docs/site-convention.md +++ b/docs/site-convention.md @@ -67,6 +67,12 @@ unrecorded time counts as nothing. Training is strict: it uses only minutes without the mark. This is an estimate from received measurements, not a correction of device clocks. +While the sun is below the horizon at the site's map pin, an unknown PV source +counts as 0 W instead of its last value, and an inverter the watchdog took +offline keeps its place in the topology. Past civil dusk (sun 6° down) the +0 W is physics and carries no mark, so the night also trains the load models. +In twilight it is marked. Without a pin nothing is inferred. + In self-consumption mode, the controller drives the site meter toward the grid target (normally 0 W): it charges from live surplus and may discharge to cover local load. It must not intentionally export via the battery; export diff --git a/go/cmd/ftw/forecast_site.go b/go/cmd/ftw/forecast_site.go index 008e3706..06719754 100644 --- a/go/cmd/ftw/forecast_site.go +++ b/go/cmd/ftw/forecast_site.go @@ -88,7 +88,7 @@ func (s *forecastSiteConfig) Snapshot() forecastSite { } func (s *forecastSiteConfig) Configure(cfg *config.Config, catalog []drivers.CatalogEntry) bool { - v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: localzone.Name(), Options: forecastMeasurementOptions(cfg, catalog), HistoryOptions: historyMeasurementOptions(cfg, catalog)} + v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: localzone.Name(), Options: forecastMeasurementOptions(cfg, catalog), HistoryOptions: historyMeasurementOptions(cfg, catalog), HistorySun: historySunOf(siteLocation(cfg))} if cfg.Weather != nil && cfg.Weather.Provider != "" && cfg.Weather.Provider != "none" { v.Latitude, v.Longitude = cfg.Weather.Latitude, cfg.Weather.Longitude v.HasLocation = !math.IsNaN(v.Latitude) && !math.IsNaN(v.Longitude) && math.Abs(v.Latitude) <= 90 && math.Abs(v.Longitude) <= 180 diff --git a/go/cmd/ftw/forecast_tracking.go b/go/cmd/ftw/forecast_tracking.go index 2e88aaf5..984c5fd5 100644 --- a/go/cmd/ftw/forecast_tracking.go +++ b/go/cmd/ftw/forecast_tracking.go @@ -36,6 +36,7 @@ type forecastSite struct { Timezone string Options telemetry.ForecastOptions HistoryOptions telemetry.ForecastOptions // chart and site energy; outside every revision + HistorySun historySun // where unknown PV reads as 0 W after sunset; outside every revision PVLearningStartedMS int64 LoadLearningStartedMS int64 } diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go new file mode 100644 index 00000000..f8666f6e --- /dev/null +++ b/go/cmd/ftw/history_dark_pv.go @@ -0,0 +1,67 @@ +package main + +import ( + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/sunpos" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// sunDownZenithDeg is standard sunset: the sun's upper limb below the +// horizon, refraction included. No panel then gets direct sunlight. +const sunDownZenithDeg = 90.833 + +// historySun places the site for history. The zero value has no position, +// and then nothing is inferred. +type historySun struct { + Lat, Lon float64 + Placed bool +} + +func historySunOf(lat, lon float64, ok bool) historySun { + return historySun{Lat: lat, Lon: lon, Placed: ok} +} + +func (s historySun) down(atMS int64) bool { + return s.Placed && sunpos.At(time.UnixMilli(atMS), s.Lat, s.Lon).ZenithDeg > sunDownZenithDeg +} + +// darkPV records an unknown PV source as 0 W while the sun is down at the +// site. That is physics, not interpolation, and the snapshot carries +// EstimatedLoad so it never reads as measured. Grid, battery and EV keep +// their own evidence; Load stays unknown for any cause but PV. +func darkPV(p *state.HistorySources, opts telemetry.ForecastOptions, sun historySun) bool { + if p == nil || p.Missing&state.MissingPV == 0 || !sun.down(p.AtMS) { + return false + } + p.PVW = 0 + p.Missing = p.Missing&^state.MissingPV | state.EstimatedLoad + // historySources sets MissingLoad for the first failing source or for + // site configuration. With every other source known and the + // configuration valid, PV was that source. + if opts.HouseholdInvalidReason == "" && p.Missing&(state.MissingGrid|state.MissingBattery|state.MissingEV|state.MissingV2X) == 0 { + p.Missing &^= state.MissingLoad + } + return true +} + +// sleepingPVDevices names PV devices the watchdog took offline. After sunset +// a silent inverter is asleep, not replaced, so it stays in the topology and +// its night is not cut out of the record. +func sleepingPVDevices(st *state.Store, tel *telemetry.Store, identity energyIdentityLookup) []string { + if st == nil || tel == nil || identity == nil { + return nil + } + known := st.CachedDevices() + var out []string + for _, r := range tel.ReadingsByType(telemetry.DerPV) { + if h := tel.DriverHealth(r.Driver); h != nil && h.Status != telemetry.StatusOffline { + continue + } + if id := confirmedEnergyDeviceID(identity(r.Driver), known); id != "" { + out = append(out, string(state.AssetPV)+":"+id) + } + } + return out +} diff --git a/go/cmd/ftw/history_dark_pv_test.go b/go/cmd/ftw/history_dark_pv_test.go new file mode 100644 index 00000000..0a626de7 --- /dev/null +++ b/go/cmd/ftw/history_dark_pv_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "math" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// Göran's box, 19 October 2026: SMA was silent for 145 s at 02:22 local time, +// 0 W on both sides, and the day's Home use became unknown. +func TestHistorySunIsDownAtNightOnly(t *testing.T) { + stockholm := time.FixedZone("CEST", 2*3600) + sun := historySun{Lat: 59.6, Lon: 16.5, Placed: true} + for _, c := range []struct { + at time.Time + sun historySun + down bool + }{ + {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), sun, true}, + {time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun, false}, + // Sunset in Västerås on 19 October is at 17:38. + {time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun, false}, + {time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun, true}, + {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), historySun{}, false}, + } { + if got := c.sun.down(c.at.UnixMilli()); got != c.down { + t.Errorf("%v placed=%v: down=%v, want %v", c.at, c.sun.Placed, got, c.down) + } + } + for _, c := range []struct { + lat, lon float64 + ok bool + }{{0, 0, false}, {unsetPinLat, unsetPinLon, false}, {59.6, 16.5, true}} { + if got := historySunOf(siteLocation(&config.Config{Weather: &config.Weather{Latitude: c.lat, Longitude: c.lon}})).Placed; got != c.ok { + t.Errorf("pin %v,%v placed=%v, want %v", c.lat, c.lon, got, c.ok) + } + } +} + +// Two hours of a 1000 W house sampled every 5 s; PV is unknown for 5 minutes +// one hour in. On the equator the window is local midnight or local noon in +// every season. +func TestUnknownPVAfterSunsetKeepsHomeUse(t *testing.T) { + const tick, hour = int64(5000), int64(3600000) + midnight := historySun{Lat: 1, Lon: -15, Placed: true} + noon := historySun{Lat: 1, Lon: 165, Placed: true} + for _, c := range []struct { + name string + sun historySun + pvW float64 + opts telemetry.ForecastOptions + other state.HistoryMissing + known bool + }{ + {name: "night", sun: midnight, known: true}, + {name: "day", sun: noon, pvW: -2000}, + {name: "night without a position"}, + {name: "night with an unmeasured PV array", sun: midnight, opts: telemetry.ForecastOptions{HouseholdInvalidReason: "configured_pv_without_measurement_source"}}, + {name: "night with the battery also unknown", sun: midnight, other: state.MissingBattery}, + } { + t.Run(c.name, func(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + if err := st.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 24*hour + for at := base; at <= base+2*hour; at += tick { + off := at - base - hour + r := telemetry.ForecastReading{ + At: time.UnixMilli(at), + Known: map[telemetry.DerType]bool{telemetry.DerMeter: true, telemetry.DerPV: true, telemetry.DerBattery: true, telemetry.DerEV: true, telemetry.DerV2X: true}, + Sources: map[string]telemetry.ForecastSource{ + "grid:meter": {Flow: telemetry.ForecastFlow{Driver: "grid", DerType: telemetry.DerMeter}}, + "sma:pv": {Flow: telemetry.ForecastFlow{Driver: "sma", DerType: telemetry.DerPV}}, + }, + GridW: 1000 + c.pvW, PVW: c.pvW, + } + if off > 0 && off <= 300000 { + r.Known[telemetry.DerPV], r.Reason = false, "offline:sma:pv" + if c.other == state.MissingBattery { + r.Known[telemetry.DerBattery] = false + } + } + v := historySources(r) + darkPV(v, c.opts, c.sun) + if err := st.RecordTickWithEnergy(state.HistoryPoint{TsMs: at, Sources: v}, nil, nil); err != nil { + t.Fatal(err) + } + } + if err := st.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } + d, err := st.DailyEnergy(base, base+2*hour) + if err != nil { + t.Fatal(err) + } + if got := d.Missing&state.MissingLoad == 0; got != c.known { + t.Fatalf("Home use known=%v, want %v (quality %s)", got, c.known, d.Missing.Quality()) + } + if c.pvW == 0 && math.Abs(d.ImportWh-2000) > 1e-6 || c.pvW != 0 && math.Abs(d.ExportWh-2000) > 1e-6 { + t.Fatalf("grid import %v, export %v Wh: the meter's own record changed", d.ImportWh, d.ExportWh) + } + if !c.known { + return + } + if q := d.Missing.Quality(); q != "estimated" { + t.Fatalf("quality %s, want estimated", q) + } + if d.Missing&state.MissingPV != 0 || d.PVWh != 0 { + t.Fatalf("PV %v Wh, missing %v; want a known 0 Wh after sunset", d.PVWh, d.Missing&state.MissingPV != 0) + } + if math.Abs(d.LoadWh-2000) > 1e-6 { + t.Fatalf("Home use %v Wh, want 2000", d.LoadWh) + } + }) + } +} + +// The watchdog drops an offline driver's device from the energy record. After +// sunset the sleeping inverter keeps its place in the history topology, so the +// writer sees one topology and does not cut the night out. +func TestSleepingInverterKeepsItsDeviceAfterSunset(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + if _, err := st.RegisterDevice(state.Device{DriverName: "sma", Make: "SMA", Serial: "PV-7"}); err != nil { + t.Fatal(err) + } + identity := testEnergyIdentity(st) + tel := telemetry.NewStore() + tel.Update("sma", telemetry.DerPV, 0, nil, nil) + tel.RecordDriverSuccess("sma") + pvDevices := func(observations []state.EnergyObservation) []string { + var out []string + for _, o := range observations { + if o.AssetKind == state.AssetPV && o.DeviceID != "" { + out = append(out, string(o.AssetKind)+":"+o.DeviceID) + } + } + return out + } + awake := pvDevices(buildEnergyObservations(st, tel, tickPersistControl{}, state.HistoryPoint{}, identity)) + if len(awake) != 1 || len(sleepingPVDevices(st, tel, identity)) != 0 { + t.Fatalf("awake inverter devices %v, sleeping %v", awake, sleepingPVDevices(st, tel, identity)) + } + tel.DriverHealthMut("sma").SetOffline() + if got := pvDevices(buildEnergyObservations(st, tel, tickPersistControl{}, state.HistoryPoint{}, identity)); len(got) != 0 { + t.Fatalf("offline inverter still observed: %v", got) + } + if got := sleepingPVDevices(st, tel, identity); !slices.Equal(got, awake) { + t.Fatalf("sleeping inverter devices %v, want %v", got, awake) + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index d50b719a..40a69e58 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2951,7 +2951,7 @@ func main() { // charger standdowns, self-tuning or normal physical dispatch. lpMgr.RollSchedules(tickNow.UTC()) carsRT.Tick() - if _, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions); err != nil { + if _, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot()); err != nil { slog.Warn("setup measurement persistence failed", "err", err) } continue @@ -3051,7 +3051,7 @@ func main() { // ctrl, so the stored tick has to show the hold already // released rather than one the blocked tick never executed. clearBatteryManualHoldForDispatchBlock(ctrl, ctrlMu) - sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions) + sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot()) if err != nil { slog.Warn("tick persistence failed", "samples", sampleCount, "err", err) } @@ -3308,7 +3308,7 @@ func main() { // ---- Persist the tick: history snapshot + flushed metrics ---- // One transaction for both — separate commits doubled the WAL // commit rate for no isolation benefit (SD-card wear). - sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions) + sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot()) if err != nil { slog.Warn("tick persistence failed", "samples", sampleCount, "err", err) } @@ -4000,8 +4000,14 @@ func snapshotTickPersistControl(ctrl *control.State, ctrlMu *sync.Mutex) tickPer } } -func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersistControl, nowMs int64, historyMaxAge time.Duration, identity energyIdentityLookup, options ...telemetry.ForecastOptions) (int, error) { - hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options...) +func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersistControl, nowMs int64, historyMaxAge time.Duration, identity energyIdentityLookup, site ...forecastSite) (int, error) { + var options telemetry.ForecastOptions + var sun historySun + if len(site) > 0 { + options, sun = site[0].HistoryOptions, site[0].HistorySun + } + hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options) + dark := darkPV(hp.Sources, options, sun) if hp.Sources != nil { // Live Load is not an energy observation. The source interval writer // owns historical Load; the chart placeholder stays explicitly unknown. @@ -4032,6 +4038,9 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersis devices = append(devices, string(o.AssetKind)+":"+o.DeviceID) } } + if dark { + devices = append(devices, sleepingPVDevices(st, tel, identity)...) + } sort.Strings(devices) hp.Sources.Topology += "\ndevices:" + strings.Join(devices, "\n") } diff --git a/go/internal/state/history_sources.go b/go/internal/state/history_sources.go index 1599a167..1d913648 100644 --- a/go/internal/state/history_sources.go +++ b/go/internal/state/history_sources.go @@ -12,6 +12,8 @@ import ( // HistorySources carries independently fresh site-side powers, never the live // Load result. Topology identifies the electrical flows included in the sum. +// EstimatedLoad in Missing marks a snapshot that holds an inferred value, such +// as PV read as 0 W in twilight; every interval it bounds keeps the marker. type HistorySources struct { AtMS int64 Topology string diff --git a/go/internal/state/history_sources_test.go b/go/internal/state/history_sources_test.go index 99120ff3..362d4604 100644 --- a/go/internal/state/history_sources_test.go +++ b/go/internal/state/history_sources_test.go @@ -144,6 +144,31 @@ func TestSourceLoadKeepsUnknownDistinctFromZero(t *testing.T) { } } +// A snapshot holding an inferred value, such as PV read as 0 W after sunset, +// marks every interval it bounds and no other, so storage keeps it apart from +// measurement. +func TestSourceLoadMarksIntervalsOfAnEstimatedSnapshot(t *testing.T) { + s := sourceStore(t) + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + for at := base; at <= base+180000; at += 5000 { + v := HistorySources{GridW: 1000} + if at > base+60000 && at <= base+120000 { + v.Missing = EstimatedLoad + } + sourceTick(t, s, at, v) + } + b, err := s.sourceLoadBuckets(context.Background(), base, base+180000, 60000) + if err != nil || len(b) != 3 { + t.Fatal(err, len(b)) + } + for i, estimated := range []bool{false, true, true} { + if b[i].missing&MissingPower != 0 || (b[i].missing&EstimatedLoad != 0) != estimated { + t.Fatalf("minute %d missing %b, want estimated=%v", i, b[i].missing, estimated) + } + near(t, "minute Wh", b[i].wh, 1000.0/60) + } +} + func TestSourceLoadUsesSnapshotTimeAndMarksRestartBridge(t *testing.T) { path := filepath.Join(t.TempDir(), "state.db") s, err := Open(path) From 741a6c972a40e9b8eabe16ae9462608188ad8512 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 19 Oct 2026 20:35:44 +0200 Subject: [PATCH 2/5] fix(history): count PV past civil dusk as measured 0 W MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Load models train only on fully measured intervals (#1754). Marking every dark PV snapshot as an estimate would drop each night with a sleeping inverter from training. Past civil dusk (zenith over 96°) no panel produces, so an unknown PV source now reads as a known 0 W without EstimatedLoad: the night counts in Home use and in training. Between sunset and civil dusk diffuse light remains, and the 0 W stays a marked estimate. No new bit records the physical zero. Co-Authored-By: Claude Opus 5.5 --- .changeset/unknown-pv-after-sunset.md | 2 +- go/cmd/ftw/history_dark_pv.go | 37 ++++++++++++++++++------- go/cmd/ftw/history_dark_pv_test.go | 40 +++++++++++++++++++-------- 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/.changeset/unknown-pv-after-sunset.md b/.changeset/unknown-pv-after-sunset.md index a93efb61..58d603ed 100644 --- a/.changeset/unknown-pv-after-sunset.md +++ b/.changeset/unknown-pv-after-sunset.md @@ -2,4 +2,4 @@ "ftw": patch --- -Keep the day's Home use when a PV inverter goes silent at night. While the sun is below the horizon at the site's map pin, an unknown PV source counts as 0 W, marked as an estimate. Without a pin, or by day, the gap stays unknown as before. +Keep the day's Home use when a PV inverter goes silent at night. While the sun is below the horizon at the site's map pin, an unknown PV source counts as 0 W: past civil dusk as a measurement, so the night also trains the load models, and in twilight as a marked estimate. Without a pin, or by day, the gap stays unknown as before. diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go index f8666f6e..30d120ed 100644 --- a/go/cmd/ftw/history_dark_pv.go +++ b/go/cmd/ftw/history_dark_pv.go @@ -8,9 +8,13 @@ import ( "github.com/srcfl/ftw/go/internal/telemetry" ) -// sunDownZenithDeg is standard sunset: the sun's upper limb below the -// horizon, refraction included. No panel then gets direct sunlight. -const sunDownZenithDeg = 90.833 +// Sun positions as zenith angles. After sunset (upper limb below the horizon, +// refraction included) no panel gets direct sunlight, but diffuse light +// remains until civil dusk, when the sun is 6° down. +const ( + sunsetZenithDeg = 90.833 + civilDuskZenithDeg = 96 +) // historySun places the site for history. The zero value has no position, // and then nothing is inferred. @@ -23,20 +27,33 @@ func historySunOf(lat, lon float64, ok bool) historySun { return historySun{Lat: lat, Lon: lon, Placed: ok} } -func (s historySun) down(atMS int64) bool { - return s.Placed && sunpos.At(time.UnixMilli(atMS), s.Lat, s.Lon).ZenithDeg > sunDownZenithDeg +func (s historySun) zenith(atMS int64) (float64, bool) { + if !s.Placed { + return 0, false + } + return sunpos.At(time.UnixMilli(atMS), s.Lat, s.Lon).ZenithDeg, true } // darkPV records an unknown PV source as 0 W while the sun is down at the -// site. That is physics, not interpolation, and the snapshot carries -// EstimatedLoad so it never reads as measured. Grid, battery and EV keep -// their own evidence; Load stays unknown for any cause but PV. +// site: physics, not interpolation. Past civil dusk the 0 W is as good as a +// reading and carries no marker, so a night with a sleeping inverter also +// trains the load models; no separate bit records it. In twilight the +// snapshot carries EstimatedLoad: Home use counts it, training does not. +// Grid, battery and EV keep their own evidence; Load stays unknown for any +// cause but PV. func darkPV(p *state.HistorySources, opts telemetry.ForecastOptions, sun historySun) bool { - if p == nil || p.Missing&state.MissingPV == 0 || !sun.down(p.AtMS) { + if p == nil || p.Missing&state.MissingPV == 0 { + return false + } + z, placed := sun.zenith(p.AtMS) + if !placed || z <= sunsetZenithDeg { return false } p.PVW = 0 - p.Missing = p.Missing&^state.MissingPV | state.EstimatedLoad + p.Missing &^= state.MissingPV + if z <= civilDuskZenithDeg { + p.Missing |= state.EstimatedLoad + } // historySources sets MissingLoad for the first failing source or for // site configuration. With every other source known and the // configuration valid, PV was that source. diff --git a/go/cmd/ftw/history_dark_pv_test.go b/go/cmd/ftw/history_dark_pv_test.go index 0a626de7..aaec9ec5 100644 --- a/go/cmd/ftw/history_dark_pv_test.go +++ b/go/cmd/ftw/history_dark_pv_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "math" "path/filepath" "slices" @@ -14,24 +15,37 @@ import ( ) // Göran's box, 19 October 2026: SMA was silent for 145 s at 02:22 local time, -// 0 W on both sides, and the day's Home use became unknown. -func TestHistorySunIsDownAtNightOnly(t *testing.T) { +// 0 W on both sides, and the day's Home use became unknown. In Västerås that +// day the sun sets at 17:38 and civil dusk is at about 18:20. +func TestDarkPVByTimeOfDay(t *testing.T) { stockholm := time.FixedZone("CEST", 2*3600) sun := historySun{Lat: 59.6, Lon: 16.5, Placed: true} + const unchanged, estimate, known = "unchanged", "estimate", "known" for _, c := range []struct { at time.Time sun historySun - down bool + want string }{ - {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), sun, true}, - {time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun, false}, - // Sunset in Västerås on 19 October is at 17:38. - {time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun, false}, - {time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun, true}, - {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), historySun{}, false}, + {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), sun, known}, + {time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun, unchanged}, + {time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun, unchanged}, + {time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun, estimate}, + {time.Date(2026, 10, 19, 18, 45, 0, 0, stockholm), sun, known}, + {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), historySun{}, unchanged}, } { - if got := c.sun.down(c.at.UnixMilli()); got != c.down { - t.Errorf("%v placed=%v: down=%v, want %v", c.at, c.sun.Placed, got, c.down) + p := &state.HistorySources{AtMS: c.at.UnixMilli(), PVW: 0, Missing: state.MissingPV | state.MissingLoad} + applied := darkPV(p, telemetry.ForecastOptions{}, c.sun) + got := unchanged + switch { + case applied && p.Missing == state.EstimatedLoad: + got = estimate + case applied && p.Missing == 0: + got = known + case applied || p.Missing != state.MissingPV|state.MissingLoad: + got = fmt.Sprintf("missing %b", p.Missing) + } + if got != c.want { + t.Errorf("%s placed=%v: %s, want %s", c.at.Format("15:04"), c.sun.Placed, got, c.want) } } for _, c := range []struct { @@ -93,7 +107,9 @@ func TestUnknownPVAfterSunsetKeepsHomeUse(t *testing.T) { } } v := historySources(r) - darkPV(v, c.opts, c.sun) + if darkPV(v, c.opts, c.sun) && v.Missing&state.EstimatedLoad != 0 { + t.Fatal("0 W past civil dusk marked as an estimate") + } if err := st.RecordTickWithEnergy(state.HistoryPoint{TsMs: at, Sources: v}, nil, nil); err != nil { t.Fatal(err) } From bbf02705cb51e9689efa0b4384d843d1230e760d Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 19 Oct 2026 20:48:20 +0200 Subject: [PATCH 3/5] test(history): a full night with a sleeping inverter stays measured Rebased on the rewritten #1753, whose writer marks an interval as an estimate at an unknown source, a stop, a long gap or a topology change. A night past civil dusk therefore trains only if every snapshot is fully known and the sleeping inverter keeps its device in the topology. The devices line moves into historyDevices so the test drives the same code as persistTelemetryTick. The test runs eight hours of a sleeping inverter through it, with and without a map pin. TestDarkPVByTimeOfDay now also checks that PV reads 0 W and that Load stays unknown for any cause but PV; it replaces the two-hour series test, whose day-level outcome the new writer decides. Co-Authored-By: Claude Opus 5.5 --- go/cmd/ftw/history_dark_pv.go | 39 +++-- go/cmd/ftw/history_dark_pv_test.go | 173 +++++++++------------- go/cmd/ftw/main.go | 12 +- go/internal/state/history_sources_test.go | 2 +- 4 files changed, 92 insertions(+), 134 deletions(-) diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go index 30d120ed..cce91473 100644 --- a/go/cmd/ftw/history_dark_pv.go +++ b/go/cmd/ftw/history_dark_pv.go @@ -1,6 +1,8 @@ package main import ( + "sort" + "strings" "time" "github.com/srcfl/ftw/go/internal/state" @@ -63,22 +65,29 @@ func darkPV(p *state.HistorySources, opts telemetry.ForecastOptions, sun history return true } -// sleepingPVDevices names PV devices the watchdog took offline. After sunset -// a silent inverter is asleep, not replaced, so it stays in the topology and -// its night is not cut out of the record. -func sleepingPVDevices(st *state.Store, tel *telemetry.Store, identity energyIdentityLookup) []string { - if st == nil || tel == nil || identity == nil { - return nil - } - known := st.CachedDevices() - var out []string - for _, r := range tel.ReadingsByType(telemetry.DerPV) { - if h := tel.DriverHealth(r.Driver); h != nil && h.Status != telemetry.StatusOffline { - continue +// historyDevices names the hardware behind a history snapshot, as the +// topology's devices line. The watchdog drops an offline driver from the +// energy record. After sunset a silent inverter is asleep, not replaced, so it +// keeps its place: the night stays one topology, and its minutes stay +// measured. +func historyDevices(st *state.Store, tel *telemetry.Store, identity energyIdentityLookup, observations []state.EnergyObservation, dark bool) string { + var devices []string + for _, o := range observations { + if o.DeviceID != "" { + devices = append(devices, string(o.AssetKind)+":"+o.DeviceID) } - if id := confirmedEnergyDeviceID(identity(r.Driver), known); id != "" { - out = append(out, string(state.AssetPV)+":"+id) + } + if dark && st != nil && tel != nil && identity != nil { + known := st.CachedDevices() + for _, r := range tel.ReadingsByType(telemetry.DerPV) { + if h := tel.DriverHealth(r.Driver); h != nil && h.Status != telemetry.StatusOffline { + continue + } + if id := confirmedEnergyDeviceID(identity(r.Driver), known); id != "" { + devices = append(devices, string(state.AssetPV)+":"+id) + } } } - return out + sort.Strings(devices) + return "\ndevices:" + strings.Join(devices, "\n") } diff --git a/go/cmd/ftw/history_dark_pv_test.go b/go/cmd/ftw/history_dark_pv_test.go index aaec9ec5..b111ac41 100644 --- a/go/cmd/ftw/history_dark_pv_test.go +++ b/go/cmd/ftw/history_dark_pv_test.go @@ -2,10 +2,8 @@ package main import ( "context" - "fmt" "math" "path/filepath" - "slices" "testing" "time" @@ -20,32 +18,31 @@ import ( func TestDarkPVByTimeOfDay(t *testing.T) { stockholm := time.FixedZone("CEST", 2*3600) sun := historySun{Lat: 59.6, Lon: 16.5, Placed: true} - const unchanged, estimate, known = "unchanged", "estimate", "known" + night := time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm) + const unknown = state.MissingPV | state.MissingLoad for _, c := range []struct { - at time.Time - sun historySun - want string + name string + at time.Time + sun historySun + opts telemetry.ForecastOptions + missing state.HistoryMissing + applied bool + want state.HistoryMissing }{ - {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), sun, known}, - {time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun, unchanged}, - {time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun, unchanged}, - {time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun, estimate}, - {time.Date(2026, 10, 19, 18, 45, 0, 0, stockholm), sun, known}, - {time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm), historySun{}, unchanged}, + {name: "night", at: night, sun: sun, missing: unknown, applied: true}, + {name: "after dusk", at: time.Date(2026, 10, 19, 18, 45, 0, 0, stockholm), sun: sun, missing: unknown, applied: true}, + {name: "twilight", at: time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun: sun, missing: unknown, applied: true, want: state.EstimatedLoad}, + {name: "noon", at: time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun: sun, missing: unknown, want: unknown}, + {name: "before sunset", at: time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun: sun, missing: unknown, want: unknown}, + {name: "no pin", at: night, missing: unknown, want: unknown}, + // Load stays unknown for any cause but PV. + {name: "battery also unknown", at: night, sun: sun, missing: unknown | state.MissingBattery, applied: true, want: state.MissingBattery | state.MissingLoad}, + {name: "PV array without a measurement source", at: night, sun: sun, opts: telemetry.ForecastOptions{HouseholdInvalidReason: "configured_pv_without_measurement_source"}, missing: unknown, applied: true, want: state.MissingLoad}, } { - p := &state.HistorySources{AtMS: c.at.UnixMilli(), PVW: 0, Missing: state.MissingPV | state.MissingLoad} - applied := darkPV(p, telemetry.ForecastOptions{}, c.sun) - got := unchanged - switch { - case applied && p.Missing == state.EstimatedLoad: - got = estimate - case applied && p.Missing == 0: - got = known - case applied || p.Missing != state.MissingPV|state.MissingLoad: - got = fmt.Sprintf("missing %b", p.Missing) - } - if got != c.want { - t.Errorf("%s placed=%v: %s, want %s", c.at.Format("15:04"), c.sun.Placed, got, c.want) + p := &state.HistorySources{AtMS: c.at.UnixMilli(), PVW: -300, Missing: c.missing} + applied := darkPV(p, c.opts, c.sun) + if applied != c.applied || p.Missing != c.want || applied && p.PVW != 0 { + t.Errorf("%s: applied=%v missing=%b PV=%v W, want applied=%v missing=%b", c.name, applied, p.Missing, p.PVW, c.applied, c.want) } } for _, c := range []struct { @@ -58,26 +55,21 @@ func TestDarkPVByTimeOfDay(t *testing.T) { } } -// Two hours of a 1000 W house sampled every 5 s; PV is unknown for 5 minutes -// one hour in. On the equator the window is local midnight or local noon in -// every season. -func TestUnknownPVAfterSunsetKeepsHomeUse(t *testing.T) { +// A full night on the equator: 20:00–04:00 UTC at 15° W is past civil dusk in +// every season. The inverter reports 0 W for ten minutes, then sleeps, and a +// minute later the watchdog takes it offline. With a pin every snapshot is +// fully known and of one topology, so the writer records measured minutes that +// train the load models. Without one, PV stays unknown for the writer to +// hold, and the topology changes when the device drops out. +func TestSleepingInverterNightIsMeasured(t *testing.T) { const tick, hour = int64(5000), int64(3600000) - midnight := historySun{Lat: 1, Lon: -15, Placed: true} - noon := historySun{Lat: 1, Lon: 165, Placed: true} for _, c := range []struct { - name string - sun historySun - pvW float64 - opts telemetry.ForecastOptions - other state.HistoryMissing - known bool + name string + sun historySun + measured bool }{ - {name: "night", sun: midnight, known: true}, - {name: "day", sun: noon, pvW: -2000}, - {name: "night without a position"}, - {name: "night with an unmeasured PV array", sun: midnight, opts: telemetry.ForecastOptions{HouseholdInvalidReason: "configured_pv_without_measurement_source"}}, - {name: "night with the battery also unknown", sun: midnight, other: state.MissingBattery}, + {"with a pin", historySun{Lat: 1, Lon: -15, Placed: true}, true}, + {"without a pin", historySun{}, false}, } { t.Run(c.name, func(t *testing.T) { st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) @@ -88,9 +80,23 @@ func TestUnknownPVAfterSunsetKeepsHomeUse(t *testing.T) { if err := st.EnableHistoryAggregation(); err != nil { t.Fatal(err) } - base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 24*hour - for at := base; at <= base+2*hour; at += tick { - off := at - base - hour + if _, err := st.RegisterDevice(state.Device{DriverName: "sma", Make: "SMA", Serial: "PV-7"}); err != nil { + t.Fatal(err) + } + identity := testEnergyIdentity(st) + ctrl := tickPersistControl{SiteMeterDriver: "grid"} + tel := telemetry.NewStore() + tel.Update("grid", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("grid") + tel.Update("sma", telemetry.DerPV, 0, nil, nil) + tel.RecordDriverSuccess("sma") + base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 28*hour + topologies := map[string]bool{} + for at := base; at <= base+8*hour; at += tick { + off := at - base + if off == 11*60000 { + tel.DriverHealthMut("sma").SetOffline() + } r := telemetry.ForecastReading{ At: time.UnixMilli(at), Known: map[telemetry.DerType]bool{telemetry.DerMeter: true, telemetry.DerPV: true, telemetry.DerBattery: true, telemetry.DerEV: true, telemetry.DerV2X: true}, @@ -98,17 +104,17 @@ func TestUnknownPVAfterSunsetKeepsHomeUse(t *testing.T) { "grid:meter": {Flow: telemetry.ForecastFlow{Driver: "grid", DerType: telemetry.DerMeter}}, "sma:pv": {Flow: telemetry.ForecastFlow{Driver: "sma", DerType: telemetry.DerPV}}, }, - GridW: 1000 + c.pvW, PVW: c.pvW, + GridW: 1000, } - if off > 0 && off <= 300000 { + if off > 10*60000 { r.Known[telemetry.DerPV], r.Reason = false, "offline:sma:pv" - if c.other == state.MissingBattery { - r.Known[telemetry.DerBattery] = false - } } v := historySources(r) - if darkPV(v, c.opts, c.sun) && v.Missing&state.EstimatedLoad != 0 { - t.Fatal("0 W past civil dusk marked as an estimate") + dark := darkPV(v, telemetry.ForecastOptions{}, c.sun) + v.Topology += historyDevices(st, tel, identity, buildEnergyObservations(st, tel, ctrl, state.HistoryPoint{}, identity), dark) + topologies[v.Topology] = true + if known := v.Missing == 0; known != (c.measured || off <= 10*60000) { + t.Fatalf("%d min: snapshot missing %b", off/60000, v.Missing) } if err := st.RecordTickWithEnergy(state.HistoryPoint{TsMs: at, Sources: v}, nil, nil); err != nil { t.Fatal(err) @@ -117,66 +123,19 @@ func TestUnknownPVAfterSunsetKeepsHomeUse(t *testing.T) { if err := st.FlushHistory(context.Background()); err != nil { t.Fatal(err) } - d, err := st.DailyEnergy(base, base+2*hour) - if err != nil { - t.Fatal(err) + if one := len(topologies) == 1; one != c.measured { + t.Fatalf("%d topologies over the night", len(topologies)) } - if got := d.Missing&state.MissingLoad == 0; got != c.known { - t.Fatalf("Home use known=%v, want %v (quality %s)", got, c.known, d.Missing.Quality()) - } - if c.pvW == 0 && math.Abs(d.ImportWh-2000) > 1e-6 || c.pvW != 0 && math.Abs(d.ExportWh-2000) > 1e-6 { - t.Fatalf("grid import %v, export %v Wh: the meter's own record changed", d.ImportWh, d.ExportWh) - } - if !c.known { + if !c.measured { return } - if q := d.Missing.Quality(); q != "estimated" { - t.Fatalf("quality %s, want estimated", q) - } - if d.Missing&state.MissingPV != 0 || d.PVWh != 0 { - t.Fatalf("PV %v Wh, missing %v; want a known 0 Wh after sunset", d.PVWh, d.Missing&state.MissingPV != 0) + d, err := st.DailyEnergy(base, base+8*hour) + if err != nil { + t.Fatal(err) } - if math.Abs(d.LoadWh-2000) > 1e-6 { - t.Fatalf("Home use %v Wh, want 2000", d.LoadWh) + if d.Missing&state.MissingLoad != 0 || math.Abs(d.LoadWh-8000) > 1e-6 { + t.Fatalf("Home use %v Wh, missing %b; want 8000 Wh", d.LoadWh, d.Missing) } }) } } - -// The watchdog drops an offline driver's device from the energy record. After -// sunset the sleeping inverter keeps its place in the history topology, so the -// writer sees one topology and does not cut the night out. -func TestSleepingInverterKeepsItsDeviceAfterSunset(t *testing.T) { - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - if _, err := st.RegisterDevice(state.Device{DriverName: "sma", Make: "SMA", Serial: "PV-7"}); err != nil { - t.Fatal(err) - } - identity := testEnergyIdentity(st) - tel := telemetry.NewStore() - tel.Update("sma", telemetry.DerPV, 0, nil, nil) - tel.RecordDriverSuccess("sma") - pvDevices := func(observations []state.EnergyObservation) []string { - var out []string - for _, o := range observations { - if o.AssetKind == state.AssetPV && o.DeviceID != "" { - out = append(out, string(o.AssetKind)+":"+o.DeviceID) - } - } - return out - } - awake := pvDevices(buildEnergyObservations(st, tel, tickPersistControl{}, state.HistoryPoint{}, identity)) - if len(awake) != 1 || len(sleepingPVDevices(st, tel, identity)) != 0 { - t.Fatalf("awake inverter devices %v, sleeping %v", awake, sleepingPVDevices(st, tel, identity)) - } - tel.DriverHealthMut("sma").SetOffline() - if got := pvDevices(buildEnergyObservations(st, tel, tickPersistControl{}, state.HistoryPoint{}, identity)); len(got) != 0 { - t.Fatalf("offline inverter still observed: %v", got) - } - if got := sleepingPVDevices(st, tel, identity); !slices.Equal(got, awake) { - t.Fatalf("sleeping inverter devices %v, want %v", got, awake) - } -} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 40a69e58..2a45ff7d 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -4032,17 +4032,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersis } energyObservations := buildEnergyObservations(st, tel, ctrl, hp, identity) if hp.Sources != nil { - var devices []string - for _, o := range energyObservations { - if o.DeviceID != "" { - devices = append(devices, string(o.AssetKind)+":"+o.DeviceID) - } - } - if dark { - devices = append(devices, sleepingPVDevices(st, tel, identity)...) - } - sort.Strings(devices) - hp.Sources.Topology += "\ndevices:" + strings.Join(devices, "\n") + hp.Sources.Topology += historyDevices(st, tel, identity, energyObservations, dark) } filtered := energyObservations[:0] for _, observation := range energyObservations { diff --git a/go/internal/state/history_sources_test.go b/go/internal/state/history_sources_test.go index 362d4604..55f76128 100644 --- a/go/internal/state/history_sources_test.go +++ b/go/internal/state/history_sources_test.go @@ -144,7 +144,7 @@ func TestSourceLoadKeepsUnknownDistinctFromZero(t *testing.T) { } } -// A snapshot holding an inferred value, such as PV read as 0 W after sunset, +// A snapshot holding an inferred value, such as PV read as 0 W in twilight, // marks every interval it bounds and no other, so storage keeps it apart from // measurement. func TestSourceLoadMarksIntervalsOfAnEstimatedSnapshot(t *testing.T) { From cc75e99b23bb6d3a89dea96e53eb3a2da08308a0 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 19 Oct 2026 20:53:26 +0200 Subject: [PATCH 4/5] refactor(history): let the dark rule touch only PV #1753 (85100d65) no longer marks Load unknown when a single source fails, so darkPV need not clear it. It now changes only the PV flow and its marker; a configuration fault keeps Load unknown on its own. The night test without a pin checks Home use again: the writer holds PV's last value as an estimate. Co-Authored-By: Claude Opus 5.5 --- go/cmd/ftw/history_dark_pv.go | 11 ++-------- go/cmd/ftw/history_dark_pv_test.go | 32 +++++++++++++----------------- go/cmd/ftw/main.go | 2 +- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go index cce91473..cc47c2fa 100644 --- a/go/cmd/ftw/history_dark_pv.go +++ b/go/cmd/ftw/history_dark_pv.go @@ -41,9 +41,8 @@ func (s historySun) zenith(atMS int64) (float64, bool) { // reading and carries no marker, so a night with a sleeping inverter also // trains the load models; no separate bit records it. In twilight the // snapshot carries EstimatedLoad: Home use counts it, training does not. -// Grid, battery and EV keep their own evidence; Load stays unknown for any -// cause but PV. -func darkPV(p *state.HistorySources, opts telemetry.ForecastOptions, sun historySun) bool { +// Every other flow, and Load, keeps its own evidence. +func darkPV(p *state.HistorySources, sun historySun) bool { if p == nil || p.Missing&state.MissingPV == 0 { return false } @@ -56,12 +55,6 @@ func darkPV(p *state.HistorySources, opts telemetry.ForecastOptions, sun history if z <= civilDuskZenithDeg { p.Missing |= state.EstimatedLoad } - // historySources sets MissingLoad for the first failing source or for - // site configuration. With every other source known and the - // configuration valid, PV was that source. - if opts.HouseholdInvalidReason == "" && p.Missing&(state.MissingGrid|state.MissingBattery|state.MissingEV|state.MissingV2X) == 0 { - p.Missing &^= state.MissingLoad - } return true } diff --git a/go/cmd/ftw/history_dark_pv_test.go b/go/cmd/ftw/history_dark_pv_test.go index b111ac41..f2a4ea85 100644 --- a/go/cmd/ftw/history_dark_pv_test.go +++ b/go/cmd/ftw/history_dark_pv_test.go @@ -19,28 +19,26 @@ func TestDarkPVByTimeOfDay(t *testing.T) { stockholm := time.FixedZone("CEST", 2*3600) sun := historySun{Lat: 59.6, Lon: 16.5, Placed: true} night := time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm) - const unknown = state.MissingPV | state.MissingLoad for _, c := range []struct { name string at time.Time sun historySun - opts telemetry.ForecastOptions missing state.HistoryMissing applied bool want state.HistoryMissing }{ - {name: "night", at: night, sun: sun, missing: unknown, applied: true}, - {name: "after dusk", at: time.Date(2026, 10, 19, 18, 45, 0, 0, stockholm), sun: sun, missing: unknown, applied: true}, - {name: "twilight", at: time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun: sun, missing: unknown, applied: true, want: state.EstimatedLoad}, - {name: "noon", at: time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun: sun, missing: unknown, want: unknown}, - {name: "before sunset", at: time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun: sun, missing: unknown, want: unknown}, - {name: "no pin", at: night, missing: unknown, want: unknown}, - // Load stays unknown for any cause but PV. - {name: "battery also unknown", at: night, sun: sun, missing: unknown | state.MissingBattery, applied: true, want: state.MissingBattery | state.MissingLoad}, - {name: "PV array without a measurement source", at: night, sun: sun, opts: telemetry.ForecastOptions{HouseholdInvalidReason: "configured_pv_without_measurement_source"}, missing: unknown, applied: true, want: state.MissingLoad}, + {name: "night", at: night, sun: sun, missing: state.MissingPV, applied: true}, + {name: "after dusk", at: time.Date(2026, 10, 19, 18, 45, 0, 0, stockholm), sun: sun, missing: state.MissingPV, applied: true}, + {name: "twilight", at: time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun: sun, missing: state.MissingPV, applied: true, want: state.EstimatedLoad}, + {name: "noon", at: time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun: sun, missing: state.MissingPV, want: state.MissingPV}, + {name: "before sunset", at: time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun: sun, missing: state.MissingPV, want: state.MissingPV}, + {name: "no pin", at: night, missing: state.MissingPV, want: state.MissingPV}, + // Other flows, and a configuration fault, keep their own evidence. + {name: "battery also unknown", at: night, sun: sun, missing: state.MissingPV | state.MissingBattery, applied: true, want: state.MissingBattery}, + {name: "PV array without a measurement source", at: night, sun: sun, missing: state.MissingPV | state.MissingLoad, applied: true, want: state.MissingLoad}, } { p := &state.HistorySources{AtMS: c.at.UnixMilli(), PVW: -300, Missing: c.missing} - applied := darkPV(p, c.opts, c.sun) + applied := darkPV(p, c.sun) if applied != c.applied || p.Missing != c.want || applied && p.PVW != 0 { t.Errorf("%s: applied=%v missing=%b PV=%v W, want applied=%v missing=%b", c.name, applied, p.Missing, p.PVW, c.applied, c.want) } @@ -59,8 +57,9 @@ func TestDarkPVByTimeOfDay(t *testing.T) { // every season. The inverter reports 0 W for ten minutes, then sleeps, and a // minute later the watchdog takes it offline. With a pin every snapshot is // fully known and of one topology, so the writer records measured minutes that -// train the load models. Without one, PV stays unknown for the writer to -// hold, and the topology changes when the device drops out. +// train the load models. Without one, the writer holds PV's last value as an +// estimate, and the topology changes when the device drops out. Home use is +// known in both. func TestSleepingInverterNightIsMeasured(t *testing.T) { const tick, hour = int64(5000), int64(3600000) for _, c := range []struct { @@ -110,7 +109,7 @@ func TestSleepingInverterNightIsMeasured(t *testing.T) { r.Known[telemetry.DerPV], r.Reason = false, "offline:sma:pv" } v := historySources(r) - dark := darkPV(v, telemetry.ForecastOptions{}, c.sun) + dark := darkPV(v, c.sun) v.Topology += historyDevices(st, tel, identity, buildEnergyObservations(st, tel, ctrl, state.HistoryPoint{}, identity), dark) topologies[v.Topology] = true if known := v.Missing == 0; known != (c.measured || off <= 10*60000) { @@ -126,9 +125,6 @@ func TestSleepingInverterNightIsMeasured(t *testing.T) { if one := len(topologies) == 1; one != c.measured { t.Fatalf("%d topologies over the night", len(topologies)) } - if !c.measured { - return - } d, err := st.DailyEnergy(base, base+8*hour) if err != nil { t.Fatal(err) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 2a45ff7d..dedbb874 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -4007,7 +4007,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersis options, sun = site[0].HistoryOptions, site[0].HistorySun } hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options) - dark := darkPV(hp.Sources, options, sun) + dark := darkPV(hp.Sources, sun) if hp.Sources != nil { // Live Load is not an energy observation. The source interval writer // owns historical Load; the chart placeholder stays explicitly unknown. From 37d52b597a10f371676f3c98eb69a4277c669e46 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 20 Oct 2026 06:49:56 +0200 Subject: [PATCH 5/5] fix(telemetry): name offline drivers through a health-gated accessor historyDevices read cached PV readings directly to find a sleeping inverter. Move the health check into telemetry.OfflineDrivers, which returns only driver names, so no caller handles a stale cached value. Co-Authored-By: Claude Opus 5.5 --- go/cmd/ftw/history_dark_pv.go | 7 ++----- go/internal/telemetry/offline.go | 23 +++++++++++++++++++++++ go/internal/telemetry/offline_test.go | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 go/internal/telemetry/offline.go create mode 100644 go/internal/telemetry/offline_test.go diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go index cc47c2fa..909b2476 100644 --- a/go/cmd/ftw/history_dark_pv.go +++ b/go/cmd/ftw/history_dark_pv.go @@ -72,11 +72,8 @@ func historyDevices(st *state.Store, tel *telemetry.Store, identity energyIdenti } if dark && st != nil && tel != nil && identity != nil { known := st.CachedDevices() - for _, r := range tel.ReadingsByType(telemetry.DerPV) { - if h := tel.DriverHealth(r.Driver); h != nil && h.Status != telemetry.StatusOffline { - continue - } - if id := confirmedEnergyDeviceID(identity(r.Driver), known); id != "" { + for _, driver := range tel.OfflineDrivers(telemetry.DerPV) { + if id := confirmedEnergyDeviceID(identity(driver), known); id != "" { devices = append(devices, string(state.AssetPV)+":"+id) } } diff --git a/go/internal/telemetry/offline.go b/go/internal/telemetry/offline.go new file mode 100644 index 00000000..94aafe50 --- /dev/null +++ b/go/internal/telemetry/offline.go @@ -0,0 +1,23 @@ +package telemetry + +import "sort" + +// OfflineDrivers names the drivers with a reading of kind whose telemetry is +// offline or has no health yet. Only names leave the store, never a cached +// value, so a stale reading cannot be mistaken for a live one. +func (s *Store) OfflineDrivers(kind DerType) []string { + s.mu.RLock() + defer s.mu.RUnlock() + var out []string + for _, r := range s.readings { + if r.DerType != kind { + continue + } + if h := s.health[r.Driver]; h != nil && h.Status != StatusOffline { + continue + } + out = append(out, r.Driver) + } + sort.Strings(out) + return out +} diff --git a/go/internal/telemetry/offline_test.go b/go/internal/telemetry/offline_test.go new file mode 100644 index 00000000..7b047a02 --- /dev/null +++ b/go/internal/telemetry/offline_test.go @@ -0,0 +1,21 @@ +package telemetry + +import ( + "slices" + "testing" +) + +func TestOfflineDriversNamesOnlySilentDriversOfOneKind(t *testing.T) { + s := NewStore() + s.Update("live", DerPV, -1200, nil, nil) + s.RecordDriverSuccess("live") + s.Update("asleep", DerPV, 0, nil, nil) + s.RecordDriverSuccess("asleep") + s.DriverHealthMut("asleep").SetOffline() + s.Update("new", DerPV, 0, nil, nil) // no health yet + s.Update("battery", DerBattery, 0, nil, nil) + s.DriverHealthMut("battery").SetOffline() + if got := s.OfflineDrivers(DerPV); !slices.Equal(got, []string{"asleep", "new"}) { + t.Fatalf("offline PV drivers = %v", got) + } +}