diff --git a/.changeset/unknown-pv-after-sunset.md b/.changeset/unknown-pv-after-sunset.md new file mode 100644 index 00000000..58d603ed --- /dev/null +++ b/.changeset/unknown-pv-after-sunset.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Keep the day's Home use when a PV inverter goes silent at night. While the sun is below the horizon at the site's map pin, an unknown PV source counts as 0 W: past civil dusk as a measurement, so the night also trains the load models, and in twilight as a marked estimate. Without a pin, or by day, the gap stays unknown as before. diff --git a/docs/site-convention.md b/docs/site-convention.md index 72872e5f..9b65d6e6 100644 --- a/docs/site-convention.md +++ b/docs/site-convention.md @@ -67,6 +67,12 @@ unrecorded time counts as nothing. Training is strict: it uses only minutes without the mark. This is an estimate from received measurements, not a correction of device clocks. +While the sun is below the horizon at the site's map pin, an unknown PV source +counts as 0 W instead of its last value, and an inverter the watchdog took +offline keeps its place in the topology. Past civil dusk (sun 6° down) the +0 W is physics and carries no mark, so the night also trains the load models. +In twilight it is marked. Without a pin nothing is inferred. + In self-consumption mode, the controller drives the site meter toward the grid target (normally 0 W): it charges from live surplus and may discharge to cover local load. It must not intentionally export via the battery; export diff --git a/go/cmd/ftw/forecast_site.go b/go/cmd/ftw/forecast_site.go index 008e3706..06719754 100644 --- a/go/cmd/ftw/forecast_site.go +++ b/go/cmd/ftw/forecast_site.go @@ -88,7 +88,7 @@ func (s *forecastSiteConfig) Snapshot() forecastSite { } func (s *forecastSiteConfig) Configure(cfg *config.Config, catalog []drivers.CatalogEntry) bool { - v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: localzone.Name(), Options: forecastMeasurementOptions(cfg, catalog), HistoryOptions: historyMeasurementOptions(cfg, catalog)} + v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: localzone.Name(), Options: forecastMeasurementOptions(cfg, catalog), HistoryOptions: historyMeasurementOptions(cfg, catalog), HistorySun: historySunOf(siteLocation(cfg))} if cfg.Weather != nil && cfg.Weather.Provider != "" && cfg.Weather.Provider != "none" { v.Latitude, v.Longitude = cfg.Weather.Latitude, cfg.Weather.Longitude v.HasLocation = !math.IsNaN(v.Latitude) && !math.IsNaN(v.Longitude) && math.Abs(v.Latitude) <= 90 && math.Abs(v.Longitude) <= 180 diff --git a/go/cmd/ftw/forecast_tracking.go b/go/cmd/ftw/forecast_tracking.go index 2e88aaf5..984c5fd5 100644 --- a/go/cmd/ftw/forecast_tracking.go +++ b/go/cmd/ftw/forecast_tracking.go @@ -36,6 +36,7 @@ type forecastSite struct { Timezone string Options telemetry.ForecastOptions HistoryOptions telemetry.ForecastOptions // chart and site energy; outside every revision + HistorySun historySun // where unknown PV reads as 0 W after sunset; outside every revision PVLearningStartedMS int64 LoadLearningStartedMS int64 } diff --git a/go/cmd/ftw/history_dark_pv.go b/go/cmd/ftw/history_dark_pv.go new file mode 100644 index 00000000..909b2476 --- /dev/null +++ b/go/cmd/ftw/history_dark_pv.go @@ -0,0 +1,83 @@ +package main + +import ( + "sort" + "strings" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/sunpos" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// Sun positions as zenith angles. After sunset (upper limb below the horizon, +// refraction included) no panel gets direct sunlight, but diffuse light +// remains until civil dusk, when the sun is 6° down. +const ( + sunsetZenithDeg = 90.833 + civilDuskZenithDeg = 96 +) + +// historySun places the site for history. The zero value has no position, +// and then nothing is inferred. +type historySun struct { + Lat, Lon float64 + Placed bool +} + +func historySunOf(lat, lon float64, ok bool) historySun { + return historySun{Lat: lat, Lon: lon, Placed: ok} +} + +func (s historySun) zenith(atMS int64) (float64, bool) { + if !s.Placed { + return 0, false + } + return sunpos.At(time.UnixMilli(atMS), s.Lat, s.Lon).ZenithDeg, true +} + +// darkPV records an unknown PV source as 0 W while the sun is down at the +// site: physics, not interpolation. Past civil dusk the 0 W is as good as a +// reading and carries no marker, so a night with a sleeping inverter also +// trains the load models; no separate bit records it. In twilight the +// snapshot carries EstimatedLoad: Home use counts it, training does not. +// Every other flow, and Load, keeps its own evidence. +func darkPV(p *state.HistorySources, sun historySun) bool { + if p == nil || p.Missing&state.MissingPV == 0 { + return false + } + z, placed := sun.zenith(p.AtMS) + if !placed || z <= sunsetZenithDeg { + return false + } + p.PVW = 0 + p.Missing &^= state.MissingPV + if z <= civilDuskZenithDeg { + p.Missing |= state.EstimatedLoad + } + return true +} + +// historyDevices names the hardware behind a history snapshot, as the +// topology's devices line. The watchdog drops an offline driver from the +// energy record. After sunset a silent inverter is asleep, not replaced, so it +// keeps its place: the night stays one topology, and its minutes stay +// measured. +func historyDevices(st *state.Store, tel *telemetry.Store, identity energyIdentityLookup, observations []state.EnergyObservation, dark bool) string { + var devices []string + for _, o := range observations { + if o.DeviceID != "" { + devices = append(devices, string(o.AssetKind)+":"+o.DeviceID) + } + } + if dark && st != nil && tel != nil && identity != nil { + known := st.CachedDevices() + for _, driver := range tel.OfflineDrivers(telemetry.DerPV) { + if id := confirmedEnergyDeviceID(identity(driver), known); id != "" { + devices = append(devices, string(state.AssetPV)+":"+id) + } + } + } + sort.Strings(devices) + return "\ndevices:" + strings.Join(devices, "\n") +} diff --git a/go/cmd/ftw/history_dark_pv_test.go b/go/cmd/ftw/history_dark_pv_test.go new file mode 100644 index 00000000..f2a4ea85 --- /dev/null +++ b/go/cmd/ftw/history_dark_pv_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "math" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// Göran's box, 19 October 2026: SMA was silent for 145 s at 02:22 local time, +// 0 W on both sides, and the day's Home use became unknown. In Västerås that +// day the sun sets at 17:38 and civil dusk is at about 18:20. +func TestDarkPVByTimeOfDay(t *testing.T) { + stockholm := time.FixedZone("CEST", 2*3600) + sun := historySun{Lat: 59.6, Lon: 16.5, Placed: true} + night := time.Date(2026, 10, 19, 2, 22, 0, 0, stockholm) + for _, c := range []struct { + name string + at time.Time + sun historySun + missing state.HistoryMissing + applied bool + want state.HistoryMissing + }{ + {name: "night", at: night, sun: sun, missing: state.MissingPV, applied: true}, + {name: "after dusk", at: time.Date(2026, 10, 19, 18, 45, 0, 0, stockholm), sun: sun, missing: state.MissingPV, applied: true}, + {name: "twilight", at: time.Date(2026, 10, 19, 17, 45, 0, 0, stockholm), sun: sun, missing: state.MissingPV, applied: true, want: state.EstimatedLoad}, + {name: "noon", at: time.Date(2026, 10, 19, 12, 0, 0, 0, stockholm), sun: sun, missing: state.MissingPV, want: state.MissingPV}, + {name: "before sunset", at: time.Date(2026, 10, 19, 17, 30, 0, 0, stockholm), sun: sun, missing: state.MissingPV, want: state.MissingPV}, + {name: "no pin", at: night, missing: state.MissingPV, want: state.MissingPV}, + // Other flows, and a configuration fault, keep their own evidence. + {name: "battery also unknown", at: night, sun: sun, missing: state.MissingPV | state.MissingBattery, applied: true, want: state.MissingBattery}, + {name: "PV array without a measurement source", at: night, sun: sun, missing: state.MissingPV | state.MissingLoad, applied: true, want: state.MissingLoad}, + } { + p := &state.HistorySources{AtMS: c.at.UnixMilli(), PVW: -300, Missing: c.missing} + applied := darkPV(p, c.sun) + if applied != c.applied || p.Missing != c.want || applied && p.PVW != 0 { + t.Errorf("%s: applied=%v missing=%b PV=%v W, want applied=%v missing=%b", c.name, applied, p.Missing, p.PVW, c.applied, c.want) + } + } + for _, c := range []struct { + lat, lon float64 + ok bool + }{{0, 0, false}, {unsetPinLat, unsetPinLon, false}, {59.6, 16.5, true}} { + if got := historySunOf(siteLocation(&config.Config{Weather: &config.Weather{Latitude: c.lat, Longitude: c.lon}})).Placed; got != c.ok { + t.Errorf("pin %v,%v placed=%v, want %v", c.lat, c.lon, got, c.ok) + } + } +} + +// A full night on the equator: 20:00–04:00 UTC at 15° W is past civil dusk in +// every season. The inverter reports 0 W for ten minutes, then sleeps, and a +// minute later the watchdog takes it offline. With a pin every snapshot is +// fully known and of one topology, so the writer records measured minutes that +// train the load models. Without one, the writer holds PV's last value as an +// estimate, and the topology changes when the device drops out. Home use is +// known in both. +func TestSleepingInverterNightIsMeasured(t *testing.T) { + const tick, hour = int64(5000), int64(3600000) + for _, c := range []struct { + name string + sun historySun + measured bool + }{ + {"with a pin", historySun{Lat: 1, Lon: -15, Placed: true}, true}, + {"without a pin", historySun{}, false}, + } { + t.Run(c.name, func(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + if err := st.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + if _, err := st.RegisterDevice(state.Device{DriverName: "sma", Make: "SMA", Serial: "PV-7"}); err != nil { + t.Fatal(err) + } + identity := testEnergyIdentity(st) + ctrl := tickPersistControl{SiteMeterDriver: "grid"} + tel := telemetry.NewStore() + tel.Update("grid", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("grid") + tel.Update("sma", telemetry.DerPV, 0, nil, nil) + tel.RecordDriverSuccess("sma") + base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 28*hour + topologies := map[string]bool{} + for at := base; at <= base+8*hour; at += tick { + off := at - base + if off == 11*60000 { + tel.DriverHealthMut("sma").SetOffline() + } + r := telemetry.ForecastReading{ + At: time.UnixMilli(at), + Known: map[telemetry.DerType]bool{telemetry.DerMeter: true, telemetry.DerPV: true, telemetry.DerBattery: true, telemetry.DerEV: true, telemetry.DerV2X: true}, + Sources: map[string]telemetry.ForecastSource{ + "grid:meter": {Flow: telemetry.ForecastFlow{Driver: "grid", DerType: telemetry.DerMeter}}, + "sma:pv": {Flow: telemetry.ForecastFlow{Driver: "sma", DerType: telemetry.DerPV}}, + }, + GridW: 1000, + } + if off > 10*60000 { + r.Known[telemetry.DerPV], r.Reason = false, "offline:sma:pv" + } + v := historySources(r) + dark := darkPV(v, c.sun) + v.Topology += historyDevices(st, tel, identity, buildEnergyObservations(st, tel, ctrl, state.HistoryPoint{}, identity), dark) + topologies[v.Topology] = true + if known := v.Missing == 0; known != (c.measured || off <= 10*60000) { + t.Fatalf("%d min: snapshot missing %b", off/60000, v.Missing) + } + if err := st.RecordTickWithEnergy(state.HistoryPoint{TsMs: at, Sources: v}, nil, nil); err != nil { + t.Fatal(err) + } + } + if err := st.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } + if one := len(topologies) == 1; one != c.measured { + t.Fatalf("%d topologies over the night", len(topologies)) + } + d, err := st.DailyEnergy(base, base+8*hour) + if err != nil { + t.Fatal(err) + } + if d.Missing&state.MissingLoad != 0 || math.Abs(d.LoadWh-8000) > 1e-6 { + t.Fatalf("Home use %v Wh, missing %b; want 8000 Wh", d.LoadWh, d.Missing) + } + }) + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index d50b719a..dedbb874 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2951,7 +2951,7 @@ func main() { // charger standdowns, self-tuning or normal physical dispatch. lpMgr.RollSchedules(tickNow.UTC()) carsRT.Tick() - if _, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions); err != nil { + if _, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot()); err != nil { slog.Warn("setup measurement persistence failed", "err", err) } continue @@ -3051,7 +3051,7 @@ func main() { // ctrl, so the stored tick has to show the hold already // released rather than one the blocked tick never executed. clearBatteryManualHoldForDispatchBlock(ctrl, ctrlMu) - sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions) + sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot()) if err != nil { slog.Warn("tick persistence failed", "samples", sampleCount, "err", err) } @@ -3308,7 +3308,7 @@ func main() { // ---- Persist the tick: history snapshot + flushed metrics ---- // One transaction for both — separate commits doubled the WAL // commit rate for no isolation benefit (SD-card wear). - sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions) + sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot()) if err != nil { slog.Warn("tick persistence failed", "samples", sampleCount, "err", err) } @@ -4000,8 +4000,14 @@ func snapshotTickPersistControl(ctrl *control.State, ctrlMu *sync.Mutex) tickPer } } -func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersistControl, nowMs int64, historyMaxAge time.Duration, identity energyIdentityLookup, options ...telemetry.ForecastOptions) (int, error) { - hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options...) +func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersistControl, nowMs int64, historyMaxAge time.Duration, identity energyIdentityLookup, site ...forecastSite) (int, error) { + var options telemetry.ForecastOptions + var sun historySun + if len(site) > 0 { + options, sun = site[0].HistoryOptions, site[0].HistorySun + } + hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options) + dark := darkPV(hp.Sources, sun) if hp.Sources != nil { // Live Load is not an energy observation. The source interval writer // owns historical Load; the chart placeholder stays explicitly unknown. @@ -4026,14 +4032,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersis } energyObservations := buildEnergyObservations(st, tel, ctrl, hp, identity) if hp.Sources != nil { - var devices []string - for _, o := range energyObservations { - if o.DeviceID != "" { - devices = append(devices, string(o.AssetKind)+":"+o.DeviceID) - } - } - sort.Strings(devices) - hp.Sources.Topology += "\ndevices:" + strings.Join(devices, "\n") + hp.Sources.Topology += historyDevices(st, tel, identity, energyObservations, dark) } filtered := energyObservations[:0] for _, observation := range energyObservations { diff --git a/go/internal/state/history_sources.go b/go/internal/state/history_sources.go index 1599a167..1d913648 100644 --- a/go/internal/state/history_sources.go +++ b/go/internal/state/history_sources.go @@ -12,6 +12,8 @@ import ( // HistorySources carries independently fresh site-side powers, never the live // Load result. Topology identifies the electrical flows included in the sum. +// EstimatedLoad in Missing marks a snapshot that holds an inferred value, such +// as PV read as 0 W in twilight; every interval it bounds keeps the marker. type HistorySources struct { AtMS int64 Topology string diff --git a/go/internal/state/history_sources_test.go b/go/internal/state/history_sources_test.go index 99120ff3..55f76128 100644 --- a/go/internal/state/history_sources_test.go +++ b/go/internal/state/history_sources_test.go @@ -144,6 +144,31 @@ func TestSourceLoadKeepsUnknownDistinctFromZero(t *testing.T) { } } +// A snapshot holding an inferred value, such as PV read as 0 W in twilight, +// marks every interval it bounds and no other, so storage keeps it apart from +// measurement. +func TestSourceLoadMarksIntervalsOfAnEstimatedSnapshot(t *testing.T) { + s := sourceStore(t) + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + for at := base; at <= base+180000; at += 5000 { + v := HistorySources{GridW: 1000} + if at > base+60000 && at <= base+120000 { + v.Missing = EstimatedLoad + } + sourceTick(t, s, at, v) + } + b, err := s.sourceLoadBuckets(context.Background(), base, base+180000, 60000) + if err != nil || len(b) != 3 { + t.Fatal(err, len(b)) + } + for i, estimated := range []bool{false, true, true} { + if b[i].missing&MissingPower != 0 || (b[i].missing&EstimatedLoad != 0) != estimated { + t.Fatalf("minute %d missing %b, want estimated=%v", i, b[i].missing, estimated) + } + near(t, "minute Wh", b[i].wh, 1000.0/60) + } +} + func TestSourceLoadUsesSnapshotTimeAndMarksRestartBridge(t *testing.T) { path := filepath.Join(t.TempDir(), "state.db") s, err := Open(path) diff --git a/go/internal/telemetry/offline.go b/go/internal/telemetry/offline.go new file mode 100644 index 00000000..94aafe50 --- /dev/null +++ b/go/internal/telemetry/offline.go @@ -0,0 +1,23 @@ +package telemetry + +import "sort" + +// OfflineDrivers names the drivers with a reading of kind whose telemetry is +// offline or has no health yet. Only names leave the store, never a cached +// value, so a stale reading cannot be mistaken for a live one. +func (s *Store) OfflineDrivers(kind DerType) []string { + s.mu.RLock() + defer s.mu.RUnlock() + var out []string + for _, r := range s.readings { + if r.DerType != kind { + continue + } + if h := s.health[r.Driver]; h != nil && h.Status != StatusOffline { + continue + } + out = append(out, r.Driver) + } + sort.Strings(out) + return out +} diff --git a/go/internal/telemetry/offline_test.go b/go/internal/telemetry/offline_test.go new file mode 100644 index 00000000..7b047a02 --- /dev/null +++ b/go/internal/telemetry/offline_test.go @@ -0,0 +1,21 @@ +package telemetry + +import ( + "slices" + "testing" +) + +func TestOfflineDriversNamesOnlySilentDriversOfOneKind(t *testing.T) { + s := NewStore() + s.Update("live", DerPV, -1200, nil, nil) + s.RecordDriverSuccess("live") + s.Update("asleep", DerPV, 0, nil, nil) + s.RecordDriverSuccess("asleep") + s.DriverHealthMut("asleep").SetOffline() + s.Update("new", DerPV, 0, nil, nil) // no health yet + s.Update("battery", DerBattery, 0, nil, nil) + s.DriverHealthMut("battery").SetOffline() + if got := s.OfflineDrivers(DerPV); !slices.Equal(got, []string{"asleep", "new"}) { + t.Fatalf("offline PV drivers = %v", got) + } +}