fix(history): keep Home use through device dropouts and restarts

.changeset/bridge-short-history-gaps.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Keep the day's Home use when a device drops out or Core restarts. An unknown PV, battery or EV source holds its last known value, and the meter or a restart holds for up to two minutes, marked as estimated. A day's total is unknown only when more than a tenth of it is missing. Days cached under the old rule are recomputed once.

docs/site-convention.md

@@ -56,8 +56,16 @@ holds each qualified snapshot forward until the next safe snapshot, at most
 the sums when reading a chart interval, day or tariff period. A negative
 minute may contribute to a valid longer interval; it is never clipped first.
 Chart means need complete interval coverage. Missing sources, writer gaps,
-process stops and topology changes cannot become measured zero. This is an
-estimate from received measurements, not a correction of device clocks.
+process stops and topology changes cannot become measured zero.
+
+History has two layers. Display is forgiving: an unknown PV, battery or EV
+source holds its last known value, and the site meter, a writer pause or a
+Core restart holds for at most two minutes. Every such minute is marked
+`EstimatedLoad`, and readers never clear the mark. A day's flow is unknown only
+when it is unknown or unrecorded for more than a tenth of the range;
+unrecorded time counts as nothing. Training is strict: it uses only minutes
+without the mark. This is an estimate from received measurements, not a
+correction of device clocks.
 
 In self-consumption mode, the controller drives the site meter toward the
 grid target (normally 0 W): it charges from live surplus and may discharge

go/cmd/ftw/history_source_load_test.go

@@ -57,6 +57,48 @@ func TestHistoryLoadIndependentReportingCadence(t *testing.T) {
 	}
 }
 
+// A silent inverter is unknown on its own. Its balance with the meter is
+// held, not voided, so the household keeps its estimated energy.
+func TestHistoryHoldsASilentSourceThroughTheBalance(t *testing.T) {
+	st, err := state.Open(filepath.Join(t.TempDir(), "state.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer st.Close()
+	if err := st.EnableHistoryAggregation(); err != nil {
+		t.Fatal(err)
+	}
+	tel := telemetry.NewStore()
+	tel.Update("grid", telemetry.DerMeter, 1000, nil, nil)
+	tel.RecordDriverSuccess("grid")
+	tel.Update("sma", telemetry.DerPV, 0, nil, nil)
+	tel.RecordDriverSuccess("sma")
+	base := time.Now().Add(-20 * time.Minute).Truncate(5 * time.Minute).UnixMilli()
+	for i := int64(0); i <= 60; i++ {
+		if i == 6 {
+			tel.Get("sma", telemetry.DerPV).UpdatedAt = time.Now().Add(-10 * time.Minute)
+		}
+		p, _ := buildHistoryPoint(tel, tickPersistControl{SiteMeterDriver: "grid"}, base+i*10000, 90*time.Second)
+		if i >= 6 && p.Sources.Missing != state.MissingPV {
+			t.Fatalf("silent inverter marked %b, want only its own flow", p.Sources.Missing)
+		}
+		p.Sources.AtMS = base + i*10000
+		if err := st.RecordTickWithEnergy(p, nil, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := st.FlushHistory(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	d, err := st.DailyEnergy(base, base+600000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d.Missing&state.MissingLoad != 0 || math.Abs(d.LoadWh-1000.0/6) > 0.001 {
+		t.Fatalf("household lost through a silent inverter: %v Wh, quality %s", d.LoadWh, d.Missing.Quality())
+	}
+}
+
 func TestHistorySourceQualificationDoesNotInventMissingFlows(t *testing.T) {
 	for _, tc := range []struct {
 		reason string
@@ -65,8 +107,11 @@ func TestHistorySourceQualificationDoesNotInventMissingFlows(t *testing.T) {
 		want   state.HistoryMissing
 	}{
 		{"missing_config", 0, true, state.MissingLoad},
+		{"duplicate_charger:easee", 0, true, state.MissingLoad},
 		{"invalid_balance", 1000, true, 0},
 		{"nonfinite_sum", math.Inf(1), false, state.MissingLoad | state.MissingEV},
+		{"stale:easee:ev", 0, false, state.MissingEV},
+		{"offline:easee:ev", 0, false, state.MissingEV},
 	} {
 		known := map[telemetry.DerType]bool{telemetry.DerMeter: true, telemetry.DerPV: true, telemetry.DerBattery: true, telemetry.DerEV: tc.known, telemetry.DerV2X: true}
 		p := historySources(telemetry.ForecastReading{At: time.Now(), Known: known, Reason: tc.reason, EVW: tc.watts})

go/cmd/ftw/main.go

@@ -4161,8 +4161,9 @@ func historySources(reading telemetry.ForecastReading) *state.HistorySources {
 		}
 	}
 	// A negative instantaneous residual does not invalidate its independent
-	// sources. Missing configuration and duplicate electrical flows do.
-	if reading.Reason != "" && reading.Reason != "invalid_balance" {
+	// sources. Missing configuration and duplicate electrical flows do. A
+	// failed source is unknown on its own flag, so history may hold it.
+	if reading.Reason != "" && reading.Reason != "invalid_balance" && !telemetry.SourceFailure(reading.Reason) {
 		p.Missing |= state.MissingLoad
 	}
 	for flag, w := range map[state.HistoryMissing]*float64{state.MissingGrid: &p.GridW, state.MissingPV: &p.PVW, state.MissingBattery: &p.BatteryW, state.MissingEV: &p.EVW, state.MissingV2X: &p.V2XW} {

go/internal/state/history_dashboard.go

@@ -273,18 +273,23 @@ func (s *Store) walkSiteEnergy(ctx context.Context, from, to int64, visit func(s
 	return rows.Err()
 }
 
+// maxSourceDayUnknownShare is the share of a source-era range a flow may be
+// unknown or unrecorded before its total is unknown. Below it the total is
+// an estimate of what was recorded; unrecorded time counts as nothing.
+const maxSourceDayUnknownShare = 10 // percent
+
 func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEnergy, error) {
+	cut, hasSources, err := s.sourceEnergyFrom(ctx)
+	if err != nil {
+		return DayEnergy{}, err
+	}
 	var out DayEnergy
 	var originalLoad float64
 	var originalMissing HistoryMissing
 	var estimatedMS, coveredMS int64
-	var sourceMS int64
-	var first, last int64
-	err := s.walkSiteEnergy(ctx, from, to, func(r siteEnergyRow) error {
-		if first == 0 || r.first < first {
-			first = r.first
-		}
-		last = max(last, r.last)
+	var sourceMS, sourceEraMS int64
+	unknownMS := map[HistoryMissing]int64{}
+	err = s.walkSiteEnergy(ctx, from, to, func(r siteEnergyRow) error {
 		out.ImportWh += r.imp
 		out.ExportWh += r.exp
 		out.PVWh += r.pv
@@ -293,8 +298,8 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner
 		originalLoad += r.load
 		originalMissing |= r.missing
 		coveredMS += r.covered
-		if r.source {
-			sourceMS += r.covered
+		if hasSources && r.first >= cut {
+			sourceEraMS += r.covered
 		}
 		if r.missing&^MissingSoC == MissingLoad|BalanceMismatch || r.missing&^MissingSoC == MissingLoad|MissingEV {
 			if r.estimateCovered > 0 && r.estimateCovered == r.covered {
@@ -304,12 +309,25 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner
 		}
 		out.LoadWh += r.load
 		out.Intervals += r.n
-		out.Missing |= r.missing
 		if r.covered > 0 {
 			out.observed |= MissingPower &^ r.missing
 		}
+		if r.source {
+			// Source rows decide each flow by its unknown time below.
+			sourceMS += r.covered
+			for _, flag := range []HistoryMissing{MissingGrid, MissingPV, MissingBattery, MissingLoad, MissingEV, MissingV2X} {
+				if r.missing&flag != 0 {
+					unknownMS[flag] += r.covered
+				}
+			}
+			r.missing &^= MissingPower
+		}
+		out.Missing |= r.missing
 		return nil
 	})
+	if err != nil {
+		return out, err
+	}
 	if sourceMS == 0 && (out.LoadWh < 0 || (estimatedMS > 120000 && estimatedMS > coveredMS/20)) {
 		out.LoadWh = originalLoad
 		out.Missing = originalMissing
@@ -318,21 +336,21 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner
 		out.LoadWh = 0
 		out.Missing |= MissingLoad
 	}
-	cut, hasSources, coverageErr := s.sourceEnergyFrom(ctx)
-	if coverageErr != nil {
-		return out, coverageErr
-	}
 	if hasSources && to > cut {
-		expected, err := s.sourcePeriodCoverage(ctx, max(from, cut), to)
+		// The writer's last receipt ends the range, so a live tail is not a gap.
+		through, _, err := s.HistorySourceProgress(ctx)
 		if err != nil {
 			return out, err
 		}
-		if sourceMS < expected {
-			out.Missing |= MissingLoad
-		}
-		if coveredMS < last-first {
-			out.Missing |= MissingLoad
+		span := max(0, min(to, through)-max(from, cut))
+		// Unrecorded time leaves the other totals as recorded, as before;
+		// the household balance needs it covered.
+		unknownMS[MissingLoad] += max(0, span-sourceEraMS)
+		for flag, ms := range unknownMS {
+			if ms*100 > span*maxSourceDayUnknownShare {
+				out.Missing |= flag
+			}
 		}
 	}
-	return out, err
+	return out, nil
 }

go/internal/state/history_quality.go

@@ -70,4 +70,4 @@ func putHistoryQuality(ctx context.Context, tx *sql.Tx, kind string, start, widt
 
 // SQL has no bitwise-OR aggregate. MAX per bit preserves unknown evidence
 // across every source row, including mixed complete/incomplete buckets.
-const historyMissingAggregate = `MAX(missing&1)|MAX(missing&2)|MAX(missing&4)|MAX(missing&8)|MAX(missing&16)|MAX(missing&32)|MAX(missing&64)|MAX(missing&128)`
+const historyMissingAggregate = `MAX(missing&1)|MAX(missing&2)|MAX(missing&4)|MAX(missing&8)|MAX(missing&16)|MAX(missing&32)|MAX(missing&64)|MAX(missing&128)|MAX(missing&256)`