fix(history): keep Home use when at most 1 % of the day is unknown

.changeset/home-use-unknown-share.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Show the day's Home use again when at most 1 % of the day is unknown, as before source history. The total sums the known time, marked as an estimate. Days cached as unknown since the change are recounted once after the update.

docs/site-convention.md

@@ -62,6 +62,10 @@ marked estimate: measured sources keep their snapshots, and an unknown source
 or a stopped Core holds the last known values. Longer gaps stay unknown. This
 is an estimate from received measurements, not a correction of device clocks.
 
+A day or other period keeps its Home use when at most 1 % of its time is
+unknown. The total is then the sum of the known intervals, marked as an
+estimate; unknown time counts as nothing. Other flows keep their own unknowns.
+
 In self-consumption mode, the controller drives the site meter toward the
 grid target (normally 0 W): it charges from live surplus and may discharge
 to cover local load. It must not intentionally export via the battery; export

go/internal/state/history_dashboard.go

@@ -273,12 +273,16 @@ func (s *Store) walkSiteEnergy(ctx context.Context, from, to int64, visit func(s
 	return rows.Err()
 }
 
+// maxUnknownHomeUsePercent is the share of a period's time that may be
+// unknown while its Home use stays known as an estimate.
+const maxUnknownHomeUsePercent = 1
+
 func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEnergy, error) {
 	var out DayEnergy
 	var originalLoad float64
 	var originalMissing HistoryMissing
 	var estimatedMS, coveredMS int64
-	var sourceMS int64
+	var sourceMS, knownMS int64
 	var first, last int64
 	err := s.walkSiteEnergy(ctx, from, to, func(r siteEnergyRow) error {
 		if first == 0 || r.first < first {
@@ -295,6 +299,9 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner
 		coveredMS += r.covered
 		if r.source {
 			sourceMS += r.covered
+			if r.missing&MissingPower == 0 {
+				knownMS += r.covered
+			}
 		}
 		if r.missing&^MissingSoC == MissingLoad|BalanceMismatch || r.missing&^MissingSoC == MissingLoad|MissingEV {
 			if r.estimateCovered > 0 && r.estimateCovered == r.covered {
@@ -314,7 +321,8 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner
 		out.LoadWh = originalLoad
 		out.Missing = originalMissing
 	}
-	if out.LoadWh < 0 {
+	negative := out.LoadWh < 0
+	if negative {
 		out.LoadWh = 0
 		out.Missing |= MissingLoad
 	}
@@ -333,6 +341,45 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner
 		if coveredMS < last-first {
 			out.Missing |= MissingLoad
 		}
+		// A short unknown share keeps Home use as an estimate: the sum of the
+		// fully known intervals, with unknown time counted as nothing, as
+		// before source history. Other flows keep their own unknowns.
+		span := max(last-first, expected)
+		if !negative && sourceMS == coveredMS && knownMS > 0 && (span-knownMS)*100 <= span*maxUnknownHomeUsePercent {
+			out.Missing = out.Missing&^MissingLoad | EstimatedLoad
+		}
 	}
 	return out, err
 }
+
+// homeUseShareRecount names the one-time recount of cached source days, so
+// days closed under the stricter rule show their Home use too.
+const homeUseShareRecount = "home-use-unknown-share-v1"
+
+func (s *Store) recountSourceDays(ctx context.Context) error {
+	tx, err := s.history.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	var done bool
+	if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM history_migrations WHERE name=?)`, homeUseShareRecount).Scan(&done); err != nil || done {
+		return err
+	}
+	var low string
+	err = tx.QueryRowContext(ctx, `SELECT date(CAST(value AS INTEGER)/1000,'unixepoch','-1 day') FROM history_source_cursor WHERE key='from'`).Scan(&low)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+	if low != "" {
+		for _, table := range []string{"energy_daily", "energy_daily_partial", "history_daily_quality"} {
+			if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE day>=?`, low); err != nil {
+				return err
+			}
+		}
+	}
+	if _, err := tx.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES(?)`, homeUseShareRecount); err != nil {
+		return err
+	}
+	return tx.Commit()
+}

go/internal/state/history_home_use_share_test.go

@@ -0,0 +1,125 @@
+package state
+
+import (
+	"testing"
+	"time"
+)
+
+// Six hours of a 1000 W house, sampled every 10 s, with one disturbance three
+// hours in. Before source history a day summed what it had; since #1741 any
+// unknown time made the whole day's Home use unknown. At most 1 % unknown time
+// (3.6 minutes here) keeps it as an estimate, with unknown time counted as
+// nothing. Other flows keep their own unknowns.
+func TestHomeUseSurvivesSmallUnknownShare(t *testing.T) {
+	const tick, hour = int64(10000), int64(3600000)
+	for _, c := range []struct {
+		name string
+		// skip leaves the writer without snapshots for this long.
+		skip    int64
+		missing HistoryMissing
+		// missingMS keeps the flag on this long; the interval after it and
+		// every partly unknown minute stay unknown too.
+		missingMS int64
+		gridW     float64
+		known     bool
+		loadWh    float64
+	}{
+		{name: "no disturbance", known: true, loadWh: 6000},
+		{name: "battery unknown 2 min", missing: MissingBattery, missingMS: 120000, known: true, loadWh: 6000 - 3*1000.0/60},
+		{name: "writer gap 3 min", skip: 180000, known: true, loadWh: 6000 - 3*1000.0/60},
+		{name: "writer gap 5 min", skip: 300000},
+		{name: "battery unknown 10 min", missing: MissingBattery, missingMS: 600000},
+		{name: "negative known total", missing: MissingBattery, missingMS: 120000, gridW: -500},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			s := sourceStore(t)
+			base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 24*hour
+			grid := 1000.0
+			if c.gridW != 0 {
+				grid = c.gridW
+			}
+			for at := base; at <= base+6*hour; at += tick {
+				off := at - base - 3*hour
+				if off > 0 && off < c.skip {
+					continue
+				}
+				v := HistorySources{GridW: grid}
+				if off > 0 && off <= c.missingMS {
+					v.Missing = c.missing
+				}
+				sourceTick(t, s, at, v)
+			}
+			d, err := s.DailyEnergy(base, base+6*hour)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if c.gridW == 0 {
+				near(t, "import Wh", d.ImportWh, 6000-1000*float64(c.skip)/3600000)
+			}
+			if got := d.Missing&MissingLoad == 0; got != c.known {
+				t.Fatalf("Home use known=%v, want %v (quality %s)", got, c.known, d.Missing.Quality())
+			}
+			if (d.Missing&MissingBattery != 0) != (c.missing == MissingBattery) {
+				t.Fatal("battery totals changed state")
+			}
+			if !c.known {
+				return
+			}
+			if d.Missing&EstimatedLoad == 0 {
+				t.Fatal("Home use not marked as an estimate")
+			}
+			near(t, "Home use Wh", d.LoadWh, c.loadWh)
+		})
+	}
+}
+
+// Days cached under the stricter rule are recounted once after the update.
+// Days before source history and later cache entries are left alone.
+func TestSourceDaysAreRecountedOnce(t *testing.T) {
+	s := sourceStore(t)
+	const hour = int64(3600000)
+	base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 24*hour
+	for at := base; at <= base+hour; at += 10000 {
+		sourceTick(t, s, at, HistorySources{GridW: 1000})
+	}
+	day := time.UnixMilli(base).UTC().Format("2006-01-02")
+	old := time.UnixMilli(base).UTC().AddDate(0, 0, -10).Format("2006-01-02")
+	stale := DayEnergy{Missing: MissingLoad, Intervals: 1}
+	for _, d := range []string{day, old} {
+		if err := s.SaveDailyEnergyForSourceMode(d, stale, true); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := s.history.Exec(`DELETE FROM history_migrations WHERE name=?`, homeUseShareRecount); err != nil {
+		t.Fatal(err)
+	}
+	reopen := func() {
+		t.Helper()
+		path := s.mainDBPath
+		if err := s.Close(); err != nil {
+			t.Fatal(err)
+		}
+		var err error
+		if s, err = Open(path); err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { s.Close() })
+		if err := s.EnableHistoryAggregation(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	reopen()
+	if _, present, err := s.LoadDailyEnergy(day); err != nil || present {
+		t.Fatal("source day still cached under the stricter rule", err)
+	}
+	if _, present, err := s.LoadDailyEnergy(old); err != nil || !present {
+		t.Fatal("day before source history was recounted", err)
+	}
+	if err := s.SaveDailyEnergyForSourceMode(day, stale, true); err != nil {
+		t.Fatal(err)
+	}
+	reopen()
+	if _, present, err := s.LoadDailyEnergy(day); err != nil || !present {
+		t.Fatal("recount ran twice", err)
+	}
+}

go/internal/state/history_source_bridge_test.go

@@ -42,11 +42,13 @@ func TestSourceLoadDaySurvivesShortGap(t *testing.T) {
 		{name: "PV unknown 145 s", receipt: unknownFor(MissingPV, 145000), importWh: 2000},
 		{name: "EV unknown 5 min", receipt: unknownFor(MissingEV, 300000), importWh: 2000},
 		{name: "3 min restart", stopMS: 180000},
+		// Each meter measured its own time. Only the 5 s between them is
+		// unknown, under 1 % of the period, and counts as nothing.
 		{name: "meter replaced", receipt: func(off int64, v *HistorySources) {
 			if off > 0 {
 				v.Topology = "new meter+charger"
 			}
-		}},
+		}, known: true, loadWh: 2000 - 1000*5.0/3600, importWh: 2000 - 1000*5.0/3600},
 	} {
 		t.Run(c.name, func(t *testing.T) {
 			s := sourceStore(t)

go/internal/state/history_sqlite.go

@@ -153,6 +153,9 @@ func (s *Store) openHistory() error {
 			return err
 		}
 	}
+	if err := s.recountSourceDays(context.Background()); err != nil {
+		return err
+	}
 	// A process stop has no continuous sampling proof, even when it was brief.
 	// Keep completed energy and quality. Source history may bridge a brief
 	// stop only as a marked estimate; the older paths never bridge it.