From ee5ac85ef5d6a0c448b6eececc41de99018c6f63 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 19 Oct 2026 20:13:35 +0200 Subject: [PATCH] fix(history): keep Home use when at most 1 % of the day is unknown Since #1741 any unknown time made the whole day's load_wh null. Before source history a day summed what it had, and nobody missed a number. A period now keeps its Home use, marked EstimatedLoad, when at most 1 % of its time is unknown. The total is the sum of the fully known source intervals; unknown time counts as nothing, so the number never changes, only whether it is shown. A negative known total stays unknown. Other flows keep their own unknowns. A replaced meter no longer hides the day either: each meter measured its own time and only the snapshot interval between them is unknown, so the bridge test's meter-replaced case now expects an estimate. Closed days are cached. A one-time history migration deletes cached days from the source cutover on, so days closed under the stricter rule are recounted. Relates to #1750. Co-Authored-By: Claude Opus 5.5 --- .changeset/home-use-unknown-share.md | 5 + docs/site-convention.md | 4 + go/internal/state/history_dashboard.go | 51 ++++++- .../state/history_home_use_share_test.go | 125 ++++++++++++++++++ .../state/history_source_bridge_test.go | 4 +- go/internal/state/history_sqlite.go | 3 + 6 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 .changeset/home-use-unknown-share.md create mode 100644 go/internal/state/history_home_use_share_test.go diff --git a/.changeset/home-use-unknown-share.md b/.changeset/home-use-unknown-share.md new file mode 100644 index 00000000..6bafd655 --- /dev/null +++ b/.changeset/home-use-unknown-share.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Show the day's Home use again when at most 1 % of the day is unknown, as before source history. The total sums the known time, marked as an estimate. Days cached as unknown since the change are recounted once after the update. diff --git a/docs/site-convention.md b/docs/site-convention.md index a699e798..574b4a35 100644 --- a/docs/site-convention.md +++ b/docs/site-convention.md @@ -62,6 +62,10 @@ marked estimate: measured sources keep their snapshots, and an unknown source or a stopped Core holds the last known values. Longer gaps stay unknown. This is an estimate from received measurements, not a correction of device clocks. +A day or other period keeps its Home use when at most 1 % of its time is +unknown. The total is then the sum of the known intervals, marked as an +estimate; unknown time counts as nothing. Other flows keep their own unknowns. + In self-consumption mode, the controller drives the site meter toward the grid target (normally 0 W): it charges from live surplus and may discharge to cover local load. It must not intentionally export via the battery; export diff --git a/go/internal/state/history_dashboard.go b/go/internal/state/history_dashboard.go index 01230129..77af75bd 100644 --- a/go/internal/state/history_dashboard.go +++ b/go/internal/state/history_dashboard.go @@ -273,12 +273,16 @@ func (s *Store) walkSiteEnergy(ctx context.Context, from, to int64, visit func(s return rows.Err() } +// maxUnknownHomeUsePercent is the share of a period's time that may be +// unknown while its Home use stays known as an estimate. +const maxUnknownHomeUsePercent = 1 + func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEnergy, error) { var out DayEnergy var originalLoad float64 var originalMissing HistoryMissing var estimatedMS, coveredMS int64 - var sourceMS int64 + var sourceMS, knownMS int64 var first, last int64 err := s.walkSiteEnergy(ctx, from, to, func(r siteEnergyRow) error { if first == 0 || r.first < first { @@ -295,6 +299,9 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner coveredMS += r.covered if r.source { sourceMS += r.covered + if r.missing&MissingPower == 0 { + knownMS += r.covered + } } if r.missing&^MissingSoC == MissingLoad|BalanceMismatch || r.missing&^MissingSoC == MissingLoad|MissingEV { if r.estimateCovered > 0 && r.estimateCovered == r.covered { @@ -314,7 +321,8 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner out.LoadWh = originalLoad out.Missing = originalMissing } - if out.LoadWh < 0 { + negative := out.LoadWh < 0 + if negative { out.LoadWh = 0 out.Missing |= MissingLoad } @@ -333,6 +341,45 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner if coveredMS < last-first { out.Missing |= MissingLoad } + // A short unknown share keeps Home use as an estimate: the sum of the + // fully known intervals, with unknown time counted as nothing, as + // before source history. Other flows keep their own unknowns. + span := max(last-first, expected) + if !negative && sourceMS == coveredMS && knownMS > 0 && (span-knownMS)*100 <= span*maxUnknownHomeUsePercent { + out.Missing = out.Missing&^MissingLoad | EstimatedLoad + } } return out, err } + +// homeUseShareRecount names the one-time recount of cached source days, so +// days closed under the stricter rule show their Home use too. +const homeUseShareRecount = "home-use-unknown-share-v1" + +func (s *Store) recountSourceDays(ctx context.Context) error { + tx, err := s.history.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var done bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM history_migrations WHERE name=?)`, homeUseShareRecount).Scan(&done); err != nil || done { + return err + } + var low string + err = tx.QueryRowContext(ctx, `SELECT date(CAST(value AS INTEGER)/1000,'unixepoch','-1 day') FROM history_source_cursor WHERE key='from'`).Scan(&low) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if low != "" { + for _, table := range []string{"energy_daily", "energy_daily_partial", "history_daily_quality"} { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE day>=?`, low); err != nil { + return err + } + } + } + if _, err := tx.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES(?)`, homeUseShareRecount); err != nil { + return err + } + return tx.Commit() +} diff --git a/go/internal/state/history_home_use_share_test.go b/go/internal/state/history_home_use_share_test.go new file mode 100644 index 00000000..75e81aae --- /dev/null +++ b/go/internal/state/history_home_use_share_test.go @@ -0,0 +1,125 @@ +package state + +import ( + "testing" + "time" +) + +// Six hours of a 1000 W house, sampled every 10 s, with one disturbance three +// hours in. Before source history a day summed what it had; since #1741 any +// unknown time made the whole day's Home use unknown. At most 1 % unknown time +// (3.6 minutes here) keeps it as an estimate, with unknown time counted as +// nothing. Other flows keep their own unknowns. +func TestHomeUseSurvivesSmallUnknownShare(t *testing.T) { + const tick, hour = int64(10000), int64(3600000) + for _, c := range []struct { + name string + // skip leaves the writer without snapshots for this long. + skip int64 + missing HistoryMissing + // missingMS keeps the flag on this long; the interval after it and + // every partly unknown minute stay unknown too. + missingMS int64 + gridW float64 + known bool + loadWh float64 + }{ + {name: "no disturbance", known: true, loadWh: 6000}, + {name: "battery unknown 2 min", missing: MissingBattery, missingMS: 120000, known: true, loadWh: 6000 - 3*1000.0/60}, + {name: "writer gap 3 min", skip: 180000, known: true, loadWh: 6000 - 3*1000.0/60}, + {name: "writer gap 5 min", skip: 300000}, + {name: "battery unknown 10 min", missing: MissingBattery, missingMS: 600000}, + {name: "negative known total", missing: MissingBattery, missingMS: 120000, gridW: -500}, + } { + t.Run(c.name, func(t *testing.T) { + s := sourceStore(t) + base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 24*hour + grid := 1000.0 + if c.gridW != 0 { + grid = c.gridW + } + for at := base; at <= base+6*hour; at += tick { + off := at - base - 3*hour + if off > 0 && off < c.skip { + continue + } + v := HistorySources{GridW: grid} + if off > 0 && off <= c.missingMS { + v.Missing = c.missing + } + sourceTick(t, s, at, v) + } + d, err := s.DailyEnergy(base, base+6*hour) + if err != nil { + t.Fatal(err) + } + if c.gridW == 0 { + near(t, "import Wh", d.ImportWh, 6000-1000*float64(c.skip)/3600000) + } + if got := d.Missing&MissingLoad == 0; got != c.known { + t.Fatalf("Home use known=%v, want %v (quality %s)", got, c.known, d.Missing.Quality()) + } + if (d.Missing&MissingBattery != 0) != (c.missing == MissingBattery) { + t.Fatal("battery totals changed state") + } + if !c.known { + return + } + if d.Missing&EstimatedLoad == 0 { + t.Fatal("Home use not marked as an estimate") + } + near(t, "Home use Wh", d.LoadWh, c.loadWh) + }) + } +} + +// Days cached under the stricter rule are recounted once after the update. +// Days before source history and later cache entries are left alone. +func TestSourceDaysAreRecountedOnce(t *testing.T) { + s := sourceStore(t) + const hour = int64(3600000) + base := time.Now().UTC().Truncate(24*time.Hour).UnixMilli() - 24*hour + for at := base; at <= base+hour; at += 10000 { + sourceTick(t, s, at, HistorySources{GridW: 1000}) + } + day := time.UnixMilli(base).UTC().Format("2006-01-02") + old := time.UnixMilli(base).UTC().AddDate(0, 0, -10).Format("2006-01-02") + stale := DayEnergy{Missing: MissingLoad, Intervals: 1} + for _, d := range []string{day, old} { + if err := s.SaveDailyEnergyForSourceMode(d, stale, true); err != nil { + t.Fatal(err) + } + } + if _, err := s.history.Exec(`DELETE FROM history_migrations WHERE name=?`, homeUseShareRecount); err != nil { + t.Fatal(err) + } + reopen := func() { + t.Helper() + path := s.mainDBPath + if err := s.Close(); err != nil { + t.Fatal(err) + } + var err error + if s, err = Open(path); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if err := s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + } + reopen() + if _, present, err := s.LoadDailyEnergy(day); err != nil || present { + t.Fatal("source day still cached under the stricter rule", err) + } + if _, present, err := s.LoadDailyEnergy(old); err != nil || !present { + t.Fatal("day before source history was recounted", err) + } + if err := s.SaveDailyEnergyForSourceMode(day, stale, true); err != nil { + t.Fatal(err) + } + reopen() + if _, present, err := s.LoadDailyEnergy(day); err != nil || !present { + t.Fatal("recount ran twice", err) + } +} diff --git a/go/internal/state/history_source_bridge_test.go b/go/internal/state/history_source_bridge_test.go index ea92d4d6..96a1b477 100644 --- a/go/internal/state/history_source_bridge_test.go +++ b/go/internal/state/history_source_bridge_test.go @@ -42,11 +42,13 @@ func TestSourceLoadDaySurvivesShortGap(t *testing.T) { {name: "PV unknown 145 s", receipt: unknownFor(MissingPV, 145000), importWh: 2000}, {name: "EV unknown 5 min", receipt: unknownFor(MissingEV, 300000), importWh: 2000}, {name: "3 min restart", stopMS: 180000}, + // Each meter measured its own time. Only the 5 s between them is + // unknown, under 1 % of the period, and counts as nothing. {name: "meter replaced", receipt: func(off int64, v *HistorySources) { if off > 0 { v.Topology = "new meter+charger" } - }}, + }, known: true, loadWh: 2000 - 1000*5.0/3600, importWh: 2000 - 1000*5.0/3600}, } { t.Run(c.name, func(t *testing.T) { s := sourceStore(t) diff --git a/go/internal/state/history_sqlite.go b/go/internal/state/history_sqlite.go index 2eb2be37..c5d57c25 100644 --- a/go/internal/state/history_sqlite.go +++ b/go/internal/state/history_sqlite.go @@ -153,6 +153,9 @@ func (s *Store) openHistory() error { return err } } + if err := s.recountSourceDays(context.Background()); err != nil { + return err + } // A process stop has no continuous sampling proof, even when it was brief. // Keep completed energy and quality. Source history may bridge a brief // stop only as a marked estimate; the older paths never bridge it.