From 4093375204d9477dadaa50a412139a1dfb821268 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 8 Sep 2026 14:02:11 +0100 Subject: [PATCH 1/4] Retain unmatched accounting records in site sync view Use an outer join when combining summary and sync record data so that rows are retained even when they only exist in one source. This exposes a wider range of accounting and publication issues, including records that are missing from either VSuperSummaries or VSyncRecords. --- monitoring/db_update_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index bf90547..4d2e1c6 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -286,7 +286,7 @@ def refresh_gridsitesync(): df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], - how='inner' + how='outer' ) fetchset = df_all.to_dict('index') From cfe063cfa9b9bcb0d81e9d10d9e1e621b7b9750e Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 8 Sep 2026 16:46:46 +0100 Subject: [PATCH 2/4] Allow missing dates for unmatched sync records Some records can exist in VSyncRecords without a corresponding published summary record. Allow RecordStart and RecordEnd to be null so these records can be displayed. --- monitoring/publishing/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 27de057..b8deb7a 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -30,8 +30,8 @@ class GridSiteSync(models.Model): YearMonth = models.CharField(max_length=255) Year = models.IntegerField() Month = models.IntegerField() - RecordStart = models.DateTimeField() - RecordEnd = models.DateTimeField() + RecordStart = models.DateTimeField(null=True) + RecordEnd = models.DateTimeField(null=True) RecordCountPublished = models.IntegerField() RecordCountInDb = models.IntegerField() SyncStatus = models.CharField(max_length=255) From a1bcad67eb0cf70c7083296dbff36caeb659e689 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 8 Sep 2026 16:51:09 +0100 Subject: [PATCH 3/4] Handle NaN values in merged site sync data Convert Pandas NaN and NaT values to None for dates and zero for counts before storing records and add a helper function to centralise the conversion logic. This prevents invalid values being passed to Django model fields and ensures unmatched records are displayed correctly. --- monitoring/db_update_sqlite.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 4d2e1c6..1d32737 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -145,8 +145,9 @@ def determine_sync_status(f): RecordCountPublished = f.get("RecordCountPublished") RecordCountInDb = f.get("RecordCountInDb") - # catches None or zero - if not RecordCountPublished or not RecordCountInDb: + # catches None or nan or zero + if (not RecordCountPublished or not RecordCountInDb or + pd.isna(RecordCountPublished) or pd.isna(RecordCountInDb)): return "WARNING [ Invalid record counts ]" diff = abs(RecordCountPublished - RecordCountInDb) @@ -297,10 +298,10 @@ def refresh_gridsitesync(): # Combined primary keys outside the default dict GridSiteSync.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), - 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), + 'RecordStart': none_if_missing(f.get("RecordStart")), + 'RecordEnd': none_if_missing(f.get("RecordEnd")), + 'RecordCountPublished': zero_if_missing(f.get("RecordCountPublished")), + 'RecordCountInDb': zero_if_missing(f.get("RecordCountInDb")), 'SyncStatus': f.get("SyncStatus"), }, YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), @@ -539,6 +540,12 @@ def none_if_missing(value): """ return None if pd.isna(value) else value +def zero_if_missing(value): + """ + Return zero when the value is missing (NaN/NaT), otherwise return it unchanged. + """ + return 0 if pd.isna(value) else value + if __name__ == "__main__": log.info('=====================') From 9cc8e63d26aafefcd6b081bd79d2cfcb700fdf65 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 9 Sep 2026 11:15:40 +0100 Subject: [PATCH 4/4] Treat summary-only records as valid accounting data Avoid flagging records as invalid solely because sync record is unavailable. Fix indentation --- monitoring/db_update_sqlite.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 1d32737..4535476 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -146,10 +146,12 @@ def determine_sync_status(f): RecordCountInDb = f.get("RecordCountInDb") # catches None or nan or zero - if (not RecordCountPublished or not RecordCountInDb or - pd.isna(RecordCountPublished) or pd.isna(RecordCountInDb)): + if not RecordCountPublished or pd.isna(RecordCountPublished): return "WARNING [ Invalid record counts ]" + if not RecordCountInDb or pd.isna(RecordCountInDb): + return "OK [ No matching sync record ]" + diff = abs(RecordCountPublished - RecordCountInDb) rel_diff1 = diff/RecordCountInDb rel_diff2 = diff/RecordCountPublished