External sites update in place instead of reloading the whole page

While a scan or a telemetry read was running, the site list, a site's page
and a scan's page reloaded themselves every fifteen seconds, which jumped the
page and lost what was ticked.  They now fetch themselves in the background
and swap only the parts that change: on a site, the scanning banner, the
counters, the scan list and the AI read; on the list, each site's telemetry
and scan status and the counters; on a scan, its status, counts and what has
run.  When nothing is running any more the page reloads once, so the results,
the screenshot and the buttons come in.  Nothing changes while a dialog is
open, and Stop and Delete keep asking before they act.

Author: hackrange

app/web/static/external.js

@@ -226,18 +226,66 @@
   Array.prototype.forEach.call(pickers, setup);
 })();
 
-/* Pages that are waiting on something reload themselves, but never while a
- * dialog is open: that would throw away whatever somebody was typing. */
+/* Pages that are waiting on something keep themselves up to date without
+ * reloading.  Every few seconds the page is fetched again in the background
+ * and only the parts marked data-live are swapped for their new versions, so
+ * the rest of the page, what is ticked and where it is scrolled to, stay put.
+ *
+ * When the new copy says nothing is running any more, the page reloads once,
+ * so the finished results, the screenshot and the buttons all come in.  It
+ * never touches the page while a dialog is open. */
 (function () {
   "use strict";
   var meta = document.querySelector('meta[name="gcr-refresh"]');
   if (!meta) { return; }
   var seconds = Math.max(5, parseInt(meta.content, 10) || 15);
-  setInterval(function () {
-    var open = Array.prototype.some.call(
+  var busy = false;
+
+  function dialogOpen() {
+    return Array.prototype.some.call(
       document.querySelectorAll(".modal-backdrop"), function (n) { return !n.hidden; });
-    if (!open && !document.hidden) { window.location.reload(); }
-  }, seconds * 1000);
+  }
+
+  function swap(fresh) {
+    document.querySelectorAll("[data-live]").forEach(function (old) {
+      var key = old.getAttribute("data-live");
+      var next = fresh.querySelector('[data-live="' + CSS.escape(key) + '"]');
+      if (!next || next.innerHTML === old.innerHTML) { return; }
+      old.innerHTML = next.innerHTML;
+      old.className = next.className;
+      /* What came in was not there when the page set up its confirmations. */
+      old.querySelectorAll("form[data-confirm]").forEach(function (f) { f.setAttribute("data-swapped", "1"); });
+    });
+  }
+
+  function tick() {
+    if (busy || document.hidden || dialogOpen()) { return; }
+    busy = true;
+    fetch(window.location.pathname + window.location.search,
+          {credentials: "same-origin", headers: {"X-Requested-With": "live"}})
+      .then(function (r) { if (!r.ok) { throw new Error(r.status); } return r.text(); })
+      .then(function (html) {
+        var fresh = new DOMParser().parseFromString(html, "text/html");
+        if (!fresh.querySelector('meta[name="gcr-refresh"]')) {
+          window.location.reload();
+          return;
+        }
+        swap(fresh);
+      })
+      .catch(function () { /* try again next time */ })
+      .then(function () { busy = false; });
+  }
+
+  /* The confirmation a swapped in Stop or Delete button needs. */
+  document.addEventListener("submit", function (event) {
+    var form = event.target;
+    if (form.getAttribute && form.getAttribute("data-swapped") === "1" &&
+        !window.confirm(form.getAttribute("data-confirm"))) {
+      event.preventDefault();
+    }
+  }, true);
+
+  setInterval(tick, seconds * 1000);
 })();
 
 /* The list: tick all, a count of what is ticked, actions that need a tick,

app/web/templates/external.html

@@ -1,7 +1,7 @@
 {% extends "base.html" %}
 {% from "_external_fields.html" import detail_fields %}
 {% block title %}External sites · Git Code Review{% endblock %}
-{% block head %}{% if running or scanning %}<meta name="gcr-refresh" content="15">{% endif %}{% endblock %}
+{% block head %}{% if running or scanning %}<meta name="gcr-refresh" content="10">{% endif %}{% endblock %}
 {% block content %}
 <div class="page-head">
   <div><h1>External sites</h1>
@@ -30,7 +30,7 @@
 </div>
 
 {% set here = base ~ '/external' %}
-<div class="stats">
+<div class="stats" data-live="list-stats">
   <div class="stat"><div class="n">{{ totals.sites }}</div><div class="l">Sites</div></div>
   <div class="stat"><div class="n">{{ totals.live }}</div><div class="l">Answering</div></div>
   <a class="stat warn" href="{{ here }}?dast=never"><div class="n">{{ totals.never }}</div><div class="l">Never scanned</div></a>
@@ -39,8 +39,10 @@
   <a class="stat" href="{{ here }}?scope=mapped"><div class="n">{{ totals.mapped }}</div><div class="l">Mapped to code</div></a>
   <a class="stat" href="{{ here }}?scope=no-scan"><div class="n">{{ totals.no_scan }}</div><div class="l">No scan</div></a>
 </div>
-{% if running %}<div class="notice info"><div>Reading telemetry for {{ running }} site{{ '' if running == 1 else 's' }}.  This page refreshes until they finish.</div></div>{% endif %}
-{% if scanning %}<div class="notice info"><div>Scanning {{ scanning }} site{{ '' if scanning == 1 else 's' }} now.  {{ max_concurrent }} engine scan{{ '' if max_concurrent == 1 else 's' }} run at a time, set in Settings; the rest wait their turn.  This page refreshes until they finish.</div></div>{% endif %}
+<div data-live="list-notices">
+{% if running %}<div class="notice info"><div>Reading telemetry for {{ running }} site{{ '' if running == 1 else 's' }}.  The list updates as they finish.</div></div>{% endif %}
+{% if scanning %}<div class="notice info"><div>Scanning {{ scanning }} site{{ '' if scanning == 1 else 's' }} now.  {{ max_concurrent }} engine scan{{ '' if max_concurrent == 1 else 's' }} run at a time, set in Settings; the rest wait their turn.  The list updates as they run.</div></div>{% endif %}
+</div>
 
 <form method="get" action="{{ here }}" class="filters">
   <div class="field" style="min-width:240px"><label for="q">Search</label>
@@ -103,7 +105,7 @@
               {% if a.external_auth_needed %}<span class="badge info" title="External auth needed">ext auth</span>{% endif %}
               {% if a.whitelist_needed %}<span class="badge pending" title="Whitelist needed">whitelist</span>{% endif %}</div>
             {% if a.executive_owner or a.technical_owner %}<div class="small faint">{{ a.executive_owner }}{% if a.executive_owner and a.technical_owner %}; {% endif %}{{ a.technical_owner }}</div>{% endif %}</td>
-          <td class="small">
+          <td class="small" data-live="tele-{{ a.id }}">
             {% if a.telemetry_status == 'running' %}<span class="badge pending">reading</span>
             {% elif not a.telemetry_at %}<span class="faint">not read yet</span>
             {% else %}
@@ -117,7 +119,7 @@
               <div class="faint">read {{ a.telemetry_at | dt('%Y-%m-%d') }}</div>
             {% endif %}
             {% if a.telemetry_status == 'failed' %}<div class="faint" title="{{ a.telemetry_error }}">last read failed</div>{% endif %}</td>
-          <td class="small">
+          <td class="small" data-live="scan-{{ a.id }}">
             {% if r.scan %}
               <div style="margin-bottom:.35rem"><a class="badge pending" href="{{ base }}/external/{{ a.id }}/scans/{{ r.scan.id }}">{{ 'queued' if r.scan.status == 'queued' else 'scanning' }}{% if r.scan.profile %}, {{ r.scan.profile }}{% endif %}</a>
                 {% if r.scan.step %}<div class="faint">{{ r.scan.step }}</div>{% endif %}

app/web/templates/external_scan.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 {% block title %}Scan of {{ asset.host }} · Git Code Review{% endblock %}
-{% block head %}{% if busy %}<meta name="gcr-refresh" content="15">{% endif %}{% endblock %}
+{% block head %}{% if busy %}<meta name="gcr-refresh" content="8">{% endif %}{% endblock %}
 {% block content %}
 {% set here = base ~ '/external/' ~ asset.id ~ '/scans/' ~ scan.id %}
 <div class="page-head">
@@ -34,19 +34,22 @@
   </div>
 </div>
 
+<div data-live="scan-status">
 {% if scan.status in ('queued','running') %}
-<div class="notice info"><div>{{ scan.status | capitalize }}{% if scan.step %}: {{ scan.step }}{% endif %}.  This page refreshes while it runs.</div></div>
+<div class="notice info"><div>{{ scan.status | capitalize }}{% if scan.step %}: {{ scan.step }}{% endif %}.  This updates as it runs.</div></div>
 {% elif scan.status == 'failed' %}
 <div class="notice bad"><div>{{ scan.error or 'It failed.' }}</div></div>
 {% endif %}
+</div>
 
-<div class="stats">
+<div class="stats" data-live="scan-stats">
   <div class="stat {{ 'bad' if view.severity in ('critical','high') else ('ok' if scan.status == 'done' and view.severity in ('', 'none') else '') }}"><div class="n" style="{{ 'font-size:1.1rem' if view.severity in ('', 'none') }}">{% if scan.status != 'done' %}&ndash;{% elif view.severity in ('', 'none') %}no findings{% else %}{{ view.severity }}{% endif %}</div><div class="l">Worst found</div></div>
   {% for level in ['critical','high','medium','low','info'] %}
   <a class="stat {{ 'bad' if level in ('critical','high') and view.counts[level] else '' }}" href="{{ here }}?sev={{ level }}"><div class="n">{{ view.counts[level] }}</div><div class="l">{{ level }}</div></a>
   {% endfor %}
 </div>
 
+<div data-live="scan-engines">
 {% if view.engines %}
 <div class="card">
   <div class="card-head">What ran</div>
@@ -63,6 +66,7 @@
 </div>
 {% endif %}
 
+</div>
 {% set ai = view.ai %}
 {% if ai.status %}
 <div class="card">

app/web/templates/external_site.html

@@ -1,7 +1,7 @@
 {% extends "base.html" %}
 {% from "_external_fields.html" import detail_fields %}
 {% block title %}{{ asset.host }} · Git Code Review{% endblock %}
-{% block head %}{% if busy %}<meta name="gcr-refresh" content="15">{% endif %}{% endblock %}
+{% block head %}{% if busy %}<meta name="gcr-refresh" content="8">{% endif %}{% endblock %}
 {% block content %}
 {% set tele_headers = tele.headers or {} %}
 <div class="page-head">
@@ -27,10 +27,11 @@
   </div>
 </div>
 
+<div data-live="site-notices">
 {% if active_scan %}
 <div class="notice info"><div style="display:flex;gap:.6rem;align-items:center;flex-wrap:wrap">
   <span><strong>{{ 'Queued' if active_scan.status == 'queued' else 'Scanning' }}</strong>{% if active_scan.profile %}, {{ active_scan.profile }} profile{% endif %}
-    on {{ active_scan.server_name }}{% if active_scan.step %}: {{ active_scan.step }}{% endif %}.  This page refreshes until it ends.</span>
+    on {{ active_scan.server_name }}{% if active_scan.step %}: {{ active_scan.step }}{% endif %}.  This updates as it runs.</span>
   <a class="btn sm" href="{{ base }}/external/{{ asset.id }}/scans/{{ active_scan.id }}">Watch it</a>
   {% if active_scan.source != 'recorded' %}
   <form method="post" action="{{ base }}/external/{{ asset.id }}/scans/{{ active_scan.id }}/stop" style="display:inline"
@@ -40,8 +41,9 @@
     <button class="btn sm danger" type="submit">Stop</button></form>{% endif %}
 </div></div>
 {% endif %}
-{% if asset.telemetry_status == 'running' %}<div class="notice info"><div>Reading telemetry.  This page refreshes until it is done.</div></div>{% endif %}
+{% if asset.telemetry_status == 'running' %}<div class="notice info"><div>Reading telemetry.  The page shows it when it is done.</div></div>{% endif %}
 {% if asset.telemetry_status == 'failed' %}<div class="notice bad"><div>The last telemetry read failed: {{ asset.telemetry_error }}</div></div>{% endif %}
+</div>
 
 {# The front page first, large, the way the site report opened each host. #}
 <div class="card">
@@ -60,7 +62,7 @@
   </div>
 </div>
 
-<div class="stats">
+<div class="stats" data-live="site-stats">
   <div class="stat"><div class="n">{% if asset.grade and asset.grade != 'N/A' %}<span class="grade {{ asset.grade }}">{{ asset.grade }}</span>{% else %}&ndash;{% endif %}</div><div class="l">Web hygiene{% if asset.score %} ({{ asset.score }}){% endif %}</div></div>
   <div class="stat {{ 'bad' if asset.last_dast_severity in ('critical','high') else ('ok' if asset.last_dast_at and asset.last_dast_severity in ('', 'none') else '') }}"><div class="n" style="{{ 'font-size:1.1rem' if not asset.last_dast_at or asset.last_dast_severity in ('', 'none') }}">{% if not asset.last_dast_at %}never{% elif asset.last_dast_severity in ('', 'none') %}no findings{% else %}{{ asset.last_dast_severity }}{% endif %}</div><div class="l">Last dynamic scan{% if asset.last_dast_at %}, {{ asset.last_dast_at | dt('%Y-%m-%d') }}{% endif %}</div></div>
   <div class="stat {{ 'bad' if risk.level in ('critical','high') else ('warn' if risk.level == 'medium' else '') }}"><div class="n">{{ risk.level }}</div><div class="l">Combined risk</div></div>
@@ -156,7 +158,7 @@
   </div>
 
   <div>
-    <div class="card">
+    <div class="card" data-live="site-ai">
       <div class="card-head">AI risk read
         {% if asset.ai_input_tokens or asset.ai_output_tokens %}<span class="faint small" style="font-weight:400">{{ asset.ai_input_tokens | thousands }} in, {{ asset.ai_output_tokens | thousands }} out{% if asset.ai_cost %}, {{ asset.ai_cost }}{% endif %} so far, counted on the <a href="{{ base }}/ai?tab=cost">AI cost page</a></span>{% endif %}</div>
       <div class="card-body">
@@ -176,7 +178,7 @@
       </div>
     </div>
 
-    <div class="card">
+    <div class="card" data-live="site-scans">
       <div class="card-head">Dynamic scans
         {% if scan_total %}<span class="faint small" style="font-weight:400">{{ scan_total }} in all, newest first</span>{% endif %}</div>
       {% if scan_views %}