Give a scanner of this site its own rate budget, and check address settings

A web scan aimed at the site arrives through the load balancer.  When the
balancer is not trusted, every caller shares its request budget, so the scan
spent it and sign in answered 429 for everyone.

Scanner addresses on the Sign in tab gives a named address its own budget, ten
times the general one and never unlimited, so a scan can finish without
swamping the app.  The balancer itself is refused there, on save and again at
request time, since exempting it would lift the limit for everybody behind it.
Ranges wider than a /24 or /64 are refused.  Load balancer addresses now
rejects anything that is not an address or range with a 400 that names it,
instead of skipping it silently.

The load balancer guide says how to set both and that 0.0.0.0 trusts nothing.

Author: hackrange

app/core/live.py

@@ -138,3 +138,8 @@ def timezone_name() -> str:
 
 def trusted_proxies() -> list:
     return list(_get("security.trusted_proxies", env_settings.trusted_proxies) or [])
+
+
+def rate_limit_exempt() -> list:
+    """Addresses the request rate limit leaves alone.  Nothing by default."""
+    return list(_get("security.rate_limit_exempt", []) or [])

app/core/settings_store.py

@@ -307,6 +307,10 @@
     "security.force_https":    (True, False, "Mark cookies secure and send HSTS. Leave on behind TLS."),
     "security.trusted_proxies": ([], False,
                                 "Addresses the load balancer connects from. X-Forwarded-For is believed from these and nowhere else."),
+    "security.rate_limit_exempt": ([], False,
+                                "Addresses that get their own, ten times wider request "
+                                "budget, such as the scanner that tests this site. Never "
+                                "the load balancer."),
 
     # ---- what the systems being scanned hold ----
     "sensitivity.labels":      (["PCI", "HIPAA", "SOX", "Customer Data", "PII",
@@ -549,7 +553,7 @@
     "oidc.default_role",
     "session.hours", "session.idle_minutes", "login.attempt_limit",
     "login.lockout_minutes", "security.force_https", "security.trusted_proxies",
-    "ui.timezone",
+    "security.rate_limit_exempt", "ui.timezone",
 ]
 
 
@@ -582,7 +586,8 @@
                  "oidc.default_role"]),
     ("Sessions", ["session.hours", "session.idle_minutes",
                   "login.attempt_limit", "login.lockout_minutes",
-                  "security.force_https", "security.trusted_proxies"]),
+                  "security.force_https", "security.trusted_proxies",
+                  "security.rate_limit_exempt"]),
     ("Display", ["ui.timezone"]),
     ("Fidelity", ["code.muted_rules"]),
     ("Sensitivity labels", ["sensitivity.labels"]),

app/web/routes_api.py

@@ -1824,6 +1824,31 @@ async def scan_one_repository(repo_id: int, request: Request,
 # ---------------------------------------------------------------------------
 # Settings
 # ---------------------------------------------------------------------------
+def _address_list(raw: str, label: str, narrow: bool = False) -> list:
+    """A box of addresses and ranges, each checked, or a 400 that names the bad one.
+
+    narrow refuses anything wider than a /24 (IPv4) or a /64 (IPv6).  An
+    exemption is meant for one scanner, not a slice of the internet.
+    """
+    import ipaddress
+
+    text = (raw or "").replace("\r", "\n").replace("\n", ",")
+    items = [item.strip() for item in text.split(",") if item.strip()]
+    if len(items) > 50:
+        raise HTTPException(status_code=400, detail=f"{label}: at most 50 entries.")
+    for item in items:
+        try:
+            network = ipaddress.ip_network(item, strict=False)
+        except ValueError:
+            raise HTTPException(status_code=400, detail=(
+                f"{label}: {item[:60]!r} is not an IP address or range."))
+        if narrow and network.prefixlen < (24 if network.version == 4 else 64):
+            raise HTTPException(status_code=400, detail=(
+                f"{label}: {item[:60]} is too wide.  Name the scanner's own "
+                "address, or a /24 at most."))
+    return items
+
+
 def _csv_list(raw: str) -> list:
     """Split a comma or newline separated box into a clean list.
 
@@ -2221,7 +2246,21 @@ def put(key, value):
             put("ias.enabled", ias_on)
 
         put("security.force_https", _checked(form, "security.force_https"))
-        put("security.trusted_proxies", _csv_list(text("security.trusted_proxies")))
+        proxies = _address_list(text("security.trusted_proxies"),
+                                "Load balancer addresses")
+        put("security.trusted_proxies", proxies)
+        # Only when the form sent it.  A page without the box must not empty it.
+        if "security.rate_limit_exempt" in form:
+            exempt = _address_list(text("security.rate_limit_exempt"),
+                                   "Scanner addresses", narrow=True)
+            from app.web.security import parse_networks
+            lbs = parse_networks(proxies or live.trusted_proxies())
+            for entry in parse_networks(exempt):
+                if any(entry.version == lb.version and entry.overlaps(lb) for lb in lbs):
+                    raise HTTPException(status_code=400, detail=(
+                        "%s is a load balancer address.  Exempting it would switch the "
+                        "rate limit off for everybody behind it." % entry))
+            put("security.rate_limit_exempt", exempt)
 
         # Check what they just asked for before it takes effect.  Turning on a
         # provider that cannot answer would lock everybody out of their own

app/web/security.py

@@ -51,6 +51,25 @@ def public_base(request: Request) -> str:
     return base
 
 
+def parse_networks(entries) -> list:
+    """Addresses and ranges from a settings list.  Anything unreadable is skipped."""
+    networks = []
+    for entry in entries or []:
+        try:
+            networks.append(ipaddress.ip_network(str(entry).strip(), strict=False))
+        except ValueError:
+            continue
+    return networks
+
+
+def address_in(address: str, networks) -> bool:
+    try:
+        parsed = ipaddress.ip_address(address)
+    except ValueError:
+        return False
+    return any(parsed in network for network in networks)
+
+
 def client_ip(request: Request) -> str:
     """The caller's address.
 
@@ -63,19 +82,10 @@ def client_ip(request: Request) -> str:
     if not proxies:
         return peer
 
-    networks = []
-    for entry in proxies:
-        try:
-            networks.append(ipaddress.ip_network(entry, strict=False))
-        except ValueError:
-            continue
+    networks = parse_networks(proxies)
 
     def trusted(address: str) -> bool:
-        try:
-            parsed = ipaddress.ip_address(address)
-        except ValueError:
-            return False
-        return any(parsed in network for network in networks)
+        return address_in(address, networks)
 
     if not trusted(peer):
         return peer
@@ -246,6 +256,25 @@ def _allowed(self, key: str, limit: int) -> bool:
         bucket.append(now)
         return True
 
+    # How many times the general budget a named scanner gets.  3000 a minute
+    # at the defaults: enough for a scan to finish, not enough to swamp a
+    # single process.
+    SCANNER_FACTOR = 10
+
+    @staticmethod
+    def _exempt(ip: str) -> bool:
+        """An address an administrator named, so a scan of this site runs whole.
+
+        Never a load balancer.  When X-Forwarded-For is not believed, every
+        caller wears the balancer's address, and exempting it would switch the
+        limiter off for the whole internet.  The save refuses that pairing, and
+        this refuses it again in case the lists were changed some other way.
+        """
+        exempt = live.rate_limit_exempt()
+        if not exempt or not address_in(ip, parse_networks(exempt)):
+            return False
+        return not address_in(ip, parse_networks(live.trusted_proxies()))
+
     async def dispatch(self, request: Request, call_next):
         path = request.url.path
         if path.endswith(("/healthz", "/readyz")):
@@ -257,7 +286,12 @@ async def dispatch(self, request: Request, call_next):
         # same budget as somebody clicking around the pages would lock the
         # pages out.  Its own bucket, and a wider one.
         is_api = "/api/v1/" in path
-        if is_api:
+        if self._exempt(ip):
+            # A scanner an administrator named gets its own, much wider
+            # budget, but still a budget.  Unlimited would trade a locked out
+            # sign in page for a scanner that can flood the app itself.
+            limit, kind = self.general * self.SCANNER_FACTOR, "scan"
+        elif is_api:
             limit, kind = self.general * 4, "api"
         elif is_auth_path:
             limit, kind = self.auth, "auth"

app/web/templates/settings.html

@@ -422,6 +422,18 @@
         it came from the balancer, so one person getting their password wrong
         locks out everybody.</span>
     </label>
+    <label class="wide">Scanner addresses
+      <input type="text" name="security.rate_limit_exempt"
+             value="{{ values['security.rate_limit_exempt'] | join(', ') }}"
+             placeholder="203.0.113.10">
+      <span class="faint small">For a scanner testing this site.  It gets its
+        own budget, ten times the normal one, and never uses up anybody
+        else's.  When this
+        site scans itself, that is this server's public address.  Set the load
+        balancer addresses above first, or the scanner looks like the balancer
+        and gets limited along with everyone else.  The balancer itself is
+        refused here.</span>
+    </label>
     <label class="check">
       <input type="checkbox" name="security.force_https" value="1"
              {{ 'checked' if values['security.force_https'] }}> Behind TLS

documentation/CHANGELOG.md

@@ -6,6 +6,16 @@ moved.
 
 ## 2026-11-10
 
+**A scan of this site no longer locks everybody out.**  A web scan aimed at the
+site came in through the load balancer, and when the balancer was not listed as
+trusted every person shared the balancer's request budget, so the scan used it
+up and sign in answered 429 for everyone.  The **Load balancer addresses** box
+now refuses anything that is not an address or range, and a new **Scanner
+addresses** box on the Sign in tab gives a named scanner its own budget, ten
+times the normal one and never unlimited.  The balancer itself cannot be listed
+there.  The load balancer guide says how to set both, and that `0.0.0.0` trusts
+nothing.
+
 **Sites link to the code that serves them, from the code itself.**  Every scan now
 reads each repository's default branch for the hosts it answers on: ingress and
 route hosts, Helm values, nginx, Apache, Traefik and Caddy, a `CNAME` file, CDN

documentation/load-balancer.md

@@ -43,6 +43,25 @@ sees, list both. Extra entries cost nothing.
 
 CIDR works: `GCR_TRUSTED_PROXIES=10.0.0.0/24`.
 
+`0.0.0.0` is not a wildcard.  It is one address that never connects, so it
+trusts nothing and puts everybody back in one shared budget.
+
+The same list is on Settings, Sign in tab, under **Load balancer addresses**.
+A value saved there wins over `.env` and takes effect within about 15 seconds,
+with no restart, so running scans keep going.
+
+### Scanning this site through the balancer
+
+A web scan of this site, including one it runs on itself, sends thousands of
+requests a minute.  With the balancer trusted, those count against the
+scanner's own address and nobody else's.  To give the scanner room to finish,
+put its address under **Scanner addresses** on the same tab.  When the site
+scans itself, that is the server's own public address.  A listed address gets
+ten times the normal budget, never an unlimited one, so a scan cannot swamp
+the app either.  Ranges wider than a /24 are refused, and so is any balancer
+address, because exempting the balancer would lift the limit for everybody
+behind it.
+
 ### TLS stops at the balancer
 
 The app speaks plain HTTP. Keep `GCR_FORCE_HTTPS=true`, which marks session