Add exposure check config: a non allow listed proxy, and validate proxy

The DAST admin settings gain a proxy that leaves from an IP the targets do
not allow list, so a site can be seen the way the public sees it, next to the
scanner's own allow listed IP.  The proxy address carries basic auth as
user:password@host and is held encrypted.

Validate proxy asks an IP echo service what address this server and the proxy
each leave from, so an admin can confirm the proxy works and is a different
vantage.  The network calls run off the event loop.

A toggle is added to compare each site from the non allow listed IP on a
refresh.  The second screenshot and the open to the world posture it drives
come in the next change; the toggle has no effect until then.

Author: hackrange

app/core/external.py

@@ -128,6 +128,43 @@ def options(db) -> dict:
             "whitebox": bool(s.get(db, "external.whitebox"))}
 
 
+def exposure_proxy(db) -> str:
+    """The saved non allow listed proxy, decrypted, or empty."""
+    from app.core import settings_store as s
+
+    return (s.get(db, "dast.unwhitelisted_proxy") or "").strip()
+
+
+def egress_ip(proxy: str = "", echo_url: str = "", allow_private: bool = False) -> dict:
+    """Ask a service that echoes the caller's IP what address a request leaves
+    from, optionally through a proxy.  -> {ok, ip, detail}."""
+    import ipaddress
+
+    from app.core import external_probe
+
+    url = (echo_url or "https://api.ipify.org").strip()
+    try:
+        with external_probe.via_proxy(proxy or ""):
+            got = external_probe.fetch(url, allow_private=allow_private, body_limit=256)
+    except Exception as exc:  # noqa: BLE001 - every failure is an answer
+        return {"ok": False, "ip": "", "detail": f"{type(exc).__name__}: {str(exc)[:160]}"}
+    body = (got.get("body") or "").strip()
+    candidate = body.split()[0] if body else ""
+    try:
+        return {"ok": True, "ip": str(ipaddress.ip_address(candidate)), "detail": ""}
+    except ValueError:
+        detail = got.get("error") or (f"the service answered {got['status']}: {body[:80]}"
+                                      if got.get("status") else "no answer from the echo service")
+        return {"ok": False, "ip": "", "detail": detail}
+
+
+def validate_proxy(echo_url: str, proxy: str, allow_private: bool = False) -> dict:
+    """The egress IP this server shows directly, and the one the proxy shows.
+    -> {direct: {...}, through: {...}}."""
+    return {"direct": egress_ip("", echo_url, allow_private),
+            "through": egress_ip(proxy, echo_url, allow_private)}
+
+
 def data_dir() -> Path:
     path = app_settings.data_dir / "external"
     path.mkdir(parents=True, exist_ok=True)

app/core/settings_store.py

@@ -388,6 +388,25 @@
                                 "can be made to fetch a URL will hit it here, which proves "
                                 "server side request forgery.  Leave empty to switch the "
                                 "out of band checks off; they are then skipped."),
+    "dast.exposure_check":     (False, False,
+                                "On a telemetry refresh, also load each site through the "
+                                "non allow listed proxy below and take a second "
+                                "screenshot, to tell a site that only its allow listed "
+                                "callers reach from one that is open to the world."),
+    "dast.whitelisted_ip":     ("", False,
+                                "This scanner's own public IP, the one a target's allow "
+                                "list lets in.  Shown for reference when the proxy is "
+                                "validated.  Optional."),
+    "dast.unwhitelisted_proxy": ("", True,
+                                "A proxy that leaves from an IP the targets do not allow "
+                                "list, so a site can be seen as the public sees it.  A "
+                                "Squid address such as http://proxy.host:3128, or "
+                                "http://user:password@proxy.host:3128 when it needs a "
+                                "sign in.  Held encrypted."),
+    "dast.ip_echo_url":        ("https://api.ipify.org", False,
+                                "A service that answers with the caller's IP, used by "
+                                "Validate proxy to show the address the proxy leaves "
+                                "from.  It must return the plain IP."),
 
     # ---- whose sites these are ----
     "web.company_domains":     ([], False,

app/web/routes_external.py

@@ -1555,6 +1555,10 @@ def _rule_context(db, user):
                    scan_profiles=saved, profile_rulesets=profile_rulesets,
                    available_rulesets=_available_rulesets(db),
                    oob_url=settings_store.get(db, "dast.oob_url") or "",
+                   whitelisted_ip=settings_store.get(db, "dast.whitelisted_ip") or "",
+                   ip_echo_url=settings_store.get(db, "dast.ip_echo_url") or "https://api.ipify.org",
+                   exposure_check=bool(settings_store.get(db, "dast.exposure_check")),
+                   has_exposure_proxy=bool((settings_store.get(db, "dast.unwhitelisted_proxy") or "").strip()),
                    drafts=drafts, drafting=any(d.status == "working" for d in drafts),
                    values=settings_store.all_values(db), ias=_ias_context(),
                    ai_on=ai.enabled(ai.config(db)))
@@ -1658,18 +1662,74 @@ async def save_dast_settings(request: Request, user: User = Depends(current_user
     from app.core import settings_store
 
     oob = str(form.get("oob_url") or "").strip()[:500]
-    if oob and not oob.startswith(("http://", "https://")):
-        return redirect(_back(back, "The callback address must start with http:// "
-                              "or https://, or be left empty.", "error"))
+    whitelisted = str(form.get("whitelisted_ip") or "").strip()[:60]
+    echo = str(form.get("ip_echo_url") or "").strip()[:500]
+    exposure = _checked(form, "exposure_check")
+    proxy_typed = str(form.get("unwhitelisted_proxy") or "").strip()[:1000]
+    proxy_clear = _checked(form, "proxy_clear")
+    for label, value in (("callback address", oob), ("IP echo service", echo),
+                         ("proxy address", proxy_typed)):
+        if value and not value.startswith(("http://", "https://")):
+            return redirect(_back(back, f"The {label} must start with http:// or "
+                                  "https://, or be left empty.", "error"))
     with session_scope() as db:
         _guard(db, user)
         settings_store.set_value(db, "dast.oob_url", oob, user.username)
+        settings_store.set_value(db, "dast.whitelisted_ip", whitelisted, user.username)
+        settings_store.set_value(db, "dast.ip_echo_url",
+                                 echo or "https://api.ipify.org", user.username)
+        settings_store.set_value(db, "dast.exposure_check", exposure, user.username)
+        if proxy_clear:
+            settings_store.set_value(db, "dast.unwhitelisted_proxy", "", user.username)
+        elif proxy_typed:
+            settings_store.set_value(db, "dast.unwhitelisted_proxy", proxy_typed, user.username)
         record(db, "dast.settings_saved", actor=user.username,
                detail="oob_url set" if oob else "oob_url cleared",
                source_ip=client_ip(request))
     return redirect(_back(back, "Saved DAST settings."))
 
 
+@router.post("/dast-admin/validate-proxy")
+async def validate_proxy(request: Request, user: User = Depends(current_user)):
+    """Show the egress IP this server presents directly and through the saved
+    non allow listed proxy, so an admin can confirm the proxy works and leaves
+    from a different address.  The network calls run off the event loop."""
+    from starlette.concurrency import run_in_threadpool
+
+    from app.core import settings_store
+
+    await check_csrf(request)
+    back = "/dast-admin#tab-settings"
+    with session_scope() as db:
+        _guard(db, user)
+        proxy = external.exposure_proxy(db)
+        echo = settings_store.get(db, "dast.ip_echo_url") or "https://api.ipify.org"
+        allow_private = external.options(db)["allow_private"]
+    if not proxy:
+        return redirect(_back(back, "Set and save the non allow listed proxy first.", "warn"))
+    got = await run_in_threadpool(external.validate_proxy, echo, proxy, allow_private)
+    direct, through = got["direct"], got["through"]
+    if direct["ok"] and through["ok"]:
+        if direct["ip"] == through["ip"]:
+            msg = (f"Direct egress IP {direct['ip']}.  Through the proxy {through['ip']}.  "
+                   "Warning: the proxy leaves from the same IP as this server, so it is "
+                   "not a different vantage.")
+            kind = "warn"
+        else:
+            msg = (f"Direct egress IP {direct['ip']}.  Through the proxy {through['ip']}.  "
+                   "The proxy leaves from a different IP, which is what the exposure "
+                   "check needs.")
+            kind = "ok"
+    elif direct["ok"]:
+        msg = f"Direct egress IP {direct['ip']}, but the proxy did not answer: {through['detail']}"
+        kind = "error"
+    else:
+        msg = (f"Could not read the egress IP.  Direct: {direct['detail']}.  "
+               f"Proxy: {through['detail']}")
+        kind = "error"
+    return redirect(_back(back, msg, kind))
+
+
 @router.post("/dast-admin/profiles")
 async def save_scan_profile(request: Request, user: User = Depends(current_user)):
     await check_csrf(request)

app/web/templates/dast_admin.html

@@ -305,8 +305,39 @@
             URL will hit it here, which proves server side request forgery.  It
             must be reachable from the sites you scan.  Leave empty to switch the
             out of band checks off.</span></label>
+
+        <div class="card-head" style="margin-top:1rem">Exposure check</div>
+        <label style="margin-top:.4rem">This scanner's allow listed IP
+          <input type="text" name="whitelisted_ip" maxlength="60" value="{{ whitelisted_ip }}"
+                 placeholder="203.0.113.10">
+          <span class="faint small">The public IP a target's allow list lets in,
+            shown for reference.  Optional.</span></label>
+        <label style="margin-top:.6rem">Non allow listed proxy (Squid)
+          <input type="password" name="unwhitelisted_proxy" maxlength="1000" autocomplete="off"
+                 placeholder="{{ 'saved, leave empty to keep' if has_exposure_proxy else 'http://proxy.host:3128 or http://user:password@proxy.host:3128' }}">
+          <span class="faint small">Leaves from an IP the targets do not allow list,
+            so a site can be seen the way the public sees it.  Basic auth goes in the
+            address as user:password@host.  Held encrypted.</span></label>
+        {% if has_exposure_proxy %}<label style="margin-top:.4rem" class="faint small">
+          <input type="checkbox" name="proxy_clear" value="1"> Remove the saved proxy</label>{% endif %}
+        <label style="margin-top:.6rem">IP echo service
+          <input type="url" name="ip_echo_url" maxlength="500" value="{{ ip_echo_url }}"
+                 placeholder="https://api.ipify.org">
+          <span class="faint small">Answers with the caller's IP, used by Validate
+            proxy to show the address the proxy leaves from.</span></label>
+        <label style="margin-top:.6rem">
+          <input type="checkbox" name="exposure_check" value="1" {{ 'checked' if exposure_check }}>
+          Compare each site from a non allow listed IP on refresh
+          <span class="faint small">Also loads each site through the proxy and takes a
+            second screenshot, to tell an allow listed site from one open to the
+            world.</span></label>
+
         <div style="margin-top:.5rem"><button class="btn primary" type="submit">Save</button></div>
       </form>
+      {% if has_exposure_proxy %}<form method="post" action="{{ base }}/dast-admin/validate-proxy" style="margin-top:.5rem">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+        <button class="btn" type="submit">Validate proxy</button>
+        <span class="faint small">Shows the egress IP this server and the proxy each present.</span></form>{% endif %}
       <p class="faint small" style="margin-top:.8rem">More dynamic scanning
         settings will move here from the general Settings page over time.</p>
     </div>