Skip to content

Commit 448cdcd

Browse files
committed
Leave out findings a firewall's bot check answered instead of the site
Some firewalls, Oracle Zenedge among them, answer every address with a 200 "Are you human?" page whose size changes on every request. The not found check compares each page to the answer for an address that cannot exist, so it never matched, and nikto reported hundreds of files that were not there. Curated rules said GraphQL, Swagger and ColdFusion pages were confirmed against a control on the same page. The bot check test that already named the firewall is now used where a finding is judged. A path finding answered by a bot check page is left out, the path inventory does not count it as reached, and a curated rule never confirms on one, the JWT and two account checks included. The scan page and the PDF say how many were left out and that the scan did not reach the application. A curated rule's evidence now gives the control's real status instead of always saying the control answered differently.
1 parent 8faa8ac commit 448cdcd

6 files changed

Lines changed: 82 additions & 13 deletions

File tree

‎app/core/dast_checks/engine.py‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,11 @@ def run_rule(spec: dict, ctx: dict) -> dict:
415415
last = control = None
416416
for _attempt in range(retest):
417417
last = _run_probe(spec, client, route, oob_value)
418+
if _bot_checked(last):
419+
# A firewall answered instead of the site, with a page that
420+
# differs on every request, so any differential would pass.
421+
out["detail"] = f"{_bot_checked(last)}'s bot check answered, not the site"
422+
break
418423
control = (_control_response(client)
419424
if spec["safety"] != "oob" else {})
420425
oob_hit = bool(oob and token and oob.hit(token))
@@ -424,7 +429,7 @@ def run_rule(spec: dict, ctx: dict) -> dict:
424429
else:
425430
break
426431
if confirmations == retest and confirmations > 0:
427-
out["findings"].append(_finding(spec, client, last, route))
432+
out["findings"].append(_finding(spec, client, last, route, control))
428433
break # one solid finding per rule; the rest is noise
429434
except Refused as exc:
430435
out["status"] = "refused"
@@ -547,7 +552,7 @@ def _run_jwt(spec: dict, ctx: dict, domains, budget: Budget) -> list:
547552
# The endpoint must really be behind the session; the differential then
548553
# confirms, per attempt, that the forgery got the signed in response and
549554
# the signed out one did not, so a public page stays silent.
550-
if resp_authed.get("status") != 200:
555+
if resp_authed.get("status") != 200 or _bot_checked(resp_authed):
551556
continue
552557
resp_anon = anon.request("GET", path)
553558
for mode in modes:
@@ -623,13 +628,13 @@ def live(client, who, url):
623628
resp_b = live(b, "b", url)
624629
# It must be the second account's real object: a 200 that is not a
625630
# sign in or session expired wall.
626-
if resp_b.get("status") != 200 or _looks_authless(resp_b):
631+
if resp_b.get("status") != 200 or _looks_authless(resp_b) or _bot_checked(resp_b):
627632
break
628633
resp_a = live(a, "a", url)
629634
resp_anon = anon.request("GET", url)
630635
# The first account must have reached the real object too; a sign in
631636
# wall is not access and there would be nothing to compare.
632-
if _looks_authless(resp_a):
637+
if _looks_authless(resp_a) or _bot_checked(resp_a):
633638
resp_a = None
634639
break
635640
env = _env_for(resp_a, {}, False,
@@ -976,7 +981,17 @@ def _run_probe(spec: dict, client: Client, route: str, oob_value: str) -> dict:
976981
return last
977982

978983

979-
def _finding(spec: dict, client: Client, response: dict, route: str) -> dict:
984+
def _bot_checked(resp) -> str:
985+
"""Whose bot check answered instead of the site, or empty."""
986+
from app.core.external_probe import bot_check
987+
resp = resp or {}
988+
if resp.get("status") != 200:
989+
return ""
990+
return bot_check(resp.get("body") or "", 200)
991+
992+
993+
def _finding(spec: dict, client: Client, response: dict, route: str,
994+
control: dict = None) -> dict:
980995
"""A finding that stands on evidence, with how to repeat it and how to fix."""
981996
from app.core import dast_engines
982997

@@ -985,9 +1000,16 @@ def _finding(spec: dict, client: Client, response: dict, route: str) -> dict:
9851000
evidence = ("Confirmed by an out of band callback: the site was made to "
9861001
"reach our unique address, which proves it, and re-tested.")
9871002
else:
1003+
got, ctl = response.get("status"), (control or {}).get("status")
1004+
if ctl and ctl != got:
1005+
told = f"and {ctl} for the control"
1006+
elif ctl:
1007+
told = "here, with a different page than the control's"
1008+
else:
1009+
told = "here"
9881010
evidence = (f"Confirmed against a control (a request that cannot "
9891011
f"succeed) and re-tested. The site answered status "
990-
f"{response.get('status')} here but not for the control.")
1012+
f"{got} {told}.")
9911013
f = dast_engines._finding(
9921014
tool=f"rule:{spec['id']}", title=spec.get("title", spec["id"]),
9931015
severity=spec.get("severity_floor", "medium"), url=url,

‎app/core/dast_paths.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,11 @@ def verify(records: list, controls: dict, fetch, first=()) -> int:
208208
status = int(got.get("status") or 0)
209209
if rec.get("s") is None and status:
210210
rec["s"] = status
211+
# A firewall's bot check changes on every request, so it never matches
212+
# the control's page; it is still not the site's.
213+
from app.core.external_probe import bot_check
211214
rec["r"] = (status == 200 and bool(got.get("body")) and
215+
not bot_check(got["body"], status) and
212216
(control["status"] != 200 or fingerprint(got["body"]) != control["fp"]))
213217
rec.pop("v", None)
214218
return done

‎app/core/dast_report.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,9 @@ def build(db, asset, scan, user=None, organization: str = "") -> dict:
422422
for e in skipped:
423423
limitations.append(f"{e.get('name') or e.get('tool')} did not complete: "
424424
f"{e.get('detail') or e.get('status')}.")
425+
for e in engines:
426+
if e.get("status") == "warn" and e.get("tool") == "verify":
427+
limitations.append(f"{e.get('name')}: {e.get('detail')}")
425428
if scan.authenticated:
426429
kinds = {"header": "a header", "cookie": "a session cookie", "form": "a form login"}
427430
signin = next((e for e in engines if e.get("tool") == "signin"), None)

‎app/core/external.py‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2697,13 +2697,25 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
26972697
# Nothing an engine echoed back of the sign in is kept.
26982698
findings = external_auth.redact(findings, secrets)
26992699
engines = external_auth.redact(engines, secrets)
2700-
findings, dropped = drop_soft_404(findings, url, opts["allow_private"])
2701-
if dropped:
2700+
soft_stats = {}
2701+
findings, dropped = drop_soft_404(findings, url, opts["allow_private"], soft_stats)
2702+
challenged = soft_stats.get("challenged", 0)
2703+
if dropped - challenged:
2704+
n = dropped - challenged
27022705
engines.append({"tool": "verify", "name": "Soft 404 check", "status": "ok",
27032706
"seconds": 0,
2704-
"detail": f"left out {dropped} finding{'s' if dropped != 1 else ''} "
2707+
"detail": f"left out {n} finding{'s' if n != 1 else ''} "
27052708
"for paths that answer not found, or with the same "
27062709
"page as one that cannot exist"})
2710+
if challenged:
2711+
engines.append({"tool": "verify", "name": "Bot check", "status": "warn",
2712+
"seconds": 0,
2713+
"detail": f"left out {challenged} finding"
2714+
f"{'s' if challenged != 1 else ''} for paths that "
2715+
f"answered with {soft_stats['bot_check']}'s bot check "
2716+
"page instead of the site. The scan did not reach "
2717+
"the application there. Let the scanner through the "
2718+
"firewall, or scan the origin, for a real result."})
27072719
csp = (base["telemetry"].get("headers") or {}).get("content-security-policy", "")
27082720
findings, lowered = soften_blocked_xss(findings, csp)
27092721
if lowered:
@@ -2763,7 +2775,8 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
27632775
r"|CWE-79\b|CWE-113\b", re.I)
27642776

27652777

2766-
def drop_soft_404(findings: list, base_url: str, allow_private: bool = False) -> tuple:
2778+
def drop_soft_404(findings: list, base_url: str, allow_private: bool = False,
2779+
stats: dict = None) -> tuple:
27672780
"""Leave out path findings whose page is the site's not found page.
27682781
27692782
One request to an address that cannot exist gives the site's answer for
@@ -2828,13 +2841,27 @@ def same(a: str, a_cut: bool, b: str, b_cut: bool) -> bool:
28282841
body, cut = got["body"], len(got["body"]) >= SOFT_404_BODY
28292842
loc = got["headers"].get("location", "")
28302843
to_login = 300 <= got["status"] < 400 and bool(_LOGIN_URL.search(loc))
2831-
seen[url] = (got["status"], to_login, got["status"] == 200 and any(
2844+
# A firewall's bot check ("Are you human?") answers in place of
2845+
# the site, with a page that changes on every request, so it never
2846+
# matches the control. The path was not reached at all.
2847+
challenge = (external_probe.bot_check(body, got["status"])
2848+
if got["status"] == 200 else "")
2849+
seen[url] = (got["status"], to_login, challenge, got["status"] == 200 and any(
28322850
same(norm(body, *echoed), cut, mark, mark_cut)
28332851
# Bodies cut at the limit all have its length, so only a
28342852
# whole page can be matched on size.
28352853
or html and not (cut or mark_cut) and abs(len(body) - size) < 16
28362854
for mark, size, mark_cut in controls))
2837-
status, to_login, soft = seen[url]
2855+
status, to_login, challenge, soft = seen[url]
2856+
if challenge:
2857+
# Nothing the scanner said about this address is about the site:
2858+
# not that a file is there, and not what the page holds, which is
2859+
# the firewall's own page.
2860+
dropped += 1
2861+
if stats is not None:
2862+
stats["bot_check"] = challenge
2863+
stats["challenged"] = stats.get("challenged", 0) + 1
2864+
continue
28382865
# Nothing is there: a claim that a file or page was found is only the
28392866
# site's not found page. What the page itself carries stays: input
28402867
# it echoes back, or a key in it, is on the 404 page too.

‎app/web/templates/external_scan.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
<tbody>
7070
{% for e in view.engines %}
7171
<tr><td>{{ e.name }}</td>
72-
<td class="small"><span class="badge {{ {'ok':'ok','skipped':'waived','failed':'critical','timeout':'pending','stopped':'pending'}.get(e.status, 'info') }}">{{ e.status }}</span> {{ e.detail }}{% if e.reached %}<span class="faint">, {{ e.reached | thousands }} address{{ '' if e.reached == 1 else 'es' }} answered</span>{% endif %}{% if e.requests %}<span class="faint">, {{ e.requests | thousands }} request{{ '' if e.requests == 1 else 's' }} sent</span>{% endif %}
72+
<td class="small"><span class="badge {{ {'ok':'ok','skipped':'waived','failed':'critical','timeout':'pending','stopped':'pending','warn':'high'}.get(e.status, 'info') }}">{{ e.status }}</span> {{ e.detail }}{% if e.reached %}<span class="faint">, {{ e.reached | thousands }} address{{ '' if e.reached == 1 else 'es' }} answered</span>{% endif %}{% if e.requests %}<span class="faint">, {{ e.requests | thousands }} request{{ '' if e.requests == 1 else 's' }} sent</span>{% endif %}
7373
{% for r in e.repos or [] %}<div class="faint">{{ r.repository }}{% if r.branch %} ({{ r.branch }}){% endif %}: {{ r.routes }} routes{% if r.host_in_code %}, host named in {{ r.host_in_code | join(', ') }}{% endif %}{% if r.error %}, {{ r.error }}{% endif %}</div>{% endfor %}</td>
7474
<td class="small nowrap">{{ e.seconds // 60 }}m {{ e.seconds % 60 }}s</td></tr>
7575
{% endfor %}

‎documentation/CHANGELOG.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,19 @@ Newest first. Written for somebody deciding whether to upgrade, so every entry
44
says what is different for a person using the product rather than which files
55
moved.
66

7+
## 2026-11-11
8+
9+
**A firewall's bot check is no longer reported as findings.** Some firewalls,
10+
Oracle Zenedge among them, answer every address with a 200 "Are you human?"
11+
page that changes size on every request. The not found check never matched
12+
it, so nikto reported hundreds of files that were not there, and curated rules
13+
called GraphQL, Swagger and ColdFusion pages confirmed. Now an address that
14+
answers with a known bot check page is left out, it is not counted as reached,
15+
and a curated rule never confirms on one. The scan page and the PDF say how
16+
many were left out and that the scan did not reach the application. A
17+
curated rule's evidence now gives the control's real status. Scans that ran
18+
before this keep their findings; scan the site again.
19+
720
## 2026-11-10
821

922
**Record a session.** A site behind a bot check, or behind single sign on,

0 commit comments

Comments
 (0)