|
29 | 29 | import re |
30 | 30 | import shutil |
31 | 31 | import threading |
| 32 | +import time |
32 | 33 | from concurrent.futures import ThreadPoolExecutor |
33 | 34 | from datetime import datetime, timedelta, timezone |
34 | 35 | from decimal import Decimal |
@@ -1297,7 +1298,8 @@ def _auth_view(asset) -> dict: |
1297 | 1298 |
|
1298 | 1299 | def scan_dict(scan: ExternalDastScan, findings: bool = False) -> dict: |
1299 | 1300 | out = {"id": scan.id, "asset_id": scan.asset_id, "server": scan.server_name, |
1300 | | - "source": scan.source, "status": scan.status, "profile": scan.profile or "", |
| 1301 | + "source": scan.source, "provider": scan.provider or "", |
| 1302 | + "status": scan.status, "profile": scan.profile or "", |
1301 | 1303 | "authenticated": bool(scan.authenticated), "auth_kind": scan.auth_kind or "", |
1302 | 1304 | "step": scan.step or "", "remote_id": scan.remote_id or "", |
1303 | 1305 | "started_at": _iso(scan.started_at), "scanned_at": _iso(scan.scanned_at), |
@@ -1478,13 +1480,19 @@ def busy(self) -> int: |
1478 | 1480 |
|
1479 | 1481 |
|
1480 | 1482 | def start_scan(asset_id: int, profile: str, actor: str, ai_triage: bool = False, |
1481 | | - server_id: int = None, authenticated: bool = None) -> tuple: |
1482 | | - """Start a dynamic scan here or on a server. -> (scan id, problem) |
| 1483 | + server_id: int = None, authenticated: bool = None, |
| 1484 | + ias_config: str = "") -> tuple: |
| 1485 | + """Start a dynamic scan here, on a server, or on InsightAppSec. -> (scan id, problem) |
1483 | 1486 |
|
1484 | | - authenticated None means sign in when the site has a sign in set. |
| 1487 | + authenticated None means sign in when the site has a sign in set. A |
| 1488 | + scan config id in ias_config means InsightAppSec, which signs in with |
| 1489 | + whatever its scan config says, never with the sign in saved here. |
1485 | 1490 | """ |
1486 | 1491 | from app.core import dast_engines, external_auth |
1487 | 1492 |
|
| 1493 | + if ias_config: |
| 1494 | + return _start_ias(asset_id, ias_config, actor, ai_triage) |
| 1495 | + |
1488 | 1496 | with session_scope() as db: |
1489 | 1497 | if not enabled(db): |
1490 | 1498 | return None, "External sites are switched off in Settings." |
@@ -1542,10 +1550,16 @@ def stop_scan(scan_id: int) -> bool: |
1542 | 1550 | scan = db.get(ExternalDastScan, scan_id) |
1543 | 1551 | if scan is None or scan.status not in ("queued", "running"): |
1544 | 1552 | return False |
1545 | | - if scan.source == "remote": |
| 1553 | + if scan.source == "remote" and scan.provider == "insightappsec": |
| 1554 | + remote_id = scan.remote_id |
| 1555 | + elif scan.source == "remote": |
1546 | 1556 | scan.status = "failed" |
1547 | 1557 | scan.error = "stopped here; the server may still be running it" |
1548 | 1558 | return True |
| 1559 | + else: |
| 1560 | + remote_id = None |
| 1561 | + if remote_id is not None: |
| 1562 | + return _stop_ias(scan_id, remote_id) |
1549 | 1563 | _stopping.add(scan_id) |
1550 | 1564 | _update(scan_id, step="stopping") |
1551 | 1565 | dast_engines.stop(scan_id) |
@@ -2219,17 +2233,154 @@ def _start_remote(scan_id: int) -> None: |
2219 | 2233 | if "body" in locals() else f"could not start it: {type(exc).__name__}") |
2220 | 2234 |
|
2221 | 2235 |
|
| 2236 | +# ------------------------------------------------------------------ InsightAppSec |
| 2237 | + |
| 2238 | +# Paused and blacked out scans can wait a day there, and a long site takes |
| 2239 | +# hours, so this is longer than a DAST server gets. |
| 2240 | +IAS_GIVE_UP_HOURS = 72 |
| 2241 | +_ias_backoff = {"until": 0.0} |
| 2242 | + |
| 2243 | + |
| 2244 | +def ias_running(db) -> int: |
| 2245 | + return (db.query(ExternalDastScan) |
| 2246 | + .filter(ExternalDastScan.provider == "insightappsec", |
| 2247 | + ExternalDastScan.status.in_(("queued", "running"))).count()) |
| 2248 | + |
| 2249 | + |
| 2250 | +def _start_ias(asset_id: int, config_id: str, actor: str, ai_triage: bool) -> tuple: |
| 2251 | + from app.core import insightappsec as ias |
| 2252 | + |
| 2253 | + with session_scope() as db: |
| 2254 | + if not enabled(db): |
| 2255 | + return None, "External sites are switched off in Settings." |
| 2256 | + cfg = ias.config(db) |
| 2257 | + if not cfg["ready"]: |
| 2258 | + return None, ("InsightAppSec is not set up. Turn it on and give the API " |
| 2259 | + "key on the Integrations tab in Settings.") |
| 2260 | + asset = db.get(ExternalAsset, asset_id) |
| 2261 | + if asset is None: |
| 2262 | + return None, "No such site." |
| 2263 | + why = blocked(db, asset) |
| 2264 | + if why: |
| 2265 | + return None, f"{asset.host} may not be scanned: {why}." |
| 2266 | + if not ias.is_uuid(config_id): |
| 2267 | + return None, "Pick an InsightAppSec scan config." |
| 2268 | + if db.query(ExternalDastScan).filter( |
| 2269 | + ExternalDastScan.asset_id == asset_id, |
| 2270 | + ExternalDastScan.status.in_(("queued", "running"))).first(): |
| 2271 | + return None, f"A scan of {asset.host} is already running." |
| 2272 | + if ias_running(db) >= cfg["max_concurrent"]: |
| 2273 | + return None, (f"{cfg['max_concurrent']} InsightAppSec scan" |
| 2274 | + f"{'s are' if cfg['max_concurrent'] != 1 else ' is'} already " |
| 2275 | + f"running from here, the most the Integrations tab allows.") |
| 2276 | + known = ias.find_config(config_id) |
| 2277 | + asset.ias_scan_config_id = config_id |
| 2278 | + scan = ExternalDastScan( |
| 2279 | + asset_id=asset_id, server_id=None, server_name=ias.NAME, |
| 2280 | + source="remote", provider=ias.PROVIDER, status="queued", |
| 2281 | + profile="", started_at=utcnow(), recorded_by=(actor or "")[:120], |
| 2282 | + ai_status="wanted" if ai_triage else "", authenticated=False, |
| 2283 | + step=(f"starting {known['name']}" if known else "starting")[:120], |
| 2284 | + note=(f"InsightAppSec scan config {known['name']} ({known['app']})" |
| 2285 | + if known else f"InsightAppSec scan config {config_id}")[:500]) |
| 2286 | + db.add(scan) |
| 2287 | + db.flush() |
| 2288 | + scan_id = scan.id |
| 2289 | + region, key = cfg["region"], cfg["key"] |
| 2290 | + app_id = (known or {}).get("app_id", "") |
| 2291 | + try: |
| 2292 | + remote = ias.start(region, key, config_id) |
| 2293 | + except ias.IASError as exc: |
| 2294 | + # Kept as a failed scan, so the site's history shows the attempt. |
| 2295 | + _update(scan_id, status="failed", step="", error=f"could not start it: {exc}"[:500]) |
| 2296 | + return None, f"InsightAppSec would not start it: {exc}" |
| 2297 | + _update(scan_id, status="running", remote_id=remote, step="queued in InsightAppSec", |
| 2298 | + result_url=ias.scan_link(region, app_id, config_id, remote)) |
| 2299 | + return scan_id, "" |
| 2300 | + |
| 2301 | + |
| 2302 | +def _stop_ias(scan_id: int, remote_id: str) -> bool: |
| 2303 | + """Ask InsightAppSec to stop it. STOP keeps what it found so far, and the |
| 2304 | + poll reads that back when the scan reaches its end.""" |
| 2305 | + from app.core import insightappsec as ias |
| 2306 | + |
| 2307 | + cfg = ias.config() |
| 2308 | + if not remote_id: |
| 2309 | + _update(scan_id, status="failed", step="", error="stopped before it started") |
| 2310 | + return True |
| 2311 | + try: |
| 2312 | + ias.act(cfg["region"], cfg["key"], remote_id, "STOP") |
| 2313 | + except ias.IASError as exc: |
| 2314 | + _update(scan_id, step=f"could not stop it: {exc}"[:120]) |
| 2315 | + return False |
| 2316 | + _update(scan_id, step="stopping in InsightAppSec") |
| 2317 | + return True |
| 2318 | + |
| 2319 | + |
| 2320 | +def _poll_ias(scan_id: int, remote_id: str, started) -> int: |
| 2321 | + """One InsightAppSec scan: its status, and its findings once it is done.""" |
| 2322 | + from app.core import insightappsec as ias |
| 2323 | + |
| 2324 | + if _ias_backoff["until"] > time.time(): |
| 2325 | + return 0 |
| 2326 | + cfg = ias.config() |
| 2327 | + if not cfg["key"]: |
| 2328 | + _update(scan_id, status="failed", step="", |
| 2329 | + error="the InsightAppSec key was removed from Settings") |
| 2330 | + return 0 |
| 2331 | + if not remote_id: |
| 2332 | + _update(scan_id, status="failed", step="", error="InsightAppSec gave it no id") |
| 2333 | + return 0 |
| 2334 | + region, key = cfg["region"], cfg["key"] |
| 2335 | + try: |
| 2336 | + state = ias.status(region, key, remote_id) |
| 2337 | + now = str(state.get("status") or "") |
| 2338 | + if now == "COMPLETE": |
| 2339 | + found = ias.findings(region, key, remote_id) |
| 2340 | + app_id = (state.get("app") or {}).get("id", "") |
| 2341 | + config_id = (state.get("scan_config") or {}).get("id", "") |
| 2342 | + finish(scan_id, merge(found), |
| 2343 | + result_url=ias.scan_link(region, app_id, config_id, remote_id)) |
| 2344 | + with session_scope() as db: |
| 2345 | + want_ai = db.get(ExternalDastScan, scan_id).ai_status == "wanted" |
| 2346 | + if want_ai: |
| 2347 | + triage(scan_id) |
| 2348 | + return 1 |
| 2349 | + if now == "FAILED": |
| 2350 | + why = ias.failure(state) |
| 2351 | + event = ias.last_event(region, key, remote_id) |
| 2352 | + _update(scan_id, status="failed", step="", |
| 2353 | + error=(why + (f". Last event: {event}" if event else ""))[:500]) |
| 2354 | + return 0 |
| 2355 | + _update(scan_id, step=f"{now}: {ias.STATUS_WORDS.get(now, now.lower())}"[:120]) |
| 2356 | + if started and _aware(started) < utcnow() - timedelta(hours=IAS_GIVE_UP_HOURS): |
| 2357 | + _update(scan_id, status="failed", step="", |
| 2358 | + error=f"no result after {IAS_GIVE_UP_HOURS} hours; it may still be " |
| 2359 | + f"running in InsightAppSec") |
| 2360 | + except ias.IASError as exc: |
| 2361 | + if exc.retry_at: |
| 2362 | + _ias_backoff["until"] = exc.retry_at |
| 2363 | + elif exc.status == 404: |
| 2364 | + _update(scan_id, status="failed", step="", |
| 2365 | + error="InsightAppSec no longer has this scan, or the key cannot see it") |
| 2366 | + log.warning("polling InsightAppSec scan %s: %s", scan_id, exc) |
| 2367 | + return 0 |
| 2368 | + |
| 2369 | + |
2222 | 2370 | def poll_remote() -> int: |
2223 | 2371 | """Read back every remote scan still running. The scheduler calls this.""" |
2224 | 2372 | with session_scope() as db: |
2225 | 2373 | if not enabled(db): |
2226 | 2374 | return 0 |
2227 | | - rows = [(s.id, s.remote_id, s.server_id, s.started_at) for s in |
| 2375 | + rows = [(s.id, s.remote_id, s.server_id, s.started_at, s.provider) for s in |
2228 | 2376 | db.query(ExternalDastScan).filter( |
2229 | 2377 | ExternalDastScan.source == "remote", |
2230 | 2378 | ExternalDastScan.status == "running")] |
2231 | 2379 | done = 0 |
2232 | | - for scan_id, remote_id, server_id, started in rows: |
| 2380 | + for scan_id, remote_id, server_id, started, provider in rows: |
| 2381 | + if provider == "insightappsec": |
| 2382 | + done += _poll_ias(scan_id, remote_id, started) |
| 2383 | + continue |
2233 | 2384 | try: |
2234 | 2385 | with session_scope() as db: |
2235 | 2386 | server = db.get(DastServer, server_id) if server_id else None |
|
0 commit comments