|
| 1 | +"""Scheduled scans of external sites. |
| 2 | +
|
| 3 | +A schedule says when, in the time zone set in Settings: once at a date and |
| 4 | +time, or every day, every week on a weekday, or every month on a day, at a |
| 5 | +time. The scheduler looks once a minute for schedules whose next run has |
| 6 | +come, starts the scan the way the Scan button would, and works out the next |
| 7 | +run. Missed runs while the server was down are not made up one by one: the |
| 8 | +schedule runs once and moves on to its next time. |
| 9 | +""" |
| 10 | + |
| 11 | +from __future__ import annotations |
| 12 | + |
| 13 | +import calendar |
| 14 | +import logging |
| 15 | +import re |
| 16 | +from datetime import datetime, timedelta, timezone |
| 17 | + |
| 18 | +from app.core.db import session_scope |
| 19 | +from app.core.models import ExternalAsset, ExternalScanSchedule, utcnow |
| 20 | + |
| 21 | +log = logging.getLogger("gcr.external") |
| 22 | + |
| 23 | +FREQUENCIES = {"once": "Once", "daily": "Every day", "weekly": "Every week", |
| 24 | + "monthly": "Every month"} |
| 25 | +WEEKDAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] |
| 26 | +MAX_PER_SITE = 10 |
| 27 | + |
| 28 | + |
| 29 | +def zone(): |
| 30 | + from app.core.scheduler import _zone |
| 31 | + |
| 32 | + return _zone() |
| 33 | + |
| 34 | + |
| 35 | +def _aware(value): |
| 36 | + return value if value is None or value.tzinfo else value.replace(tzinfo=timezone.utc) |
| 37 | + |
| 38 | + |
| 39 | +def next_run(sched, after: datetime, tz=None) -> datetime | None: |
| 40 | + """The first run strictly after `after`, in UTC. None when there is none.""" |
| 41 | + tz = tz or zone() |
| 42 | + after = _aware(after) |
| 43 | + local = after.astimezone(tz) |
| 44 | + hour, minute = (int(x) for x in (sched.time_of_day or "00:00").split(":")) |
| 45 | + |
| 46 | + def at(day): |
| 47 | + return datetime(day.year, day.month, day.day, hour, minute, tzinfo=tz) |
| 48 | + |
| 49 | + if sched.frequency == "once": |
| 50 | + try: |
| 51 | + when = datetime.strptime(sched.run_on, "%Y-%m-%dT%H:%M").replace(tzinfo=tz) |
| 52 | + except (TypeError, ValueError): |
| 53 | + return None |
| 54 | + return when.astimezone(timezone.utc) if when > after else None |
| 55 | + if sched.frequency == "daily": |
| 56 | + candidate = at(local) |
| 57 | + if candidate <= after: |
| 58 | + candidate = at(local + timedelta(days=1)) |
| 59 | + return candidate.astimezone(timezone.utc) |
| 60 | + if sched.frequency == "weekly": |
| 61 | + ahead = (int(sched.weekday or 0) - local.weekday()) % 7 |
| 62 | + candidate = at(local + timedelta(days=ahead)) |
| 63 | + if candidate <= after: |
| 64 | + candidate = at(local + timedelta(days=ahead + 7)) |
| 65 | + return candidate.astimezone(timezone.utc) |
| 66 | + if sched.frequency == "monthly": |
| 67 | + year, month = local.year, local.month |
| 68 | + for _ in range(3): |
| 69 | + last = calendar.monthrange(year, month)[1] |
| 70 | + day = min(int(sched.day_of_month or 1), last) |
| 71 | + candidate = datetime(year, month, day, hour, minute, tzinfo=tz) |
| 72 | + if candidate > after: |
| 73 | + return candidate.astimezone(timezone.utc) |
| 74 | + month += 1 |
| 75 | + if month > 12: |
| 76 | + month, year = 1, year + 1 |
| 77 | + return None |
| 78 | + |
| 79 | + |
| 80 | +def clean(data: dict, profiles, site_has_auth: bool) -> dict: |
| 81 | + """A schedule as a form or the API sends it, checked. Raises ValueError.""" |
| 82 | + frequency = str(data.get("frequency") or "").strip().lower() |
| 83 | + if frequency not in FREQUENCIES: |
| 84 | + raise ValueError("How often is once, daily, weekly or monthly.") |
| 85 | + time_of_day = str(data.get("time_of_day") or "").strip() |
| 86 | + if not re.fullmatch(r"([01]\d|2[0-3]):[0-5]\d", time_of_day): |
| 87 | + raise ValueError("Give the time as HH:MM, such as 02:30.") |
| 88 | + out = {"frequency": frequency, "time_of_day": time_of_day, "run_on": "", |
| 89 | + "weekday": 0, "day_of_month": 1} |
| 90 | + if frequency == "once": |
| 91 | + day = str(data.get("run_on") or "").strip()[:10] |
| 92 | + try: |
| 93 | + datetime.strptime(day, "%Y-%m-%d") |
| 94 | + except ValueError: |
| 95 | + raise ValueError("Give the date to run on, as YYYY-MM-DD.") |
| 96 | + out["run_on"] = f"{day}T{time_of_day}" |
| 97 | + elif frequency == "weekly": |
| 98 | + try: |
| 99 | + weekday = int(data.get("weekday")) |
| 100 | + except (TypeError, ValueError): |
| 101 | + raise ValueError("Pick the day of the week.") |
| 102 | + if not 0 <= weekday <= 6: |
| 103 | + raise ValueError("Pick the day of the week.") |
| 104 | + out["weekday"] = weekday |
| 105 | + elif frequency == "monthly": |
| 106 | + try: |
| 107 | + day = int(data.get("day_of_month")) |
| 108 | + except (TypeError, ValueError): |
| 109 | + raise ValueError("Pick the day of the month.") |
| 110 | + if not 1 <= day <= 31: |
| 111 | + raise ValueError("The day of the month is 1 to 31.") |
| 112 | + out["day_of_month"] = day |
| 113 | + profile = str(data.get("profile") or "").strip().lower() |
| 114 | + if profile and profile not in profiles: |
| 115 | + raise ValueError("Pick a scan profile.") |
| 116 | + out["profile"] = profile |
| 117 | + for key in ("authenticated", "ai_triage"): |
| 118 | + value = data.get(key) |
| 119 | + if isinstance(value, str): |
| 120 | + value = value.lower() in ("1", "on", "true", "yes") |
| 121 | + out[key] = bool(value) |
| 122 | + if out["authenticated"] and not site_has_auth: |
| 123 | + raise ValueError("This site has no scan sign in to use. Set one in Edit first.") |
| 124 | + return out |
| 125 | + |
| 126 | + |
| 127 | +def describe(sched) -> str: |
| 128 | + """The schedule in words, for the page and the API.""" |
| 129 | + if sched.frequency == "once": |
| 130 | + return f"Once, on {sched.run_on.replace('T', ' at ')}" |
| 131 | + if sched.frequency == "daily": |
| 132 | + return f"Every day at {sched.time_of_day}" |
| 133 | + if sched.frequency == "weekly": |
| 134 | + return f"Every {WEEKDAYS[int(sched.weekday or 0)]} at {sched.time_of_day}" |
| 135 | + day = int(sched.day_of_month or 1) |
| 136 | + suffix = "th" if 10 <= day % 100 <= 20 else {1: "st", 2: "nd", 3: "rd"}.get(day % 10, "th") |
| 137 | + tail = ", or the last day of a shorter month" if day > 28 else "" |
| 138 | + return f"Every month on the {day}{suffix} at {sched.time_of_day}{tail}" |
| 139 | + |
| 140 | + |
| 141 | +def as_dict(sched) -> dict: |
| 142 | + return {"id": sched.id, "asset_id": sched.asset_id, "frequency": sched.frequency, |
| 143 | + "when": describe(sched), "run_on": sched.run_on or None, |
| 144 | + "time_of_day": sched.time_of_day, "weekday": sched.weekday, |
| 145 | + "day_of_month": sched.day_of_month, "profile": sched.profile or "", |
| 146 | + "authenticated": bool(sched.authenticated), "ai_triage": bool(sched.ai_triage), |
| 147 | + "enabled": bool(sched.enabled), |
| 148 | + "next_run_at": _aware(sched.next_run_at).isoformat() if sched.next_run_at else None, |
| 149 | + "last_run_at": _aware(sched.last_run_at).isoformat() if sched.last_run_at else None, |
| 150 | + "last_result": sched.last_result or "", "last_scan_id": sched.last_scan_id, |
| 151 | + "created_by": sched.created_by or ""} |
| 152 | + |
| 153 | + |
| 154 | +def create(db, asset: ExternalAsset, data: dict, actor: str) -> ExternalScanSchedule: |
| 155 | + from app.core import dast_engines |
| 156 | + |
| 157 | + if db.query(ExternalScanSchedule).filter_by(asset_id=asset.id).count() >= MAX_PER_SITE: |
| 158 | + raise ValueError(f"A site can have at most {MAX_PER_SITE} schedules.") |
| 159 | + fields = clean(data, dast_engines.PROFILES, bool(asset.auth_kind)) |
| 160 | + sched = ExternalScanSchedule(asset_id=asset.id, created_by=(actor or "")[:120], **fields) |
| 161 | + sched.next_run_at = next_run(sched, utcnow()) |
| 162 | + if sched.next_run_at is None: |
| 163 | + raise ValueError("That time has already passed.") |
| 164 | + db.add(sched) |
| 165 | + db.flush() |
| 166 | + return sched |
| 167 | + |
| 168 | + |
| 169 | +def run_due(now: datetime = None) -> int: |
| 170 | + """Start every scan whose time has come. The scheduler calls this each minute.""" |
| 171 | + from app.core import external |
| 172 | + |
| 173 | + now = now or utcnow() |
| 174 | + with session_scope() as db: |
| 175 | + if not external.enabled(db): |
| 176 | + return 0 |
| 177 | + due = [s.id for s in db.query(ExternalScanSchedule).filter( |
| 178 | + ExternalScanSchedule.enabled.is_(True), |
| 179 | + ExternalScanSchedule.next_run_at.isnot(None), |
| 180 | + ExternalScanSchedule.next_run_at <= now)] |
| 181 | + started = 0 |
| 182 | + for sched_id in due: |
| 183 | + with session_scope() as db: |
| 184 | + sched = db.get(ExternalScanSchedule, sched_id) |
| 185 | + asset = db.get(ExternalAsset, sched.asset_id) if sched else None |
| 186 | + if sched is None or asset is None: |
| 187 | + continue |
| 188 | + args = dict(asset_id=asset.id, |
| 189 | + profile=sched.profile or external.profile_for(asset), |
| 190 | + actor=sched.created_by, ai=sched.ai_triage, |
| 191 | + auth=sched.authenticated) |
| 192 | + obsolete = asset.status == "obsolete" |
| 193 | + if obsolete: |
| 194 | + scan_id, problem = None, "the site is obsolete" |
| 195 | + else: |
| 196 | + scan_id, problem = external.start_scan(args["asset_id"], args["profile"], |
| 197 | + args["actor"], ai_triage=args["ai"], |
| 198 | + authenticated=args["auth"]) |
| 199 | + with session_scope() as db: |
| 200 | + sched = db.get(ExternalScanSchedule, sched_id) |
| 201 | + if sched is None: |
| 202 | + continue |
| 203 | + sched.last_run_at = now |
| 204 | + sched.last_scan_id = scan_id |
| 205 | + sched.last_result = (f"started a {args['profile']} scan" if scan_id |
| 206 | + else f"skipped: {problem}")[:300] |
| 207 | + sched.next_run_at = next_run(sched, now) |
| 208 | + if sched.frequency == "once": |
| 209 | + sched.enabled = False |
| 210 | + started += 1 if scan_id else 0 |
| 211 | + log.info("scheduled scan %s of site %s: %s", sched_id, args["asset_id"], |
| 212 | + "started" if scan_id else f"skipped, {problem}") |
| 213 | + return started |
| 214 | + |
| 215 | + |
| 216 | +def refresh_all() -> None: |
| 217 | + """Work every next run out again, after the time zone changed.""" |
| 218 | + with session_scope() as db: |
| 219 | + for sched in db.query(ExternalScanSchedule).filter(ExternalScanSchedule.enabled.is_(True)): |
| 220 | + sched.next_run_at = next_run(sched, utcnow()) |
0 commit comments