Skip to content

Commit d336e72

Browse files
committed
Import web applications, and fill change frequency on the sites CSV
The Repositories page could export the web applications but not take them back. Now it can. The file is the whole list: rows add or update, and a web application missing from the file is taken off with its addresses cleared. A preview shows every change first, and a removal has to be ticked. A cell that matches what applies, even from the connection, changes nothing, so a round trip does not copy the connection's owner onto every repository. A bad row keeps its repository, an empty file is refused, and the plan is worked out again on apply and refused if anything moved in between. The External sites CSV left change_frequency_days empty unless somebody typed it. It now falls back to the newest change on a linked repository's default branch, the same number the Web Applications tab shows. A typed value still wins.
1 parent d30806f commit d336e72

7 files changed

Lines changed: 480 additions & 2 deletions

File tree

‎app/core/tagging.py‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -613,3 +613,224 @@ def from_csv(session, text: str, actor: str = "") -> dict:
613613
session.flush()
614614
log.info("tagging import: %s", counts)
615615
return {**counts, "problems": problems}
616+
617+
618+
# The columns the web applications import reads. The rest of what the export
619+
# writes (visibility, last change, last scan and so on) is what a scan found,
620+
# so it is ignored on the way back in.
621+
WEB_IMPORT_FIELDS = ("application_name", "application_owner",
622+
"technical_owner", "sensitivity")
623+
624+
# Small enough to ride along in the confirm page's form, which is how the
625+
# apply step gets exactly the file that was shown.
626+
MAX_WEB_IMPORT_BYTES = 512 * 1024
627+
628+
629+
def _web_cell(field: str, value: str) -> str:
630+
"""A cell the way the repository stores it, so a match reads as a match."""
631+
if field in OWNERS:
632+
return _owners(value)
633+
if field == "sensitivity":
634+
return ", ".join(split_labels(value))
635+
return value
636+
637+
638+
def web_apps_plan(session, text: str, base=None) -> dict:
639+
"""What a web applications CSV would change. Nothing is written.
640+
641+
The file is the whole list. A repository in it is marked as part of a
642+
web application with the addresses it gives, and one marked now that is
643+
not in it is taken off, addresses and all. Only repositories `base`
644+
lets this person see are touched either way.
645+
646+
The export writes what applies, which may be the connection's answer.
647+
So a cell that matches what applies is left alone, and only one that
648+
differs becomes the repository's own value. Otherwise every export and
649+
import would copy the connection's owner onto each repository, and
650+
nobody could tell later which ones had been meant.
651+
652+
-> {"rows", "adds", "updates", "removes", "same", "unknown",
653+
"problems", "refused"}. Each change is {"id", "repository",
654+
"set": {field: (old, new)}}.
655+
"""
656+
import csv
657+
import io
658+
659+
out = {"rows": 0, "adds": [], "updates": [], "removes": [], "same": 0,
660+
"unknown": [], "problems": [], "refused": ""}
661+
662+
try:
663+
reader = csv.DictReader(io.StringIO(text))
664+
header = [(name or "").strip().lower()
665+
for name in (reader.fieldnames or [])]
666+
except (csv.Error, ValueError) as exc:
667+
out["refused"] = f"could not read the file: {exc}"
668+
return out
669+
if "repository" not in header or "web_urls" not in header:
670+
out["refused"] = ("that file needs repository and web_urls columns; "
671+
"export the web applications first to see the shape")
672+
return out
673+
674+
query = base if base is not None else session.query(Repository)
675+
visible = {row.full_name: row for row in query.all()}
676+
applies = effective_many(session, list(visible.values()))
677+
offered = labels(session)
678+
for repo in visible.values():
679+
offered += [x for x in split_labels(repo.sensitivity or "")
680+
if x not in offered]
681+
by_case = {label.casefold(): label for label in offered}
682+
683+
listed = set()
684+
try:
685+
for raw in reader:
686+
out["rows"] += 1
687+
if out["rows"] > MAX_IMPORT_ROWS:
688+
out["refused"] = f"more than {MAX_IMPORT_ROWS} rows"
689+
return out
690+
row = {(k or "").strip().lower():
691+
_unquote((str(v) if v is not None else "").strip())
692+
for k, v in raw.items() if k}
693+
name = row.get("repository", "")
694+
if not name:
695+
continue
696+
repo = visible.get(name)
697+
if repo is None:
698+
if name not in out["unknown"]:
699+
out["unknown"].append(name)
700+
continue
701+
if name in listed:
702+
out["problems"].append(f"{name} is in the file twice; the "
703+
"second row was left out")
704+
continue
705+
# Listed, even when the row turns out bad below. A typo in one
706+
# row must not read as "take this application off".
707+
listed.add(name)
708+
709+
why, change = _web_row(repo, row, header,
710+
applies.get(repo.id, {}), by_case, offered)
711+
if why:
712+
out["problems"].append(f"{name}: {why}")
713+
continue
714+
if not repo.web_app:
715+
out["adds"].append(change)
716+
elif change["set"]:
717+
out["updates"].append(change)
718+
else:
719+
out["same"] += 1
720+
except csv.Error as exc:
721+
out["refused"] = f"could not read the file: {exc}"
722+
return out
723+
724+
for name, repo in sorted(visible.items()):
725+
if repo.web_app and name not in listed:
726+
out["removes"].append({
727+
"id": repo.id, "repository": name,
728+
"set": {"web_app": (True, False),
729+
"web_app_urls": (repo.web_app_urls or "", "")}})
730+
731+
# An empty or wrong file would otherwise take every web application off.
732+
if out["removes"] and not listed:
733+
out["refused"] = ("the file names no web application you can see, so "
734+
"importing it would take every one off; nothing "
735+
"was changed")
736+
return out
737+
738+
739+
def _web_row(repo, row, header, said, by_case, offered) -> tuple:
740+
"""One row of the web applications CSV, checked. -> (why, change)."""
741+
change = {"id": repo.id, "repository": repo.full_name, "set": {}}
742+
743+
given = row.get("web_urls", "")
744+
urls, why = web_urls(given.replace("|", ","))
745+
if why:
746+
return f"web urls: {why}", None
747+
if not urls:
748+
return "a web application needs its web urls", None
749+
new = ", ".join(urls)
750+
if len(new) > LIMITS["web_app_urls"]:
751+
return f"web urls is longer than {LIMITS['web_app_urls']}", None
752+
if not repo.web_app:
753+
change["set"]["web_app"] = (False, True)
754+
if web_urls(repo.web_app_urls)[0] != urls:
755+
change["set"]["web_app_urls"] = (repo.web_app_urls or "", new)
756+
757+
after = {}
758+
for field in WEB_IMPORT_FIELDS:
759+
if field not in header:
760+
continue
761+
given = row.get(field, "")
762+
if given == "-":
763+
given = ""
764+
if field in OWNERS:
765+
bad = bad_address(given.replace("|", ","))
766+
if bad:
767+
return (f"{field.replace('_', ' ')} has to be an email "
768+
f"address, and {bad!r} is not one"), None
769+
elif field == "sensitivity":
770+
held = []
771+
for one in split_labels(given):
772+
label = by_case.get(one.casefold())
773+
if label is None:
774+
return (f"{one!r} is not one of the labels on the "
775+
f"Settings page ({', '.join(offered)})"), None
776+
if label not in held:
777+
held.append(label)
778+
given = ", ".join(held)
779+
given = _web_cell(field, given)
780+
if len(given) > LIMITS[field]:
781+
return (f"{field.replace('_', ' ')} is longer than "
782+
f"{LIMITS[field]}"), None
783+
applied = _web_cell(field, (said.get(field) or {}).get("value", ""))
784+
own = (getattr(repo, field, "") or "").strip()
785+
if given == applied:
786+
continue
787+
# An empty cell over an inherited answer, with nothing set here, is
788+
# already as empty as this repository can make it.
789+
if not given and not own:
790+
continue
791+
change["set"][field] = (own, given)
792+
after[field] = given
793+
794+
# The same rule the repository's form holds to: a notify tick needs an
795+
# owner to send to, its own or the connection's.
796+
for flag, field in (("notify_application_owner", "application_owner"),
797+
("notify_technical_owner", "technical_owner")):
798+
if field not in after or after[field]:
799+
continue
800+
if (getattr(repo, flag, False)
801+
and (said.get(field) or {}).get("from") != "connection"):
802+
return (f"{flag.replace('_', ' ')} is on, but the file leaves no "
803+
f"{field.replace('_', ' ')} to send it to"), None
804+
return "", change
805+
806+
807+
def web_apps_digest(plan: dict) -> str:
808+
"""A short fingerprint of a plan, so apply can tell it still holds."""
809+
import hashlib
810+
import json
811+
812+
body = json.dumps({side: [[c["id"], sorted(c["set"].items())]
813+
for c in plan[side]]
814+
for side in ("adds", "updates", "removes")},
815+
sort_keys=True, default=list)
816+
return hashlib.sha256(body.encode("utf-8")).hexdigest()
817+
818+
819+
def web_apps_apply(session, plan: dict, actor: str = "") -> dict:
820+
"""Write what web_apps_plan worked out. -> counts by side."""
821+
ids = {c["id"]: c for side in ("adds", "updates", "removes")
822+
for c in plan[side]}
823+
rows = {row.id: row for row in session.query(Repository)
824+
.filter(Repository.id.in_(list(ids))).all()} if ids else {}
825+
for repo_id, change in ids.items():
826+
repo = rows.get(repo_id)
827+
if repo is None:
828+
continue
829+
for field, (_old, new) in change["set"].items():
830+
setattr(repo, field, new)
831+
repo.tagged_by = actor or repo.tagged_by or ""
832+
repo.tagged_at = utcnow()
833+
session.flush()
834+
counts = {side: len(plan[side]) for side in ("adds", "updates", "removes")}
835+
log.info("web applications import: %s", counts)
836+
return counts

‎app/web/routes_api.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4170,6 +4170,55 @@ async def import_tagging(request: Request, user: User = Depends(current_user)):
41704170
+ "&msg=" + quote(said[:290]))
41714171

41724172

4173+
@router.post("/repositories/web-apps/import/apply")
4174+
async def apply_web_apps_import(request: Request,
4175+
user: User = Depends(current_user)):
4176+
"""The web applications CSV, as the confirm page showed it.
4177+
4178+
The file is planned again from scratch. If that plan is not the one the
4179+
page showed, because somebody changed a repository in the meantime,
4180+
nothing is written and the person is sent back to look again.
4181+
"""
4182+
await check_csrf(request)
4183+
rbac.require(user, "repo.write")
4184+
4185+
from app.core import tagging
4186+
4187+
form = await request.form()
4188+
text = str(form.get("file_text") or "")
4189+
shown = str(form.get("digest") or "")
4190+
if not text or len(text.encode("utf-8")) > 2 * tagging.MAX_WEB_IMPORT_BYTES:
4191+
raise HTTPException(status_code=400, detail="Choose a file first.")
4192+
4193+
def work():
4194+
with session_scope() as db:
4195+
plan = tagging.web_apps_plan(
4196+
db, text, base=rbac.visible_repositories(db, user))
4197+
if plan["refused"]:
4198+
return None, plan["refused"]
4199+
if tagging.web_apps_digest(plan) != shown:
4200+
return None, ("Something changed since that page was shown. "
4201+
"Import the file again to see what it does now.")
4202+
counts = tagging.web_apps_apply(db, plan, actor=user.username)
4203+
names = lambda side: ", ".join(c["repository"]
4204+
for c in plan[side])
4205+
record(db, "webapps.imported", actor=user.username,
4206+
detail=(f"{counts['adds']} added, {counts['updates']} "
4207+
f"updated, {counts['removes']} removed. "
4208+
f"Added: {names('adds') or '-'}. "
4209+
f"Removed: {names('removes') or '-'}")[:4000],
4210+
outcome="warn" if plan["problems"] else "ok",
4211+
source_ip=client_ip(request))
4212+
return counts, ""
4213+
4214+
counts, why = await run_in_threadpool(work)
4215+
if counts is None:
4216+
return redirect(_back_with("/repositories", why, "warn"))
4217+
said = (f"Web applications imported: {counts['adds']} added, "
4218+
f"{counts['updates']} updated, {counts['removes']} removed.")
4219+
return redirect(_back_with("/repositories", said, "ok"))
4220+
4221+
41734222
@router.post("/blast-radius/repo/{repo_id}/describe")
41744223
async def describe_repository(repo_id: int, request: Request,
41754224
user: User = Depends(current_user)):

‎app/web/routes_external.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -306,22 +306,33 @@ def export_csv(request: Request, user: User = Depends(current_user)):
306306
# One read of each site's newest counted scan, not one per column.
307307
scoped = external.scoping_by_asset(db)
308308
from app.core import dast_paths as _dast_paths
309+
from app.core import webapps as _webapps
309310
for a in db.query(ExternalAsset).order_by(ExternalAsset.host):
310311
values = external.field_values(a)
311312
scope = (scoped.get(a.id) or {}).get("scope")
312-
repos = []
313+
repos, moved = [], []
313314
for l in mapper.links(a):
314315
if l["linked"]:
315316
repos += [f"{l['full_name']}@{b}" for b in l.get("branches") or []] \
316317
or [l["full_name"]]
318+
days = _webapps.changed(l["repo"])["days"]
319+
if days is not None:
320+
moved.append(days)
321+
# Typed on the site wins. Otherwise the newest change on any
322+
# linked repository's default branch, the same number the Web
323+
# Applications tab shows on an application's line.
324+
if a.change_frequency_days:
325+
frequency = a.change_frequency_days
326+
else:
327+
frequency = min(moved) if moved else ""
317328
writer.writerow([_quote_formula(str(c)) for c in [
318329
a.host, a.kind, a.parent_domain, a.application, "; ".join(a.tag_list()),
319330
envs.get(a.environment, a.environment or ""),
320331
external.STATUSES.get(a.status, a.status or ""),
321332
a.executive_owner or "", a.technical_owner or "",
322333
"yes" if a.external_auth_needed else "no",
323334
"yes" if a.whitelist_needed else "no",
324-
a.change_frequency_days or "",
335+
frequency,
325336
"yes" if a.no_scan else "", a.no_scan_reason or "",
326337
a.telemetry_at.strftime("%Y-%m-%d %H:%M") if a.telemetry_at else "",
327338
a.last_dast_at.strftime("%Y-%m-%d") if a.last_dast_at else "",

‎app/web/routes_pages.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2344,6 +2344,71 @@ def export_web_apps(request: Request, user: User = Depends(current_user)):
23442344
f'attachment; filename="web-applications-{stamp}.csv"'})
23452345

23462346

2347+
def _web_import_text(raw: bytes) -> str:
2348+
"""An uploaded file as text, however the spreadsheet saved it."""
2349+
try:
2350+
return raw.decode("utf-8-sig")
2351+
except UnicodeDecodeError:
2352+
return raw.decode("latin-1", "replace")
2353+
2354+
2355+
def _web_import_plan(user, text: str):
2356+
"""The plan for a web applications file, worked out off the event loop."""
2357+
from app.core import tagging
2358+
2359+
with session_scope() as db:
2360+
plan = tagging.web_apps_plan(db, text,
2361+
base=rbac.visible_repositories(db, user))
2362+
return plan, tagging.web_apps_digest(plan)
2363+
2364+
2365+
@router.get("/repositories/web-apps/import")
2366+
def web_apps_import_page(request: Request, user: User = Depends(current_user)):
2367+
"""Where the web applications CSV goes back in."""
2368+
rbac.require(user, "repo.write")
2369+
with session_scope() as db:
2370+
context = _shared(db, user)
2371+
context.update(active_page="repositories")
2372+
return page(request, "web_apps_import.html", **context)
2373+
2374+
2375+
@router.post("/repositories/web-apps/import")
2376+
async def web_apps_import_check(request: Request,
2377+
user: User = Depends(current_user)):
2378+
"""What a web applications CSV would add, change and take off.
2379+
2380+
Nothing is written here. The confirm page carries the file itself and a
2381+
fingerprint of what it showed, so apply can refuse if anything moved in
2382+
between.
2383+
"""
2384+
from starlette.concurrency import run_in_threadpool
2385+
2386+
from app.core import tagging
2387+
from app.web.security import check_csrf
2388+
2389+
form = await request.form()
2390+
await check_csrf(request)
2391+
rbac.require(user, "repo.write")
2392+
upload = form.get("file")
2393+
if upload is None or not hasattr(upload, "read"):
2394+
raise HTTPException(status_code=400, detail="Choose a file first.")
2395+
raw = await upload.read()
2396+
if len(raw) > tagging.MAX_WEB_IMPORT_BYTES:
2397+
raise HTTPException(
2398+
status_code=413,
2399+
detail=f"That file is larger than "
2400+
f"{tagging.MAX_WEB_IMPORT_BYTES // 1024}KB.")
2401+
text = _web_import_text(raw)
2402+
plan, digest = await run_in_threadpool(_web_import_plan, user, text)
2403+
with session_scope() as db:
2404+
context = _shared(db, user)
2405+
context.update(active_page="repositories", plan=plan, digest=digest,
2406+
file_text=text,
2407+
changes=(len(plan["adds"]) + len(plan["updates"])
2408+
+ len(plan["removes"])))
2409+
return page(request, "web_apps_import_confirm.html", **context)
2410+
2411+
23472412
@router.get("/repositories/{repo_id:int}")
23482413
def repository_about(repo_id: int, request: Request, msg: str = "",
23492414
kind: str = "", user: User = Depends(current_user)):

‎app/web/templates/repositories.html‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
{% endif %}
1919
<a class="btn" href="{{ base }}/repositories/web-apps.csv"
2020
title="Every repository marked as part of a web application, with its addresses and owners">Export web applications</a>
21+
{% if can_tag %}
22+
<a class="btn" href="{{ base }}/repositories/web-apps/import"
23+
title="Bring the web applications file back. Rows update what is there, and web applications not in the file are taken off">Import web applications</a>
24+
{% endif %}
2125
{% if certified and not certified_filter %}
2226
<a class="btn" href="#certifications">Certified branches ({{ certified | length }})</a>
2327
{% endif %}</div></div>

0 commit comments

Comments
 (0)