@@ -613,3 +613,224 @@ def from_csv(session, text: str, actor: str = "") -> dict:
613613 session .flush ()
614614 log .info ("tagging import: %s" , counts )
615615 return {** counts , "problems" : problems }
616+
617+
618+ # The columns the web applications import reads. The rest of what the export
619+ # writes (visibility, last change, last scan and so on) is what a scan found,
620+ # so it is ignored on the way back in.
621+ WEB_IMPORT_FIELDS = ("application_name" , "application_owner" ,
622+ "technical_owner" , "sensitivity" )
623+
624+ # Small enough to ride along in the confirm page's form, which is how the
625+ # apply step gets exactly the file that was shown.
626+ MAX_WEB_IMPORT_BYTES = 512 * 1024
627+
628+
629+ def _web_cell (field : str , value : str ) -> str :
630+ """A cell the way the repository stores it, so a match reads as a match."""
631+ if field in OWNERS :
632+ return _owners (value )
633+ if field == "sensitivity" :
634+ return ", " .join (split_labels (value ))
635+ return value
636+
637+
638+ def web_apps_plan (session , text : str , base = None ) -> dict :
639+ """What a web applications CSV would change. Nothing is written.
640+
641+ The file is the whole list. A repository in it is marked as part of a
642+ web application with the addresses it gives, and one marked now that is
643+ not in it is taken off, addresses and all. Only repositories `base`
644+ lets this person see are touched either way.
645+
646+ The export writes what applies, which may be the connection's answer.
647+ So a cell that matches what applies is left alone, and only one that
648+ differs becomes the repository's own value. Otherwise every export and
649+ import would copy the connection's owner onto each repository, and
650+ nobody could tell later which ones had been meant.
651+
652+ -> {"rows", "adds", "updates", "removes", "same", "unknown",
653+ "problems", "refused"}. Each change is {"id", "repository",
654+ "set": {field: (old, new)}}.
655+ """
656+ import csv
657+ import io
658+
659+ out = {"rows" : 0 , "adds" : [], "updates" : [], "removes" : [], "same" : 0 ,
660+ "unknown" : [], "problems" : [], "refused" : "" }
661+
662+ try :
663+ reader = csv .DictReader (io .StringIO (text ))
664+ header = [(name or "" ).strip ().lower ()
665+ for name in (reader .fieldnames or [])]
666+ except (csv .Error , ValueError ) as exc :
667+ out ["refused" ] = f"could not read the file: { exc } "
668+ return out
669+ if "repository" not in header or "web_urls" not in header :
670+ out ["refused" ] = ("that file needs repository and web_urls columns; "
671+ "export the web applications first to see the shape" )
672+ return out
673+
674+ query = base if base is not None else session .query (Repository )
675+ visible = {row .full_name : row for row in query .all ()}
676+ applies = effective_many (session , list (visible .values ()))
677+ offered = labels (session )
678+ for repo in visible .values ():
679+ offered += [x for x in split_labels (repo .sensitivity or "" )
680+ if x not in offered ]
681+ by_case = {label .casefold (): label for label in offered }
682+
683+ listed = set ()
684+ try :
685+ for raw in reader :
686+ out ["rows" ] += 1
687+ if out ["rows" ] > MAX_IMPORT_ROWS :
688+ out ["refused" ] = f"more than { MAX_IMPORT_ROWS } rows"
689+ return out
690+ row = {(k or "" ).strip ().lower ():
691+ _unquote ((str (v ) if v is not None else "" ).strip ())
692+ for k , v in raw .items () if k }
693+ name = row .get ("repository" , "" )
694+ if not name :
695+ continue
696+ repo = visible .get (name )
697+ if repo is None :
698+ if name not in out ["unknown" ]:
699+ out ["unknown" ].append (name )
700+ continue
701+ if name in listed :
702+ out ["problems" ].append (f"{ name } is in the file twice; the "
703+ "second row was left out" )
704+ continue
705+ # Listed, even when the row turns out bad below. A typo in one
706+ # row must not read as "take this application off".
707+ listed .add (name )
708+
709+ why , change = _web_row (repo , row , header ,
710+ applies .get (repo .id , {}), by_case , offered )
711+ if why :
712+ out ["problems" ].append (f"{ name } : { why } " )
713+ continue
714+ if not repo .web_app :
715+ out ["adds" ].append (change )
716+ elif change ["set" ]:
717+ out ["updates" ].append (change )
718+ else :
719+ out ["same" ] += 1
720+ except csv .Error as exc :
721+ out ["refused" ] = f"could not read the file: { exc } "
722+ return out
723+
724+ for name , repo in sorted (visible .items ()):
725+ if repo .web_app and name not in listed :
726+ out ["removes" ].append ({
727+ "id" : repo .id , "repository" : name ,
728+ "set" : {"web_app" : (True , False ),
729+ "web_app_urls" : (repo .web_app_urls or "" , "" )}})
730+
731+ # An empty or wrong file would otherwise take every web application off.
732+ if out ["removes" ] and not listed :
733+ out ["refused" ] = ("the file names no web application you can see, so "
734+ "importing it would take every one off; nothing "
735+ "was changed" )
736+ return out
737+
738+
739+ def _web_row (repo , row , header , said , by_case , offered ) -> tuple :
740+ """One row of the web applications CSV, checked. -> (why, change)."""
741+ change = {"id" : repo .id , "repository" : repo .full_name , "set" : {}}
742+
743+ given = row .get ("web_urls" , "" )
744+ urls , why = web_urls (given .replace ("|" , "," ))
745+ if why :
746+ return f"web urls: { why } " , None
747+ if not urls :
748+ return "a web application needs its web urls" , None
749+ new = ", " .join (urls )
750+ if len (new ) > LIMITS ["web_app_urls" ]:
751+ return f"web urls is longer than { LIMITS ['web_app_urls' ]} " , None
752+ if not repo .web_app :
753+ change ["set" ]["web_app" ] = (False , True )
754+ if web_urls (repo .web_app_urls )[0 ] != urls :
755+ change ["set" ]["web_app_urls" ] = (repo .web_app_urls or "" , new )
756+
757+ after = {}
758+ for field in WEB_IMPORT_FIELDS :
759+ if field not in header :
760+ continue
761+ given = row .get (field , "" )
762+ if given == "-" :
763+ given = ""
764+ if field in OWNERS :
765+ bad = bad_address (given .replace ("|" , "," ))
766+ if bad :
767+ return (f"{ field .replace ('_' , ' ' )} has to be an email "
768+ f"address, and { bad !r} is not one" ), None
769+ elif field == "sensitivity" :
770+ held = []
771+ for one in split_labels (given ):
772+ label = by_case .get (one .casefold ())
773+ if label is None :
774+ return (f"{ one !r} is not one of the labels on the "
775+ f"Settings page ({ ', ' .join (offered )} )" ), None
776+ if label not in held :
777+ held .append (label )
778+ given = ", " .join (held )
779+ given = _web_cell (field , given )
780+ if len (given ) > LIMITS [field ]:
781+ return (f"{ field .replace ('_' , ' ' )} is longer than "
782+ f"{ LIMITS [field ]} " ), None
783+ applied = _web_cell (field , (said .get (field ) or {}).get ("value" , "" ))
784+ own = (getattr (repo , field , "" ) or "" ).strip ()
785+ if given == applied :
786+ continue
787+ # An empty cell over an inherited answer, with nothing set here, is
788+ # already as empty as this repository can make it.
789+ if not given and not own :
790+ continue
791+ change ["set" ][field ] = (own , given )
792+ after [field ] = given
793+
794+ # The same rule the repository's form holds to: a notify tick needs an
795+ # owner to send to, its own or the connection's.
796+ for flag , field in (("notify_application_owner" , "application_owner" ),
797+ ("notify_technical_owner" , "technical_owner" )):
798+ if field not in after or after [field ]:
799+ continue
800+ if (getattr (repo , flag , False )
801+ and (said .get (field ) or {}).get ("from" ) != "connection" ):
802+ return (f"{ flag .replace ('_' , ' ' )} is on, but the file leaves no "
803+ f"{ field .replace ('_' , ' ' )} to send it to" ), None
804+ return "" , change
805+
806+
807+ def web_apps_digest (plan : dict ) -> str :
808+ """A short fingerprint of a plan, so apply can tell it still holds."""
809+ import hashlib
810+ import json
811+
812+ body = json .dumps ({side : [[c ["id" ], sorted (c ["set" ].items ())]
813+ for c in plan [side ]]
814+ for side in ("adds" , "updates" , "removes" )},
815+ sort_keys = True , default = list )
816+ return hashlib .sha256 (body .encode ("utf-8" )).hexdigest ()
817+
818+
819+ def web_apps_apply (session , plan : dict , actor : str = "" ) -> dict :
820+ """Write what web_apps_plan worked out. -> counts by side."""
821+ ids = {c ["id" ]: c for side in ("adds" , "updates" , "removes" )
822+ for c in plan [side ]}
823+ rows = {row .id : row for row in session .query (Repository )
824+ .filter (Repository .id .in_ (list (ids ))).all ()} if ids else {}
825+ for repo_id , change in ids .items ():
826+ repo = rows .get (repo_id )
827+ if repo is None :
828+ continue
829+ for field , (_old , new ) in change ["set" ].items ():
830+ setattr (repo , field , new )
831+ repo .tagged_by = actor or repo .tagged_by or ""
832+ repo .tagged_at = utcnow ()
833+ session .flush ()
834+ counts = {side : len (plan [side ]) for side in ("adds" , "updates" , "removes" )}
835+ log .info ("web applications import: %s" , counts )
836+ return counts
0 commit comments