From 008e61d081f89ce5e256199dbcb619fae7aa2181 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 10:22:26 +0100 Subject: [PATCH 1/9] scripts/g.extension.all: fix reinstall complex type addons When reinstalled addon is complex type addon e.g. wx.metadata which have multiple modules e.g. g.gui.cswbrowser and etc. for every single module is return wx.metadata addon name to avoid install individual modules which isn't possible and causes an error. For finding correct addon name is using modules.xml file, which have to exists on the osgeo server for download. --- scripts/g.extension.all/g.extension.all.py | 131 ++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index 24ba3a72b30..026da32b259 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -36,7 +36,10 @@ # % label: Force operation (required for removal) # % end from __future__ import print_function +import http +import json import os +import re import sys try: @@ -44,9 +47,17 @@ except ImportError: import elementtree.ElementTree as etree # Python <= 2.4 +from six.moves.urllib import request as urlrequest +from six.moves.urllib.error import HTTPError, URLError + import grass.script as gscript from grass.exceptions import CalledModuleError +HEADERS = { + "User-Agent": "Mozilla/5.0", +} +HTTP_STATUS_CODES = list(http.HTTPStatus) + def get_extensions(): addon_base = os.getenv("GRASS_ADDON_BASE") @@ -77,6 +88,124 @@ def get_extensions(): return ret +def urlopen(url, *args, **kwargs): + """Wrapper around urlopen. Same function as 'urlopen', but with the + ability to define headers. + """ + request = urlrequest.Request(url, headers=HEADERS) + return urlrequest.urlopen(request, *args, **kwargs) + + +def download_modules_xml_file(url, response_format, *args, **kwargs): + """Generates JSON file containing the download URLs of the official + Addons + + :param str url: url address + :param str response_format: content type + + :return response: urllib.request.urlopen response object or None + """ + try: + response = urlopen(url, *args, **kwargs) + + if not response.code == 200: + index = HTTP_STATUS_CODES.index(response.code) + desc = HTTP_STATUS_CODES[index].description + gscript.fatal( + _( + "Download file from <{url}>, " + "return status code {code}, " + "{desc}".format( + url=url, + code=response.code, + desc=desc, + ), + ), + ) + if response_format not in response.getheader("Content-Type"): + gscript.fatal( + _( + "Wrong downloaded file format. " + "Check url <{url}>. Allowed file format is " + "{response_format}.".format( + url=url, + response_format=response_format, + ), + ), + ) + return response + + except HTTPError as err: + if err.code == 404: + gscript.fatal( + _( + "The download of the modules.xml file " + "from the osgeo server wasn't successful. " + " File on the server <{url}> doesn't " + " exists.".format(url=url), + ), + ) + else: + return download_modules_xml_file( + url=url, + response_format=response_format, + ) + except URLError: + gscript.fatal( + _( + "Download file from <{url}>, " + "failed. Check internet connection.".format( + url=url, + ), + ), + ) + + +def find_addon_name(addons): + """Find correct addon name if addon is complex type addon + e.g. wx.metadata has multiple modules g.gui.cswbrowser etc. + + for g.gui.cswbrowser module is return wx.metadata addon name + for r.info.iso module is return wx.metadata addon name + etc. + + + :return set result: set of unique addons names to + reinstall + """ + grass_version = os.getenv("GRASS_VERSION", "unknown") + if grass_version != "unknown": + major, minor, patch = grass_version.split(".") + else: + gscript.fatal(_("Unable to get GRASS GIS version.")) + url = "https://grass.osgeo.org/addons/grass{major}/modules.xml".format( + major=major, + ) + response = download_modules_xml_file( + url=url, + response_format="application/xml", + ) + tree = etree.fromstring(response.read()) + result = [] + for addon in addons: + found = False + for i in tree: + for f in i.findall(".//binary/file"): + if re.match(".*.{}$".format(addon), f.text): + result.append(i.attrib["name"]) + found = True + break + if not found: + gscript.warning( + _( + "The {} addon cannot be reinstalled. " + "Addon wasn't found among the official " + "addons.".format(addon) + ), + ) + return set(result) + + def main(): remove = options["operation"] == "remove" if remove or flags["f"]: @@ -103,7 +232,7 @@ def main(): ) return 0 - for ext in extensions: + for ext in find_addon_name(addons=extensions): gscript.message("-" * 60) if remove: gscript.message(_("Removing extension <%s>...") % ext) From 8b2f9f1d9e3becf940b41d8e35584abe499b1ee2 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 10:58:11 +0100 Subject: [PATCH 2/9] Fix func doc --- scripts/g.extension.all/g.extension.all.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index 026da32b259..6329ef9e74e 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -169,9 +169,9 @@ def find_addon_name(addons): for r.info.iso module is return wx.metadata addon name etc. + :param list addons: list of individual addons modules to reinstall - :return set result: set of unique addons names to - reinstall + :return set result: set of unique addons names to reinstall """ grass_version = os.getenv("GRASS_VERSION", "unknown") if grass_version != "unknown": From fdcb94abbeeebe7e45d1b183b31b9bb0e88a433f Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 11:41:09 +0100 Subject: [PATCH 3/9] Incorporate suggestions --- scripts/g.extension.all/g.extension.all.py | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index 6329ef9e74e..ac64087b3a8 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -140,9 +140,9 @@ def download_modules_xml_file(url, response_format, *args, **kwargs): gscript.fatal( _( "The download of the modules.xml file " - "from the osgeo server wasn't successful. " - " File on the server <{url}> doesn't " - " exists.".format(url=url), + "from the server grass.osgeo.org was not successful. " + "File on the server <{url}> doesn't " + "exists.".format(url=url), ), ) else: @@ -162,16 +162,19 @@ def download_modules_xml_file(url, response_format, *args, **kwargs): def find_addon_name(addons): - """Find correct addon name if addon is complex type addon - e.g. wx.metadata has multiple modules g.gui.cswbrowser etc. - - for g.gui.cswbrowser module is return wx.metadata addon name - for r.info.iso module is return wx.metadata addon name + """Find correct addon name if addon is a multi-addon + e.g. wx.metadata contains multiple modules g.gui.cswbrowser etc. + + Examples: + - for the g.gui.cswbrowser module the wx.metadata addon name is + returned + - for the i.sentinel.download module the i.sentinel.download addon + name is returned etc. - :param list addons: list of individual addons modules to reinstall + :param list addons: list of individual addon modules to be reinstalled - :return set result: set of unique addons names to reinstall + :return set result: set of unique addon names to be reinstalled """ grass_version = os.getenv("GRASS_VERSION", "unknown") if grass_version != "unknown": From 2c170deb7a5512a2d26b814a41bdea9c24542cb3 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 11:45:29 +0100 Subject: [PATCH 4/9] Fix fatal message --- scripts/g.extension.all/g.extension.all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index ac64087b3a8..dba4bd92387 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -140,7 +140,7 @@ def download_modules_xml_file(url, response_format, *args, **kwargs): gscript.fatal( _( "The download of the modules.xml file " - "from the server grass.osgeo.org was not successful. " + "from the server was not successful. " "File on the server <{url}> doesn't " "exists.".format(url=url), ), From d962ea80f3d0c1f8cd709c3379fdc53531f4d78a Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 13:31:04 +0100 Subject: [PATCH 5/9] Remove unnecessary module import --- scripts/g.extension.all/g.extension.all.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index dba4bd92387..c3a22df784b 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -37,7 +37,6 @@ # % end from __future__ import print_function import http -import json import os import re import sys From 2b383979a38a676851c082c2b13f777d6a5f366a Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 13:41:58 +0100 Subject: [PATCH 6/9] Fix warning message --- scripts/g.extension.all/g.extension.all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index c3a22df784b..0868ab6259e 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -200,7 +200,7 @@ def find_addon_name(addons): if not found: gscript.warning( _( - "The {} addon cannot be reinstalled. " + "The <{}> addon cannot be reinstalled. " "Addon wasn't found among the official " "addons.".format(addon) ), From 6b3fcc445f2856726abffba1fc4683d86b5bc0a4 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 12 Jan 2022 19:09:59 +0100 Subject: [PATCH 7/9] Incorporate suggestions --- scripts/g.extension.all/g.extension.all.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index 0868ab6259e..d846bce10df 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -124,7 +124,7 @@ def download_modules_xml_file(url, response_format, *args, **kwargs): if response_format not in response.getheader("Content-Type"): gscript.fatal( _( - "Wrong downloaded file format. " + "Wrong file format downloaded. " "Check url <{url}>. Allowed file format is " "{response_format}.".format( url=url, @@ -167,8 +167,8 @@ def find_addon_name(addons): Examples: - for the g.gui.cswbrowser module the wx.metadata addon name is returned - - for the i.sentinel.download module the i.sentinel.download addon - name is returned + - for the i.sentinel.download module the i.sentinel addon name is + returned etc. :param list addons: list of individual addon modules to be reinstalled From ae062a87c21d488d8cb55b89a893fd259fcf5d1f Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Thu, 13 Jan 2022 12:51:15 +0100 Subject: [PATCH 8/9] Fix regular expression pattern --- scripts/g.extension.all/g.extension.all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index d846bce10df..3ed7f7863bf 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -193,7 +193,7 @@ def find_addon_name(addons): found = False for i in tree: for f in i.findall(".//binary/file"): - if re.match(".*.{}$".format(addon), f.text): + if re.match(r".*{}$".format(addon), f.text): result.append(i.attrib["name"]) found = True break From f94e7c5bf8d67184ddb7db812f6f53a2c54e0afe Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Fri, 14 Jan 2022 06:36:56 +0100 Subject: [PATCH 9/9] Move addon regular expression pattern --- scripts/g.extension.all/g.extension.all.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index 3ed7f7863bf..c3aeebf0cc2 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -191,9 +191,10 @@ def find_addon_name(addons): result = [] for addon in addons: found = False + addon_pattern = re.compile(r".*{}$".format(addon)) for i in tree: for f in i.findall(".//binary/file"): - if re.match(r".*{}$".format(addon), f.text): + if re.match(addon_pattern, f.text): result.append(i.attrib["name"]) found = True break