From 12333f8ce75e869c65216cbf2bef791a6dd946ce Mon Sep 17 00:00:00 2001 From: Tomas Zigo <50632337+tmszi@users.noreply.github.com> Date: Sun, 30 Apr 2023 06:19:22 +0200 Subject: [PATCH] g.extension: fix addon SOURCE_URL env var value for official GitHub addons (#2936) --- scripts/g.extension/g.extension.py | 50 +++++++++++++++---- .../testsuite/test_addons_download.py | 32 ++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 5de246cf5da..2d13549d0e5 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -174,7 +174,7 @@ from subprocess import PIPE from urllib import request as urlrequest from urllib.error import HTTPError, URLError -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse # Get the XML parsing exceptions to catch. The behavior changed with Python 2.7 # and ElementTree 1.3. @@ -196,7 +196,7 @@ "User-Agent": "Mozilla/5.0", } HTTP_STATUS_CODES = list(http.HTTPStatus) -GIT_URL = "https://github.com/OSGeo/grass-addons" +GIT_URL = "https://github.com/OSGeo/grass-addons/" # MAKE command # GRASS Makefile are type of GNU Make and not BSD Make @@ -445,6 +445,26 @@ def fetch_addons(self, addon_list, all_addons=False): cwd=self.local_copy, ) + def get_addons_src_code_git_repo_url_path(self): + """Get addons official GitHub repository source code URL path + + :return dict addons_url: dictionary of addons official GitHub + repository source code URL path + """ + addons_url = {} + for addon in self.addons: + addons_url[addon] = urljoin( + self.url, + urljoin( + "tree/", + urljoin( + f"{self.branch}/", + self.addons[addon], + ), + ), + ) + return addons_url + def replace_shebang_win(python_file): """ @@ -1785,9 +1805,11 @@ def download_source_code_official_github(url, name, branch, directory=None): :param directory: directory where the source code will be downloaded (default is the current directory with name attached) - :returns: full path to the directory with the source code - (useful when you not specify directory, if *directory* is specified - the return value is equal to it) + :return str, str: full path to the directory with the source code + (useful when you not specify directory, if + *directory* is specified the return value is equal + to it), + addon official GitHub repository source code URL path """ try: @@ -1804,7 +1826,10 @@ def download_source_code_official_github(url, name, branch, directory=None): ga.fetch_addons([name]) - return str(ga.local_copy / ga.addons[name]) + return ( + str(ga.local_copy / ga.addons[name]), + ga.get_addons_src_code_git_repo_url_path()[name], + ) def move_extracted_files(extract_dir, target_dir, files): @@ -1918,7 +1943,12 @@ def extract_tar(name, directory, tmpdir): def download_source_code( source, url, name, outdev, directory=None, tmpdir=None, branch=None ): - """Get source code to a local directory for compilation""" + """Get source code to a local directory for compilation + + :return dictionary, url: addon source code directory path, + addon official GitHub repository source code + URL path + """ gs.verbose(_("Type of source identified as '{source}'.").format(source=source)) if source in ("official", "official_fork"): gs.message( @@ -1926,7 +1956,7 @@ def download_source_code( name=name, url=url ) ) - directory = download_source_code_official_github( + directory, url = download_source_code_official_github( url, name, branch, directory=directory ) elif source == "svn": @@ -1999,7 +2029,7 @@ def download_source_code( ).format(source) ) assert os.path.isdir(directory) - return directory + return directory, url def install_extension_std_platforms(name, source, url, branch): @@ -2015,7 +2045,7 @@ def install_extension_std_platforms(name, source, url, branch): os.chdir(TMPDIR) # this is just to not leave something behind srcdir = os.path.join(TMPDIR, name) - srcdir = download_source_code( + srcdir, url = download_source_code( source, url, name, diff --git a/scripts/g.extension/testsuite/test_addons_download.py b/scripts/g.extension/testsuite/test_addons_download.py index 38d76d8581e..7b804232d38 100644 --- a/scripts/g.extension/testsuite/test_addons_download.py +++ b/scripts/g.extension/testsuite/test_addons_download.py @@ -17,6 +17,7 @@ import unittest from pathlib import Path +from urllib import request as urlrequest from grass.gunittest.case import TestCase from grass.gunittest.gmodules import SimpleModule @@ -41,6 +42,10 @@ class TestModuleDownloadFromDifferentSources(TestCase): install_prefix / "docs" / "html" / "r.example.plus.html", ] + request_headers = { + "User-Agent": "Mozilla/5.0", + } + def setUp(self): """Make sure we are not dealing with some old files""" if self.install_prefix.exists(): @@ -185,6 +190,33 @@ def test_github_download_official_module_src_code_only(self): self.assertTrue(ext_path.exists()) self.assertIn(ext_path / "Makefile", list(ext_path.iterdir())) + def test_github_official_module_man_page_src_code_links_exists(self): + """Test if the installed extension HTML manual page from the + official GitHub repository contains the source code link and + the source code history link and if they exists + """ + extension = "db.join" + self.assertModule( + "g.extension", + extension=extension, + prefix=str(self.install_prefix), + ) + html_man_page = self.install_prefix / "docs" / "html" / "db.join.html" + self.assertFileExists(str(html_man_page)) + with open(html_man_page) as f: + content = f.read() + for link_name in [f"{extension} source code", "history"]: + url = re.search(rf"{link_name}", content).group(1) + self.assertTrue(url) + try: + request = urlrequest.Request(url, headers=self.request_headers) + response = urlrequest.urlopen(request).code + except urlrequest.HTTPError as e: + response = e.code + except urlrequest.URLError as e: + response = e.args + self.assertEqual(response, 200) + if __name__ == "__main__": test()