From 7ed6088ece964a329f90fff435bd2d0711cd6a38 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Sun, 29 Nov 2020 23:19:55 +0100 Subject: [PATCH 01/10] add branch option --- scripts/g.extension/g.extension.py | 51 +++++++++++++++++++----------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 9eb54396d80..9e74b3194f9 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -68,6 +68,15 @@ #% required: no #% multiple: yes #%end +#%option +#% key: branch +#% type: string +#% key_desc: branch +#% description: Specific branch to fetch addon from (only used when fetching from git) +#% required: no +#% multiple: no +#% answer: master +#%end #%flag #% key: l @@ -826,7 +835,7 @@ def write_xml_toolboxes(name, tree=None): file_.close() -def install_extension(source, url, xmlurl): +def install_extension(source, url, xmlurl, branch): """Install extension (e.g. one module) or a toolbox (list of modules)""" gisbase = os.getenv('GISBASE') if not gisbase: @@ -867,7 +876,7 @@ def install_extension(source, url, xmlurl): ret1, new_modules_ext, new_files_ext = install_extension_win(extension) else: ret1, new_modules_ext, new_files_ext, tmp_dir = install_extension_std_platforms(extension, - source=source, url=url) + source=source, url=url, branch=branch) if not flags['d'] and not flags['i']: edict[extension]['mlist'].extend(new_modules_ext) edict[extension]['flist'].extend(new_files_ext) @@ -1485,10 +1494,10 @@ def download_source_code(source, url, name, outdev, assert os.path.isdir(directory) -def install_extension_std_platforms(name, source, url): +def install_extension_std_platforms(name, source, url, branch): """Install extension on standard platforms""" gisbase = os.getenv('GISBASE') - source_url = 'https://github.com/OSGeo/grass-addons/tree/master/grass7/' + source_url = 'https://github.com/OSGeo/grass-addons/tree/{branch}/grass7/'.format(branch=branch) if source == 'official': gscript.message(_("Fetching <%s> from " @@ -2017,21 +2026,21 @@ def resolve_xmlurl_prefix(url, source=None): 'ignored_suffixes': ['.zip', '.tar.gz'], 'possible_starts': ['', 'https://', 'http://'], 'url_start': 'https://', - 'url_end': '/archive/master.zip', + 'url_end': '/archive/{branch}.zip', }, 'GitLab': { 'domain': 'gitlab.com', 'ignored_suffixes': ['.zip', '.tar.gz', '.tar.bz2', '.tar'], 'possible_starts': ['', 'https://', 'http://'], 'url_start': 'https://', - 'url_end': '/-/archive/master/{name}-master.zip', + 'url_end': '/-/archive/{branch}/{name}-{branch}.zip', }, 'Bitbucket': { 'domain': 'bitbucket.org', 'ignored_suffixes': ['.zip', '.tar.gz', '.gz', '.bz2'], 'possible_starts': ['', 'https://', 'http://'], 'url_start': 'https://', - 'url_end': '/get/master.zip', + 'url_end': '/get/{branch}.zip', }, } @@ -2039,7 +2048,7 @@ def resolve_xmlurl_prefix(url, source=None): # https://gitlab.com/user/reponame/repository/archive.zip?ref=b%C3%A9po -def resolve_known_host_service(url, name): +def resolve_known_host_service(url, name, branch): """Determine source type and full URL for known hosting service If the service is not determined from the provided URL, tuple with @@ -2069,10 +2078,13 @@ def resolve_known_host_service(url, name): actual_start = match['url_start'] else: actual_start = '' + if 'branch' in match['url_end']: + suffix = match['url_end'].format(name=name, branch=branch) + else: + suffix = match['url_end'].format(name=name) url = '{prefix}{base}{suffix}'.format(prefix=actual_start, base=url.rstrip('/'), - suffix=match['url_end'].format( - name=name)) + suffix=suffix) gscript.verbose(_("Will use the following URL for download: {0}") .format(url)) return 'remote_zip', url @@ -2081,7 +2093,7 @@ def resolve_known_host_service(url, name): # TODO: add also option to enforce the source type -def resolve_source_code(url=None, name=None): +def resolve_source_code(url=None, name=None, branch=None): """Return type and URL or path of the source code Local paths are not presented as URLs to be usable in standard functions. @@ -2089,7 +2101,7 @@ def resolve_source_code(url=None, name=None): has the unfortunate consequence that the not existing files are evaluated as remote URLs. When path is not evaluated, Subversion is assumed for backwards compatibility. When GitHub repository is specified, ZIP file - link is returned. The ZIP is for master branch, not the default one because + link is returned. The ZIP is for {branch} branch, not the default one because GitHub does not provide the default branch in the URL (July 2015). :returns: tuple with type of source and full URL or path @@ -2202,7 +2214,7 @@ def resolve_source_code(url=None, name=None): return suffix, os.path.abspath(url) # Handle remote URLs else: - source, resolved_url = resolve_known_host_service(url, name) + source, resolved_url = resolve_known_host_service(url, name, branch) if source: return source, resolved_url # we allow URL to end with =zip or ?zip and not only .zip @@ -2217,7 +2229,7 @@ def resolve_source_code(url=None, name=None): return 'svn', url -def get_addons_paths(gg_addons_base_dir): +def get_addons_paths(gg_addons_base_dir, branch): """Get and save extensions paths as 'extensions_paths.json' json file in the $GRASS_ADDON_BASE dir. The file serves as a list of all addons, and their paths (mkhmtl.py tool) @@ -2225,7 +2237,7 @@ def get_addons_paths(gg_addons_base_dir): get_addons_paths.json_file = 'addons_paths.json' url = 'https://api.github.com/repos/OSGeo/grass-addons/git/trees/'\ - 'master?recursive=1' + '{branch}?recursive=1' response = download_addons_paths_file( url=url, response_format='application/json', @@ -2243,6 +2255,7 @@ def main(): check_progs() original_url = options['url'] + branch = options['branch'] # manage proxies global PROXIES @@ -2268,7 +2281,8 @@ def main(): # but will work only as long as the function does not check # if the URL is actually valid or something source, url = resolve_source_code(name='dummy', - url=original_url) + url=original_url, + branch=branch) xmlurl = resolve_xmlurl_prefix(original_url, source=source) list_available_extensions(xmlurl) return 0 @@ -2295,9 +2309,10 @@ def main(): """ get_addons_paths(gg_addons_base_dir=options['prefix']) source, url = resolve_source_code(name=options['extension'], - url=original_url) + url=original_url, + branch=branch) xmlurl = resolve_xmlurl_prefix(original_url, source=source) - install_extension(source=source, url=url, xmlurl=xmlurl) + install_extension(source=source, url=url, xmlurl=xmlurl, branch=branch) else: # remove remove_extension(force=flags['f']) From bd219fb251f5ee74a62c422596a1f7e7639339ec Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Nov 2020 09:18:31 +0100 Subject: [PATCH 02/10] use main as default branch --- scripts/g.extension/g.extension.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 9e74b3194f9..554899048ee 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -75,7 +75,7 @@ #% description: Specific branch to fetch addon from (only used when fetching from git) #% required: no #% multiple: no -#% answer: master +#% answer: main #%end #%flag @@ -1497,7 +1497,7 @@ def download_source_code(source, url, name, outdev, def install_extension_std_platforms(name, source, url, branch): """Install extension on standard platforms""" gisbase = os.getenv('GISBASE') - source_url = 'https://github.com/OSGeo/grass-addons/tree/{branch}/grass7/'.format(branch=branch) + source_url = 'https://github.com/OSGeo/grass-addons/tree/master/grass7/' if source == 'official': gscript.message(_("Fetching <%s> from " @@ -2229,7 +2229,7 @@ def resolve_source_code(url=None, name=None, branch=None): return 'svn', url -def get_addons_paths(gg_addons_base_dir, branch): +def get_addons_paths(gg_addons_base_dir): """Get and save extensions paths as 'extensions_paths.json' json file in the $GRASS_ADDON_BASE dir. The file serves as a list of all addons, and their paths (mkhmtl.py tool) @@ -2237,7 +2237,7 @@ def get_addons_paths(gg_addons_base_dir, branch): get_addons_paths.json_file = 'addons_paths.json' url = 'https://api.github.com/repos/OSGeo/grass-addons/git/trees/'\ - '{branch}?recursive=1' + 'master?recursive=1' response = download_addons_paths_file( url=url, response_format='application/json', From 16cfa7f85d26da03780e72c29b9604e84c4ed279 Mon Sep 17 00:00:00 2001 From: Tomas Zigo <50632337+tmszi@users.noreply.github.com> Date: Mon, 30 Nov 2020 20:07:39 +0100 Subject: [PATCH 03/10] try download add-on from the master branch if the main branch does not exist fix install add-on e.g.: g.extension r.example.plus url='https://github.com/wenzeslaus/r.example.plus' --- scripts/g.extension/g.extension.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 554899048ee..59fd09c4207 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1464,7 +1464,11 @@ def download_source_code(source, url, name, outdev, try: response = urlopen(url) except URLError: - grass.fatal(_("Extension <%s> not found") % name) + # Try download add-on from 'master' branch + try: + response = urlopen(url.replace('main', 'master')) + except URLError: + grass.fatal(_("Extension <%s> not found") % name) with open(zip_name, 'wb') as out_file: shutil.copyfileobj(response, out_file) extract_zip(name=zip_name, directory=directory, tmpdir=tmpdir) From 1def8d3b20b8deafdb6829fdb21a3990f4f41608 Mon Sep 17 00:00:00 2001 From: Tomas Zigo <50632337+tmszi@users.noreply.github.com> Date: Mon, 30 Nov 2020 21:11:01 +0100 Subject: [PATCH 04/10] try download add-ons files paths json file from the master branch if the main branch doesn't exist e.g. g.extension -j --- scripts/g.extension/g.extension.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 59fd09c4207..8f31d2a5fa4 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -255,14 +255,9 @@ def download_addons_paths_file( ), ) else: - gscript.fatal( - _( - "Download file from <{url}>, " - "return status code {code}, ".format( - url=url, - code=err, - ), - ), + return download_addons_paths_file( + url=url.replace('main', 'master'), + response_format=response_format, ) except URLError: gscript.fatal( @@ -2241,7 +2236,7 @@ def get_addons_paths(gg_addons_base_dir): get_addons_paths.json_file = 'addons_paths.json' url = 'https://api.github.com/repos/OSGeo/grass-addons/git/trees/'\ - 'master?recursive=1' + 'main?recursive=1' response = download_addons_paths_file( url=url, response_format='application/json', From 2fbf51ab408cda67ff9ba5003a01a666e1745796 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Tue, 1 Dec 2020 12:50:18 +0100 Subject: [PATCH 05/10] add example with branch --- scripts/g.extension/g.extension.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/g.extension/g.extension.html b/scripts/g.extension/g.extension.html index 19ff64befb5..96376bfaba6 100644 --- a/scripts/g.extension/g.extension.html +++ b/scripts/g.extension/g.extension.html @@ -185,7 +185,13 @@

Installing from various online repositories: GitHub, GitLab, Bitbucket

Simple URL to GitHub, GitLab, Bitbucket repositories: 
-g.extension r.example url=github.com/johnsmith/r.example
+g.extension r.example.plus url="https://github.com/wenzeslaus/r.example.plus"
+
+ +Simple URL to GitHub, GitLab, Bitbucket repositories from a specific (e.g. development) branch: + +
+g.extension r.example.plus url="https://github.com/wenzeslaus/r.example.plus" branch=master
Simple URL to OSGeo Trac (downloads a ZIP file, requires download to be enabled in Trac): From a4524a64f6f4f981183f8f1e2bb7dcf81f9b3400 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Tue, 1 Dec 2020 23:13:45 +0100 Subject: [PATCH 06/10] consolidate messages --- scripts/g.extension/g.extension.py | 43 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 8f31d2a5fa4..834b0c1ea3c 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1232,7 +1232,8 @@ def install_extension_win(name): os.chdir(TMPDIR) # this is just to not leave something behind srcdir = os.path.join(TMPDIR, name) download_source_code(source=source, url=url, name=name, - outdev=outdev, directory=srcdir, tmpdir=TMPDIR) + outdev=outdev, directory=srcdir, tmpdir=TMPDIR, + branch=branch) # collect module names and file names module_list = list() @@ -1444,26 +1445,40 @@ def extract_tar(name, directory, tmpdir): def download_source_code(source, url, name, outdev, - directory=None, tmpdir=None): + directory=None, tmpdir=None, branch=None): """Get source code to a local directory for compilation""" - gscript.verbose("Downloading source code for <{name}> from <{url}>" - " which is identified as '{source}' type of source..." - .format(source=source, url=url, name=name)) + gscript.verbose(_("Type of source identified as '{source}'.") + .format(source=source)) if source == 'official': + gscript.message(_("Fetching <%s> from " + "GRASS GIS Addons repository (be patient)...") % name) download_source_code_official_github(url, name, outdev, directory) elif source == 'svn': + gscript.message(_("Fetching <{name}> from " + "<{url}> (be patient)...").format(name=name, url=url)) download_source_code_svn(url, name, outdev, directory) elif source in ['remote_zip']: # , 'official' + gscript.message(_("Fetching <{name}> from " + "<{url}> (be patient)...").format(name=name, url=url)) # we expect that the module.zip file is not by chance in the archive zip_name = os.path.join(tmpdir, 'extension.zip') try: response = urlopen(url) except URLError: - # Try download add-on from 'master' branch - try: - response = urlopen(url.replace('main', 'master')) - except URLError: + # Try download add-on from 'master' branch if default "main" fails + if branch == "main": + try: + url = url.replace('main', 'master') + gscript.message(_("Failed with default branch. " + "Try again from <{url}>...") + .format(url=url)) + response = urlopen(url) + except URLError: + grass.fatal(_("Extension <%s> not found. Please check " + "'url' and 'branch' options".format(name))) + else: grass.fatal(_("Extension <%s> not found") % name) + with open(zip_name, 'wb') as out_file: shutil.copyfileobj(response, out_file) extract_zip(name=zip_name, directory=directory, tmpdir=tmpdir) @@ -1498,13 +1513,6 @@ def install_extension_std_platforms(name, source, url, branch): gisbase = os.getenv('GISBASE') source_url = 'https://github.com/OSGeo/grass-addons/tree/master/grass7/' - if source == 'official': - gscript.message(_("Fetching <%s> from " - "GRASS GIS Addons repository (be patient)...") % name) - else: - gscript.message(_("Fetching <{name}> from " - "<{url}> (be patient)...").format(name=name, url=url)) - # to hide non-error messages from subprocesses if grass.verbosity() <= 2: outdev = open(os.devnull, 'w') @@ -1514,7 +1522,8 @@ def install_extension_std_platforms(name, source, url, branch): os.chdir(TMPDIR) # this is just to not leave something behind srcdir = os.path.join(TMPDIR, name) download_source_code(source=source, url=url, name=name, - outdev=outdev, directory=srcdir, tmpdir=TMPDIR) + outdev=outdev, directory=srcdir, tmpdir=TMPDIR, + branch=branch) os.chdir(srcdir) # collect module names From 36914f0dfa7d49bceea73abfba0c533f66275d5b Mon Sep 17 00:00:00 2001 From: ninsbl Date: Tue, 1 Dec 2020 23:18:36 +0100 Subject: [PATCH 07/10] fix a flake8 issue --- scripts/g.extension/g.extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 834b0c1ea3c..86e81abe521 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1474,7 +1474,7 @@ def download_source_code(source, url, name, outdev, .format(url=url)) response = urlopen(url) except URLError: - grass.fatal(_("Extension <%s> not found. Please check " + grass.fatal(_("Extension <{name}> not found. Please check " "'url' and 'branch' options".format(name))) else: grass.fatal(_("Extension <%s> not found") % name) From a273ea0ebe58be094df874969bf25405c0bd8973 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Tue, 1 Dec 2020 23:23:05 +0100 Subject: [PATCH 08/10] use format correctly --- scripts/g.extension/g.extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 86e81abe521..7d2fb0e6a7c 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1475,7 +1475,7 @@ def download_source_code(source, url, name, outdev, response = urlopen(url) except URLError: grass.fatal(_("Extension <{name}> not found. Please check " - "'url' and 'branch' options".format(name))) + "'url' and 'branch' options".format(name=name))) else: grass.fatal(_("Extension <%s> not found") % name) From f2116f977a4249155571224ad4f315d108202d8f Mon Sep 17 00:00:00 2001 From: Stefan Blumentrath Date: Wed, 2 Dec 2020 18:50:48 +0100 Subject: [PATCH 09/10] improve message Co-authored-by: Markus Neteler --- scripts/g.extension/g.extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 7d2fb0e6a7c..d7233c177d9 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1470,7 +1470,7 @@ def download_source_code(source, url, name, outdev, try: url = url.replace('main', 'master') gscript.message(_("Failed with default branch. " - "Try again from <{url}>...") + "Trying again from <{url}>...") .format(url=url)) response = urlopen(url) except URLError: From 8ee1ba1547ad08b1ec1228f86f9649aa359cdacb Mon Sep 17 00:00:00 2001 From: Stefan Blumentrath Date: Wed, 2 Dec 2020 18:52:11 +0100 Subject: [PATCH 10/10] improve message Co-authored-by: Markus Neteler --- scripts/g.extension/g.extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index d7233c177d9..a1636e8b5b1 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1469,7 +1469,7 @@ def download_source_code(source, url, name, outdev, if branch == "main": try: url = url.replace('main', 'master') - gscript.message(_("Failed with default branch. " + gscript.message(_("Expected default branch not found. " "Trying again from <{url}>...") .format(url=url)) response = urlopen(url)