diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index 9f51d761c22..0ee6028a7ee 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -77,7 +77,6 @@ # % description: Specific branch to fetch addon from (only used when fetching from git) # % required: no # % multiple: no -# % answer: main # %end # %flag @@ -169,6 +168,7 @@ from six.moves.urllib import request as urlrequest from six.moves.urllib.error import HTTPError, URLError +from six.moves.urllib.parse import urlparse # Get the XML parsing exceptions to catch. The behavior changed with Python 2.7 # and ElementTree 1.3. @@ -191,6 +191,7 @@ "User-Agent": "Mozilla/5.0", } HTTP_STATUS_CODES = list(http.HTTPStatus) +GIT_URL = "https://github.com/OSGeo/grass-addons" def replace_shebang_win(python_file): @@ -234,6 +235,78 @@ def urlopen(url, *args, **kwargs): return urlrequest.urlopen(request, *args, **kwargs) +def get_version_branch(major_version): + """Check if version branch for the current GRASS version exists, + if not, take branch for the previous version + For the official repo we assume that at least one version branch is present""" + version_branch = f"grass{major_version}" + try: + urlrequest.urlopen(f"{GIT_URL}/tree/{version_branch}/src") + except URLError: + version_branch = "grass{}".format(int(major_version) - 1) + return version_branch + + +def get_github_branches( + github_api_url="https://api.github.com/repos/OSGeo/grass-addons/branches", + version_only=True, +): + """Get ordered list of branch names in repo using github API + For the official repo we assume that at least one version branch is present + Due to strict rate limits in the github API (60 calls per hour) this function + is currently not used.""" + req = urlrequest.urlopen(github_api_url) + content = json.loads(req.read()) + branches = [repo_branch["name"] for repo_branch in content] + if version_only: + branches = [ + version_branch + for version_branch in branches + if version_branch.startswith("grass") + ] + branches.sort() + return branches + + +def get_default_branch(full_url): + """Get default branch for repository in known hosting services + (currently only implemented for github, gitlab and bitbucket API) + In all other cases "main" is used as default""" + # Parse URL + url_parts = urlparse(full_url) + # Get organization and repository component + try: + organization, repository = url_parts.path.split("/")[1:3] + except URLError: + gscript.fatal( + _( + "Cannot retrieve organization and repository from URL: <{}>.".format( + full_url + ) + ) + ) + # Construct API call and retrieve default branch + api_calls = { + "github.com": f"https://api.github.com/repos/{organization}/{repository}", + "gitlab.com": f"https://gitlab.com/api/v4/projects/{organization}%2F{repository}", + "bitbucket.org": f"https://api.bitbucket.org/2.0/repositories/{organization}/{repository}/branching-model?", + } + # Try to get default branch via API. The API call is known to fail a) if the full_url + # does not belong to an implemented hosting service or b) if the rate limit of the + # API is exceeded + try: + req = urlrequest.urlopen(api_calls.get(url_parts.netloc)) + content = json.loads(req.read()) + # For github and gitlab + default_branch = content.get("default_branch") + # For bitbucket + if not default_branch: + default_branch = content.get("development").get("name") + except URLError: + default_branch = "main" + return default_branch + + def download_addons_paths_file(url, response_format, *args, **kwargs): """Generates JSON file containing the download URLs of the official Addons @@ -1403,8 +1476,6 @@ def download_source_code_official_github(url, name, outdev, directory=None): """ if not directory: directory = os.path.join(os.getcwd, name) - classchar = name.split(".", 1)[0] - moduleclass = expand_module_class_name(classchar) if grass.call(["svn", "export", url, directory], stdout=outdev) != 0: grass.fatal(_("GRASS Addons <%s> not found") % name) return directory @@ -1555,7 +1626,7 @@ def download_source_code( response = urlopen(url) except URLError: # Try download add-on from 'master' branch if default "main" fails - if branch == "main": + if not branch: try: url = url.replace("main", "master") gscript.message( @@ -1611,8 +1682,6 @@ def download_source_code( def install_extension_std_platforms(name, source, url, branch): """Install extension on standard platforms""" gisbase = os.getenv("GISBASE") - # TODO: workaround, https://github.com/OSGeo/grass-addons/issues/528 - source_url = "https://github.com/OSGeo/grass-addons/tree/master/grass7/" # to hide non-error messages from subprocesses if grass.verbosity() <= 2: @@ -1694,7 +1763,7 @@ def install_extension_std_platforms(name, source, url, branch): "SCRIPTDIR=%s" % dirs["script"], "STRINGDIR=%s" % dirs["string"], "ETC=%s" % os.path.join(dirs["etc"]), - "SOURCE_URL=%s" % source_url, + "SOURCE_URL=%s" % url, ] install_cmd = [ @@ -2140,7 +2209,10 @@ def resolve_xmlurl_prefix(url, source=None): gscript.debug("resolve_xmlurl_prefix(url={0}, source={1})".format(url, source)) if source == "official": # use pregenerated modules XML file - url = "https://grass.osgeo.org/addons/grass%s/" % version[0] + # Define branch to fetch from (latest or current version) + version_branch = get_version_branch(version[0]) + + url = "https://grass.osgeo.org/addons/{}/".format(version_branch) # else try to get extensions XMl from SVN repository (provided URL) # the exact action depends on subsequent code (somewhere) @@ -2218,7 +2290,10 @@ def resolve_known_host_service(url, name, branch): else: actual_start = "" if "branch" in match["url_end"]: - suffix = match["url_end"].format(name=name, branch=branch) + suffix = match["url_end"].format( + name=name, + branch=branch if branch else get_default_branch(url), + ) else: suffix = match["url_end"].format(name=name) url = "{prefix}{base}{suffix}".format( @@ -2299,47 +2374,32 @@ def resolve_source_code(url=None, name=None, branch=None, fork=False): >>> resolve_source_code('https://bitbucket.org/joe-user/grass-module') # doctest: +SKIP ('remote_zip', 'https://bitbucket.org/joe-user/grass-module/get/default.zip') """ - if not url and name: - module_class = get_module_class_name(name) - # note: 'trunk' is required to make URL usable for 'svn export' call - git_url = ( - "https://github.com/OSGeo/grass-addons/trunk/" - "grass{version}/{module_class}/{module_name}".format( - version=version[0], module_class=module_class, module_name=name - ) - ) - # trac_url = 'https://trac.osgeo.org/grass/browser/grass-addons/' \ - # 'grass{version}/{module_class}/{module_name}?format=zip' \ - # .format(version=version[0], - # module_class=module_class, module_name=name) - # return 'official', trac_url - return "official", git_url - - if url and fork: + # Handle URL for the offical repo + if name and (not url or fork): module_class = get_module_class_name(name) - # note: 'trunk' is required to make URL usable for 'svn export' call - if branch in ["master", "main"]: - svn_reference = "trunk" + # and fetches the default branch + if not branch: + # Fetch from default branch + version_branch = get_version_branch(version[0]) + try: + url = url.rstrip("/") if url else GIT_URL + urlrequest.urlopen(f"{url}/tree/{version_branch}/src") + svn_reference = "branches/{}".format(version_branch) + except URLError: + svn_reference = "trunk" else: svn_reference = "branches/{}".format(branch) - git_url = ( - "{url}/{branch}/" - "grass{version}/{module_class}/{module_name}".format( - url=url, - version=version[0], - module_class=module_class, - module_name=name, - branch=svn_reference, - ) - ) - # trac_url = 'https://trac.osgeo.org/grass/browser/grass-addons/' \ - # 'grass{version}/{module_class}/{module_name}?format=zip' \ - # .format(version=version[0], - # module_class=module_class, module_name=name) - # return 'official', trac_url - return "official_fork", git_url + if not url: + # Set URL for the given GRASS version + git_url = f"{GIT_URL}/{svn_reference}/src/{module_class}/{name}" + return "official", git_url + else: + # Forks from the official repo should reflect the current structure + url = url.rstrip("/") + git_url = f"{url}/{svn_reference}/src/{module_class}/{name}" + return "official_fork", git_url # Check if URL can be found # Catch corner case if local URL is given starting with file:// @@ -2351,20 +2411,20 @@ def resolve_source_code(url=None, name=None, branch=None, fork=False): open_url = urlopen(url) open_url.close() url_validated = True - except: + except URLError: pass else: try: open_url = urlopen("http://" + url) open_url.close() url_validated = True - except: + except URLError: pass try: open_url = urlopen("https://" + url) open_url.close() url_validated = True - except: + except URLError: pass if not url_validated: @@ -2402,10 +2462,9 @@ def get_addons_paths(gg_addons_base_dir): and their paths (mkhmtl.py tool) """ get_addons_paths.json_file = "addons_paths.json" - - url = ( - "https://api.github.com/repos/OSGeo/grass-addons/git/trees/" "main?recursive=1" - ) + # Define branch to fetch from (latest or current version) + addons_branch = get_version_branch(version[0]) + url = f"https://api.github.com/repos/OSGeo/grass-addons/git/trees/{addons_branch}?recursive=1" response = download_addons_paths_file( url=url, @@ -2505,10 +2564,7 @@ def main(): grass_version = grass.version() version = grass_version["version"].split(".") - # TODO: update temporary workaround of using grass7 subdir of addon-repo, see - # https://github.com/OSGeo/grass-addons/issues/528 - version[0] = 7 - version[1] = 9 + build_platform = grass_version["build_platform"].split("-", 1)[0] sys.exit(main())