Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

g.extension: add branch option #1130

Merged
merged 10 commits into from
Dec 2, 2020
Merged

g.extension: add branch option #1130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7ed6088
add branch option
Nov 29, 2020
bd219fb
use main as default branch
Nov 30, 2020
16cfa7f
try download add-on from the master branch if the main branch does no…
tmszi Nov 30, 2020
1def8d3
try download add-ons files paths json file from the master branch if …
tmszi Nov 30, 2020
2fbf51a
add example with branch
Dec 1, 2020
a4524a6
consolidate messages
Dec 1, 2020
36914f0
fix a flake8 issue
Dec 1, 2020
a273ea0
use format correctly
Dec 1, 2020
f2116f9
improve message
ninsbl Dec 2, 2020
8ee1ba1
improve message
ninsbl Dec 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion scripts/g.extension/g.extension.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,13 @@ <h3>Installing from various online repositories: GitHub, GitLab, Bitbucket</h3>
Simple URL to GitHub, GitLab, Bitbucket repositories:

<div class="code"><pre>
g.extension r.example url=github.com/johnsmith/r.example
g.extension r.example.plus url="https://github.com/wenzeslaus/r.example.plus"
</pre></div>

Simple URL to GitHub, GitLab, Bitbucket repositories from a specific (e.g. development) branch:

<div class="code"><pre>
g.extension r.example.plus url="https://github.com/wenzeslaus/r.example.plus" branch=master
</pre></div>

Simple URL to OSGeo Trac (downloads a ZIP file, requires download to be enabled in Trac):
Expand Down
99 changes: 61 additions & 38 deletions scripts/g.extension/g.extension.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,15 @@
#% required: no
#% multiple: yes
#%end
#%option
#% key: branch
#% type: string
#% key_desc: branch
#% description: Specific branch to fetch addon from (only used when fetching from git)
#% required: no
#% multiple: no
#% answer: main
#%end

#%flag
#% key: l
Expand Down Expand Up @@ -246,14 +255,9 @@ def download_addons_paths_file(
),
)
else:
gscript.fatal(
_(
"Download file from <{url}>, "
"return status code {code}, ".format(
url=url,
code=err,
),
),
return download_addons_paths_file(
url=url.replace('main', 'master'),
response_format=response_format,
)
except URLError:
gscript.fatal(
Expand Down Expand Up @@ -826,7 +830,7 @@ def write_xml_toolboxes(name, tree=None):
file_.close()


def install_extension(source, url, xmlurl):
def install_extension(source, url, xmlurl, branch):
"""Install extension (e.g. one module) or a toolbox (list of modules)"""
gisbase = os.getenv('GISBASE')
if not gisbase:
Expand Down Expand Up @@ -867,7 +871,7 @@ def install_extension(source, url, xmlurl):
ret1, new_modules_ext, new_files_ext = install_extension_win(extension)
else:
ret1, new_modules_ext, new_files_ext, tmp_dir = install_extension_std_platforms(extension,
source=source, url=url)
source=source, url=url, branch=branch)
if not flags['d'] and not flags['i']:
edict[extension]['mlist'].extend(new_modules_ext)
edict[extension]['flist'].extend(new_files_ext)
Expand Down Expand Up @@ -1228,7 +1232,8 @@ def install_extension_win(name):
os.chdir(TMPDIR) # this is just to not leave something behind
srcdir = os.path.join(TMPDIR, name)
download_source_code(source=source, url=url, name=name,
outdev=outdev, directory=srcdir, tmpdir=TMPDIR)
outdev=outdev, directory=srcdir, tmpdir=TMPDIR,
branch=branch)

# collect module names and file names
module_list = list()
Expand Down Expand Up @@ -1440,22 +1445,40 @@ def extract_tar(name, directory, tmpdir):


def download_source_code(source, url, name, outdev,
directory=None, tmpdir=None):
directory=None, tmpdir=None, branch=None):
"""Get source code to a local directory for compilation"""
gscript.verbose("Downloading source code for <{name}> from <{url}>"
" which is identified as '{source}' type of source..."
.format(source=source, url=url, name=name))
gscript.verbose(_("Type of source identified as '{source}'.")
.format(source=source))
if source == 'official':
gscript.message(_("Fetching <%s> from "
"GRASS GIS Addons repository (be patient)...") % name)
download_source_code_official_github(url, name, outdev, directory)
elif source == 'svn':
gscript.message(_("Fetching <{name}> from "
"<{url}> (be patient)...").format(name=name, url=url))
download_source_code_svn(url, name, outdev, directory)
elif source in ['remote_zip']: # , 'official'
gscript.message(_("Fetching <{name}> from "
"<{url}> (be patient)...").format(name=name, url=url))
# we expect that the module.zip file is not by chance in the archive
zip_name = os.path.join(tmpdir, 'extension.zip')
try:
response = urlopen(url)
except URLError:
grass.fatal(_("Extension <%s> not found") % name)
# Try download add-on from 'master' branch if default "main" fails
if branch == "main":
try:
url = url.replace('main', 'master')
gscript.message(_("Expected default branch not found. "
"Trying again from <{url}>...")
.format(url=url))
response = urlopen(url)
except URLError:
grass.fatal(_("Extension <{name}> not found. Please check "
"'url' and 'branch' options".format(name=name)))
else:
grass.fatal(_("Extension <%s> not found") % name)

with open(zip_name, 'wb') as out_file:
shutil.copyfileobj(response, out_file)
extract_zip(name=zip_name, directory=directory, tmpdir=tmpdir)
Expand Down Expand Up @@ -1485,18 +1508,11 @@ def download_source_code(source, url, name, outdev,
assert os.path.isdir(directory)


def install_extension_std_platforms(name, source, url):
def install_extension_std_platforms(name, source, url, branch):
"""Install extension on standard platforms"""
gisbase = os.getenv('GISBASE')
source_url = 'https://github.com/OSGeo/grass-addons/tree/master/grass7/'

if source == 'official':
gscript.message(_("Fetching <%s> from "
"GRASS GIS Addons repository (be patient)...") % name)
else:
gscript.message(_("Fetching <{name}> from "
"<{url}> (be patient)...").format(name=name, url=url))

# to hide non-error messages from subprocesses
if grass.verbosity() <= 2:
outdev = open(os.devnull, 'w')
Expand All @@ -1506,7 +1522,8 @@ def install_extension_std_platforms(name, source, url):
os.chdir(TMPDIR) # this is just to not leave something behind
srcdir = os.path.join(TMPDIR, name)
download_source_code(source=source, url=url, name=name,
outdev=outdev, directory=srcdir, tmpdir=TMPDIR)
outdev=outdev, directory=srcdir, tmpdir=TMPDIR,
branch=branch)
os.chdir(srcdir)

# collect module names
Expand Down Expand Up @@ -2017,29 +2034,29 @@ def resolve_xmlurl_prefix(url, source=None):
'ignored_suffixes': ['.zip', '.tar.gz'],
'possible_starts': ['', 'https://', 'http://'],
'url_start': 'https://',
'url_end': '/archive/master.zip',
'url_end': '/archive/{branch}.zip',
},
'GitLab': {
'domain': 'gitlab.com',
'ignored_suffixes': ['.zip', '.tar.gz', '.tar.bz2', '.tar'],
'possible_starts': ['', 'https://', 'http://'],
'url_start': 'https://',
'url_end': '/-/archive/master/{name}-master.zip',
'url_end': '/-/archive/{branch}/{name}-{branch}.zip',
},
'Bitbucket': {
'domain': 'bitbucket.org',
'ignored_suffixes': ['.zip', '.tar.gz', '.gz', '.bz2'],
'possible_starts': ['', 'https://', 'http://'],
'url_start': 'https://',
'url_end': '/get/master.zip',
'url_end': '/get/{branch}.zip',
},
}

# TODO: support ZIP URLs which don't end with zip
# https://gitlab.com/user/reponame/repository/archive.zip?ref=b%C3%A9po


def resolve_known_host_service(url, name):
def resolve_known_host_service(url, name, branch):
"""Determine source type and full URL for known hosting service

If the service is not determined from the provided URL, tuple with
Expand Down Expand Up @@ -2069,10 +2086,13 @@ def resolve_known_host_service(url, name):
actual_start = match['url_start']
else:
actual_start = ''
if 'branch' in match['url_end']:
suffix = match['url_end'].format(name=name, branch=branch)
else:
suffix = match['url_end'].format(name=name)
url = '{prefix}{base}{suffix}'.format(prefix=actual_start,
base=url.rstrip('/'),
suffix=match['url_end'].format(
name=name))
suffix=suffix)
gscript.verbose(_("Will use the following URL for download: {0}")
.format(url))
return 'remote_zip', url
Expand All @@ -2081,15 +2101,15 @@ def resolve_known_host_service(url, name):


# TODO: add also option to enforce the source type
def resolve_source_code(url=None, name=None):
def resolve_source_code(url=None, name=None, branch=None):
"""Return type and URL or path of the source code

Local paths are not presented as URLs to be usable in standard functions.
Path is identified as local path if the directory of file exists which
has the unfortunate consequence that the not existing files are evaluated
as remote URLs. When path is not evaluated, Subversion is assumed for
backwards compatibility. When GitHub repository is specified, ZIP file
link is returned. The ZIP is for master branch, not the default one because
link is returned. The ZIP is for {branch} branch, not the default one because
GitHub does not provide the default branch in the URL (July 2015).

:returns: tuple with type of source and full URL or path
Expand Down Expand Up @@ -2202,7 +2222,7 @@ def resolve_source_code(url=None, name=None):
return suffix, os.path.abspath(url)
# Handle remote URLs
else:
source, resolved_url = resolve_known_host_service(url, name)
source, resolved_url = resolve_known_host_service(url, name, branch)
if source:
return source, resolved_url
# we allow URL to end with =zip or ?zip and not only .zip
Expand All @@ -2225,7 +2245,7 @@ def get_addons_paths(gg_addons_base_dir):
get_addons_paths.json_file = 'addons_paths.json'

url = 'https://api.github.com/repos/OSGeo/grass-addons/git/trees/'\
'master?recursive=1'
'main?recursive=1'

response = download_addons_paths_file(
url=url, response_format='application/json',
Expand All @@ -2243,6 +2263,7 @@ def main():
check_progs()

original_url = options['url']
branch = options['branch']

# manage proxies
global PROXIES
Expand All @@ -2268,7 +2289,8 @@ def main():
# but will work only as long as the function does not check
# if the URL is actually valid or something
source, url = resolve_source_code(name='dummy',
url=original_url)
url=original_url,
branch=branch)
xmlurl = resolve_xmlurl_prefix(original_url, source=source)
list_available_extensions(xmlurl)
return 0
Expand All @@ -2295,9 +2317,10 @@ def main():
"""
get_addons_paths(gg_addons_base_dir=options['prefix'])
source, url = resolve_source_code(name=options['extension'],
url=original_url)
url=original_url,
branch=branch)
xmlurl = resolve_xmlurl_prefix(original_url, source=source)
install_extension(source=source, url=url, xmlurl=xmlurl)
install_extension(source=source, url=url, xmlurl=xmlurl, branch=branch)
else: # remove
remove_extension(force=flags['f'])

Expand Down