From 01401c45c6dd1cf6b575e253d350531b8c7d4cb9 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Mon, 7 Dec 2020 14:53:10 -0500 Subject: [PATCH 1/4] sort issues --- skare3_tools/github/scripts/milestone_issues.py | 1 + 1 file changed, 1 insertion(+) diff --git a/skare3_tools/github/scripts/milestone_issues.py b/skare3_tools/github/scripts/milestone_issues.py index 06cf362..d212b7a 100755 --- a/skare3_tools/github/scripts/milestone_issues.py +++ b/skare3_tools/github/scripts/milestone_issues.py @@ -30,6 +30,7 @@ def main(): args = get_parser().parse_args() github.init() issues = milestone_issues(args.milestone) + issues = sorted(issues, key=lambda i: int(i["number"])) for issue in issues: print(f'Fixes #{issue["number"]}') From b0d006fedcece253a6ba906a49593781f72578fe Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Mon, 7 Dec 2020 14:54:12 -0500 Subject: [PATCH 2/4] allow a list of channels as argument to get_conda_pkg_info --- skare3_tools/packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skare3_tools/packages.py b/skare3_tools/packages.py index 955cd5a..07eef41 100755 --- a/skare3_tools/packages.py +++ b/skare3_tools/packages.py @@ -484,6 +484,8 @@ def get_conda_pkg_info(conda_package, cmd = ['conda', 'search', conda_package, '--override-channels', '--json'] if conda_channel is None: conda_channels = CONFIG['conda_channels']['main'] + elif type(conda_channel) is list: + conda_channels = conda_channel elif conda_channel in CONFIG['conda_channels']: conda_channels = CONFIG['conda_channels'][conda_channel] else: From 5afc5f9296aab878b404783843bdc7d24cdd3350 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Tue, 17 Nov 2020 09:26:18 -0500 Subject: [PATCH 3/4] parse conda package deps using space between package name and version --- skare3_tools/scripts/skare3_update_summary.py | 192 +++++++++++------- 1 file changed, 115 insertions(+), 77 deletions(-) diff --git a/skare3_tools/scripts/skare3_update_summary.py b/skare3_tools/scripts/skare3_update_summary.py index 2988357..711b5cd 100755 --- a/skare3_tools/scripts/skare3_update_summary.py +++ b/skare3_tools/scripts/skare3_update_summary.py @@ -25,11 +25,11 @@ class ArgumentException(Exception): pass -def repository_change_summary(packages, initial_versions='flight', final_versions='last_tag'): +def repository_change_summary(pkgs_repo_info, initial_versions='flight', final_versions='last_tag'): """ - Assemble a list of all PR merges that occured between initial_version and final_version. + Assemble a list of all PR merges that occurred between initial_version and final_version. - :param packages: + :param pkgs_repo_info: dictionary with github repository information :param initial_versions: str or dict if this is a string, it must be one of 'flight', 'matlab', 'last_tag' @@ -39,48 +39,65 @@ def repository_change_summary(packages, initial_versions='flight', final_version if this is a dictionary, it must me of the form {name: version} :return: """ - packages = packages.copy() - for p in packages: - p['full_name'] = f"{p['owner']}/{p['name']}" - - summary = [] - packages = [p for p in packages - if p['full_name'] in final_versions and p['full_name'] in initial_versions - and final_versions[p['full_name']]] - for p in packages: - p.update({'version_1': initial_versions[p['full_name']], - 'version_2': final_versions[p['full_name']]}) - if p['version_2'] != p['version_1']: - releases = [r['release_tag'] for r in p['release_info']] - if p['version_1'] not in releases: - logging.warning(f" - Initial version of {p['full_name']} is not in release list:" - f" {p['version_1']}, {releases}") - if len(releases) == 1 and releases[0] == '': - logging.warning(f'Package {p["name"]} has no releases?') - continue - - if p['version_1'] in releases and p['version_1']: - releases = releases[releases.index(p['version_2']):releases.index(p['version_1'])] - else: - releases = releases[releases.index(p['version_2']):] - release_info = {r['release_tag']: r['merges'] for r in p['release_info']} - merges = [] - for merge in sum([release_info[k] for k in releases], []): - pr = merge['pr_number'] - url = f'{p["owner"]}/{p["name"]}/pull/{pr}' if merge['pr_number'] else '' - merges.append({ - 'PR': pr, - 'url': url, - 'description': merge['title'] - }) - summary.append({ - 'name': p['full_name'], - 'version_2': p['version_2'], - 'version_1': p['version_1'], - 'versions': [p['version_1']] + releases[::-1], - 'merges': merges[::-1] - }) - summary = sorted(summary, key=lambda pkg: pkg['name'].lower()) + pkg_name_map = packages.get_package_list() + package_to_repo = {n['package']: n['repository'] for n in pkg_name_map + if n['repository'] and n['package']} + + pkgs_repo_info = {f"{p['owner']}/{p['name']}": p for p in pkgs_repo_info} + summary = {'updates': [], 'new': [], 'removed': []} + package_names = sorted(set(list(final_versions) + list(initial_versions))) + for package_name in package_names: + if package_name not in initial_versions: + summary['new'].append( + {'name': package_name, 'version': final_versions[package_name]} + ) + elif package_name not in final_versions or not final_versions[package_name]: + summary['removed'].append(package_name) + else: + version_1 = initial_versions[package_name] + version_2 = final_versions[package_name] + if version_2 != version_1: + update_info = { + 'name': package_name, + 'version_2': version_2, + 'version_1': version_1, + } + if package_name in package_to_repo: + full_name = package_to_repo[package_name] + if full_name in pkgs_repo_info: + p = pkgs_repo_info[full_name] + releases = [r['release_tag'] for r in p['release_info']] + if version_1 not in releases: + logging.warning(f" - Initial version of {full_name} is not in release list:" + f" {version_1}, {releases}") + if len(releases) == 1 and releases[0] == '': + logging.warning(f'Package {p["name"]} has no releases?') + continue + + if version_1 in releases and version_1: + releases = releases[releases.index(version_2):releases.index(version_1)] + else: + releases = releases[releases.index(version_2):] + release_info = {r['release_tag']: r['merges'] for r in p['release_info']} + merges = [] + for merge in sum([release_info[k] for k in releases], []): + pr = merge['pr_number'] + url = f'{p["owner"]}/{p["name"]}/pull/{pr}' if merge['pr_number'] else '' + merges.append({ + 'PR': pr, + 'url': url, + 'description': merge['title'] + }) + update_info.update({ + 'versions': [version_1] + releases[::-1], + 'merges': merges[::-1] + }) + summary['updates'].append(update_info) + + summary['removed'] = sorted(summary['removed']) + summary['new'] = sorted(summary['new'], key=lambda pkg: pkg['name'].lower()) + summary['updates'] = sorted(summary['updates'], key=lambda pkg: pkg['name'].lower()) + return summary @@ -92,24 +109,35 @@ def write_conda_pkg_change_summary(change_summary): the summary :return: """ - for p in change_summary: - print('**{name}: {version_1} -> {version_2}**'.format(**p), - f'({" -> ".join(p["versions"])})') - for merge in p['merges']: - print(' - [PR {PR}](https://github.com/{url}): {description}'.format(**merge)) - print('') + import jinja2 + template = jinja2.Template(PKG_SUMMARY_MD) + print(template.render(summary=change_summary)) # an alternative using jinja2 PKG_SUMMARY_MD = """ -{% for package in summary -%} -**{{ package.name }}:** {{ package.version_1 }} -> {{ package.version_2 }} ( -{%- for v in package.versions -%} +## {{ summary.package }} changes ({{ summary.initial_version }} -> {{ summary.final_version }}) + +{%if 'new' in summary %}### New Packages{% endif %} +{% for package in summary.new -%} +- **{{ package.name }}: {{ package.version }}** +{% endfor %} + +{% if 'removed' in summary and summary.removed|length > 0 %}### Removed Packages{% endif %} +{% for package in summary.removed %} +- **{{ package }}** +{%- endfor %} + +### Updated Packages + +{% for package in summary.updates -%} +- **{{ package.name }}:** {{ package.version_1 }} -> {{ package.version_2 }} +{%- if 'versions' in package %} ({% for v in package.versions -%} {{ v }}{{ " -> " if not loop.last }} -{%- endfor %}) -{% for merge in package.merges -%} +{%- endfor %}){% endif %} +{%- for merge in package.merges %} - [PR {{ merge.PR }}](https://github.com/{{ merge.url }}): {{ merge.description }} -{% endfor %} +{%- endfor %} {% endfor %} """ @@ -121,7 +149,7 @@ def parser(): parse.add_argument('--final-version', default='last_tag', help='Either a string or a json file with dictionary of package/versions.') parse.add_argument('--meta-package', default='ska3-flight') - parse.add_argument('--conda-channel', default='test') + parse.add_argument('--conda-channel', action='append', default=[]) parse.add_argument('--token', help='Github token, or name of file that contains token') return parse @@ -148,29 +176,40 @@ def _get_versions(version, repository_info, conda_info): return version +def split_versions(depends): + result = {} + for depend in depends: + v = depend.split('==') if '==' in depend else depend.split() + if len(v) > 2: + raise Exception(f'Version spec got split into too many parts: {depend}') + p_name = v[0].strip() + p_version = v[1].strip() if len(v) == 2 else '---' + result[p_name] = p_version + return result + + def main(): parse = parser() args = parse.parse_args() + + if len(args.conda_channel) == 0: + args.conda_channel = 'test' + elif len(args.conda_channel) == 1: + args.conda_channel = args.conda_channel[0] + github.init(token=args.token) try: - pkg_name_map = packages.get_package_list() - repo_to_package = {n['repository']: n['package'] for n in pkg_name_map} - package_to_repo = {n['package']: n['repository'] for n in pkg_name_map} repository_info = packages.get_repositories_info() conda_info = packages.get_conda_pkg_info(args.meta_package, conda_channel=args.conda_channel) - conda_info = collections.OrderedDict([(i['version'], i) for i in conda_info[args.meta_package]]) - - # change names in conda_info to repository names + conda_info = collections.OrderedDict( + [(i['version'], i) for i in conda_info[args.meta_package]] + ) for version in conda_info: - depends = [v.split('==') for v in conda_info[version]['depends']] - depends = {v[0].strip(): v[1].strip() for v in depends} - conda_info[version]['depends'] = {package_to_repo[k]: v - for k, v in depends.items() - if k in package_to_repo} + conda_info[version]['depends'] = split_versions(conda_info[version]['depends']) # get the version sets (they can come from file, from repository_info or conda_info) initial_version = _get_versions(args.initial_version, repository_info, conda_info) @@ -178,14 +217,13 @@ def main(): final_version = _get_versions(args.final_version, repository_info, conda_info) change_summary = repository_change_summary(repository_info['packages'], - initial_versions=initial_version, - final_versions=final_version) - - # change the name so the one reported is the conda package name and not the repository - for p in change_summary: - if p['name'] not in pkg_name_map: - continue - p['name'] = repo_to_package[p['name']] + initial_versions=initial_version, + final_versions=final_version) + change_summary.update({ + 'package': args.meta_package, + 'initial_version': args.initial_version, + 'final_version': args.final_version, + }) write_conda_pkg_change_summary(change_summary) except ArgumentException as e: From f948cfbb89339d64b761a312021b59d405cfb5c6 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Mon, 7 Dec 2020 15:36:02 -0500 Subject: [PATCH 4/4] add author to PR merge summary --- skare3_tools/github/graphql.py | 5 +++++ skare3_tools/packages.py | 6 ++++++ skare3_tools/scripts/skare3_update_summary.py | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/skare3_tools/github/graphql.py b/skare3_tools/github/graphql.py index 774952b..33caa70 100644 --- a/skare3_tools/github/graphql.py +++ b/skare3_tools/github/graphql.py @@ -308,6 +308,11 @@ class AuthException(Exception): nodes { oid message + author { + user { + login + } + } } } } diff --git a/skare3_tools/packages.py b/skare3_tools/packages.py index 07eef41..7e0511c 100755 --- a/skare3_tools/packages.py +++ b/skare3_tools/packages.py @@ -318,6 +318,11 @@ def _get_tag_target(tag): oid message pushedDate + author { + user { + login + } + } } } } @@ -406,6 +411,7 @@ def _get_repository_info_v4(owner_repo, commit['message']) if match: merge = match.groupdict() + merge['author'] = commit['author']['user']['login'] merge["pr_number"] = int(merge["pr_number"]) if use_pr_titles: if merge["pr_number"] in all_pull_requests: diff --git a/skare3_tools/scripts/skare3_update_summary.py b/skare3_tools/scripts/skare3_update_summary.py index 711b5cd..131109e 100755 --- a/skare3_tools/scripts/skare3_update_summary.py +++ b/skare3_tools/scripts/skare3_update_summary.py @@ -86,7 +86,8 @@ def repository_change_summary(pkgs_repo_info, initial_versions='flight', final_v merges.append({ 'PR': pr, 'url': url, - 'description': merge['title'] + 'description': merge['title'], + 'author': merge['author'] }) update_info.update({ 'versions': [version_1] + releases[::-1], @@ -136,7 +137,7 @@ def write_conda_pkg_change_summary(change_summary): {{ v }}{{ " -> " if not loop.last }} {%- endfor %}){% endif %} {%- for merge in package.merges %} - - [PR {{ merge.PR }}](https://github.com/{{ merge.url }}): {{ merge.description }} + - [PR {{ merge.PR }}](https://github.com/{{ merge.url }}) ({{ merge.author }}): {{ merge.description }} {%- endfor %} {% endfor %} """