Improve info provided by ska3_update_summary

skare3_tools/github/graphql.py

@@ -308,6 +308,11 @@ class AuthException(Exception):
             nodes {
               oid
               message
+              author {
+                user {
+                  login
+                }
+              }
             }
           }
         }

skare3_tools/github/scripts/milestone_issues.py

@@ -30,6 +30,7 @@ def main():
     args = get_parser().parse_args()
     github.init()
     issues = milestone_issues(args.milestone)
+    issues = sorted(issues, key=lambda i: int(i["number"]))
     for issue in issues:
         print(f'Fixes #{issue["number"]}')
 

skare3_tools/packages.py

@@ -318,6 +318,11 @@ def _get_tag_target(tag):
               oid
               message
               pushedDate
+              author {
+                user {
+                  login
+                }
+              }
             }
           }
         }
@@ -406,6 +411,7 @@ def _get_repository_info_v4(owner_repo,
             commit['message'])
         if match:
             merge = match.groupdict()
+            merge['author'] = commit['author']['user']['login']
             merge["pr_number"] = int(merge["pr_number"])
             if use_pr_titles:
                 if merge["pr_number"] in all_pull_requests:
@@ -484,6 +490,8 @@ def get_conda_pkg_info(conda_package,
     cmd = ['conda', 'search', conda_package, '--override-channels', '--json']
     if conda_channel is None:
         conda_channels = CONFIG['conda_channels']['main']
+    elif type(conda_channel) is list:
+        conda_channels = conda_channel
     elif conda_channel in CONFIG['conda_channels']:
         conda_channels = CONFIG['conda_channels'][conda_channel]
     else:

skare3_tools/scripts/skare3_update_summary.py

@@ -25,11 +25,11 @@ class ArgumentException(Exception):
     pass
 
 
-def repository_change_summary(packages, initial_versions='flight', final_versions='last_tag'):
+def repository_change_summary(pkgs_repo_info, initial_versions='flight', final_versions='last_tag'):
     """
-    Assemble a list of all PR merges that occured between initial_version and final_version.
+    Assemble a list of all PR merges that occurred between initial_version and final_version.
 
-    :param packages:
+    :param pkgs_repo_info:
         dictionary with github repository information
     :param initial_versions: str or dict
         if this is a string, it must be one of 'flight', 'matlab', 'last_tag'
@@ -39,48 +39,66 @@ def repository_change_summary(packages, initial_versions='flight', final_version
         if this is a dictionary, it must me of the form {name: version}
     :return:
     """
-    packages = packages.copy()
-    for p in packages:
-        p['full_name'] = f"{p['owner']}/{p['name']}"
-
-    summary = []
-    packages = [p for p in packages
-                if p['full_name'] in final_versions and p['full_name'] in initial_versions
-                and final_versions[p['full_name']]]
-    for p in packages:
-        p.update({'version_1': initial_versions[p['full_name']],
-                  'version_2': final_versions[p['full_name']]})
-        if p['version_2'] != p['version_1']:
-            releases = [r['release_tag'] for r in p['release_info']]
-            if p['version_1'] not in releases:
-                logging.warning(f" - Initial version of {p['full_name']} is not in release list:"
-                                f" {p['version_1']}, {releases}")
-            if len(releases) == 1 and releases[0] == '':
-                logging.warning(f'Package {p["name"]} has no releases?')
-                continue
-
-            if p['version_1'] in releases and p['version_1']:
-                releases = releases[releases.index(p['version_2']):releases.index(p['version_1'])]
-            else:
-                releases = releases[releases.index(p['version_2']):]
-            release_info = {r['release_tag']: r['merges'] for r in p['release_info']}
-            merges = []
-            for merge in sum([release_info[k] for k in releases], []):
-                pr = merge['pr_number']
-                url = f'{p["owner"]}/{p["name"]}/pull/{pr}' if merge['pr_number'] else ''
-                merges.append({
-                    'PR': pr,
-                    'url': url,
-                    'description': merge['title']
-                })
-            summary.append({
-                'name': p['full_name'],
-                'version_2': p['version_2'],
-                'version_1': p['version_1'],
-                'versions': [p['version_1']] + releases[::-1],
-                'merges': merges[::-1]
-            })
-    summary = sorted(summary, key=lambda pkg: pkg['name'].lower())
+    pkg_name_map = packages.get_package_list()
+    package_to_repo = {n['package']: n['repository'] for n in pkg_name_map
+                       if n['repository'] and n['package']}
+
+    pkgs_repo_info = {f"{p['owner']}/{p['name']}": p for p in pkgs_repo_info}
+    summary = {'updates': [], 'new': [], 'removed': []}
+    package_names = sorted(set(list(final_versions) + list(initial_versions)))
+    for package_name in package_names:
+        if package_name not in initial_versions:
+            summary['new'].append(
+                {'name': package_name, 'version': final_versions[package_name]}
+            )
+        elif package_name not in final_versions or not final_versions[package_name]:
+            summary['removed'].append(package_name)
+        else:
+            version_1 = initial_versions[package_name]
+            version_2 = final_versions[package_name]
+            if version_2 != version_1:
+                update_info = {
+                    'name': package_name,
+                    'version_2': version_2,
+                    'version_1': version_1,
+                }
+                if package_name in package_to_repo:
+                    full_name = package_to_repo[package_name]
+                    if full_name in pkgs_repo_info:
+                        p = pkgs_repo_info[full_name]
+                        releases = [r['release_tag'] for r in p['release_info']]
+                        if version_1 not in releases:
+                            logging.warning(f" - Initial version of {full_name} is not in release list:"
+                                            f" {version_1}, {releases}")
+                        if len(releases) == 1 and releases[0] == '':
+                            logging.warning(f'Package {p["name"]} has no releases?')
+                            continue
+
+                        if version_1 in releases and version_1:
+                            releases = releases[releases.index(version_2):releases.index(version_1)]
+                        else:
+                            releases = releases[releases.index(version_2):]
+                        release_info = {r['release_tag']: r['merges'] for r in p['release_info']}
+                        merges = []
+                        for merge in sum([release_info[k] for k in releases], []):
+                            pr = merge['pr_number']
+                            url = f'{p["owner"]}/{p["name"]}/pull/{pr}' if merge['pr_number'] else ''
+                            merges.append({
+                                'PR': pr,
+                                'url': url,
+                                'description': merge['title'],
+                                'author': merge['author']
+                            })
+                        update_info.update({
+                            'versions': [version_1] + releases[::-1],
+                            'merges': merges[::-1]
+                        })
+                summary['updates'].append(update_info)
+
+    summary['removed'] = sorted(summary['removed'])
+    summary['new'] = sorted(summary['new'], key=lambda pkg: pkg['name'].lower())
+    summary['updates'] = sorted(summary['updates'], key=lambda pkg: pkg['name'].lower())
+
     return summary
 
 
@@ -92,24 +110,35 @@ def write_conda_pkg_change_summary(change_summary):
         the summary
     :return:
     """
-    for p in change_summary:
-        print('**{name}: {version_1} -> {version_2}**'.format(**p),
-              f'({" -> ".join(p["versions"])})')
-        for merge in p['merges']:
-            print('  - [PR {PR}](https://github.com/{url}): {description}'.format(**merge))
-        print('')
+    import jinja2
+    template = jinja2.Template(PKG_SUMMARY_MD)
+    print(template.render(summary=change_summary))
 
 
 # an alternative using jinja2
 PKG_SUMMARY_MD = """
-{% for package in summary -%}
-**{{ package.name }}:** {{ package.version_1 }} -> {{ package.version_2 }} (
-{%- for v in package.versions -%}
-{{ v }}{{ " -> " if not loop.last }}
-{%- endfor %})
-{% for merge in package.merges -%}
-  - [PR {{ merge.PR }}](https://github.com/{{ merge.url }}): {{ merge.description }}
+## {{ summary.package }} changes ({{ summary.initial_version }} -> {{ summary.final_version }})
+
+{%if 'new' in summary %}### New Packages{% endif %}
+{% for package in summary.new -%}
+- **{{ package.name }}: {{ package.version }}**
 {% endfor %}
+
+{% if 'removed' in summary and summary.removed|length > 0 %}### Removed Packages{% endif %}
+{% for package in summary.removed %}
+- **{{ package }}**
+{%- endfor %}
+
+### Updated Packages
+
+{% for package in summary.updates -%}
+- **{{ package.name }}:** {{ package.version_1 }} -> {{ package.version_2 }}
+{%- if 'versions' in package %} ({% for v in package.versions -%}
+{{ v }}{{ " -> " if not loop.last }}
+{%- endfor %}){% endif %}
+{%-  for merge in package.merges %}
+  - [PR {{ merge.PR }}](https://github.com/{{ merge.url }}) ({{ merge.author }}): {{ merge.description }}
+{%- endfor %}
 {% endfor %}
 """
 
@@ -121,7 +150,7 @@ def parser():
     parse.add_argument('--final-version', default='last_tag',
                        help='Either a string or a json file with dictionary of package/versions.')
     parse.add_argument('--meta-package', default='ska3-flight')
-    parse.add_argument('--conda-channel', default='test')
+    parse.add_argument('--conda-channel', action='append', default=[])
     parse.add_argument('--token', help='Github token, or name of file that contains token')
     return parse
 
@@ -148,44 +177,54 @@ def _get_versions(version, repository_info, conda_info):
     return version
 
 
+def split_versions(depends):
+    result = {}
+    for depend in depends:
+        v = depend.split('==') if '==' in depend else depend.split()
+        if len(v) > 2:
+            raise Exception(f'Version spec got split into too many parts: {depend}')
+        p_name = v[0].strip()
+        p_version = v[1].strip() if len(v) == 2 else '---'
+        result[p_name] = p_version
+    return result
+
+
 def main():
     parse = parser()
     args = parse.parse_args()
+
+    if len(args.conda_channel) == 0:
+        args.conda_channel = 'test'
+    elif len(args.conda_channel) == 1:
+        args.conda_channel = args.conda_channel[0]
+
     github.init(token=args.token)
 
     try:
-        pkg_name_map = packages.get_package_list()
-        repo_to_package = {n['repository']: n['package'] for n in pkg_name_map}
-        package_to_repo = {n['package']: n['repository'] for n in pkg_name_map}
 
         repository_info = packages.get_repositories_info()
 
         conda_info = packages.get_conda_pkg_info(args.meta_package,
                                                  conda_channel=args.conda_channel)
-        conda_info = collections.OrderedDict([(i['version'], i) for i in conda_info[args.meta_package]])
-
-        # change names in conda_info to repository names
+        conda_info = collections.OrderedDict(
+            [(i['version'], i) for i in conda_info[args.meta_package]]
+        )
         for version in conda_info:
-            depends = [v.split('==') for v in conda_info[version]['depends']]
-            depends = {v[0].strip(): v[1].strip() for v in depends}
-            conda_info[version]['depends'] = {package_to_repo[k]: v
-                                              for k, v in depends.items()
-                                              if k in package_to_repo}
+            conda_info[version]['depends'] = split_versions(conda_info[version]['depends'])
 
         # get the version sets (they can come from file, from repository_info or conda_info)
         initial_version = _get_versions(args.initial_version, repository_info, conda_info)
 
         final_version = _get_versions(args.final_version, repository_info, conda_info)
 
         change_summary = repository_change_summary(repository_info['packages'],
-                                                   initial_versions=initial_version,
-                                                   final_versions=final_version)
-
-        # change the name so the one reported is the conda package name and not the repository
-        for p in change_summary:
-            if p['name'] not in pkg_name_map:
-                continue
-            p['name'] = repo_to_package[p['name']]
+                                                                  initial_versions=initial_version,
+                                                                  final_versions=final_version)
+        change_summary.update({
+            'package': args.meta_package,
+            'initial_version': args.initial_version,
+            'final_version': args.final_version,
+        })
 
         write_conda_pkg_change_summary(change_summary)
     except ArgumentException as e: