Fixes to package info summaries

scripts/update-dashboard.sh

@@ -1,8 +1,8 @@
 source $HOME/.ci-auth
 skare3-dashboard -o /proj/sot/ska/jgonzalez/index.html
 skare3-dashboard -o /proj/sot/ska/jgonzalez/packages.json
-skare3-test-dashboard -o /proj/sot/ska/jgonzalez/test_results.html --static-dir https://cxc.cfa.harvard.edu/mta/ASPECT/skare3/dashboard/static --log-dir https://icxc.cfa.harvard.edu/aspect/skare3/dashboard/tests
-skare3-test-dashboard -o /proj/sot/ska/jgonzalez/test_results.json
+skare3-test-dashboard -b -o /proj/sot/ska/jgonzalez/test_results.html --static-dir https://cxc.cfa.harvard.edu/mta/ASPECT/skare3/dashboard/static --log-dir https://icxc.cfa.harvard.edu/aspect/skare3/dashboard/tests
+skare3-test-dashboard -b -o /proj/sot/ska/jgonzalez/test_results.json
 
 # horrible hack:
 rm -f /proj/sot/ska/www/ASPECT_ICXC/skare3/dashboard/tests

skare3_tools/github/github.py

@@ -393,7 +393,7 @@ def _get_list_generator(self, url, limit=None, **kwargs):
                 yield item
                 count += 1
                 if limit and count >= limit:
-                    raise StopIteration()
+                    return
 
     def _get_list(self, *args, **kwargs):
         """
@@ -436,6 +436,7 @@ def __init__(self, repo=None, owner=None, api=None):
         self.branches = Branches(self)
         self.checks = Checks(self)
         self.pull_requests = PullRequests(self)
+        self.compare = Compare(self)
         self.merge = Merge(self)
         self.dispatch_event = DispatchEvent(self)
         self.contents = Contents(self)
@@ -615,7 +616,7 @@ def __call__(self, ref=None, **kwargs):
         json.update({k: kwargs[k] for k in optional if k in kwargs})
         kwargs = {k: v for k, v in kwargs.items() if k not in json}
         if ref is not None:
-            return self._get(
+            return self._get_list(
                 "repos/:owner/:repo/commits/:ref", ref=ref, params=json, **kwargs
             )
         return self._get_list("repos/:owner/:repo/commits", params=json, **kwargs)
@@ -778,6 +779,24 @@ def edit(self, issue_number, **kwargs):
         )
 
 
+class Compare(_EndpointGroup):
+    """Compare two commits
+
+    (`compare API docs </repos/{owner}/{repo}/compare/{basehead}>`)
+    """
+
+    def __call__(self, base, head, **kwargs):
+        """ """
+        required = []
+        json = {k: kwargs[k] for k in required}
+        kwargs = {k: v for k, v in kwargs.items() if k not in json}
+        return self._get(
+            "/repos/:owner/:repo/compare/:basehead",
+            basehead=f"{base}...{head}",
+            **kwargs,
+        )
+
+
 class PullRequests(_EndpointGroup):
     """
     Endpoints that have to do with pull requests
@@ -833,7 +852,7 @@ def __call__(self, pull_number=None, **kwargs):
         json = {k: kwargs[k] for k in required}
         json.update({k: kwargs[k] for k in optional if k in kwargs})
         kwargs = {k: v for k, v in kwargs.items() if k not in json}
-        return self._get("/repos/:owner/:repo/pulls", params=json, **kwargs)
+        return self._get_list("/repos/:owner/:repo/pulls", params=json, **kwargs)
 
     def create(self, **kwargs):
         """

skare3_tools/github/scripts/add_secrets.py

@@ -153,7 +153,7 @@ def add_secrets(repository, secrets):
 
         _driver_.find_element_by_id("secret_name").send_keys(secret)
         value = secrets[secret]
-        if type(value) == dict:
+        if type(value) is dict:
             value = json.dumps(value)
         _driver_.find_element_by_id("secret_value").send_keys(value)
 

skare3_tools/github/scripts/merge_pr.py

@@ -46,7 +46,7 @@ def main():
     kwargs["state"] = "open"
     prs = repository.pull_requests(**kwargs)
 
-    if type(prs) == dict and not prs["response"]["ok"]:
+    if type(prs) is dict and not prs["response"]["ok"]:
         print(f'Failed getting requested PR: {prs["response"]["reason"]}')
         sys.exit(1)
 

skare3_tools/github/scripts/release_merge_info.py

@@ -11,88 +11,128 @@
 import argparse
 import re
 import sys
+import numpy as np
 
+from packaging.version import Version
 from skare3_tools import github, packages
 
 
-def parser():
+def get_parser():
     parse = argparse.ArgumentParser(description=__doc__)
     parse.add_argument(
         "--repository", required=True, help="repository name. Example: sot/chandra_aca"
     )
     parse.add_argument("--sha", help="sha of the release")
+    parse.add_argument("--tag", help="tag of the release")
     parse.add_argument(
         "--token", "-t", help="Github token, or name of file that contains token"
     )
+    parse.add_argument(
+        "--stdout",
+        action="store_true",
+        help="print to stdout instead of editing release description",
+    )
     return parse
 
 
-def main():
-    args = parser().parse_args()
-    github.init(token=args.token)
-    repository = github.Repository(args.repository)
-
-    # get all releases and find the one we are working on
-    releases = repository.releases()
-    releases = [r for r in releases if not r["draft"] and not r["prerelease"]]
-    releases = sorted(releases, key=lambda r: r["tag_name"], reverse=True)
-    release_commits = [
-        packages._get_release_commit(repository, r["tag_name"]) for r in releases
-    ]
-    release_shas = [c["sha"] for c in release_commits]
-    i_1 = None
-    for i, sha in enumerate(release_shas):
-        if sha == args.sha:
-            i_1 = i
-    if i_1 is None:
-        raise Exception(f"Release with sha {args.sha} was not found")
-
-    # get all commits between this release and the previous one, if any.
-    kwargs = {
-        "until": repository.commits(ref=release_shas[i_1])["commit"]["author"]["date"]
-    }
-    if i_1 + 1 < len(releases):
-        kwargs["since"] = repository.commits(ref=release_shas[i_1 + 1])["commit"][
-            "author"
-        ]["date"]
-    commits = repository.commits(sha="master", **kwargs)[
-        :-1
-    ]  # remove the last one, the release
+def merges_in_range(repo, sha_1, sha_2):
+    commits_1 = repo.commits(sha=sha_1)
+    commits_2 = repo.commits(sha=sha_2)
+    sha_1 = [c["sha"] for c in commits_1]
+    sha_2 = [c["sha"] for c in commits_2]
+    i = np.argwhere(np.in1d(sha_2, sha_1)).flatten()[0]
+    commits = commits_2[:i]
+    assert len(commits)  # TODO: shouldn't it be possible with no commits?
 
     # get commit messages matching the standard merge commit
     merges = []
     for commit in commits:
         msg = commit["commit"]["message"]
         match = re.match(
-            r"Merge pull request (?P<pr>.+) from (?P<branch>\S+)\n\n(?P<description>.+)",
+            r"Merge pull request (?P<pr>.+) from (?P<branch>\S+)(\n\n(?P<description>.+))?",
             msg,
         )
         if match:
             msg = match.groupdict()
-            if msg["pr"][0] == "#":
-                msg["pr"] = (
-                    f'[{msg["pr"]}]'
-                    f'(https://github.com/{args.repository}/pull/{msg["pr"][1:]})'
-                )
-            merges.append(f'PR {msg["pr"]}: {msg["description"]}')
-    if merges:
+            if msg["description"] is None:
+                msg["description"] = repo.pull_requests(msg["pr"][1:])[0]["title"]
+            merges.append(msg)
+    return merges
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
+    if not args.tag and not args.sha:
+        parser.exit("Need to specify tag or sha")
+
+    github.init(token=args.token)
+
+    repo = github.Repository(args.repository)
+
+    # get all releases and their commit sha
+    releases = repo.releases()
+    releases = [r for r in releases if not r["draft"] and not r["prerelease"]]
+    releases = sorted(releases, key=lambda r: Version(r["tag_name"]), reverse=True)
+    releases = {r["tag_name"]: r for r in releases}
+    for tag, rel in releases.items():
+        releases[tag]["sha"] = packages._get_release_commit(repo, rel["tag_name"])[
+            "sha"
+        ]
+
+    release_shas = [rel["sha"] for rel in releases.values()]
+    releases_by_sha = {rel["sha"]: rel for rel in releases.values()}
+
+    # normalize and check arguments
+    if args.tag and not args.sha:
+        args.sha = releases[args.tag]["sha"]
+    elif args.sha and not args.tag:
+        args.tag = releases_by_sha[args.sha]["tag_name"]
+
+    assert args.sha in release_shas, f"Release with sha {args.sha} was not found"
+    assert args.tag in releases, f"Release with tag {args.tag} was not found"
+    assert releases[args.tag]["sha"] == args.sha, f"Inconsistent release sha and tag"
+
+    # now find all merges between the previous release and the requested one
+    # checking for commit messages matching the standard merge commit
+    merges = merges_in_range(
+        repo,
+        release_shas[release_shas.index(args.sha) + 1],  # TODO: limit check here?
+        args.sha,
+    )
+
+    msgs = []
+    for merge in merges:
+        if merge["pr"][0] == "#":
+            merge["pr"] = (
+                f'[{merge["pr"]}]'
+                f'(https://github.com/{args.repository}/pull/{merge["pr"][1:]})'
+            )
+        msgs.append(f'PR {merge["pr"]}: {merge["description"]}')
+
+    if msgs:
         # edit the release to include the merge information
-        release_id = releases[i_1]["id"]
-        body = releases[i_1]["body"]
+        release = releases[args.tag]
+        release_id = release["id"]
+        body = release["body"]
         if body:
             body += "\n\n"
         body += "Includes the following merges:\n"
-        for merge in merges:
-            body += f"- {merge}\n"
-
-        r = repository.releases.edit(release_id, body=body)
-        if not r["response"]["ok"]:
-            sys.exit(
-                (
-                    f"Failed to edit release '{releases[i_1]['name']}'"
-                    f" ({release_id}): {r['response']['reason']}"
+        for msg in msgs:
+            body += f"- {msg}\n"
+
+        if args.stdout:
+            print(body)
+        else:
+            r = repo.releases.edit(release_id, body=body)
+            if not r["response"]["ok"]:
+                sys.exit(
+                    (
+                        f"Failed to edit release '{release['name']}'"
+                        f" ({release_id}): {r['response']['reason']}"
+                    )
                 )
-            )
 
 
 if __name__ == "__main__":