notify-issues

#!/usr/bin/python3
# coding=utf-8
#
# The Qubes OS Project, https://www.qubes-os.org/
#
# Copyright (C) 2015
#       Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

from argparse import ArgumentParser
import os
import subprocess
import sys
import re
import requests

github_issues_repo = "QubesOS/qubes-issues"
github_api_prefix = "https://api.github.com"
github_repo_prefix = "QubesOS/qubes-"
github_baseurl = "https://github.com/"

fixes_re = re.compile(r"(fixes|closes)( (https://github.com/[^ ]+/|"
                      r"QubesOS/Qubes-issues#)[0-9]+)",
                      re.IGNORECASE)
issue_re = re.compile(r"QubesOS/Qubes-issues(#|/issues/)[0-9]+",
                      re.IGNORECASE)
cleanup_re = re.compile(r"[^ ]+[#/]")
release_name_re = re.compile("r[0-9.]+")
number_re = re.compile("\"number\": *([0-9]+)")

auth_headers = {}


def comment_issue(issue, message, add_labels, delete_labels,
        github_repo=github_issues_repo):
    if message:
        resp = requests.post("{}/repos/{}/issues/{}/comments".format(
            github_api_prefix,
            github_repo,
            issue),
            json={'body': message},
            headers=auth_headers)
        if not resp.ok:
            print("WARNING: failed to create comment on {}: {} {}".format(
                issue, resp.status_code, resp.content))
    for label in delete_labels:
        resp = requests.delete("{}/repos/{}/issues/{}/labels/{}".format(
            github_api_prefix,
            github_repo,
            issue,
            label
        ),
            headers=auth_headers)
        # ignore 404 error - most likely package was uploaded to testing
        # before introducing this mechanism
        if not resp.ok and resp.status_code != 404:
            print("WARNING: failed to delete {} label from issue {}: {} {}"
                .format(label, issue, resp.status_code, resp.content))
    for label in add_labels:
        resp = requests.post("{}/repos/{}/issues/{}/labels".format(
            github_api_prefix, github_repo, issue),
            json=[label],
            headers=auth_headers)
        if not resp.ok:
            print("WARNING: failed to add {} label to issue {}: {} {}".\
                format(label, issue, resp.status_code, resp.content))

def get_issue_no_from_http_resp(resp):
    # search for the first line with '"number":' and return that number
    # this intentionally avoids throwing json parser at the response (one
    #  parser less to trust, even if simple one)
    for line in resp.content.decode('ascii', errors='replace').splitlines():
        if '"number":' in line:
            match = number_re.search(line)
            if match:
                return int(match.group(1))
            # don't look at other such lines
            break
    return None

def search_or_create_issue(github_repo, release, component,
        version,
        create=True, message_template_kwargs=None):
    # first look into cache
    try:
        cachedir = os.environ['GITHUB_ISSUES_CACHE']
    except KeyError:
        try:
            cachedir = os.path.join(
                os.environ['HOME'], '.cache', 'builder-github-issues'
            )
        except KeyError:
            cachedir = None
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)
    cachefile_path = os.path.join(cachedir, '{}-{}-{}'.format(
        component, version, release))
    if os.path.exists(cachefile_path):
        with open(cachefile_path) as cachefile:
            issue_no = int(cachefile.read().strip())
        return issue_no

    # then search
    issue_title = '{component} {version} ({release})'.format(
        component=component,
        version=version,
        release=release
    )
    issue_no = None
    search_url = 'https://api.github.com/search/issues?' \
                 'q=repo:{github_repo}+"{issue_title}"&sort=created'.format(
                    github_repo=github_repo,
                    issue_title=issue_title)
    resp = requests.get(search_url, headers=auth_headers)
    if resp.ok and issue_title in resp.content.decode('ascii', errors='replace'):
        issue_no = get_issue_no_from_http_resp(resp)

    # if still nothing, create new issue
    if issue_no is None:
        # don't create if requested so
        if not create:
            return None

        if component.startswith('qubes-template'):
            message_template_path = os.path.join(
                os.path.dirname(sys.argv[0]),
                'templates/message-build-report-template'
            )
        else:
            message_template_path = os.path.join(
                os.path.dirname(sys.argv[0]),
                'templates/message-build-report'
            )
        if not os.path.exists(message_template_path):
            print("WARNING: {} does not exist".format(message_template_path))
            return None

        with open(message_template_path) as f:
            message_template = f.read()


        message = message_template. \
            replace("@COMPONENT@", component). \
            replace("@RELEASE_NAME@", release). \
            replace("@VERSION@", version)

        if message_template_kwargs is not None:
            for key, value in message_template_kwargs.items():
                message = message.replace(key, value)

        # and actually create the issue
        resp = requests.post(
            'https://api.github.com/repos/{}/issues'.format(github_repo),
            json={'title': issue_title, 'body': message},
            headers=auth_headers)
        if resp.ok:
            issue_no = get_issue_no_from_http_resp(resp)
        else:
            print("WARNING: failed to create issue: {}".format(resp.reason))

    # if anything found/created - cache this value
    if issue_no:
        with open(cachefile_path, 'w') as cachefile:
            cachefile.write(str(issue_no))

    return issue_no

def notify_closed_issues(args, repo_type,
        current_commit, previous_commit,
        add_labels, delete_labels):
    message_template_path = "{}/templates/message-{}-{}".format(
        os.path.dirname(sys.argv[0]),
        repo_type,
        args.package_set)

    if os.path.exists(message_template_path + "-" + args.dist):
        message_template_path = message_template_path + "-" + args.dist

    if not os.path.exists(message_template_path):
        print("WARNING: message template {} doesn't exist, not adding comments".format(
            message_template_path))
        message_template_path = None

    git_log_proc = subprocess.Popen(
        ['git', '-C', args.src_dir, 'log', '{}..{}'.format(previous_commit,
                                                           current_commit)],
        stdout=subprocess.PIPE)
    (git_log, _) = git_log_proc.communicate()
    closed_issues = []
    for line in git_log.decode().splitlines():
        match = fixes_re.search(line)
        if match:
            issues_string = match.group(0)
            issues_numbers = [int(cleanup_re.sub("", s))
                for s in issues_string.split()[1:]]
            closed_issues.extend(issues_numbers)

    closed_issues = set(closed_issues)

    git_log_proc = subprocess.Popen(
        ['git', '-C', args.src_dir, 'log',
         '--pretty=format:{}-{}@%h %s'.format(
             github_repo_prefix,
             os.path.basename(args.src_dir)),
         '{}..{}'.format(previous_commit, current_commit)],
        stdout=subprocess.PIPE)
    (shortlog, _) = git_log_proc.communicate()
    shortlog = shortlog.decode()

    git_url_var = 'GIT_URL_' + os.path.basename(args.src_dir).replace('-', '_')
    if git_url_var in os.environ:
        git_url = os.environ[git_url_var]
    else:
        git_url = "{base}/{prefix}{repo}".format(
            base=github_baseurl,
            prefix=github_repo_prefix,
            repo=os.path.basename(args.src_dir))
    git_log_url = \
        "{git_url}/compare/{prev_commit}...{curr_commit}".format(
            git_url=git_url,
            prev_commit=previous_commit,
            curr_commit=current_commit
        )

    for issue in closed_issues:
        print("Adding a comment to issue #{}".format(issue))
        if message_template_path:
            message = open(message_template_path, 'r').read().\
                replace("@DIST@", args.dist).\
                replace("@PACKAGE_SET@", args.package_set).\
                replace("@PACKAGE_NAME@", args.package_name).\
                replace("@COMPONENT@", os.environ['COMPONENT']).\
                replace("@REPOSITORY@", repo_type).\
                replace("@RELEASE_NAME@", args.release_name).\
                replace("@GIT_LOG@", shortlog).\
                replace("@GIT_LOG_URL@", git_log_url)
        else:
            message = None

        comment_issue(issue, message, add_labels, delete_labels)

def get_package_changes(src_dir, git_url, commit_sha, previous_commit_sha=None):
    ''' Returns a tuple of:
        - current version
        - previous version
        - git short log formatted with github links
        - referenced github issues, in github syntax
    '''
    git_proc = subprocess.Popen(
        ['git', '-C', src_dir, 'tag', '--list', '--points-at=HEAD', 'v*'],
        stdout=subprocess.PIPE)
    (version_tags, _) = git_proc.communicate()
    version = version_tags.decode().splitlines()[0]

    # get previous version
    if previous_commit_sha:
        git_proc = subprocess.Popen(
            ['git', '-C', src_dir, 'describe', '--match',
                'v*', '--exact-match', previous_commit_sha],
            stdout=subprocess.PIPE)
        (version_tags, _) = git_proc.communicate()
        if not version_tags:
            # if no tag there, point at the commit directly
            version_tags = previous_commit_sha.encode()
    else:
        git_proc = subprocess.Popen(
            ['git', '-C', src_dir, 'describe', '--match',
             'v*', '--abbrev=0', commit_sha + '~'],
            stdout=subprocess.PIPE)
        (version_tags, _) = git_proc.communicate()
    if not version_tags:
        # if no previous version tag, check from (some) root commit
        git_proc = subprocess.Popen(
            ['git', '-C', src_dir, 'rev-list', '--max-parents=0',
                commit_sha + '~'],
            stdout=subprocess.PIPE)
        (version_tags, _) = git_proc.communicate()

    if not version_tags:
        # still nothing - looks there is only one commit - no history
        return version, version, '', ''
    previous_version = version_tags.decode().splitlines()[0]

    git_log_proc = subprocess.Popen(
        ['git', '-C', src_dir, 'log', '{}..{}'.format(previous_version,
            version)],
        stdout=subprocess.PIPE)

    (git_log, _) = git_log_proc.communicate()
    git_log = git_log.decode()
    referenced_issues = []
    for line in git_log.splitlines():
        match = issue_re.search(line)
        if match:
            issues_string = match.group(0)
            issues_numbers = [int(cleanup_re.sub("", s))
                for s in issues_string.split()]
            referenced_issues.extend(issues_numbers)

    referenced_issues_txt = '\n'.join(
        'QubesOS/qubes-issues#{}'.format(x) for x in set(referenced_issues))

    github_full_repo_name = '/'.join(git_url.split('/')[-2:])

    git_log_proc = subprocess.Popen(
        ['git', '-C', src_dir, 'log',
         '--pretty=format:{}@%h %s'.format(
             github_full_repo_name),
         '{}..{}'.format(previous_version, version)],
        stdout=subprocess.PIPE)
    (shortlog, _) = git_log_proc.communicate()
    shortlog = shortlog.decode()

    return version, previous_version, shortlog, referenced_issues_txt

def notify_build_report(args, dist_label, repo_type, commit_sha,
        add_labels, delete_labels, previous_stable_commit_sha=None):
    if 'GITHUB_BUILD_REPORT_REPO' not in os.environ:
        return

    github_report_repo = os.environ['GITHUB_BUILD_REPORT_REPO']
    if args.command == 'build-start':
        report_message = None
    elif args.build_log:
        report_message = \
            "Package for {dist} was built ([build log]({build_log})) and " \
            "uploaded to {repo_type} repository".format(
                dist=dist_label,
                build_log=args.build_log,
                repo_type=repo_type
            )
    else:
        report_message = \
            "Package for {dist} was uploaded to {repo_type} repository".format(
                dist=dist_label, repo_type=repo_type)

    component = os.path.basename(args.src_dir)
    git_url_var = 'GIT_URL_' + component.replace('-', '_')
    if git_url_var in os.environ:
        git_url = os.environ[git_url_var]
    else:
        git_url = "{base}/{prefix}{repo}".format(
            base=github_baseurl,
            prefix=github_repo_prefix,
            repo=component)

    if component == 'linux-template-builder':
        # qubes-template-fedora-25-4.0.0-201710170053
        version = '-'.join(args.package_name.split('-')[-2:])
        component = '-'.join(args.package_name.split('-')[:-2])
        template_name = component.replace('qubes-template-', '')
        message_kwargs = {
            '@COMMIT_SHA@': commit_sha,
            '@GIT_URL@': git_url,
            '@TEMPLATE_NAME@': template_name,
            '@DIST@': os.getenv('DIST_ORIG_ALIAS', '(dist)'),
        }
    else:
        version, previous_version, shortlog, referenced_issues_txt = \
             get_package_changes(args.src_dir, git_url, commit_sha,
                 previous_commit_sha=previous_stable_commit_sha)

        git_log_url = \
            "{git_url}/compare/{prev_commit}...{curr_commit}".format(
                git_url=git_url,
                prev_commit=previous_version,
                curr_commit=version
            )

        message_kwargs = {
            '@COMMIT_SHA@': commit_sha,
            '@GIT_URL@': git_url,
            '@GIT_LOG@': shortlog,
            '@GIT_LOG_URL@': git_log_url,
            '@ISSUES@': referenced_issues_txt
        }

    report_issue_no = search_or_create_issue(
        github_report_repo, args.release_name, component,
        version=version,
        create=True, message_template_kwargs=message_kwargs)
    if report_issue_no and args.command == 'upload':
        comment_issue(report_issue_no, report_message,
            add_labels, delete_labels, github_repo=github_report_repo)


def main():
    global auth_headers
    epilog = "When state_file doesn't exists, no notify is sent, but the " \
             "current state is recorded"

    parser = ArgumentParser(epilog=epilog)
    parser.add_argument("--auth-token",
                        help="Github authentication token (OAuth2)")
    parser.add_argument("--build-log",
                        help="Build log name in build-logs repository")
    subparsers = parser.add_subparsers(help='command')
    upload_parser = subparsers.add_parser('upload')
    upload_parser.set_defaults(command='upload')
    upload_parser.add_argument("release_name",
                        help="Release name, for example r3.0")
    upload_parser.add_argument("src_dir",
                        help="Component sources path")
    upload_parser.add_argument("package_name",
                        help="Binary package name")
    upload_parser.add_argument("dist",
                        help="Distribution release code name")
    upload_parser.add_argument("package_set",
                        choices=['dom0', 'vm'])
    upload_parser.add_argument("repo_type",
                        help="Repository type",
                        choices=['current', 'current-testing',
                                 'security-testing', 'unstable',
                                 'templates-itl', 'templates-itl-testing',
                                 'templates-community', 'templates-community-testing'])
    upload_parser.add_argument("state_file",
                        help="File to store internal state (previous commit "
                             "id)")
    upload_parser.add_argument("stable_state_file",
                        help="File to store internal state (previous commit "
                             "id of a stable aka current package)")

    build_parser = subparsers.add_parser('build-start')
    build_parser.set_defaults(command='build-start')
    build_parser.add_argument("release_name",
                        help="Release name, for example r3.0")
    build_parser.add_argument("src_dir",
                        help="Component sources path")
    build_parser.add_argument("package_name",
                        help="Binary package name")
    build_parser.add_argument("dist",
                        help="Distribution release code name")
    build_parser.add_argument("package_set",
                        choices=['dom0', 'vm'])

    args = parser.parse_args()

    if args.auth_token:
        auth_headers['Authorization'] = \
            'token {}'.format(args.auth_token)
    elif 'GITHUB_API_KEY' in os.environ:
        auth_headers['Authorization'] = \
            'token {}'.format(os.environ['GITHUB_API_KEY'])

    dist_label = args.dist
    if args.package_set == 'dom0':
        dist_label = 'dom0'

    if args.command == 'upload':
        delete_labels = []
        add_labels = []
        repo_type = args.repo_type
        if args.repo_type == 'current':
            repo_type = 'stable'
            delete_labels = ["{}-{}-cur-test".format(args.release_name, dist_label),
                            "{}-{}-sec-test".format(args.release_name, dist_label)]
            add_labels = ["{}-{}-stable".format(args.release_name, dist_label)]
        elif args.repo_type == 'current-testing':
            add_labels = ["{}-{}-cur-test".format(args.release_name, dist_label)]
        elif args.repo_type == 'security-testing':
            add_labels = ["{}-{}-sec-test".format(args.release_name, dist_label)]
        elif args.repo_type == 'templates-itl':
            delete_labels = ["{}-testing".format(args.release_name)]
            add_labels = ["{}-stable".format(args.release_name)]
        elif args.repo_type == 'templates-itl-testing':
            add_labels = ["{}-testing".format(args.release_name)]
        elif args.repo_type == 'templates-community':
            delete_labels = ["{}-testing".format(args.release_name)]
            add_labels = ["{}-stable".format(args.release_name)]
        elif args.repo_type == 'templates-community-testing':
            add_labels = ["{}-testing".format(args.release_name)]
        else:
            print("Ignoring {}".format(args.repo_type))
            return
    else:
        add_labels = []
        delete_labels = []
        repo_type = None

    if not release_name_re.match(args.release_name):
        print("Ignoring release {}".format(args.release_name))
        return

    git_proc = subprocess.Popen(
        ['git', '-C', args.src_dir, 'log', '-n', '1', '--pretty=format:%H'],
        stdout=subprocess.PIPE)
    (current_commit, _) = git_proc.communicate()
    current_commit = current_commit.decode().strip()

    previous_stable_commit = None

    if args.command == 'upload':
        if not os.path.exists(args.state_file):
            print("WARNING: {} does not exist, initializing with the "
                  "current state".format(args.state_file))
            previous_commit = None
        else:
            with open(args.state_file, 'r') as f:
                previous_commit = f.readline().strip()

        if previous_commit is not None:
            if repo_type in ['stable', 'current-testing']:
                notify_closed_issues(args, repo_type, current_commit,
                    previous_commit, add_labels, delete_labels)

        with open(args.state_file, 'w') as f:
            f.write(current_commit)

        if os.path.exists(args.stable_state_file):
            with open(args.stable_state_file, 'r') as f:
                previous_stable_commit = f.readline().strip()

    notify_build_report(args, dist_label, repo_type, current_commit,
        add_labels, delete_labels,
        previous_stable_commit_sha=previous_stable_commit)

if __name__ == '__main__':
    main()