diff --git a/script/lib/github.py b/script/lib/github.py index aedcd0808f61..bfda3b1d0364 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -5,6 +5,8 @@ import re import requests import sys +import base64 +from util import execute, scoped_cwd REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', 'requests')) @@ -80,3 +82,199 @@ def __getattr__(self, attr): name = '%s/%s' % (self._name, attr) return _Callable(self._gh, name) + + +def get_authenticated_user_login(token): + """given a valid GitHub access token, return the associated GitHub user login""" + # for more info see: https://developer.github.com/v3/users/#get-the-authenticated-user + user = GitHub(token).user() + try: + response = user.get() + return response['login'] + except Exception as e: + print('[ERROR] ' + str(e)) + + +def parse_user_logins(token, login_csv, verbose=False): + """given a list of logins in csv format, parse into a list and validate logins""" + if login_csv is None: + return [] + login_csv = login_csv.replace(" ", "") + parsed_logins = login_csv.split(',') + + users = GitHub(token).users() + + invalid_logins = [] + + # check login/username against GitHub + # for more info see: https://developer.github.com/v3/users/#get-a-single-user + for login in parsed_logins: + try: + response = users(login).get() + if verbose: + print('[INFO] Login "' + login + '" found: ' + str(response)) + except Exception as e: + if verbose: + print('[INFO] Login "' + login + '" does not appear to be valid. ' + str(e)) + invalid_logins.append(login) + + if len(invalid_logins) > 0: + raise Exception('Invalid logins found. Are they misspelled? ' + ','.join(invalid_logins)) + + return parsed_logins + + +def parse_labels(token, repo_name, label_csv, verbose=False): + global config + if label_csv is None: + return [] + label_csv = label_csv.replace(" ", "") + parsed_labels = label_csv.split(',') + + invalid_labels = [] + + # validate labels passed in are correct + # for more info see: https://developer.github.com/v3/issues/labels/#get-a-single-label + repo = GitHub(token).repos(repo_name) + for label in parsed_labels: + try: + response = repo.labels(label).get() + if verbose: + print('[INFO] Label "' + label + '" found: ' + str(response)) + except Exception as e: + if verbose: + print('[INFO] Label "' + label + '" does not appear to be valid. ' + str(e)) + invalid_labels.append(label) + + if len(invalid_labels) > 0: + raise Exception('Invalid labels found. Are they misspelled? ' + ','.join(invalid_labels)) + + return parsed_labels + + +def get_file_contents(token, repo_name, filename, branch=None): + # NOTE: API only supports files up to 1MB in size + # for more info see: https://developer.github.com/v3/repos/contents/ + repo = GitHub(token).repos(repo_name) + get_data = {} + if branch: + get_data['ref'] = branch + file = repo.contents(filename).get(params=get_data) + if file['encoding'] == 'base64': + return base64.b64decode(file['content']) + return file['content'] + + +def add_reviewers_to_pull_request(token, repo_name, pr_number, reviewers=[], verbose=False, dryrun=False): + # add reviewers to pull request + # for more info see: https://developer.github.com/v3/pulls/review_requests/ + repo = GitHub(token).repos(repo_name) + patch_data = {} + if len(reviewers) > 0: + patch_data['reviewers'] = reviewers + if dryrun: + print('[INFO] would call `repo.pulls(' + str(pr_number) + + ').requested_reviewers.post(' + str(patch_data) + ')`') + return + response = repo.pulls(pr_number).requested_reviewers.post(data=patch_data) + if verbose: + print('repo.pulls(' + str(pr_number) + ').requested_reviewers.post(data) response:\n' + str(response)) + return response + + +def get_milestones(token, repo_name, verbose=False): + # get all milestones for a repo + # for more info see: https://developer.github.com/v3/issues/milestones/ + repo = GitHub(token).repos(repo_name) + response = repo.milestones.get() + if verbose: + print('repo.milestones.get() response:\n' + str(response)) + return response + + +def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, + open_in_browser=False, verbose=False, dryrun=False): + post_data = { + 'title': title, + 'head': branch_src, + 'base': branch_dst, + 'body': body, + 'maintainer_can_modify': True + } + # create the pull request + # for more info see: http://developer.github.com/v3/pulls + if dryrun: + print('[INFO] would call `repo.pulls.post(' + str(post_data) + ')`') + if open_in_browser: + print('[INFO] would open PR in web browser') + return 1234 + repo = GitHub(token).repos(repo_name) + response = repo.pulls.post(data=post_data) + if verbose: + print('repo.pulls.post(data) response:\n' + str(response)) + if open_in_browser: + import webbrowser + webbrowser.open(response['html_url']) + return int(response['number']) + + +def set_issue_details(token, repo_name, issue_number, milestone_number=None, + assignees=[], labels=[], verbose=False, dryrun=False): + patch_data = {} + if milestone_number: + patch_data['milestone'] = milestone_number + if len(assignees) > 0: + patch_data['assignees'] = assignees + if len(labels) > 0: + patch_data['labels'] = labels + # TODO: error if no keys in patch_data + + # add milestone and assignee to issue / pull request + # for more info see: https://developer.github.com/v3/issues/#edit-an-issue + if dryrun: + print('[INFO] would call `repo.issues(' + str(issue_number) + ').patch(' + str(patch_data) + ')`') + return + repo = GitHub(token).repos(repo_name) + response = repo.issues(issue_number).patch(data=patch_data) + if verbose: + print('repo.issues(' + str(issue_number) + ').patch(data) response:\n' + str(response)) + + +def fetch_origin_check_staged(path): + """given a path on disk (to a git repo), fetch origin and ensure there aren't unstaged files""" + with scoped_cwd(path): + execute(['git', 'fetch', 'origin']) + status = execute(['git', 'status', '-s']).strip() + if len(status) > 0: + print('[ERROR] There appear to be unstaged changes.\n' + + 'Please resolve these before running (ex: `git status`).') + return 1 + return 0 + + +def get_local_branch_name(path): + with scoped_cwd(path): + return execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + + +def get_title_from_first_commit(path, branch_to_compare): + """get the first commit subject (useful for the title of a pull request)""" + with scoped_cwd(path): + local_branch = execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + title_list = execute(['git', 'log', 'origin/' + branch_to_compare + + '..HEAD', '--pretty=format:%s', '--reverse']) + title_list = title_list.split('\n') + if len(title_list) == 0: + raise Exception('No commits found! Local branch matches "' + branch_to_compare + '"') + return title_list[0] + + +def push_branches_to_remote(path, branches_to_push, dryrun=False): + if dryrun: + print('[INFO] would push the following local branches to remote: ' + str(branches_to_push)) + else: + with scoped_cwd(path): + for branch_to_push in branches_to_push: + print('- pushing ' + branch_to_push + '...') + # TODO: if they already exist, force push?? or error?? + execute(['git', 'push', '-u', 'origin', branch_to_push]) diff --git a/script/lib/helpers.py b/script/lib/helpers.py index fb9784823ef8..de6d2be8b4f7 100644 --- a/script/lib/helpers.py +++ b/script/lib/helpers.py @@ -8,11 +8,21 @@ from .config import get_raw_version BRAVE_REPO = "brave/brave-browser" +BRAVE_CORE_REPO = "brave/brave-core" + + +def channels(): + return ['nightly', 'dev', 'beta', 'release'] def get_channel_display_name(): - d = {'beta': 'Beta', 'canary': 'Canary', 'dev': 'Developer', - 'release': 'Release'} + raw = channels() + d = { + 'canary': 'Canary', + raw[1]: 'Developer', + raw[2]: 'Beta', + raw[3]: 'Release' + } return d[release_channel()] diff --git a/script/pr.py b/script/pr.py new file mode 100755 index 000000000000..f43b4c0a622a --- /dev/null +++ b/script/pr.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import errno +import hashlib +import os +import requests +import re +import shutil +import subprocess +import sys +import tempfile +import json + +from io import StringIO +from lib.config import get_env_var, SOURCE_ROOT, BRAVE_CORE_ROOT, get_raw_version +from lib.util import execute, scoped_cwd +from lib.helpers import * +from lib.github import (GitHub, get_authenticated_user_login, parse_user_logins, + parse_labels, get_file_contents, add_reviewers_to_pull_request, + get_milestones, create_pull_request, set_issue_details, + fetch_origin_check_staged, get_local_branch_name, + get_title_from_first_commit, push_branches_to_remote) + + +# TODOs for this version +##### +# - parse out issue (so it can be included in body). ex: git log --pretty=format:%b +# - discover associated issues +# - put them in the uplift / original PR body +# - set milestone! (on the ISSUE) + + +class PrConfig: + channel_names = channels() + channels_to_process = channels() + is_verbose = False + is_dryrun = False + branches_to_push = [] + master_pr_number = -1 + github_token = None + parsed_reviewers = [] + parsed_owners = [] + milestones = None + title = None + labels = [] + + def initialize(self, args): + try: + self.is_verbose = args.verbose + self.is_dryrun = args.dry_run + self.title = args.title + # validate channel names + validate_channel(args.uplift_to) + validate_channel(args.start_from) + # read github token FIRST from CLI, then from .npmrc + self.github_token = get_env_var('GITHUB_TOKEN') + if len(self.github_token) == 0: + try: + result = execute(['npm', 'config', 'get', 'BRAVE_GITHUB_TOKEN']).strip() + if result == 'undefined': + raise Exception('`BRAVE_GITHUB_TOKEN` value not found!') + self.github_token = result + except Exception as e: + print('[ERROR] no valid GitHub token was found either in npmrc or ' + + 'via environment variables (BRAVE_GITHUB_TOKEN)') + return 1 + self.parsed_reviewers = parse_user_logins(self.github_token, args.reviewers, verbose=self.is_verbose) + # if `--owners` is not provided, fall back to user owning token + self.parsed_owners = parse_user_logins(self.github_token, args.owners, verbose=self.is_verbose) + if len(self.parsed_owners) == 0: + self.parsed_owners = [get_authenticated_user_login(self.github_token)] + self.labels = parse_labels(self.github_token, BRAVE_CORE_REPO, args.labels, verbose=self.is_verbose) + if self.is_verbose: + print('[INFO] config: ' + str(vars(self))) + return 0 + except Exception as e: + print('[ERROR] error returned from GitHub API while initializing config: ' + str(e)) + return 1 + + +config = PrConfig() + + +def is_nightly(channel): + global config + return config.channel_names[0] == channel + + +# given something like "0.60.2", return branch version ("0.60.x") +def get_current_version_branch(version): + version = str(version) + if version[0] == 'v': + version = version[1:] + parts = version.split('.', 3) + parts[2] = 'x' + return '.'.join(parts) + + +# given something like "0.60.x", get previous version ("0.59.x") +def get_previous_version_branch(version): + version = str(version) + if version[0] == 'v': + version = version[1:] + parts = version.split('.', 3) + parts[1] = str(int(parts[1]) - 1) + parts[2] = 'x' + return '.'.join(parts) + + +def get_remote_channel_branches(raw_nightly_version): + global config + nightly_version = get_current_version_branch(raw_nightly_version) + dev_version = get_previous_version_branch(nightly_version) + beta_version = get_previous_version_branch(dev_version) + release_version = get_previous_version_branch(beta_version) + return { + config.channel_names[0]: nightly_version, + config.channel_names[1]: dev_version, + config.channel_names[2]: beta_version, + config.channel_names[3]: release_version + } + + +def validate_channel(channel): + global config + try: + config.channel_names.index(channel) + except Exception as e: + raise Exception('Channel name "' + channel + '" is not valid!') + + +def parse_args(): + parser = argparse.ArgumentParser(description='create PRs for all branches given branch against master') + parser.add_argument('--reviewers', + help='comma separated list of GitHub logins to mark as reviewers', + default=None) + parser.add_argument('--owners', + help='comma seperated list of GitHub logins to mark as assignee', + default=None) + parser.add_argument('--uplift-to', + help='starting at nightly (master), how far back to uplift the changes', + default='nightly') + parser.add_argument('--uplift-using-pr', + help='link to already existing pull request (number) to use as a reference for uplifting', + default=None) + parser.add_argument('--start-from', + help='instead of starting from nightly (default), start from beta/dev/release', + default='nightly') + parser.add_argument('-v', '--verbose', action='store_true', + help='prints the output of the GitHub API calls') + parser.add_argument('-n', '--dry-run', action='store_true', + help='don\'t actually create pull requests; just show a call would be made') + parser.add_argument('--labels', + help='comma seperated list of labels to apply to each pull request', + default=None) + parser.add_argument('--title', + help='title to use (instead of inferring one from the first commit)', + default=None) + parser.add_argument('-f', '--force', action='store_true', + help='use the script forcefully; ignore warnings') + + return parser.parse_args() + + +def get_remote_version(branch_to_compare): + global config + decoded_file = get_file_contents(config.github_token, BRAVE_REPO, 'package.json', branch_to_compare) + json_file = json.loads(decoded_file) + return json_file['version'] + + +def fancy_print(text): + print('#' * len(text)) + print(text) + print('#' * len(text)) + + +def main(): + args = parse_args() + if args.verbose: + print('[INFO] args: ' + str(args)) + + global config + result = config.initialize(args) + if result != 0: + return result + + result = fetch_origin_check_staged(BRAVE_CORE_ROOT) + if result != 0: + return result + + # if starting point is NOT nightly, remove options which aren't desired + # also, find the branch which should be used for diffs (for cherry-picking) + local_version = get_raw_version() + remote_branches = get_remote_channel_branches(local_version) + top_level_base = 'master' + if not is_nightly(args.start_from): + top_level_base = remote_branches[args.start_from] + try: + start_index = config.channel_names.index(args.start_from) + config.channels_to_process = config.channel_names[start_index:] + except Exception as e: + print('[ERROR] specified `start-from` value "' + args.start_from + '" not found in channel list') + return 1 + + # optionally (instead of having a local branch), allow uplifting a specific PR + # this pulls down the pr locally (in a special branch) + if args.uplift_using_pr: + pr_number = int(args.uplift_using_pr) + repo = GitHub(config.github_token).repos(BRAVE_CORE_REPO) + try: + # get enough details from PR to check out locally + response = repo.pulls(pr_number).get() + head = response['head'] + local_branch = 'pr' + str(pr_number) + '_' + head['ref'] + head_sha = head['sha'] + top_level_base = response['base']['ref'] + + except Exception as e: + print('[ERROR] API returned an error when looking up pull request "' + str(pr_number) + '"\n' + str(e)) + return 1 + + # set starting point AHEAD of the PR provided + config.master_pr_number = pr_number + if top_level_base == 'master': + config.channels_to_process = config.channel_names[1:] + else: + branch_index = remote_branches.index(top_level_base) + config.channels_to_process = config.channel_names[branch_index:] + + # create local branch which matches the contents of the PR + with scoped_cwd(BRAVE_CORE_ROOT): + # check if branch exists already + try: + branch_sha = execute(['git', 'rev-parse', '-q', '--verify', local_branch]) + except Exception as e: + branch_sha = '' + if len(branch_sha) > 0: + # branch exists; reset it + print('branch "' + local_branch + '" exists; resetting to origin/' + head['ref'] + ' (' + head_sha + ')') + execute(['git', 'checkout', local_branch]) + execute(['git', 'reset', '--hard', head_sha]) + else: + # create the branch + print('creating branch "' + local_branch + '" using origin/' + head['ref'] + ' (' + head_sha + ')') + execute(['git', 'checkout', '-b', local_branch, head_sha]) + # now that branch exists, switch back to previous local branch + execute(['git', 'checkout', '-']) + + # get local version + latest version on remote (master) + # if they don't match, rebase is needed + remote_version = get_remote_version(top_level_base) + if local_version != remote_version: + if not args.force: + print('[ERROR] Your branch is out of sync (local=' + local_version + + ', remote=' + remote_version + '); please rebase (ex: "git rebase origin/' + + top_level_base + '"). NOTE: You can bypass this check by using -f') + return 1 + print('[WARNING] Your branch is out of sync (local=' + local_version + + ', remote=' + remote_version + '); continuing since -f was provided') + + # If title isn't set already, generate one from first commit + local_branch = get_local_branch_name(BRAVE_CORE_ROOT) + if not config.title: + config.title = get_title_from_first_commit(BRAVE_CORE_ROOT, top_level_base) + + # Create a branch for each channel + print('\nCreating branches...') + fancy_print('NOTE: Commits are being detected by diffing "' + local_branch + '" against "' + top_level_base + '"') + remote_branches = get_remote_channel_branches(local_version) + local_branches = {} + try: + for channel in config.channels_to_process: + branch = create_branch(channel, top_level_base, remote_branches[channel], local_branch) + local_branches[channel] = branch + if channel == args.uplift_to: + break + except Exception as e: + print('[ERROR] cherry-pick failed for branch "' + branch + '". Please resolve manually:\n' + str(e)) + return 1 + + print('\nPushing local branches to remote...') + push_branches_to_remote(BRAVE_CORE_ROOT, config.branches_to_push, dryrun=config.is_dryrun) + + try: + print('\nCreating the pull requests...') + for channel in config.channels_to_process: + submit_pr( + channel, + top_level_base, + remote_branches[channel], + local_branches[channel]) + if channel == args.uplift_to: + break + print('\nDone!') + except Exception as e: + print('\n[ERROR] Unhandled error while creating pull request; ' + str(e)) + return 1 + + return 0 + + +def create_branch(channel, top_level_base, remote_base, local_branch): + global config + + if is_nightly(channel): + return local_branch + + channel_branch = remote_base + '_' + local_branch + + with scoped_cwd(BRAVE_CORE_ROOT): + # get SHA for all commits (in order) + sha_list = execute(['git', 'log', 'origin/' + top_level_base + '..HEAD', '--pretty=format:%h', '--reverse']) + sha_list = sha_list.split('\n') + try: + # check if branch exists already + try: + branch_sha = execute(['git', 'rev-parse', '-q', '--verify', channel_branch]) + except Exception as e: + branch_sha = '' + + if len(branch_sha) > 0: + # branch exists; reset it + print('(' + channel + ') branch "' + channel_branch + + '" exists; resetting to origin/' + remote_base) + execute(['git', 'reset', '--hard', 'origin/' + remote_base]) + else: + # create the branch + print('(' + channel + ') creating "' + channel_branch + '" from ' + channel) + execute(['git', 'checkout', remote_base]) + execute(['git', 'checkout', '-b', channel_branch]) + + # TODO: handle errors thrown by cherry-pick + for sha in sha_list: + output = execute(['git', 'cherry-pick', sha]).split('\n') + print('- picked ' + sha + ' (' + output[0] + ')') + + finally: + # switch back to original branch + execute(['git', 'checkout', local_branch]) + execute(['git', 'reset', '--hard', sha_list[-1]]) + + config.branches_to_push.append(channel_branch) + + return channel_branch + + +def get_milestone_for_branch(channel_branch): + global config + if not config.milestones: + config.milestones = get_milestones(config.github_token, BRAVE_CORE_REPO) + for milestone in config.milestones: + if milestone['title'].startswith(channel_branch + ' - '): + return milestone['number'] + return None + + +def submit_pr(channel, top_level_base, remote_base, local_branch): + global config + + try: + milestone_number = get_milestone_for_branch(remote_base) + if milestone_number is None: + print('milestone for "' + remote_base + '"" was not found!') + return 0 + + print('(' + channel + ') creating pull request') + pr_title = config.title + pr_dst = remote_base + if is_nightly(channel): + pr_dst = 'master' + + # add uplift specific details (if needed) + if is_nightly(channel) or local_branch.startswith(top_level_base): + pr_body = 'TODO: fill me in\n(created using `npm run pr`)' + else: + pr_title += ' (uplift to ' + remote_base + ')' + pr_body = 'Uplift of #' + str(config.master_pr_number) + + number = create_pull_request(config.github_token, BRAVE_CORE_REPO, pr_title, pr_body, + branch_src=local_branch, branch_dst=pr_dst, + open_in_browser=True, verbose=config.is_verbose, dryrun=config.is_dryrun) + + # store the original PR number so that it can be referenced in uplifts + if is_nightly(channel) or local_branch.startswith(top_level_base): + config.master_pr_number = number + + # assign milestone / reviewer(s) / owner(s) + add_reviewers_to_pull_request(config.github_token, BRAVE_CORE_REPO, number, config.parsed_reviewers, + verbose=config.is_verbose, dryrun=config.is_dryrun) + set_issue_details(config.github_token, BRAVE_CORE_REPO, number, milestone_number, + config.parsed_owners, config.labels, + verbose=config.is_verbose, dryrun=config.is_dryrun) + except Exception as e: + print('[ERROR] unhandled error occurred:', str(e)) + + +if __name__ == '__main__': + import sys + sys.exit(main())