diff --git a/.gitlab/config/generate_pipelines.py b/.gitlab/config/generate_pipelines.py index 17b8068b99..15a70cfde3 100755 --- a/.gitlab/config/generate_pipelines.py +++ b/.gitlab/config/generate_pipelines.py @@ -5,11 +5,11 @@ # # generate_pipeline.py # -# Created: May 19, 2023 # Author: Vicente Adolfo Bolea Sanchez from datetime import datetime import argparse +import itertools import requests import time import re @@ -18,18 +18,43 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -def is_date_after(date, days): - deadline_sec = int(time.time()) - (days * 86400) - utc_dt = datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ') - timestamp_sec = (utc_dt - datetime(1970, 1, 1)).total_seconds() - return timestamp_sec > deadline_sec - - -def request_dict(url): +def request_as_dict(url): r = requests.get(url + '?per_page=100', verify=False) return r.json() +def add_timestamp(branch): + date_str = branch['commit']['committed_date'] + # We ignore the TZ since Gitlab/GitHub always reports in UTC + branch['dt'] = int( + datetime.strptime(date_str.split(".")[0], + '%Y-%m-%dT%H:%M:%S').timestamp()) + return branch + + +def is_recent(branch): + deadline_sec = int(time.time()) - (args.days * 86400) + return branch['dt'] > deadline_sec + + +def has_no_status(branch): + gh_commit_sha = branch['commit']['id'] + # Backported branches use the merge head + if re.fullmatch(r'^pr\d+_.*$', branch['name']): + gh_commit_sha = branch['commit']['parent_ids'][1] + + # Query GitHub for the status of this commit + commit = request_as_dict(gh_url + '/commits/' + gh_commit_sha + '/status') + if commit is None or 'sha' not in commit: + return False + + for status in commit['statuses']: + if status['context'] == args.gh_context: + return False + + return True + + parser = argparse.ArgumentParser( prog='generate_pipeline.py', description='Generate Dynamic pipelines for Gitlab') @@ -39,12 +64,12 @@ def request_dict(url): parser.add_argument( '-n', '--gh-name', required=True, help='Full name of the GitHub project. Ex: ornladios/ADIOS2') -parser.add_argument( - '-c', '--gh-context', default='OLCF Crusher (Frontier)', - help='Name of the status in GitHub (A.K.A context)') parser.add_argument( '-p', '--project_id', required=True, help='Gitlab internal project ID of the project.') +parser.add_argument( + '-c', '--gh-context', default='OLCF Crusher (Frontier)', + help='Name of the status in GitHub (A.K.A context)') parser.add_argument( '-d', '--days', type=int, default=1, help='How many days back to search for commits') @@ -57,34 +82,21 @@ def request_dict(url): args = parser.parse_args() -with open(args.template_file, "r") as fd: +gl_url = args.gl_url + '/api/v4/projects/' + str(args.project_id) +gh_url = 'https://api.github.com/repos/' + args.gh_name + +with open(args.template_file, 'r') as fd: template_str = fd.read() - gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id) - gh_url = 'https://api.github.com/repos/' + args.gh_name - branches = request_dict(gl_url + "/repository/branches") - num_pipeline = 0 + + branches = request_as_dict(gl_url + '/repository/branches') + branches = map(add_timestamp, branches) + branches = filter(is_recent, branches) + branches = filter(has_no_status, branches) + + # Select the arg.max most least recent branches + branches = sorted(branches, key=lambda x: x['dt']) + branches = itertools.islice(branches, args.max) + for branch in branches: - # Convert to ISO 8601 date format. - date_stamp = branch['commit']['committed_date'].split('.')[0] + "Z" - if num_pipeline < args.max and is_date_after(date_stamp, args.days): - commit_sha = branch['commit']['id'] - # Backported branches use the merge head - gh_commit_sha = commit_sha - if re.fullmatch(r'^pr\d+_.*$', branch['name']): - gh_commit_sha = branch['commit']['parent_ids'][1] - - # Quit if GitHub does not have the commit - if 'sha' not in request_dict(gh_url + "/commits/" + gh_commit_sha): - continue - - # Query GitHub for the status of this commit - commit = request_dict(gh_url + "/commits/" + - gh_commit_sha + "/status") - status_found = False - for status in commit['statuses']: - if status['context'] == args.gh_context: - status_found = True - if not status_found: - num_pipeline += 1 - print(template_str.format( - branch=branch['name'], commit=commit_sha)) + print(template_str.format( + branch=branch['name'], commit=branch['commit']['id']))