From f79dd4eaca43ee37add25caf2b15f7ba7003ae96 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 16:35:14 +0100 Subject: [PATCH 01/16] Ability to export a list of repositories last updated before a specified time Signed-off-by: jboursier --- src/cli.py | 86 ++++++++++++++++++++++++++++++ src/ghas_cli/utils/repositories.py | 26 ++++++++- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index 0fa738c..fd63035 100644 --- a/src/cli.py +++ b/src/cli.py @@ -371,6 +371,92 @@ def repositories_create_dep_enforcement_pr( ) +@repositories_cli.command("archivable") +@click.option( + "-f", + "--format", + prompt="Output format", + type=click.Choice( + ["human", "ghas", "json", "list"], + case_sensitive=False, + ), + default="human", +) +@click.option( + "-u", "--last_updated_before", prompt="Last updated before YYYY-MM-DD", type=str +) +@click.argument("output", type=click.File("w")) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def repositories_archivable( + last_updated_before: str, + format: str, + output: Any, + organization: str, + token: str, +) -> None: + """Find potentially archivable repositories""" + + try: + threshold_date = datetime.strptime(last_updated_before, "%Y-%m-%d") + except: + click.echo(f"Invalid time: {last_updated_before}") + return False + + # 1. Get list of non-archived repositories + res = repositories.get_org_repositories( + status="public", + organization=organization, + token=token, + language="", + default_branch="", + license="", + archived=False, + disabled=False, + ) + + for repo in res: + print(f"default branch: {repo.default_branch}") + + branch_last_commit_date = repositories.get_default_branch_last_updated( + token=token, + organization=organization, + repository_name=repo.name, + default_branch=repo.default_branch, + ) + if not branch_last_commit_date: + return False + + click.echo(f"{branch_last_commit_date}, {threshold_date}") + + if branch_last_commit_date > threshold_date: + continue + + if "human" == format: + output.write(repo + "\n") + click.echo(repo) + elif "ghas" == format: + output.write( + json.dumps([{"login": organization, "repos": repo.to_ghas()}]) + "\n" + ) + click.echo([{"login": organization, "repos": repo.to_ghas()}]) + elif "json" == format: + output.write(json.dumps(repo.to_json()) + "\n") + click.echo(repo.to_json()) + elif "list" == format: + output.write(repo.name + "\n") + click.echo(repo.name) + + ######### # Teams # ######### diff --git a/src/ghas_cli/utils/repositories.py b/src/ghas_cli/utils/repositories.py index 0d08040..ea32653 100644 --- a/src/ghas_cli/utils/repositories.py +++ b/src/ghas_cli/utils/repositories.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- #!/usr/bin/env python3 -from typing import List +from typing import List, Any import base64 import requests from . import network import time +import datetime class Repository: @@ -216,6 +217,29 @@ def get_org_repositories( return repos_list +def get_default_branch_last_updated( + token: str, organization: str, repository_name: str, default_branch: str +) -> Any: + """ + Return the latest commit date on the default branch + """ + headers = network.get_github_headers(token) + + branch_res = requests.get( + url=f"https://api.github.com/repos/{organization}/{repository_name}/branches/{default_branch}", + headers=headers, + ) + + if branch_res.status_code != 200: + return False + + branch_res = branch_res.json() + + return datetime.datetime.strptime( + branch_res["commit"]["commit"]["author"]["date"].split("T")[0], "%Y-%m-%d" + ) + + def check_dependabot_alerts_enabled( token: str, organization: str, repository_name: str ) -> bool: From 685f5c67a8d3cc1b31cc6a2cd42218636a8241d8 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 16:49:47 +0100 Subject: [PATCH 02/16] Ability to archive a repository Signed-off-by: jboursier --- src/cli.py | 26 ++++++++++++++++++++++++++ src/ghas_cli/utils/repositories.py | 17 +++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/cli.py b/src/cli.py index fd63035..d065e8e 100644 --- a/src/cli.py +++ b/src/cli.py @@ -457,6 +457,32 @@ def repositories_archivable( click.echo(repo.name) +@repositories_cli.command("archive") +@click.option( + "-r", + "--repository", + prompt="Repository name", +) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def repositories_archive( + repository: str, + organization: str, + token: str, +) -> None: + """Archive a repository""" + click.echo(repositories.archive(organization, token, repository)) + + ######### # Teams # ######### diff --git a/src/ghas_cli/utils/repositories.py b/src/ghas_cli/utils/repositories.py index ea32653..691b5a0 100644 --- a/src/ghas_cli/utils/repositories.py +++ b/src/ghas_cli/utils/repositories.py @@ -240,6 +240,23 @@ def get_default_branch_last_updated( ) +def archive(organization: str, token: str, repository: str) -> bool: + headers = network.get_github_headers(token) + + payload = {"archived": True} + + status = requests.patch( + url=f"https://api.github.com/repos/{organization}/{repository}", + headers=headers, + json=payload, + ) + + if status.status_code != 200: + return False + else: + return True + + def check_dependabot_alerts_enabled( token: str, organization: str, repository_name: str ) -> bool: From 276ba68a80af43fb6058c766f122a2799454a141 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 16:51:09 +0100 Subject: [PATCH 03/16] Catch Exception only Signed-off-by: jboursier --- src/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index d065e8e..051b1a7 100644 --- a/src/cli.py +++ b/src/cli.py @@ -408,7 +408,7 @@ def repositories_archivable( try: threshold_date = datetime.strptime(last_updated_before, "%Y-%m-%d") - except: + except Exception: click.echo(f"Invalid time: {last_updated_before}") return False From f102f43a651bce7bed282c88641b85bc34ee7866 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 16:52:09 +0100 Subject: [PATCH 04/16] Return a boolean Signed-off-by: jboursier --- src/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index 051b1a7..21de9a5 100644 --- a/src/cli.py +++ b/src/cli.py @@ -403,7 +403,7 @@ def repositories_archivable( output: Any, organization: str, token: str, -) -> None: +) -> bool: """Find potentially archivable repositories""" try: @@ -456,6 +456,8 @@ def repositories_archivable( output.write(repo.name + "\n") click.echo(repo.name) + return True + @repositories_cli.command("archive") @click.option( From 64c631be65c13d880b7b66abd755a63df1a5863c Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 16:58:24 +0100 Subject: [PATCH 05/16] Mass archive a list of repositories Signed-off-by: jboursier --- src/cli.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/cli.py b/src/cli.py index 21de9a5..9ce15df 100644 --- a/src/cli.py +++ b/src/cli.py @@ -996,5 +996,41 @@ def mass_deploy( ) +@mass_cli.command("archive") +@click.argument("input_repos_list", type=click.File("r")) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def mass_deploy( + input_repos_list: Any, + organization: str, + token: str, +) -> None: + """Mass archive a list of repositories""" + + repos_list = input_repos_list.readlines() + + for repo in repos_list: + + repo = repo.rstrip("\n") + + click.echo(f"{repo}...", nl=False) + + if repositories.archive( + organization=organization, token=token, repository=repo + ): + click.echo(" Archived.") + else: + click.echo(" Not Archived.", err=True) + + if __name__ == "__main__": main() From 8082005aabec6ba434ebf842cdaf8746cb0aa27f Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 17:18:04 +0100 Subject: [PATCH 06/16] Create an issue to inform of the upcoming archive event Signed-off-by: jboursier --- src/cli.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/cli.py b/src/cli.py index 9ce15df..c8e0b24 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1009,12 +1009,12 @@ def mass_deploy( show_envvar=True, ) @click.option("-o", "--organization", prompt="Organization name", type=str) -def mass_deploy( +def mass_issue_archive( input_repos_list: Any, organization: str, token: str, ) -> None: - """Mass archive a list of repositories""" + """Create an issue to inform that repositories will be archived at a specific date.""" repos_list = input_repos_list.readlines() @@ -1032,5 +1032,54 @@ def mass_deploy( click.echo(" Not Archived.", err=True) +@mass_cli.command("issue_upcoming_archive") +@click.argument("input_repos_list", type=click.File("r")) +@click.option( + "-u", + "--archived_date", + prompt="Target date when the repositories will be archived", + type=str, +) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def mass_archive( + input_repos_list: Any, + archived_date: str, + organization: str, + token: str, +) -> None: + """Mass archive a list of repositories""" + + repos_list = input_repos_list.readlines() + + for repo in repos_list: + + repo = repo.rstrip("\n") + + issue_creation = issues.create( + title=f"This repository will be archived on {archived_date} :warning: :wastebasket:", + content=f""" +Hello, + +Due to inactivity, this repository will be archived automatically on {archived_date}. + +If you think this is a mistake, please informthe Security team *ASAP* on Slack at `#github-appsec-security.` + +Thanks! :handshake:""", + repository=repo, + organization=organization, + token=token, + ) + + if __name__ == "__main__": main() From 05c417aa3a8785195dc7208218b4a35b756f4b34 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 17:23:12 +0100 Subject: [PATCH 07/16] Add a link to GitHub's documentation Signed-off-by: jboursier --- src/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cli.py b/src/cli.py index c8e0b24..c86e353 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1072,6 +1072,8 @@ def mass_archive( Due to inactivity, this repository will be archived automatically on {archived_date}. +For more information, see https://docs.github.com/en/repositories/archiving-a-github-repository/archiving-repositories#about-repository-archival + If you think this is a mistake, please informthe Security team *ASAP* on Slack at `#github-appsec-security.` Thanks! :handshake:""", From 6f29646e395ca8dd6a23305e8f91aa86a6edf65d Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 17:24:01 +0100 Subject: [PATCH 08/16] Remove unused variable Signed-off-by: jboursier --- src/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index c86e353..ce1f88f 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1065,7 +1065,7 @@ def mass_archive( repo = repo.rstrip("\n") - issue_creation = issues.create( + issues.create( title=f"This repository will be archived on {archived_date} :warning: :wastebasket:", content=f""" Hello, From 79e1d8a29cac65aebd86e284508c1cd5640c5262 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 17:53:56 +0100 Subject: [PATCH 09/16] Check archivable status on all repositories, not just public ones Signed-off-by: jboursier --- src/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index ce1f88f..8c19851 100644 --- a/src/cli.py +++ b/src/cli.py @@ -414,7 +414,7 @@ def repositories_archivable( # 1. Get list of non-archived repositories res = repositories.get_org_repositories( - status="public", + status="all", organization=organization, token=token, language="", From b1f7b229273f890f577733ae5d4d2d51aa8a632e Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 17:56:22 +0100 Subject: [PATCH 10/16] Remove excessive logging Signed-off-by: jboursier --- src/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cli.py b/src/cli.py index 8c19851..6a9c204 100644 --- a/src/cli.py +++ b/src/cli.py @@ -425,7 +425,6 @@ def repositories_archivable( ) for repo in res: - print(f"default branch: {repo.default_branch}") branch_last_commit_date = repositories.get_default_branch_last_updated( token=token, @@ -436,8 +435,6 @@ def repositories_archivable( if not branch_last_commit_date: return False - click.echo(f"{branch_last_commit_date}, {threshold_date}") - if branch_last_commit_date > threshold_date: continue From f167d0629e9ab3adc29a03ade316228d919450aa Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 18:01:41 +0100 Subject: [PATCH 11/16] Fix typo Signed-off-by: jboursier --- src/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index 6a9c204..9489efd 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1071,7 +1071,7 @@ def mass_archive( For more information, see https://docs.github.com/en/repositories/archiving-a-github-repository/archiving-repositories#about-repository-archival -If you think this is a mistake, please informthe Security team *ASAP* on Slack at `#github-appsec-security.` +If you think this is a mistake, please inform the Security team *ASAP* on Slack at `#github-appsec-security.` Thanks! :handshake:""", repository=repo, From bbd308929ea5a3a72b3226c4d29c161cf66505a6 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 12 Jan 2023 18:03:34 +0100 Subject: [PATCH 12/16] Improve issue creation output Signed-off-by: jboursier --- src/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index 9489efd..bbdaf02 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1062,7 +1062,7 @@ def mass_archive( repo = repo.rstrip("\n") - issues.create( + issue_res = issues.create( title=f"This repository will be archived on {archived_date} :warning: :wastebasket:", content=f""" Hello, @@ -1078,6 +1078,10 @@ def mass_archive( organization=organization, token=token, ) + if issue_res: + click.echo(f"{repo}... {issue_res}") + else: + click.echo(f"{repo}... Failure", err=True) if __name__ == "__main__": From 4a764967dcc65f5bb2b1be013bcc8916737f7024 Mon Sep 17 00:00:00 2001 From: jboursier Date: Fri, 13 Jan 2023 18:15:33 +0100 Subject: [PATCH 13/16] * Add debugging steps * Improve archiving issue body to be more explanatory * Fix rate-limit handling Signed-off-by: jboursier --- src/cli.py | 8 ++++++-- src/ghas_cli/utils/network.py | 2 +- src/ghas_cli/utils/repositories.py | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/cli.py b/src/cli.py index bbdaf02..f3ae0d4 100644 --- a/src/cli.py +++ b/src/cli.py @@ -414,7 +414,7 @@ def repositories_archivable( # 1. Get list of non-archived repositories res = repositories.get_org_repositories( - status="all", + status="internal", organization=organization, token=token, language="", @@ -424,6 +424,7 @@ def repositories_archivable( disabled=False, ) + print(len(res)) for repo in res: branch_last_commit_date = repositories.get_default_branch_last_updated( @@ -433,7 +434,8 @@ def repositories_archivable( default_branch=repo.default_branch, ) if not branch_last_commit_date: - return False + click.echo(f"No branch last commit date for {repo.name}", err=True) + continue if branch_last_commit_date > threshold_date: continue @@ -1069,6 +1071,8 @@ def mass_archive( Due to inactivity, this repository will be archived automatically on {archived_date}. +This means that it will become read-only: `git clone` will still work, and the repository can be unarchived at anytime if needed. + For more information, see https://docs.github.com/en/repositories/archiving-a-github-repository/archiving-repositories#about-repository-archival If you think this is a mistake, please inform the Security team *ASAP* on Slack at `#github-appsec-security.` diff --git a/src/ghas_cli/utils/network.py b/src/ghas_cli/utils/network.py index b7557d3..81bdb16 100644 --- a/src/ghas_cli/utils/network.py +++ b/src/ghas_cli/utils/network.py @@ -31,7 +31,7 @@ def check_rate_limit(response: Any) -> bool: print( f"Rate limit reached: {response.headers['x-ratelimit-remaining']}/{response.headers['x-ratelimit-limit']} - {reset_time}" ) - time.sleep(reset_time) + time.sleep(int(response.headers["x-ratelimit-remaining"])) return True if response.status_code == 403: diff --git a/src/ghas_cli/utils/repositories.py b/src/ghas_cli/utils/repositories.py index 691b5a0..5b43697 100644 --- a/src/ghas_cli/utils/repositories.py +++ b/src/ghas_cli/utils/repositories.py @@ -231,6 +231,9 @@ def get_default_branch_last_updated( ) if branch_res.status_code != 200: + print(default_branch) + print(branch_res.json()) + print(branch_res.status_code) return False branch_res = branch_res.json() From 39aa7d76f4c14801a9617891b2244eedbbb34fbc Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 16 Jan 2023 11:42:13 +0100 Subject: [PATCH 14/16] Use a passed repository list instead of having to generate the list everytime Signed-off-by: jboursier --- src/cli.py | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/cli.py b/src/cli.py index f3ae0d4..e32b979 100644 --- a/src/cli.py +++ b/src/cli.py @@ -377,14 +377,15 @@ def repositories_create_dep_enforcement_pr( "--format", prompt="Output format", type=click.Choice( - ["human", "ghas", "json", "list"], + ["human", "list"], case_sensitive=False, ), - default="human", + default="list", ) @click.option( "-u", "--last_updated_before", prompt="Last updated before YYYY-MM-DD", type=str ) +@click.argument("input_repos_list", type=click.File("r")) @click.argument("output", type=click.File("w")) @click.option( "-t", @@ -400,6 +401,7 @@ def repositories_create_dep_enforcement_pr( def repositories_archivable( last_updated_before: str, format: str, + input_repos_list: Any, output: Any, organization: str, token: str, @@ -412,48 +414,40 @@ def repositories_archivable( click.echo(f"Invalid time: {last_updated_before}") return False - # 1. Get list of non-archived repositories - res = repositories.get_org_repositories( - status="internal", - organization=organization, - token=token, - language="", - default_branch="", - license="", - archived=False, - disabled=False, - ) + # 1. Get list repositories passed as argument + res = input_repos_list.readlines() print(len(res)) for repo in res: + repo = repo.rstrip("\n") + + # 2. get default branch + default_branch = repositories.get_default_branch( + organization=organization, token=token, repository=repo + ) + + # 3. get default branch last commit date branch_last_commit_date = repositories.get_default_branch_last_updated( token=token, organization=organization, - repository_name=repo.name, - default_branch=repo.default_branch, + repository_name=repo, + default_branch=default_branch, ) if not branch_last_commit_date: - click.echo(f"No branch last commit date for {repo.name}", err=True) + click.echo(f"No branch last commit date for {repo}", err=True) continue + # 4. Compare with the threshold if branch_last_commit_date > threshold_date: continue if "human" == format: output.write(repo + "\n") click.echo(repo) - elif "ghas" == format: - output.write( - json.dumps([{"login": organization, "repos": repo.to_ghas()}]) + "\n" - ) - click.echo([{"login": organization, "repos": repo.to_ghas()}]) - elif "json" == format: - output.write(json.dumps(repo.to_json()) + "\n") - click.echo(repo.to_json()) elif "list" == format: - output.write(repo.name + "\n") - click.echo(repo.name) + output.write(f"{repo}, {branch_last_commit_date.strftime('%Y-%m-%d')}\n") + click.echo(f"{repo}, {branch_last_commit_date.strftime('%Y-%m-%d')}") return True From aa39e2ffea58cee954013efdc61b660be480b2cb Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 16 Jan 2023 11:43:15 +0100 Subject: [PATCH 15/16] Remove unused import Signed-off-by: jboursier --- src/ghas_cli/utils/actions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ghas_cli/utils/actions.py b/src/ghas_cli/utils/actions.py index f851c7b..af1fe79 100644 --- a/src/ghas_cli/utils/actions.py +++ b/src/ghas_cli/utils/actions.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- #!/usr/bin/env python3 -from typing import List import requests from . import network From 832933d00d3c9c91717957af1e09e07b074f21e3 Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 16 Jan 2023 11:48:02 +0100 Subject: [PATCH 16/16] Remove debug lines Signed-off-by: jboursier --- src/cli.py | 2 ++ src/ghas_cli/utils/repositories.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cli.py b/src/cli.py index e32b979..a5c338a 100644 --- a/src/cli.py +++ b/src/cli.py @@ -426,6 +426,8 @@ def repositories_archivable( default_branch = repositories.get_default_branch( organization=organization, token=token, repository=repo ) + if not default_branch: + continue # 3. get default branch last commit date branch_last_commit_date = repositories.get_default_branch_last_updated( diff --git a/src/ghas_cli/utils/repositories.py b/src/ghas_cli/utils/repositories.py index 5b43697..691b5a0 100644 --- a/src/ghas_cli/utils/repositories.py +++ b/src/ghas_cli/utils/repositories.py @@ -231,9 +231,6 @@ def get_default_branch_last_updated( ) if branch_res.status_code != 200: - print(default_branch) - print(branch_res.json()) - print(branch_res.status_code) return False branch_res = branch_res.json()