From 7e85953ba7b549f61895b48ba2e8e32fbd91a5f3 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 15 Sep 2022 15:48:37 +0200 Subject: [PATCH 1/6] Initial version: * List repositories * Gather security vulnerabilities if any --- src/github.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/github.py diff --git a/src/github.py b/src/github.py new file mode 100644 index 0000000..b011589 --- /dev/null +++ b/src/github.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +__author__ = "jboursier" +__copyright__ = "Copyright 2022, Malwarebytes" +__version__ = "0.0.1" +__maintainer__ = "jboursier" +__email__ = "jboursier@malwarebytes.com" +__status__ = "Development" + +try: + import click + import requests + import json + import time + import typing + from typing import List, Dict, Any + from datetime import datetime +except ImportError: + import sys + + print("Missing dependencies. Please reach @jboursier if needed.") + sys.exit(255) + +from click.exceptions import ClickException +from requests.exceptions import Timeout + +ORG_NAME = "" +GH_TOKEN = "" + + +def check_rate_limit(response: Any) -> bool: + if "0" == response.headers["x-ratelimit-remaining"]: + reset_time = datetime.fromtimestamp(int(response.headers["x-ratelimit-reset"])) + print( + f"Rate limit reached: {response.headers['x-ratelimit-remaining']}/{response.headers['x-ratelimit-limit']} - {reset_time}" + ) + return True + else: + return False + + +def get_org_repositories(org_name: str, exclude_archived: bool, session: Any) -> List: + + repositories = [] + page = 1 + while True: + params = { + "type": "all", + "sort": "full_name", + "per_page": 100, + "page": page, + } + repos = session.get( + url=f"https://api.github.com/orgs/{org_name}/repos", + params=params, + ) + if check_rate_limit(repos): + break + + if repos.status_code != 200: + break + for r in repos.json(): + print( + f"{page} - {r['name']} - {repos.headers['x-ratelimit-remaining']} / {repos.headers['x-ratelimit-limit']} - {repos.headers['x-ratelimit-reset']}" + ) + repositories.append(r["name"]) + + if [] == repos.json(): + break + page += 1 + + return repositories + + +def get_codeql_alerts_repo(repo_name: str, org_name: str, session: Any) -> List: + + # https://api.github.com/repos/OWNER/REPO/code-scanning/alerts + + alerts_repo = [] + page = 1 + while True: + params = {"state": "open", "per_page": 100, "page": page} + alerts = session.get( + url=f"https://api.github.com/repos/{org_name}/{repo_name}/code-scanning/alerts", + params=params, + ) + print( + f"https://api.github.com/repos/{org_name}/{repo_name}/code-scanning/alerts" + ) + + if check_rate_limit(alerts): + break + + if alerts.status_code != 200: + break + + for a in alerts.json(): + print( + f"{page} - {a} - {alerts.headers['x-ratelimit-remaining']} / {alerts.headers['x-ratelimit-limit']} - {alerts.headers['x-ratelimit-reset']}" + ) + alerts_repo.append(a) + + if [] == alerts.json(): + break + + page += 1 + + return alerts_repo + + +def output_to_csv(alerts_per_repos: Dict, location: str) -> bool: + try: + with open(location, "w") as log_file: + log_file.write(json.dumps(alerts_per_repos)) + except Exception as e: + print(str(e)) + print(f"Failure to write the output to {location}") + return False + return True + + +def main(): + + s = requests.Session() + s.headers.update( + { + "accept": "application/vnd.github+json", + "authorization": f"Bearer {GH_TOKEN}", + "User-Agent": "jboursier-mwb/fetch_org_ghas_metrics", + } + ) + + exclude_archived = False + + org_repos = get_org_repositories(ORG_NAME, exclude_archived, s) + + print(org_repos) + + alerts_per_repo = {} + output = "./codescanning_alerts.json" + + for repo in org_repos: + alerts_per_repo[repo] = get_codeql_alerts_repo(repo, ORG_NAME, s) + + print(alerts_per_repo) + + output_to_csv(alerts_per_repo, location=output) + + +if __name__ == "__main__": + main() From 3558e96853bb0e4b4373bacf721ffa03baeb1314 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 15 Sep 2022 16:23:58 +0200 Subject: [PATCH 2/6] Setup Poetry and Makefile --- Makefile | 31 +++++++++++++++++++++++++++++++ README.md | 44 +++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 38 ++++++++++++++++++++++++++++++++++++++ src/github.py | 2 -- 4 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 Makefile create mode 100644 pyproject.toml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f1d589d --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +.DEFAULT_GOAL=help +.PHONY=help + +ZIP = zip +PIP3 = python3 -m pip +PYTHON3 = python3 +POETRY = poetry + + +clean: ## clean existing builds + rm -rf ./dist || true + rm -rf ./src/ghas-cli/ghas-cli.egg-info || true + rm checksums.sha512 || true + rm checksums.sha512.asc || true + +release: ## Build a wheel + $(POETRY) build + cd dist && sha512sum * > ../checksums.sha512 + gpg --detach-sign --armor checksums.sha512 + +shell: ## Generate the shell autocompletion + _GHAS_CLI_COMPLETE=source_bash ghas-cli > ghas-cli-complete.sh || true + +deps: ## Fetch or update dependencies + $(POETRY) update --no-dev + +help: + @awk -F ':|##' '/^[^\t].+?:.*?##/ { printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF }' $(MAKEFILE_LIST) | sort + + +.PHONY: help diff --git a/README.md b/README.md index 21d38e9..bc2178c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,44 @@ # Security-ghas-cli -CLI utility to interact with GHAS + +CLI utility to interact with GHAS. + + +## Installation + +Builds are available in the [`Releases`](https://github.com/Malwarebytes/Security-ghas-cli/releases) tab. + +```bash +python -m pip install /full/path/to/ghas-cli-xxx.whl + +# e.g: python3 -m pip install Downloads/ghas-cli-0.5.0-none-any.whl +``` + +## Development + +### Build + +[Install Poetry](https://python-poetry.org/docs/#installation) first, then: + +```bash +make release +``` + +### Bump the version number + +* Update the `version` field in `pyproject.toml`. +* Update the `__version__` field in `src/cli.py`. + +### Publish a new version + +1. Bump the version number as described above +2. `make deps` to update the dependencies +3. `make release` to build the packages +4. `git commit -a -S Bump to version 1.1.2` and `git tag -s v1.1.2 -m "1.1.2"` +5. Upload `dist/*`, `checksums.sha512` and `checksums.sha512.asc` to a new release in Github. + +# Resources + +Please reach Jérôme Boursier for any issues or question: + +* jboursier@malwarebytes.com +* `jboursier` on Slack diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..402d531 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[tool.poetry] +name = "security-ghash-cli" +version = "0.0.1" +description = "Python3 command line interface to interact with Github Advanced Security." +authors = ["jboursier "] +license = "MIT" +readme = "README.md" +homepage = "https://malwarebytes.com" +repository = "https://github.com/Malwarebytes/Security-ghash-cli" +keywords = ["security", "cli", "github", "utility"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Topic :: Utilities" +] +include = ["src/cli.py"] + +[tool.poetry.dependencies] +python = ">=3.7" +click = ">=8" +requests = "*" +colorama = "*" +configparser = "*" +python-magic = "*" + +[tool.poetry.dev-dependencies] + +[tool.poetry.scripts] +ghas-cli = 'src.cli:main' + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/src/github.py b/src/github.py index b011589..02dc949 100644 --- a/src/github.py +++ b/src/github.py @@ -12,8 +12,6 @@ import click import requests import json - import time - import typing from typing import List, Dict, Any from datetime import datetime except ImportError: From e8b9d4eca1b899d46accdff56ef8b197c206e48b Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 15 Sep 2022 17:34:47 +0200 Subject: [PATCH 3/6] Initial port to Click --- src/github.py | 150 -------------------------------------------------- 1 file changed, 150 deletions(-) delete mode 100644 src/github.py diff --git a/src/github.py b/src/github.py deleted file mode 100644 index 02dc949..0000000 --- a/src/github.py +++ /dev/null @@ -1,150 +0,0 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/env python3 - -__author__ = "jboursier" -__copyright__ = "Copyright 2022, Malwarebytes" -__version__ = "0.0.1" -__maintainer__ = "jboursier" -__email__ = "jboursier@malwarebytes.com" -__status__ = "Development" - -try: - import click - import requests - import json - from typing import List, Dict, Any - from datetime import datetime -except ImportError: - import sys - - print("Missing dependencies. Please reach @jboursier if needed.") - sys.exit(255) - -from click.exceptions import ClickException -from requests.exceptions import Timeout - -ORG_NAME = "" -GH_TOKEN = "" - - -def check_rate_limit(response: Any) -> bool: - if "0" == response.headers["x-ratelimit-remaining"]: - reset_time = datetime.fromtimestamp(int(response.headers["x-ratelimit-reset"])) - print( - f"Rate limit reached: {response.headers['x-ratelimit-remaining']}/{response.headers['x-ratelimit-limit']} - {reset_time}" - ) - return True - else: - return False - - -def get_org_repositories(org_name: str, exclude_archived: bool, session: Any) -> List: - - repositories = [] - page = 1 - while True: - params = { - "type": "all", - "sort": "full_name", - "per_page": 100, - "page": page, - } - repos = session.get( - url=f"https://api.github.com/orgs/{org_name}/repos", - params=params, - ) - if check_rate_limit(repos): - break - - if repos.status_code != 200: - break - for r in repos.json(): - print( - f"{page} - {r['name']} - {repos.headers['x-ratelimit-remaining']} / {repos.headers['x-ratelimit-limit']} - {repos.headers['x-ratelimit-reset']}" - ) - repositories.append(r["name"]) - - if [] == repos.json(): - break - page += 1 - - return repositories - - -def get_codeql_alerts_repo(repo_name: str, org_name: str, session: Any) -> List: - - # https://api.github.com/repos/OWNER/REPO/code-scanning/alerts - - alerts_repo = [] - page = 1 - while True: - params = {"state": "open", "per_page": 100, "page": page} - alerts = session.get( - url=f"https://api.github.com/repos/{org_name}/{repo_name}/code-scanning/alerts", - params=params, - ) - print( - f"https://api.github.com/repos/{org_name}/{repo_name}/code-scanning/alerts" - ) - - if check_rate_limit(alerts): - break - - if alerts.status_code != 200: - break - - for a in alerts.json(): - print( - f"{page} - {a} - {alerts.headers['x-ratelimit-remaining']} / {alerts.headers['x-ratelimit-limit']} - {alerts.headers['x-ratelimit-reset']}" - ) - alerts_repo.append(a) - - if [] == alerts.json(): - break - - page += 1 - - return alerts_repo - - -def output_to_csv(alerts_per_repos: Dict, location: str) -> bool: - try: - with open(location, "w") as log_file: - log_file.write(json.dumps(alerts_per_repos)) - except Exception as e: - print(str(e)) - print(f"Failure to write the output to {location}") - return False - return True - - -def main(): - - s = requests.Session() - s.headers.update( - { - "accept": "application/vnd.github+json", - "authorization": f"Bearer {GH_TOKEN}", - "User-Agent": "jboursier-mwb/fetch_org_ghas_metrics", - } - ) - - exclude_archived = False - - org_repos = get_org_repositories(ORG_NAME, exclude_archived, s) - - print(org_repos) - - alerts_per_repo = {} - output = "./codescanning_alerts.json" - - for repo in org_repos: - alerts_per_repo[repo] = get_codeql_alerts_repo(repo, ORG_NAME, s) - - print(alerts_per_repo) - - output_to_csv(alerts_per_repo, location=output) - - -if __name__ == "__main__": - main() From cf9842bf419a0e6958fb9edca531efee0cf6175a Mon Sep 17 00:00:00 2001 From: jboursier Date: Fri, 16 Sep 2022 13:07:14 +0200 Subject: [PATCH 4/6] Re-arrange code with submodules --- src/cli.py | 105 +++++++++++++++++++++++++++++ src/ghas_cli/__init__.py | 0 src/ghas_cli/utils/__init__.py | 0 src/ghas_cli/utils/export.py | 17 +++++ src/ghas_cli/utils/network.py | 17 +++++ src/ghas_cli/utils/repositories.py | 44 ++++++++++++ src/ghas_cli/utils/vulns.py | 53 +++++++++++++++ 7 files changed, 236 insertions(+) create mode 100644 src/cli.py create mode 100644 src/ghas_cli/__init__.py create mode 100644 src/ghas_cli/utils/__init__.py create mode 100644 src/ghas_cli/utils/export.py create mode 100644 src/ghas_cli/utils/network.py create mode 100644 src/ghas_cli/utils/repositories.py create mode 100644 src/ghas_cli/utils/vulns.py diff --git a/src/cli.py b/src/cli.py new file mode 100644 index 0000000..317becf --- /dev/null +++ b/src/cli.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +__author__ = "jboursier" +__copyright__ = "Copyright 2022, Malwarebytes" +__version__ = "0.0.1" +__maintainer__ = "jboursier" +__email__ = "jboursier@malwarebytes.com" +__status__ = "Development" + +try: + import click + import requests + import json + from typing import List, Dict, Any + from datetime import datetime +except ImportError: + import sys + + print("Missing dependencies. Please reach @jboursier if needed.") + sys.exit(255) + +from click.exceptions import ClickException +from requests.exceptions import Timeout + +from ghas_cli.utils import repositories, network, vulns + + +def main() -> None: + try: + cli() + except Exception as e: + click.echo(e) + + +@click.group() +def cli() -> None: + """ghas-cli is a Python3 utility to interact with Github Advanced Security. + + Get help: `@jboursier` on Slack + """ + + +@cli.group() +def vuln_alerts() -> None: + """Manage vulnerability alerts""" + pass + + +@vuln_alerts.command("list") +@click.option( + "-s", + "--status", + prompt="Alert status", + type=click.Choice(["open", "closed", ""], case_sensitive=False), + default="open", +) +@click.option( + "-r", + "--repos", + prompt="Repositories name. Use `all` to retrieve alerts for all repos.", + type=str, + multiple=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +def vulns_alerts_list(repos: str, organization: str, status: str, token: str) -> Dict: + """Get CodeQL alerts for one or several repositories""" + + repositories_alerts = {} + if repos == ("all",): + repos = repositories.get_org_repositories( + status="all", organization=organization, token=token + ) + + repositories_alerts = vulns.get_codeql_alerts_repo( + repos, organization, status, token + ) + click.echo(repositories_alerts) + return repositories_alerts + + +@cli.group() +def secret_alerts() -> None: + """Manage Secret Scanner alerts""" + pass + + +@cli.group() +def dependabot_alerts() -> None: + """Manage Dependabot alerts""" + pass + + +if __name__ == "__main__": + main() diff --git a/src/ghas_cli/__init__.py b/src/ghas_cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ghas_cli/utils/__init__.py b/src/ghas_cli/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ghas_cli/utils/export.py b/src/ghas_cli/utils/export.py new file mode 100644 index 0000000..1f75553 --- /dev/null +++ b/src/ghas_cli/utils/export.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +from typing import List, Any +import requests +import json + + +def output_to_csv(alerts_per_repos: Dict, location: str) -> bool: + try: + with open(location, "w") as log_file: + log_file.write(json.dumps(alerts_per_repos)) + except Exception as e: + print(str(e)) + print(f"Failure to write the output to {location}") + return False + return True diff --git a/src/ghas_cli/utils/network.py b/src/ghas_cli/utils/network.py new file mode 100644 index 0000000..da609c8 --- /dev/null +++ b/src/ghas_cli/utils/network.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +from typing import List, Any +import requests +import json + + +def check_rate_limit(response: Any) -> bool: + if "0" == response.headers["x-ratelimit-remaining"]: + reset_time = datetime.fromtimestamp(int(response.headers["x-ratelimit-reset"])) + print( + f"Rate limit reached: {response.headers['x-ratelimit-remaining']}/{response.headers['x-ratelimit-limit']} - {reset_time}" + ) + return True + else: + return False diff --git a/src/ghas_cli/utils/repositories.py b/src/ghas_cli/utils/repositories.py new file mode 100644 index 0000000..31006d1 --- /dev/null +++ b/src/ghas_cli/utils/repositories.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +from typing import List +import requests +import json + +from . import network + + +def get_org_repositories(status: str, organization: str, token: str) -> List: + repositories = [] + page = 1 + + headers = { + "accept": "application/vnd.github+json", + "authorization": f"Bearer {token}", + "User-Agent": "jboursier-mwb/fetch_org_ghas_metrics", + } + while True: + params = { + "type": f"{status}", + "sort": "full_name", + "per_page": 100, + "page": page, + } + repos = requests.get( + url=f"https://api.github.com/orgs/{organization}/repos", + params=params, + headers=headers, + ) + if network.check_rate_limit(repos): + break + + if repos.status_code != 200: + break + for r in repos.json(): + repositories.append(r["name"]) + + if [] == repos.json(): + break + page += 1 + + return repositories diff --git a/src/ghas_cli/utils/vulns.py b/src/ghas_cli/utils/vulns.py new file mode 100644 index 0000000..e13574d --- /dev/null +++ b/src/ghas_cli/utils/vulns.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +from typing import List, Any, Dict +import requests +import json + +from . import network + + +def get_codeql_alerts_repo( + repos: str, organization: str, status: str, token: str +) -> Dict: + """Get CodeQL alerts for one or several repositories""" + + headers = { + "accept": "application/vnd.github+json", + "authorization": f"Bearer {token}", + "User-Agent": "jboursier-mwb/fetch_org_ghas_metrics", + } + + repositories_alerts = {} + + for repo in repos: + + alerts_repo = [] + page = 1 + + while True: + + params = {"state": "open", "per_page": 100, "page": page} + alerts = requests.get( + url=f"https://api.github.com/repos/{organization}/{repo}/code-scanning/alerts", + params=params, + headers=headers, + ) + if network.check_rate_limit(alerts): + break + + if alerts.status_code != 200: + break + + for a in alerts.json(): + alerts_repo.append(a) + + if [] == alerts.json(): + break + + page += 1 + + repositories_alerts[repo] = alerts_repo + + return repositories_alerts From bb1803dc1247b799d4c2ce0edadfc0ba8f9c0708 Mon Sep 17 00:00:00 2001 From: jboursier Date: Fri, 16 Sep 2022 16:03:21 +0200 Subject: [PATCH 5/6] Fix pyproject.toml typo Update gitignore --- .gitignore | 4 ++++ pyproject.toml | 2 +- src/__init__.py | 0 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 src/__init__.py diff --git a/.gitignore b/.gitignore index b6e4761..c53ee44 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ dmypy.json # Pyre type checker .pyre/ + +ghas-cli-complete.sh +checksums.sha512 +checksums.sha512.asc diff --git a/pyproject.toml b/pyproject.toml index 402d531..ff19022 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "security-ghash-cli" +name = "ghas-cli" version = "0.0.1" description = "Python3 command line interface to interact with Github Advanced Security." authors = ["jboursier "] diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 From 05003e49e74d246f694ce59ba05d582b7e996a3f Mon Sep 17 00:00:00 2001 From: jboursier Date: Fri, 16 Sep 2022 16:50:49 +0200 Subject: [PATCH 6/6] Update alert listing output --- src/ghas_cli/utils/vulns.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ghas_cli/utils/vulns.py b/src/ghas_cli/utils/vulns.py index e13574d..7b9202b 100644 --- a/src/ghas_cli/utils/vulns.py +++ b/src/ghas_cli/utils/vulns.py @@ -41,9 +41,17 @@ def get_codeql_alerts_repo( break for a in alerts.json(): - alerts_repo.append(a) + if not a: + continue - if [] == alerts.json(): + alert_summary = {} + alert_summary["number"] = a["number"] + alert_summary["created_at"] = a["created_at"] + alert_summary["state"] = a["state"] + alert_summary["severity"] = a["rule"]["severity"] + alerts_repo.append(alert_summary) + + if not alerts.json(): break page += 1