Skip to content

Commit

Permalink
Feat: Add randomness to the codeql scanning time
Browse files Browse the repository at this point in the history 
Why do we need this change?
=======================
If deploying across a large number of repositories having all jobs trigger at the same exact time causes a mass spike in potential workloads on self hosted runners. Change behaviors to randomize the time as we want to get the results weekly, but timing shouldn't entirely matter

What effects does this change have?
=======================
* include secrets to generate random time
* add randomness to the cron task
* various lint fixes
  • Loading branch information
@JoshuaJackson-jobvite
JoshuaJackson-jobvite committed Jan 9, 2025
1 parent 14788c6 commit 83fea4c
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 39 deletions.
48 changes: 37 additions & 11 deletions src/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,18 @@

__author__ = "jboursier"
__copyright__ = "Copyright 2024, Malwarebytes"
__version__ = "1.6.1"
__version__ = "1.6.2"
__maintainer__ = "jboursier"
__email__ = "jboursier@malwarebytes.com"
__status__ = "Production"

try:
import click
import json
from typing import Dict, Any, List
from datetime import datetime
import logging
from datetime import datetime
from typing import Any, Dict, List

import click

logging.getLogger().setLevel(level=logging.INFO)
except ImportError:
Expand All @@ -22,7 +23,16 @@
logging.error("Missing dependencies. Please reach @jboursier if needed.")
sys.exit(255)

from ghas_cli.utils import repositories, vulns, teams, issues, actions, roles, secrets, dependabot
from ghas_cli.utils import (
actions,
dependabot,
issues,
repositories,
roles,
secrets,
teams,
vulns,
)


def main() -> None:
Expand Down Expand Up @@ -223,6 +233,7 @@ def repositories_list(
output.write(r.name + "\n")
click.echo(r.name)


@repositories_cli.command("get_topics")
@click.option(
"-r",
Expand All @@ -246,7 +257,11 @@ def repositories_get_topics(
token: str,
) -> None:
"""Get a repository topics"""
click.echo(repositories.get_topics(token=token, organization=organization, repository_name=repository))
click.echo(
repositories.get_topics(
token=token, organization=organization, repository_name=repository
)
)


@repositories_cli.command("enable_ss_protection")
Expand Down Expand Up @@ -925,12 +940,15 @@ def dependabot_alerts_list(
show_envvar=True,
)
@click.option("-o", "--organization", prompt="Organization name", type=str)
def dependabot_get_dependencies(repository:str, organization:str, token:str, format: str="sbom") -> None:
def dependabot_get_dependencies(
repository: str, organization: str, token: str, format: str = "sbom"
) -> None:
"""Get a list of dependencies for a repository"""

res = dependabot.get_dependencies(repository, organization, token, format=format)
click.echo(res, nl=False)


###########
# Actions #
###########
Expand Down Expand Up @@ -1501,7 +1519,6 @@ def mass_set_developer_role(
return None



@mass_cli.command("topics")
@click.argument("input_repos_list", type=click.File("r"))
@click.option(
Expand All @@ -1526,7 +1543,12 @@ def mass_get_topics(
repo = repo.rstrip("\n")

click.echo(f"{repo},", nl=False)
click.echo(repositories.get_topics(token=token, organization=organization, repository_name=repo))
click.echo(
repositories.get_topics(
token=token, organization=organization, repository_name=repo
)
)


@mass_cli.command("dependencies")
@click.argument("input_repos_list", type=click.File("r"))
Expand Down Expand Up @@ -1563,8 +1585,12 @@ def mass_get_dependencies(
repo = repo.rstrip("\n")

click.echo(f"{repo},", nl=False)
click.echo(dependabot.get_dependencies(repository=repo, organization=organization, token=token, format=format), nl=False)

click.echo(
dependabot.get_dependencies(
repository=repo, organization=organization, token=token, format=format
),
nl=False,
)


if __name__ == "__main__":
Expand Down
69 changes: 41 additions & 28 deletions src/ghas_cli/utils/repositories.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
# -*- coding: utf-8 -*-
#!/usr/bin/env python3

from typing import List, Any
import base64
from . import network
import time
import datetime
import logging
import secrets
import time
from typing import Any, List

from . import network


class Repository:
Expand Down Expand Up @@ -73,19 +75,19 @@ def load_json(self, obj, token=None):
self.updated_at = obj["updated_at"]
try:
self.ghas = obj["security_and_analysis"]["advanced_security"]["status"]
except Exception as e:
except Exception:
self.ghas = False
try:
self.secret_scanner = obj["security_and_analysis"]["advanced_security"][
"secret_scanning"
]["status"]
except Exception as e:
except Exception:
self.secret_scanner = False
try:
self.secret_push_prot = obj["security_and_analysis"]["advanced_security"][
"secret_scanning_push_protection"
]["status"]
except Exception as e:
except Exception:
self.secret_push_prot = False
self.dependabot = False
if token:
Expand Down Expand Up @@ -236,9 +238,8 @@ def get_default_branch_last_updated(
branch_res["commit"]["commit"]["author"]["date"].split("T")[0], "%Y-%m-%d"
)

def get_topics(
token: str, organization: str, repository_name: str
) -> List:

def get_topics(token: str, organization: str, repository_name: str) -> List:
"""
Return the repository topics
"""
Expand All @@ -254,7 +255,7 @@ def get_topics(

topics_res = topic_res.json()

return topics_res['names']
return topics_res["names"]


def archive(
Expand Down Expand Up @@ -403,7 +404,9 @@ def get_languages(
)

if languages_resp.status_code != 200:
logging.warn(f"Received status code {languages_resp.status_code} while retrieving repository languages.")
logging.warn(
f"Received status code {languages_resp.status_code} while retrieving repository languages."
)
return ["default"]

languages = list()
Expand All @@ -419,18 +422,25 @@ def get_languages(
return languages


def load_codeql_base64_template(
languages: List, branches: List = ["main"]
) -> str:
with open(f"./templates/codeql-analysis-default.yml", "r") as f:
def load_codeql_base64_template(languages: List, branches: List = ["main"]) -> str:
minute = secrets.randbelow(60)
hour = secrets.randbelow(24)
day = secrets.randbelow(7)
with open("./templates/codeql-analysis-default.yml", "r") as f:
data = "".join(f.readlines())
data = data.replace("""branches: [ ]""", f"""branches: [{', '.join(f"'{branch}'" for branch in branches) }]""")
data = data.replace(
"""branches: [ ]""",
f"""branches: [{', '.join(f"'{branch}'" for branch in branches) }]""",
)
data = data.replace("""language: [ ]""", f"""language: {languages}""")
data = data.replace(
"""cron: '36 4 * * 3'""", f"""cron: '{minute} {hour} * * {day}'"""
)
return base64.b64encode(data.encode("utf-8")).decode("utf-8")


def load_codeql_config_base64_template() -> str:
with open(f"./templates/codeql-config-default.yml", "r") as f:
with open("./templates/codeql-config-default.yml", "r") as f:
template = f.read()
return base64.b64encode(template.encode(encoding="utf-8")).decode("utf-8")

Expand Down Expand Up @@ -511,11 +521,14 @@ def create_codeql_pr(
# Workflow config
template = load_codeql_base64_template(languages, [default_branch])
workflow_commit_payload = {
"message": f"Create CodeQL analysis workflow",
"message": "Create CodeQL analysis workflow",
"content": template,
"branch": target_branch,
"sha": get_file_sha(
organization, repository, headers, ".github/workflows/codeql-analysis-default.yml"
organization,
repository,
headers,
".github/workflows/codeql-analysis-default.yml",
),
}

Expand All @@ -537,7 +550,7 @@ def create_codeql_pr(
# CodeQL config file
template = load_codeql_config_base64_template()
config_commit_payload = {
"message": f"Create CodeQL config file",
"message": "Create CodeQL config file",
"content": template,
"branch": target_branch,
"sha": get_file_sha(
Expand Down Expand Up @@ -575,15 +588,15 @@ def create_codeql_pr(
if is_config_update:
logging.info(f"Updating configuration for {repository}")
pr_payload["title"] = "Security Code Scanning - updated configuration files"
pr_payload[
"body"
] = f"This PR updates the Security scanning (CodeQL) configuration files for your repository languages ({', '.join(languages)}).We also just opened an informative issue in this repository to give you the context and assistance you need. In most cases you will be able to merge this PR as is and start benefiting from security scanning right away, as a check in each PR, and in the [Security tab](https://github.com/{organization}/{repository}/security/code-scanning) of this repository. \nHowever, we encourage you to review the configuration files and tag @{organization}/security-appsec (or `#github-appsec-security` on Slack) if you have any questions.\n\nWe are here to help! :thumbsup:\n\n - Application Security team."
pr_payload["body"] = (
f"This PR updates the Security scanning (CodeQL) configuration files for your repository languages ({', '.join(languages)}).We also just opened an informative issue in this repository to give you the context and assistance you need. In most cases you will be able to merge this PR as is and start benefiting from security scanning right away, as a check in each PR, and in the [Security tab](https://github.com/{organization}/{repository}/security/code-scanning) of this repository. \nHowever, we encourage you to review the configuration files and tag @{organization}/security-appsec (or `#github-appsec-security` on Slack) if you have any questions.\n\nWe are here to help! :thumbsup:\n\n - Application Security team."
)
else:
logging.info(f"Creating configuration for {repository}")
pr_payload["title"] = "Security Code Scanning - configuration files"
pr_payload[
"body"
] = f"This PR creates the Security scanning (CodeQL) configuration files for your repository languages ({', '.join(languages)}).\n\n We also just opened an informative issue in this repository to give you the context and assistance you need. In most cases you will be able to merge this PR as is and start benefiting from security scanning right away, as a check in each PR, and in the [Security tab](https://github.com/{organization}/{repository}/security/code-scanning) of this repository. \nHowever, we encourage you to review the configuration files and tag @{organization}/security-appsec (or `#github-appsec-security` on Slack) if you have any questions.\n\nWe are here to help! :thumbsup:\n\n - Application Security team."
pr_payload["body"] = (
f"This PR creates the Security scanning (CodeQL) configuration files for your repository languages ({', '.join(languages)}).\n\n We also just opened an informative issue in this repository to give you the context and assistance you need. In most cases you will be able to merge this PR as is and start benefiting from security scanning right away, as a check in each PR, and in the [Security tab](https://github.com/{organization}/{repository}/security/code-scanning) of this repository. \nHowever, we encourage you to review the configuration files and tag @{organization}/security-appsec (or `#github-appsec-security` on Slack) if you have any questions.\n\nWe are here to help! :thumbsup:\n\n - Application Security team."
)

# Retry if rate-limited
i = 0
Expand Down Expand Up @@ -612,7 +625,7 @@ def create_codeql_pr(


def load_dependency_review_base64_template() -> str:
with open(f"./templates/dependency_enforcement.yml", "r") as f:
with open("./templates/dependency_enforcement.yml", "r") as f:
template = f.read()

return str(base64.b64encode(template.encode(encoding="utf-8")), "utf-8")
Expand Down Expand Up @@ -648,7 +661,7 @@ def create_dependency_enforcement_pr(
# # Create commit
template = load_dependency_review_base64_template()
payload = {
"message": f"Enable Dependency reviewer",
"message": "Enable Dependency reviewer",
"content": template,
"branch": target_branch,
}
Expand Down

0 comments on commit 83fea4c

Please sign in to comment.