59 support passing a gitleaks toml to allow custom rules (#65)

* feat: can pass custom toml rule files to gitleaks * feat: specified gitpython version * fix: change test requirements install * feat: 3.10 * feat: added tests for toml file feature --------- Co-authored-by: Victoria Kotiwcki <victoria.kotwicki@punksecurity.co.uk>
punk-security · Aug 3, 2023 · b548e2f · b548e2f
1 parent dda1852
commit b548e2f
Show file tree

Hide file tree

Showing 12 changed files with 264 additions and 188 deletions.
diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml
@@ -1,15 +1,15 @@
-name: bandit
-on: [pull_request]
-jobs:
-  sast:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.10.9
-      - name: Install Bandit
-        run: pip install bandit
-      - name: Run bandit
-        run: bandit -r .
+name: bandit
+on: [pull_request]
+jobs:
+  sast:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.10"
+      - name: Install Bandit
+        run: pip install bandit
+      - name: Run bandit
+        run: bandit -r .
diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml
@@ -1,27 +1,27 @@
-name: behave
-on: [pull_request]
-jobs:
-  tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Buildx
-        run: |
-          docker buildx build . \
-            --load \
-            --progress=plain \
-            --tag secret-magpie \
-            --platform linux/amd64
-      - name: Install and run behave
-        run: |
-          docker run \
-            -e SKIP_IN_RUNNER="" \
-            -e SECRETMAGPIE_GITHUB_PAT=${{ secrets.SECRETMAGPIE_GITHUB_PAT }} \
-            -e SECRETMAGPIE_ADO_PAT=${{ secrets.SECRETMAGPIE_ADO_PAT }} \
-            -e SECRETMAGPIE_GITLAB_PAT=${{ secrets.SECRETMAGPIE_GITLAB_PAT }} \
-            --entrypoint sh \
-            secret-magpie \
-            -c "pip install behave; python -m behave"
+name: behave
+on: [pull_request]
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Run Buildx
+        run: |
+          docker buildx build . \
+            --load \
+            --progress=plain \
+            --tag secret-magpie \
+            --platform linux/amd64
+      - name: Install and run behave
+        run: |
+          docker run \
+            -e SKIP_IN_RUNNER="" \
+            -e SECRETMAGPIE_GITHUB_PAT=${{ secrets.SECRETMAGPIE_GITHUB_PAT }} \
+            -e SECRETMAGPIE_ADO_PAT=${{ secrets.SECRETMAGPIE_ADO_PAT }} \
+            -e SECRETMAGPIE_GITLAB_PAT=${{ secrets.SECRETMAGPIE_GITLAB_PAT }} \
+            --entrypoint sh \
+            secret-magpie \
+            -c "pip install -r test-requirements.txt; python -m behave"
diff --git a/README.md b/README.md
@@ -157,6 +157,8 @@ options:
   --web                 Hosts a webserver on http://127.0.0.1:8080 to view the results in browser
   --to-scan-list TO_SCAN_LIST
                         The file to read the list of repositories to scan from. One repository per line, web URL to the repository.
+  --gl-config GL_CONFIG
+                        The .toml rules file to use for Gitleaks.
 
 github/azuredevops:
   --org ORG             Organisation name to target

diff --git a/argparsing.py b/argparsing.py
@@ -180,6 +180,12 @@ def error(self, message):
     help="The file to read the list of repositories to scan from. One repository per line, web URL to the repository.",
 )
 
+parser.add_argument(
+    "--gl-config",
+    type=str,
+    help="Path to toml file for custom rulesets for Gitleaks",
+)
+
 
 def parse_args():
     args = parser.parse_args()
@@ -201,4 +207,7 @@ def parse_args():
 
     if "filesystem" == args.provider and (args.path is None):
         parser.error("filesystem requires --path")
+
+    if args.gl_config is not None and args.disable_gitleaks:
+        parser.error("Gitleaks can't be disabled if passing a .toml file")
     return args
diff --git a/features/helper.py b/features/helper.py
@@ -385,6 +385,15 @@ def onerror(func, path, exc_info):
     func(path)
 
 
+@when("we run secret-magpie-cli with a gitleaks {conf} file")
+def step_impl(context, conf):
+    run_secret_magpie(
+        context,
+        engines="gitleaks",
+        args=[f"--gl-config={conf}"],
+    )
+
+
 class LocalRepos:
     def __init__(self, rules, dir):
         # Prepare the directory for repositories

diff --git a/features/secret_detection.feature b/features/secret_detection.feature
@@ -130,11 +130,46 @@ Feature: Validate secret detection against various engines.
         When we run secret-magpie-cli in multi branch mode, to scan list repos.txt, https validation enabled, ignoring commits older than None, extra context disabled, secret storing enabled, output format csv and engines: all
         Then there will be 2 secrets detected
 
-    @azuredevops.PunkSecurity
-    Scenario: Validate that repo filtering works for AzureDevOps
-        Given we have a file called repos.txt with content
+
+    @localrepos
+    @fixture.wantsAWSSecret
+    Scenario: Ensure that we only detect an AWS secret locally, when a matching rule is provided by toml file, and with only gitleaks enabled
+        Given we have a file called rules.toml with content
             """
-            https://dev.azure.com/PunkSecurity/SecretMagpie-Testing/_git/ssh_key
+            [[rules]]
+            description = "AWS"
+            id = "aws-access-token"
+            regex = '''AKIAYVP4CIPPERUVIFXG'''
+            keywords = [
+                "akia","agpa","aida","aroa","aipa","anpa","anva","asia",
+            ]
             """
-        When we run secret-magpie-cli in multi branch mode, to scan list repos.txt, https validation enabled, ignoring commits older than None, extra context disabled, secret storing enabled, output format csv and engines: all
-        Then there will be 2 secrets detected
+        When we run secret-magpie-cli with a gitleaks rules.toml file
+        Then there will be 1 secrets detected
+
+    @localrepos
+    @fixture.wantsAWSSecret
+    @fixture.wantsSSHKey
+    Scenario: Ensure that we only detect 1 AWS secret locally, and not the SSH key, when a matching rule is provided by toml file, and with only gitleaks enabled
+        Given we have a file called rules.toml with content
+            """
+            [[rules]]
+            description = "AWS"
+            id = "aws-access-token"
+            regex = '''AKIAYVP4CIPPERUVIFXG'''
+            keywords = [
+                "akia","agpa","aida","aroa","aipa","anpa","anva","asia",
+            ]
+            """
+        When we run secret-magpie-cli with a gitleaks rules.toml file
+        Then there will be 1 secrets detected
+
+    @localrepos
+    @fixture.wantsAWSSecret
+    Scenario: Ensure that we don't detect any secrets locally, when there are no matching rules provided by toml file, and with only gitleaks enabled
+        Given we have a file called rules.toml with content
+            """
+            """
+        When we run secret-magpie-cli with a gitleaks rules.toml file
+        Then there will be 0 secrets detected
+
diff --git a/features/validate_output.feature b/features/validate_output.feature
@@ -1,60 +1,68 @@
-Feature: Validate that the results files produced by secret-magpie-cli is of valid form and contains expected data.
-    @localrepos
-    @fixture.wantsSSHKey
-    Scenario Outline: Validate that the <format> output is of valid form when a repo contains multi-line secrets
-        When we run secret-magpie-cli with output format <format> and engines: all
-        Then the results file will be of valid form
-
-        Examples:
-            | format |
-            | json   |
-            | csv    |
-
-    @localrepos
-    @fixture.wantsAWSSecret
-    Scenario Outline: Ensure that the secrets column is blank when using format <format> and we disable storing secrets
-        When we run secret-magpie-cli with secret storing disabled, output format <format> and engines: all
-        Then the secret field within the output will be blank
-
-        Examples:
-            | format |
-            | json   |
-            | csv    |
-
-    @localrepos
-    Scenario: Ensure that when we run secret-magpie-cli with no engines enabled, we get the correct error
-        When we run secret-magpie-cli with engines: none
-        Then secret-magpie-cli's output will be
-            """
-            ERROR: No tools to scan with
-            """
-
-    @github.secretmagpie-testing
-    Scenario: Ensure that we clean up repos that we've cloned when using a remote
-        When we run secret-magpie-cli with engines: all
-        Then directory 7c484be0 won't exist
-        And directory 42cbad53 won't exist
-
-    @no-cleanup
-    @github.secretmagpie-testing
-    @rmtree.7c484be0
-    @rmtree.42cbad53
-    Scenario: Ensure that we clean up repos that we've cloned when using a remote
-        When we run secret-magpie-cli with engines: all
-        Then directory 7c484be0 will exist
-        And directory 42cbad53 will exist
-
-    @localrepos
-    @wantsAWSSecret
-    Scenario: Ensure that the date field within the repo is parseable in ISO8601 format.
-        When we run secret-magpie-cli with engines: all
-        Then the date column of results.csv will be ISO8601 format
-
-    @localrepos
-    @wantsAWSSecret
-    Scenario: Ensure that secret-magpie-cli gives the expected error when we run it with an invalid threshold date
-        When we run secret-magpie-cli in multi branch mode, ignoring commits older than invaliddate extra context disabled, secret storing enabled, output format csv and engines: all
-        Then secret-magpie-cli's output will be
-            """
-            ERROR: Invalid ISO format string.
-            """
+Feature: Validate that the results files produced by secret-magpie-cli is of valid form and contains expected data.
+    @localrepos
+    @fixture.wantsSSHKey
+    Scenario Outline: Validate that the <format> output is of valid form when a repo contains multi-line secrets
+        When we run secret-magpie-cli with output format <format> and engines: all
+        Then the results file will be of valid form
+
+        Examples:
+            | format |
+            | json   |
+            | csv    |
+
+    @localrepos
+    @fixture.wantsAWSSecret
+    Scenario Outline: Ensure that the secrets column is blank when using format <format> and we disable storing secrets
+        When we run secret-magpie-cli with secret storing disabled, output format <format> and engines: all
+        Then the secret field within the output will be blank
+
+        Examples:
+            | format |
+            | json   |
+            | csv    |
+
+    @localrepos
+    Scenario: Ensure that when we run secret-magpie-cli with no engines enabled, we get the correct error
+        When we run secret-magpie-cli with engines: none
+        Then secret-magpie-cli's output will be
+            """
+            ERROR: No tools to scan with
+            """
+
+    @github.secretmagpie-testing
+    Scenario: Ensure that we clean up repos that we've cloned when using a remote
+        When we run secret-magpie-cli with engines: all
+        Then directory 7c484be0 won't exist
+        And directory 42cbad53 won't exist
+
+    @no-cleanup
+    @github.secretmagpie-testing
+    @rmtree.7c484be0
+    @rmtree.42cbad53
+    Scenario: Ensure that we clean up repos that we've cloned when using a remote
+        When we run secret-magpie-cli with engines: all
+        Then directory 7c484be0 will exist
+        And directory 42cbad53 will exist
+
+    @localrepos
+    @wantsAWSSecret
+    Scenario: Ensure that the date field within the repo is parseable in ISO8601 format.
+        When we run secret-magpie-cli with engines: all
+        Then the date column of results.csv will be ISO8601 format
+
+    @localrepos
+    @wantsAWSSecret
+    Scenario: Ensure that secret-magpie-cli gives the expected error when we run it with an invalid threshold date
+        When we run secret-magpie-cli in multi branch mode, ignoring commits older than invaliddate extra context disabled, secret storing enabled, output format csv and engines: all
+        Then secret-magpie-cli's output will be
+            """
+            ERROR: Invalid ISO format string.
+            """
+
+    @localrepos
+    Scenario: Ensure that secret-magpie-cli gives the expected error when we provide an invalid gitleaks toml file
+        When we run secret-magpie-cli with a gitleaks rules_not_found.toml file
+        Then secret-magpie-cli's output will be
+            """
+            ERROR: File at rules_not_found.toml not found.
+            """
diff --git a/main.py b/main.py
@@ -25,6 +25,8 @@
     args = argparsing.parse_args()
     cleanup = not (args.no_cleanup or "filesystem" == args.provider)
 
+    conf = {"gitleaks": {}}
+
     if args.web:
         with open("template.html", "r", encoding="utf-8") as f:
             ag_grid_template = f.read()
@@ -34,6 +36,14 @@
         with open(args.to_scan_list, "r") as f:
             to_scan_list = f.read().split("\n")
 
+    if args.gl_config is not None:
+        try:
+            open(args.gl_config, "rb").close()
+        except FileNotFoundError:
+            print("ERROR: File at", args.gl_config, "not found.")
+            exit()
+        conf["gitleaks"]["config_file_path"] = args.gl_config
+
     with open(os.devnull, "wb") as devnull:
         if args.update_ca_store:
             subprocess.call(  # nosec subprocess_without_shell_equals_true start_process_with_partial_path
@@ -65,6 +75,7 @@
     f = partial(
         tasks.process_repo,
         functions=tool_list,
+        conf=conf,
         single_branch=args.single_branch,
         extra_context=args.extra_context,
         cleanup=cleanup,