Add additional rules (#193)

* Add Google OAuth Credentials rule * Add Azure Personal Access Token rule * Update CHANGELOG and README * Retrain tests
praetorian-inc · Jun 11, 2024 · b427866 · b427866
1 parent 5646f9b
commit b427866
Show file tree

Hide file tree

Showing 10 changed files with 110 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,8 +29,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
   - AWS API Credentials ([#190](https://github.com/praetorian-inc/noseyparker/pull/190))
   - AWS AppSync API Key ([#176](https://github.com/praetorian-inc/noseyparker/pull/176))
+  - Azure Personal Access Token ([#193](https://github.com/praetorian-inc/noseyparker/pull/193))
   - Base64-PEM-Encoded Private Key ([#192](https://github.com/praetorian-inc/noseyparker/pull/192))
-  - Databricks Personal Access Token ([#187](https://github.com/praetorian-inc/noseyparker/pull/187) from @@tobiasgyoerfi)
+  - Databricks Personal Access Token ([#187](https://github.com/praetorian-inc/noseyparker/pull/187) from (@tobiasgyoerfi)
+  - Google OAuth Credentials ([#193](https://github.com/praetorian-inc/noseyparker/pull/193))
   - Password Hash (Kerberos 5, etype 23, AS-REP) ([#176](https://github.com/praetorian-inc/noseyparker/pull/176))
 
 - Prebuilt releases now included separate debug symbols (.dSYM or .dwp files) ([#191](https://github.com/praetorian-inc/noseyparker/pull/191)).

diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@ Nosey Parker is a command-line tool that finds secrets and sensitive information
 
 **Key features:**
 - It can natively scan files, directories, and Git repository history
-- It uses regular expression matching with a set of [144 patterns](crates/noseyparker/data/default/builtin/rules) chosen for high signal-to-noise based on experience and feedback from offensive security engagements
+- It uses regular expression matching with a set of [146 patterns](crates/noseyparker/data/default/builtin/rules) chosen for high signal-to-noise based on experience and feedback from offensive security engagements
 - It deduplicates its findings, grouping matches together that share the same secret, which in practice can reduce review burden by 100x or more
 - It is fast: it can scan at hundreds of megabytes per second on a single core, and is able to scan 100GB of Linux kernel source history in less than 2 minutes on an older MacBook Pro
 - It scales: it has scanned inputs as large as 20TiB during security engagements

diff --git a/...oseyparker-cli/tests/rules/snapshots/test_noseyparker__rules__rules_check_builtins-2.snap b/...oseyparker-cli/tests/rules/snapshots/test_noseyparker__rules__rules_check_builtins-2.snap
@@ -2,4 +2,4 @@
 source: crates/noseyparker-cli/tests/rules/mod.rs
 expression: stdout
 ---
-144 rules and 3 rulesets: no issues detected
+146 rules and 3 rulesets: no issues detected
diff --git a/crates/noseyparker-cli/tests/rules/snapshots/test_noseyparker__rules__rules_list_json-2.snap b/crates/noseyparker-cli/tests/rules/snapshots/test_noseyparker__rules__rules_list_json-2.snap
@@ -256,12 +256,12 @@ expression: stdout
     },
     {
       "id": "np.aws.6",
-      "structural_id": "6081f1aa6f664739d2ca79e7aa1952b862dd32e9",
+      "structural_id": "39d60c56d8a84ca6ab5999de8fea93657e3cae99",
       "name": "AWS API Credentials",
       "syntax": {
         "name": "AWS API Credentials",
         "id": "np.aws.6",
-        "pattern": "(?x)\n(?m)\n\\b\n((?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})     (?# API key )\n\\b\n(?: (?s) .{0,40} )                                                        (?# Arbitrary intermediate stuff )\n\\b\n([A-Za-z0-9/+=]{40})                                                      (?# secret )\n(?: [^A-Za-z0-9/+=] | $ )\n",
+        "pattern": "(?x)\n\\b\n((?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})     (?# API key )\n\\b\n(?: (?s) .{0,40} )                                                        (?# Arbitrary intermediate stuff; ?s causes . to match newlines )\n\\b\n([A-Za-z0-9/+=]{40})                                                      (?# secret )\n(?: [^A-Za-z0-9/+=] | $ )\n",
         "examples": [
           "export AWS_API_KEY='A3T0ABCDEFGHIJKLMNOP'\nexport AWS_SECRET_ACCESS_KEY='ded7db27a4558eea9bbf0bf36e0e8521618f366c'\n",
           "export AWS_API_KEY='A3T0ABCDEFGHIJKLMNOP''ded7db27a4558eea9bbf0bf36e0e8521618f366c'"
@@ -324,6 +324,25 @@ expression: stdout
         ]
       }
     },
+    {
+      "id": "np.azure.3",
+      "structural_id": "357a7fb1c99c1d3ab0a7489443a564cd80cf6d67",
+      "name": "Azure Personal Access Token",
+      "syntax": {
+        "name": "Azure Personal Access Token",
+        "id": "np.azure.3",
+        "pattern": "(?x)\n(?i: ADO_PAT | pat_token | personal_?access_?token | \\$token )\n\\s* = \\s*\n[\"']\n([a-z0-9]{52})\n[\"']\n",
+        "examples": [
+          "var personalAccessToken = \"zpczok4kqgnw5prpfy3ehiylbqvgbjfkdiqkejsxqamy7qbkep7q\";  // Provide a value or retrieve it from configuration\n",
+          "$token = \"58oo4mvqr2tpw7b4w3loeckwfu5o6nw3sihfckvlwoxgqimlddza\"\n",
+          "if __name__ == \"__main__\":\n    ado_pat =  \"iyfmob6xjrfmit67anxbot64umfx2clwx7dz5ynxi4q2z3uqegvq\"\n"
+        ],
+        "negative_examples": [],
+        "references": [
+          "https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate"
+        ]
+      }
+    },
     {
       "id": "np.blynk.1",
       "structural_id": "7d7b4a3c990889ccbda6d47ec656719c22f9413c",
@@ -1271,6 +1290,22 @@ expression: stdout
         ]
       }
     },
+    {
+      "id": "np.google.6",
+      "structural_id": "4e3b74e85c70254144b1d2d08289f4d04af98f35",
+      "name": "Google OAuth Credentials",
+      "syntax": {
+        "name": "Google OAuth Credentials",
+        "id": "np.google.6",
+        "pattern": "(?x)\n\\b\n([0-9]+-[a-z0-9_]{32}\\.apps\\.googleusercontent\\.com)            (?# client ID )\n(?: (?s) .{0,40} )                                              (?# Arbitrary intermediate stuff; ?s causes . to match newlines )\n\\b\n\n(?: (GOCSPX-[a-zA-Z0-9_-]{28})                                  (?# prefixed client secret )\n  | (?: (?i) client.?secret .{0,10} \\b ([a-zA-Z0-9_-]{24}) )    (?# non-prefixed client secret )\n)\n\n(?:[^a-zA-Z0-9_-] | $)\n",
+        "examples": [
+          "const CLIENT_ID = '304167046824-45h8no7j0s38akv998nivvb7i17ckqeh.apps.googleusercontent.com';\nconst CLIENT_SECRET = '1QcFpNjHoAf4_XczYwhYicTl';\n",
+          "public static GAPIS_CREDENTIALS = {\n  // 1. Generate credentials: https://console.cloud.google.com/apis/\n  // 2. Create OAuth page and set spreadsheets and drive.metadata.readonly scopes\n  client_id: '132261435625-69ubohrvppjr9hcc5t9uighsb7j2cqhv.apps.googleusercontent.com',\n  client_secret: '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',\n  redirect_uri: `http://localhost:${Config.OAUTH_HTTP_PORT}/oauth2callback`\n};\n"
+        ],
+        "negative_examples": [],
+        "references": []
+      }
+    },
     {
       "id": "np.gradle.1",
       "structural_id": "a12f90a50f965526bfcf34016b914665483c389d",
@@ -2882,7 +2917,7 @@ expression: stdout
     {
       "id": "default",
       "name": "Nosey Parker default rules",
-      "num_rules": 123
+      "num_rules": 125
     },
     {
       "id": "np.assets",

diff --git a/...s/noseyparker-cli/tests/rules/snapshots/test_noseyparker__rules__rules_list_noargs-2.snap b/...s/noseyparker-cli/tests/rules/snapshots/test_noseyparker__rules__rules_list_noargs-2.snap
@@ -19,6 +19,7 @@ expression: stdout
  np.aws.6             AWS API Credentials 
  np.azure.1           Azure Connection String 
  np.azure.2           Azure App Configuration Connection String 
+ np.azure.3           Azure Personal Access Token 
  np.blynk.1           Blynk Device Access Token 
  np.blynk.2           Blynk Organization Access Token (URL first) 
  np.blynk.3           Blynk Organization Access Token (URL last) 
@@ -68,6 +69,7 @@ expression: stdout
  np.google.3          Google OAuth Client Secret 
  np.google.4          Google OAuth Access Token 
  np.google.5          Google API Key 
+ np.google.6          Google OAuth Credentials 
  np.gradle.1          Hardcoded Gradle Credentials 
  np.grafana.1         Grafana API Token 
  np.grafana.2         Grafana Cloud API Token 
@@ -151,6 +153,6 @@ expression: stdout
 
  Ruleset ID   Ruleset Name                         Rules 
 ─────────────────────────────────────────────────────────
- default      Nosey Parker default rules             123 
+ default      Nosey Parker default rules             125 
  np.assets    Nosey Parker asset detection rules      15 
  np.hashes    Nosey Parker password hash rules         6
diff --git a/...an/appmaker/snapshots/test_noseyparker__scan__appmaker__scan_workflow_from_git_url-4.snap b/...an/appmaker/snapshots/test_noseyparker__scan__appmaker__scan_workflow_from_git_url-4.snap
@@ -5,7 +5,7 @@ expression: read_json(report_json.path()).unwrap()
 [
   {
     "comment": null,
-    "finding_id": "43ee76ab13531cf63b65f88804fd30dffba01b54",
+    "finding_id": "4b68f1a2eebf34b00aee81bd944e3403921b8cce",
     "groups": [
       "QUtJQUpVWlBMVE5NWTJKU1hMWUE=",
       "RHZIeEhMTXNPMGZTbk5xSTgyZWZxNHNPOFFlZmRNU3ozbDJxMlhrMQ=="
@@ -60,7 +60,7 @@ expression: read_json(report_json.path()).unwrap()
           }
         ],
         "rule_name": "AWS API Credentials",
-        "rule_structural_id": "6081f1aa6f664739d2ca79e7aa1952b862dd32e9",
+        "rule_structural_id": "39d60c56d8a84ca6ab5999de8fea93657e3cae99",
         "rule_text_id": "np.aws.6",
         "score": null,
         "snippet": {
@@ -69,13 +69,13 @@ expression: read_json(report_json.path()).unwrap()
           "matching": "AKIAJUZPLTNMY2JSXLYA\nS3_OBJECT_PREFIX=flathead\nS3_SECRET=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\n"
         },
         "status": null,
-        "structural_id": "3826a0168ec2553f84592b57baa55218b004e1cb"
+        "structural_id": "4514a0252fd7c1d2b57d1e2c8ca9e38cded33685"
       }
     ],
     "mean_score": null,
     "num_matches": 1,
     "rule_name": "AWS API Credentials",
-    "rule_structural_id": "6081f1aa6f664739d2ca79e7aa1952b862dd32e9",
+    "rule_structural_id": "39d60c56d8a84ca6ab5999de8fea93657e3cae99",
     "rule_text_id": "np.aws.6",
     "statuses": []
   },

diff --git a/crates/noseyparker/data/default/builtin/rules/aws.yml b/crates/noseyparker/data/default/builtin/rules/aws.yml
@@ -348,11 +348,10 @@ rules:
 
   pattern: |
     (?x)
-    (?m)
     \b
     ((?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})     (?# API key )
     \b
-    (?: (?s) .{0,40} )                                                        (?# Arbitrary intermediate stuff )
+    (?: (?s) .{0,40} )                                                        (?# Arbitrary intermediate stuff; ?s causes . to match newlines )
     \b
     ([A-Za-z0-9/+=]{40})                                                      (?# secret )
     (?: [^A-Za-z0-9/+=] | $ )

diff --git a/crates/noseyparker/data/default/builtin/rules/azure.yml b/crates/noseyparker/data/default/builtin/rules/azure.yml
@@ -75,3 +75,28 @@ rules:
   - https://docs.microsoft.com/en-us/azure/azure-app-configuration/
   - https://docs.microsoft.com/en-us/azure/azure-app-configuration/howto-best-practices
   - https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_utils.py
+
+
+# This detects various hardcoded PATs that come from Microsoft example documentation
+- name: Azure Personal Access Token
+  id: np.azure.3
+
+  pattern: |
+    (?x)
+    (?i: ADO_PAT | pat_token | personal_?access_?token | \$token )
+    \s* = \s*
+    ["']
+    ([a-z0-9]{52})
+    ["']
+
+  examples:
+  - |
+      var personalAccessToken = "zpczok4kqgnw5prpfy3ehiylbqvgbjfkdiqkejsxqamy7qbkep7q";  // Provide a value or retrieve it from configuration
+  - |
+      $token = "58oo4mvqr2tpw7b4w3loeckwfu5o6nw3sihfckvlwoxgqimlddza"
+  - |
+      if __name__ == "__main__":
+          ado_pat =  "iyfmob6xjrfmit67anxbot64umfx2clwx7dz5ynxi4q2z3uqegvq"
+
+  references:
+  - https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate
diff --git a/crates/noseyparker/data/default/builtin/rules/google.yml b/crates/noseyparker/data/default/builtin/rules/google.yml
@@ -128,3 +128,35 @@ rules:
 
   examples:
   - 'https://storage.googleapis.com/bucket_name/object_name'
+
+
+# This matches an OAuth client ID followed closely by a client secret
+- name: Google OAuth Credentials
+  id: np.google.6
+
+  pattern: |
+    (?x)
+    \b
+    ([0-9]+-[a-z0-9_]{32}\.apps\.googleusercontent\.com)            (?# client ID )
+    (?: (?s) .{0,40} )                                              (?# Arbitrary intermediate stuff; ?s causes . to match newlines )
+    \b
+
+    (?: (GOCSPX-[a-zA-Z0-9_-]{28})                                  (?# prefixed client secret )
+      | (?: (?i) client.?secret .{0,10} \b ([a-zA-Z0-9_-]{24}) )    (?# non-prefixed client secret )
+    )
+
+    (?:[^a-zA-Z0-9_-] | $)
+
+  examples:
+  - |
+      const CLIENT_ID = '304167046824-45h8no7j0s38akv998nivvb7i17ckqeh.apps.googleusercontent.com';
+      const CLIENT_SECRET = '1QcFpNjHoAf4_XczYwhYicTl';
+
+  - |
+      public static GAPIS_CREDENTIALS = {
+        // 1. Generate credentials: https://console.cloud.google.com/apis/
+        // 2. Create OAuth page and set spreadsheets and drive.metadata.readonly scopes
+        client_id: '132261435625-69ubohrvppjr9hcc5t9uighsb7j2cqhv.apps.googleusercontent.com',
+        client_secret: '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',
+        redirect_uri: `http://localhost:${Config.OAUTH_HTTP_PORT}/oauth2callback`
+      };
diff --git a/crates/noseyparker/data/default/builtin/rulesets/default.yml b/crates/noseyparker/data/default/builtin/rulesets/default.yml
@@ -26,6 +26,7 @@ rulesets:
   - np.aws.6          # Amazon API Credentials
   - np.azure.1        # Azure Connection String
   - np.azure.2        # Azure App Configuration Connection String
+  - np.azure.3        # Azure Personal Access Token
   - np.blynk.1        # Blynk Device Access Token
   - np.blynk.2        # Blynk Organization Access Token (URL first)
   - np.blynk.3        # Blynk Organization Access Token (URL last)
@@ -71,6 +72,7 @@ rulesets:
   - np.google.3       # Google OAuth Client Secret
   - np.google.4       # Google OAuth Access Token
   - np.google.5       # Google API Key
+  - np.google.6       # Google OAuth Credentials
   - np.gradle.1       # Hardcoded Gradle Credentials
   - np.grafana.1      # Grafana API Token
   - np.grafana.2      # Grafana Cloud API Token