diff --git a/.github/workflows/javascript-test.yml b/.github/workflows/javascript-test.yml
index 19045fa..0f59a8d 100644
--- a/.github/workflows/javascript-test.yml
+++ b/.github/workflows/javascript-test.yml
@@ -41,3 +41,17 @@ jobs:
       - name: Install dependencies
         run: yarn install --frozen-lockfile
       - run: yarn run test
+
+  evaluate-config:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use node 18
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - uses: c-hive/gha-yarn-cache@v2
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Evaluates config file
+        run: yarn run cli ".github/review-bot.yml"
diff --git a/.github/workflows/review-bot.yml b/.github/workflows/review-bot.yml
index aa6b302..736bf72 100644
--- a/.github/workflows/review-bot.yml
+++ b/.github/workflows/review-bot.yml
@@ -26,7 +26,8 @@ jobs:
           private_key: ${{ secrets.TEAM_APP_KEY }}
       # !This must always point to main. 
       # Change it for the PRs but remember to change it back
-      - uses: paritytech/review-bot@main
+      - name: "Evaluates PR reviews and assigns reviewers"
+        uses: paritytech/review-bot@main
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           team-token: ${{ steps.team_token.outputs.token }}
diff --git a/.gitignore b/.gitignore
index 9efa1c6..99888df 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
 /node_modules
-/dist
+/dist*
 .env
 
 # Editor directories and files
diff --git a/README.md b/README.md
index 8470d9b..6267237 100644
--- a/README.md
+++ b/README.md
@@ -5,3 +5,11 @@
 [![Publish package to GitHub Packages](https://github.com/paritytech/review-bot/actions/workflows/publish.yml/badge.svg?branch=main)](https://github.com/paritytech/review-bot/actions/workflows/publish.yml)
 
 Have custom review rules for PRs with auto assignment.
+
+## Evaluating config
+
+If you want to evaluate the config, we have a simple `cli` to do so.
+
+```bash
+yarn run cli ".github/review-bot.yml" # set the parameter as the location of the config
+```
diff --git a/package.json b/package.json
index 4978293..0ae2537 100644
--- a/package.json
+++ b/package.json
@@ -6,6 +6,7 @@
   "scripts": {
     "start": "node dist",
     "build": "ncc build --license LICENSE",
+    "cli": "ncc build src/cli.ts -o dist-cli && node dist-cli",
     "test": "jest",
     "fix": "eslint --fix '{src,test}/**/*'",
     "lint": "eslint '{src,test}/**/*'"
diff --git a/src/cli.ts b/src/cli.ts
new file mode 100644
index 0000000..80c6033
--- /dev/null
+++ b/src/cli.ts
@@ -0,0 +1,25 @@
+import { readFileSync } from "fs";
+import { parse } from "yaml";
+
+import { ConfigurationFile } from "./file/types";
+import { validateConfig, validateRegularExpressions } from "./file/validator";
+
+const fileLocation = process.argv[2];
+
+if (!fileLocation) {
+  throw new Error("Missing file location! Write the path as a parameter. (More info in the README)");
+}
+
+console.log("Looking for config in", fileLocation);
+const configTxt = readFileSync(fileLocation, "utf8");
+console.log("Found config file");
+const config = parse(configTxt) as ConfigurationFile;
+
+const configFile = validateConfig(config);
+
+const [result, error] = validateRegularExpressions(configFile, console);
+if (!result) {
+  throw new Error(`Regular expression is invalid: ${error}`);
+}
+
+console.log("Config is valid!");