Merge pull request #600 from Thels/cli-integration-tests

Initial attempt at some basic end to end tests for the CLI.
finos · Nov 21, 2024 · 02fbfc8 · 02fbfc8
2 parents 74edd27 + a6e7c15
commit 02fbfc8
Show file tree

Hide file tree

Showing 21 changed files with 5,162 additions and 9,000 deletions.
diff --git a/.github/node-cve-ignore-list.xml b/.github/node-cve-ignore-list.xml
@@ -42,4 +42,11 @@
         <packageUrl regex="true">^pkg:npm/ejs@.*$</packageUrl>
         <vulnerabilityName>CVE-2023-29827</vulnerabilityName>
     </suppress>
+    <suppress>
+        <notes><![CDATA[
+        file name: express:4.21.1
+        ]]></notes>
+        <packageUrl regex="true">^pkg:npm/express@.*$</packageUrl>
+        <vulnerabilityName>CVE-2024-10491</vulnerabilityName>
+    </suppress>
 </suppressions>
diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml
@@ -0,0 +1,39 @@
+name: CLI
+
+on:
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - "cli/**"
+      - "shared/**"
+  push:
+    branches:
+      - "main"
+    paths:
+      - "cli/**"
+      - "shared/**"
+
+jobs:
+  cli:
+    name: Build, Test, and Lint CLI Module
+    runs-on: ubuntu-latest
+
+    steps:
+      # Ensure full PR checkout
+      - name: Checkout PR Branch
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: v20
+
+      - name: Install workspace
+        run: npm ci
+
+      - name: Build workspace
+        run: npm run build
+
+      - name: Run tests for CLI
+        run: npm run test --workspace=cli
diff --git a/.github/workflows/cve-scanning-node.yml b/.github/workflows/cve-scanning-node.yml
@@ -24,7 +24,7 @@ jobs:
     continue-on-error: false
     strategy:
       matrix:
-        module-folder: ['cli', 'docs']
+        module-folder: ['cli', 'docs', 'shared']
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -57,4 +57,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: Depcheck report ${{ env.UPNAME }}
-          path: ${{ github.workspace }}/${{ matrix.module-folder }}-reports
+          path: ${{ github.workspace }}/${{ matrix.module-folder }}-reports
diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ your project.
 | -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | [Common Architecture Language Model](./calm) | [@rocketstack-matt](https://github.com/rocketstack-matt), [@jpgough-ms](https://github.com/jpgough-ms)                                                                                               | [![Validation of CALM Samples](https://github.com/finos/architecture-as-code/actions/workflows/spectral-validation.yml/badge.svg)](https://github.com/finos/architecture-as-code/actions/workflows/spectral-validation.yml)                                                                                                                                                                                                   |
 | [Docs](./docs)                               | [@rocketstack-matt](https://github.com/rocketstack-matt)                                                                                                                                             | [![Sync Docs to S3](https://github.com/finos/architecture-as-code/actions/workflows/s3-docs-sync.yml/badge.svg)](https://github.com/finos/architecture-as-code/actions/workflows/s3-docs-sync.yml) |
-| [CLI](./cli)                                 | [@aidanm3341](https://github.com/aidanm3341), [@lbulanti-ms](https://github.com/lbulanti-ms), [@willosborne](https://github.com/willosborne), [@grahampacker-ms](https://github.com/grahampacker-ms) | [![Build CLI](https://github.com/finos/architecture-as-code/actions/workflows/cli-tests.yml/badge.svg)](https://github.com/finos/architecture-as-code/actions/workflows/cli-tests.yml)                                                                                                                                                                                                                                        |
+| [CLI](./cli) & [Shared](./shared)                                | [@aidanm3341](https://github.com/aidanm3341), [@lbulanti-ms](https://github.com/lbulanti-ms), [@willosborne](https://github.com/willosborne), [@grahampacker-ms](https://github.com/grahampacker-ms) [@Thels](https://github.com/Thels) | [![Build CLI](https://github.com/finos/architecture-as-code/actions/workflows/cli-tests.yml/badge.svg)](https://github.com/finos/architecture-as-code/actions/workflows/cli-tests.yml) [![Build Shared Module](https://github.com/finos/architecture-as-code/actions/workflows/shared-tests.yml/badge.svg?branch=main)](https://github.com/finos/architecture-as-code/actions/workflows/shared-tests.yml)                                                                                                                                                                                                                                                   |
 | [Spectral](./spectral)                       | [@willosborne](https://github.com/willosborne), [@lbulanti-ms](https://github.com/lbulanti-ms), [@grahampacker-ms](https://github.com/grahampacker-ms)                                               | [![Validation of CALM Samples](https://github.com/finos/architecture-as-code/actions/workflows/spectral-validation.yml/badge.svg)](https://github.com/finos/architecture-as-code/actions/workflows/spectral-validation.yml)                                                                                                                                                                                                   |
 | [Translators](./translator)                  | [@Budlee](https://github.com/Budlee) [@matthewgardner](https://github.com/matthewgardner) [@jpgough-ms](https://github.com/jpgough-ms)                                                               | [![Build Translators](https://github.com/finos/architecture-as-code/actions/workflows/translator.yml/badge.svg)](https://github.com/finos/architecture-as-code/actions/workflows/translator.yml)                                                                                                                                                                                                                              |
 
@@ -50,8 +50,7 @@ that are being built to utilize the CALM specification.
 
 We accept contributions via Pull Request, to make a contribution:
 
-1. Pick an issue you are interested in contributing to (or raise one yourself) and speak to the lead maintainers about
-   what you plan to do either in the issue itself or come to a meetup.
+1. Pick an issue you are interested in contributing to (or raise one yourself) and speak to the lead maintainers about what you plan to do either in the issue itself or come to a meetup. [Some issues](https://github.com/finos/architecture-as-code/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) are suggested for first time contributors.
 2. Fork the repo (<https://github.com/finos/architecture-as-code/fork>)
 3. Create your feature branch (`git checkout -b feature/fooBar`)
 4. Read our [contribution guidelines](.github/CONTRIBUTING.md)

diff --git a/cli/README.md b/cli/README.md
@@ -1,8 +1,10 @@
-## CALM CLI
+# CALM CLI
 
 A command line interface to interact with the CALM schema.
 You can use these tools to create an instantiation of an architectural pattern, validate that an instantiation conforms to a given pattern, and create visualizations of instantiations and patterns so that you can see what your architecture looks like.
 
+## Using the CLI
+
 ### Getting Started
 
 Install the CLI on to your machine with this command:
@@ -42,15 +44,15 @@ Usage: calm generate [options]
 Generate an instantiation from a CALM pattern file.
 
 Options:
-  -p, --pattern <source>        Path to the pattern file to use. May be a file path or a URL.
-  -o, --output <output>         Path location at which to output the generated file. (default: "instantiation.json")
-  -s, --schemaDirectory <path>  Path to directory containing schemas to use in instantiation
+  -p, --pattern <file>          Path to the pattern file to use. May be a file path or a URL.
+  -o, --output <file>           Path location at which to output the generated file. (default: "instantiation.json")
+  -s, --schemaDirectory <path>  Path to the directory containing the meta schemas to use.
   -v, --verbose                 Enable verbose logging. (default: false)
   -a, --instantiateAll          Instantiate all properties, ignoring the "required" field. (default: false)
   -h, --help                    display help for command
 ```
 
-The most simple way to use this command is to call it with only the pattern option, which will generate an instantiation with the default filename 'instantiation.json' in the current working directory.
+The most simple way to use this command is to call it with only the pattern option, which will generate an instantiation with the default filename `instantiation.json` in the current working directory.
 
 ```shell
 % calm generate -p calm/pattern/api-gateway.json
@@ -68,39 +70,97 @@ Usage: calm validate [options]
 Validate that an instantiation conforms to a given CALM pattern.
 
 Options:
-  -p, --pattern <pattern>                         Path to the pattern file to use. May be a file path or a URL.
-  -i, --instantiation <instantiation>             Path to the pattern instantiation file to use. May be a file path or a URL.
-  -s, --schemaDirectory <path>                    The location of the directory of the meta schemas to be loaded (default: "../calm/draft/2024-03/meta")
-  -f, --format <format>                           The format of the output (choices: "json", "junit", default: "json")
-  -o, --output <output>                           Path location at which to output the generated file.
-  -v, --verbose                                   Enable verbose logging. (default: false)
-  -h, --help                                      display help for command
+  -p, --pattern <file>          Path to the pattern file to use. May be a file path or a URL.
+  -i, --instantiation <file>    Path to the pattern instantiation file to use. May be a file path or a URL.
+  -s, --schemaDirectory <path>  Path to the directory containing the meta schemas to use. (default: "../calm/draft")
+  --strict                      When run in strict mode, the CLI will fail if any warnings are reported. (default: false)
+  -f, --format <format>         The format of the output (choices: "json", "junit", default: "json")
+  -o, --output <file>           Path location at which to output the generated file.
+  -v, --verbose                 Enable verbose logging. (default: false)
+  -h, --help                    display help for command
 ```
 
+
 This command can output warnings and errors - the command will only exit with an error code if there are errors present in the output.
 Warnings are sometimes provided as hints about how to improve the instantiation, but they are not essential for the architecture to match the pattern.
 
 If you were to try and generate an instantiation from the api-pattern, and then validate the instantation against that pattern like this
 
 ```shell
 % calm generate -p calm/pattern/api-gateway.json
-% calm validate -p calm/pattern/api-pattern.json -i instantiation.json
+% calm validate -p calm/pattern/api-gateway.json -i instantiation.json
 ```
 
 You would get an output which includes a warning like this:
 
 ```json
 ...
 {
-  "code": "instantiation-has-no-placeholder-properties-string",
-  "severity": "warning",
-  "message": "String placeholder detected in instantiated pattern.",
-  "path": "/nodes/2/interfaces/0/host"
+    "jsonSchemaValidationOutputs": [],
+    "spectralSchemaValidationOutputs": [
+        {
+            "code": "instantiation-has-no-placeholder-properties-string",
+            "severity": "warning",
+            "message": "String placeholder detected in instantiated pattern.",
+            "path": "/nodes/0/interfaces/0/host",
+            "schemaPath": "",
+            "line_start": 10,
+            "line_end": 10,
+            "character_start": 18,
+            "character_end": 30
+        },
+        {
+            "code": "instantiation-has-no-placeholder-properties-numerical",
+            "severity": "warning",
+            "message": "Numerical placeholder (-1) detected in instantiated pattern.",
+            "path": "/nodes/0/interfaces/0/port",
+            "schemaPath": "",
+            "line_start": 11,
+            "line_end": 11,
+            "character_start": 18,
+            "character_end": 20
+        },
+        {
+            "code": "instantiation-has-no-placeholder-properties-string",
+            "severity": "warning",
+            "message": "String placeholder detected in instantiated pattern.",
+            "path": "/nodes/0/well-known-endpoint",
+            "schemaPath": "",
+            "line_start": 14,
+            "line_end": 14,
+            "character_start": 29,
+            "character_end": 56
+        },
+        {
+            "code": "instantiation-has-no-placeholder-properties-string",
+            "severity": "warning",
+            "message": "String placeholder detected in instantiated pattern.",
+            "path": "/nodes/2/interfaces/0/host",
+            "schemaPath": "",
+            "line_start": 31,
+            "line_end": 31,
+            "character_start": 18,
+            "character_end": 30
+        },
+        {
+            "code": "instantiation-has-no-placeholder-properties-numerical",
+            "severity": "warning",
+            "message": "Numerical placeholder (-1) detected in instantiated pattern.",
+            "path": "/nodes/2/interfaces/0/port",
+            "schemaPath": "",
+            "line_start": 32,
+            "line_end": 32,
+            "character_start": 18,
+            "character_end": 20
+        }
+    ],
+    "hasErrors": false,
+    "hasWarnings": true
 }
 ...
 ```
 
-which is just letting you know that you have left in a placeholder value which might have been generated with the generate command.
+which is just letting you know that you have left in some placeholder values which might have been generated with the generate command.
 This isn't a full break, but it implies that you've forgotten to fill out a detail in your architecture.
 
 ### Visualizing the CALM schema
@@ -122,3 +182,37 @@ Options:
   -v, --verbose               Enable verbose logging. (default: false)
   -h, --help                  display help for command
 ```
+
+## Coding for the CLI
+
+The CLI module has its logic split into two modules, `cli` and `shared`.  Both are managed by [npm workspaces](https://docs.npmjs.com/cli/v8/using-npm/workspaces).
+
+* `cli` module is for anything pertaining to the calling of the core logic, the CLI wrapper
+* `shared` module is where the logic being delegated to actually sits, so that it can be re-used for other use-cases if required.
+
+### Getting Started
+
+Ensure you've cloned down the repository ( see root `README.md` for information on this. ) - then go to the root of the repository and execute;
+
+```shell
+# Step 1: Install all necessary dependencies for the workspace
+npm install
+
+# Step 2: Build the workspace (compiles source code for all workspaces)
+npm run build
+
+# Step 3: Link the workspace locally for testing
+npm link
+
+# Step 4: Start the watch process for live development
+npm run watch
+```
+
+
+
+### CLI Tests
+
+There are currently two types of tests;
+
+* `cli` tests - these are end-to-end and involve linking the package as part of the test so that we can assert on actual `calm X` invocations.
+* `shared` tests - these are where the core logic tests live, like how validation behaves etc.
diff --git a/cli/eslint.config.mjs b/cli/eslint.config.mjs
@@ -36,6 +36,15 @@ export default [
             "linebreak-style": ["error", "unix"],
             quotes: ["error", "single"],
             semi: ["error", "always"],
-        },
+            "no-unused-vars": "off",
+            "@typescript-eslint/no-unused-vars": [
+              "error",
+              {
+                "argsIgnorePattern": "^_",
+                "varsIgnorePattern": "^_",
+                "caughtErrorsIgnorePattern": "^_"
+              }
+            ]
+        }
     },
 ];
diff --git a/cli/jest.config.js b/cli/jest.config.js
@@ -1,9 +1,11 @@
-/** @type {import('ts-jest').JestConfigWithTsJest} */
-export default {
+module.exports =  {
     preset: 'ts-jest',
     testEnvironment: 'node',
-    testMatch: ['<rootDir>/src/**/*.spec.ts'],
-    moduleNameMapper: {
-        '^(.+)\\.js$': '$1'
-    }
+    testMatch: ['./**/*.spec.ts'],
+    transformIgnorePatterns: [
+        "<rootDir>/node_modules/(?!calm-shared)",
+        '^.+\\.js$'
+    ],
+    rootDir: '.',
+    watchPathIgnorePatterns: ['<rootDir>/../shared/']
 };