[CI] Split build and verify into parallel jobs

* Also made linter and NOTICE validation run only on Linux

Signed-off-by: Miki <miki@amazon.com>

.github/workflows/build_and_test_workflow.yml

@@ -29,11 +29,12 @@ env:
 
 jobs:
   build-lint-test:
-    name: Build and Verify on ${{ matrix.name }}
+    name: Build and Verify on ${{ matrix.name }} (ciGroup${{ matrix.group }})
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
+        group: [1, 2]
         include:
           - os: ubuntu-latest
             name: Linux
@@ -95,18 +96,23 @@ jobs:
         run: yarn osd bootstrap || yarn osd bootstrap
 
       - name: Run linter
+        # ciGroup 2 of unit-tests is shorter and Linux is faster
+        if: matrix.group == 2 && matrix.os == 'ubuntu-latest'
         id: linter
         run: yarn lint
 
       - name: Validate NOTICE file
+        # ciGroup 2 of unit-tests is shorter and Linux is faster
+        if: matrix.group == 2 && matrix.os == 'ubuntu-latest'
         id: notice-validate
         run: yarn notice:validate
 
-      - name: Run unit tests with coverage
+      - name: Run unit tests group ${{ matrix.group }} with coverage
         id: unit-tests
-        run: yarn test:jest:ci:coverage
+        run: yarn test:jest:ci:coverage  --ci-group=${{ matrix.group }}
 
       - name: Run mocha tests with coverage
+        if: matrix.group == 2
         id: mocha-tests
         run: yarn test:mocha:coverage
 
@@ -115,9 +121,10 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           directory: ./target/opensearch-dashboards-coverage
-          flags: ${{ matrix.name }}
+          flags: ${{ matrix.name }} ciGroup$${{ matrix.group }}
 
       - name: Run integration tests
+        if: matrix.group == 2
         id: integration-tests
         run: yarn test:jest_integration:ci
 

CHANGELOG.md

@@ -45,6 +45,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Upgrade the backport workflow ([#4343](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4343))
 - [Lint] Add custom stylelint rules and config to prevent unintended style overrides ([#4290](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4290))
 - [Lint] Add stylelint rule to define properties that are restricted from being used ([#4374](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4374))
+- [CI] Split build and verify into parallel jobs ([#4467](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4467))
 
 ### 📝 Documentation
 

TESTING.md

@@ -35,6 +35,8 @@ To run all the unit tests:
 `yarn test:jest`
 To run specific unit tests, pass the path to the test:
 `yarn test:jest [test path]`
+To run specific unit test groups:
+`yarn test:jest --ci-group=1 --ci-group=2`
 
 ### Integration tests
 To run all the integration tests:

src/dev/jest/config.js

@@ -30,16 +30,22 @@
 
 import { RESERVED_DIR_JEST_INTEGRATION_TESTS } from '../constants';
 
-export default {
-  rootDir: '../../..',
-  roots: [
+const rootGroups = [
+  [
+    /* CI Group 0 is left empty to make numbering natural */
+  ],
+  [
+    // CI Group 1
     '<rootDir>/src/plugins',
-    '<rootDir>/src/legacy/ui',
-    '<rootDir>/src/core',
-    '<rootDir>/src/legacy/server',
+  ],
+  [
+    // CI Group 2
     '<rootDir>/src/cli',
     '<rootDir>/src/cli_keystore',
     '<rootDir>/src/cli_plugin',
+    '<rootDir>/src/legacy/ui',
+    '<rootDir>/src/core',
+    '<rootDir>/src/legacy/server',
     '<rootDir>/packages/osd-test/target/functional_test_runner',
     '<rootDir>/src/dev',
     '<rootDir>/src/optimize',
@@ -49,6 +55,30 @@ export default {
     '<rootDir>/src/test_utils',
     '<rootDir>/test/functional/services/remote',
   ],
+];
+
+const roots = [];
+
+// Looks for --ci-group=<number> and captures the number
+const ciGroupPattern = /^--ci-group=(\d+)$/;
+const ciGroup = process.argv.reduce((acc, arg) => {
+  const match = arg.match(ciGroupPattern);
+  if (isFinite(match?.[1])) acc.push(parseInt(match[1], 10));
+  return acc;
+}, []);
+
+if (ciGroup.length > 0) {
+  console.log(`Requested group${ciGroup.length === 1 ? '' : 's'}: ${ciGroup.join(', ')}`);
+  ciGroup.forEach((id) => {
+    if (Array.isArray(rootGroups[id])) roots.push(...rootGroups[id]);
+  });
+} else {
+  roots.push(...rootGroups.flat());
+}
+
+export default {
+  rootDir: '../../..',
+  roots,
   moduleNameMapper: {
     '@elastic/eui$': '<rootDir>/node_modules/@elastic/eui/test-env',
     '@elastic/eui/lib/(.*)?': '<rootDir>/node_modules/@elastic/eui/test-env/$1',