[3.2] run nonparallel tests in parallel via separate docker containers

.github/actions/parallel-ctest-containers/action.yml

@@ -0,0 +1,14 @@
+name: 'Parallel ctest'
+description: 'Runs a set of ctests in parallel via multiple docker containers'
+inputs:
+  container:
+    required: true
+  error-log-paths:
+    required: true
+  log-tarball-prefix:
+    required: true
+  tests:
+    required: true
+runs:
+  using: 'node16'
+  main: 'dist/index.mjs'

.github/actions/parallel-ctest-containers/main.mjs

@@ -0,0 +1,63 @@
+import child_process from 'node:child_process';
+import process from 'node:process';
+import stream from 'node:stream';
+import fs from 'node:fs';
+import zlib from 'node:zlib';
+import tar from 'tar-stream';
+import core from '@actions/core'
+
+const container = core.getInput('container', {required: true});
+const error_log_paths = JSON.parse(core.getInput('error-log-paths', {required: true}));
+const log_tarball_prefix = core.getInput('log-tarball-prefix', {required: true});
+const tests = JSON.parse(core.getInput('tests', {required: true}));
+
+try {
+   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", "/__w/leap/leap", container, "tar",  "--zstd", "-xf", "/build.tar.zst"], {stdio:"inherit"}).status)
+      throw new Error("Failed to create base container");
+   if(child_process.spawnSync("docker", ["commit", "base", "baseimage"], {stdio:"inherit"}).status)
+      throw new Error("Failed to create base image");
+   if(child_process.spawnSync("docker", ["rm", "base"], {stdio:"inherit"}).status)
+      throw new Error("Failed to remove base container");
+
+   let subprocesses = [];
+   tests.forEach(t => {
+      subprocesses.push(new Promise(resolve => {
+         child_process.spawn("docker", ["run", "--name", t, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t}$'`], {stdio:"inherit"}).on('close', code => resolve(code));
+      }));
+   });
+
+   const results = await Promise.all(subprocesses);
+
+   for(let i = 0; i < results.length; ++i) {
+      if(results[i] === 0)
+         continue;
+
+      //failing test
+      core.setFailed("Some tests failed");
+
+      let extractor = tar.extract();
+      let packer = tar.pack();
+
+      extractor.on('entry', (header, stream, next) => {
+         if(!header.name.startsWith(`__w/leap/leap/build`)) {
+            stream.on('end', () => next());
+            stream.resume();
+            return;
+         }
+
+         header.name = header.name.substring(`__w/leap/leap/`.length);
+         if(header.name !== "build/" && error_log_paths.filter(p => header.name.startsWith(p)).length === 0) {
+            stream.on('end', () => next());
+            stream.resume();
+            return;
+         }
+
+         stream.pipe(packer.entry(header, next));
+      }).on('finish', () => {packer.finalize()});
+
+      child_process.spawn("docker", ["export", tests[i]]).stdout.pipe(extractor);
+      stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i]}-logs.tar.gz`));
+   }
+} catch(e) {
+   core.setFailed(`Uncaught exception ${e.message}`);
+}

.github/actions/parallel-ctest-containers/package.json

@@ -0,0 +1,7 @@
+{
+  "dependencies": {
+    "@actions/core": "^1.9.1",
+    "tar-stream": "^2.2.0"
+  }
+
+}

.github/workflows/build.yaml

@@ -66,6 +66,7 @@ jobs:
       matrix:
         platform: [ubuntu18, ubuntu20, ubuntu22]
     outputs:
+      np-tests: ${{steps.build.outputs.np-tests}}
       lr-tests: ${{steps.build.outputs.lr-tests}}
     runs-on: ["self-hosted", "enf-x86-beefy"]
     container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
@@ -83,6 +84,7 @@ jobs:
             cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -GNinja ..
             ninja
             # the correct approach is by far "--show-only=json-v1 | jq '.tests[].name' | jq -sc"  but that doesn't work on U18's cmake 3.10 since it lacks json-v1
+            echo ::set-output name=np-tests::$(ctest -L "nonparallelizable_tests" --show-only | head -n -1 | cut -d ':' -f 2 -s | jq -cnR '[inputs | select(length>0)[1:]]')
             echo ::set-output name=lr-tests::$(ctest -L "long_running_tests" --show-only | head -n -1 | cut -d ':' -f 2 -s | jq -cnR '[inputs | select(length>0)[1:]]')
             tar -pc -C .. --exclude "*.o" build | zstd --long -T0 -9 > ../build.tar.zst
         - name: Upload builddir
@@ -150,30 +152,26 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu20]
-    runs-on: ubuntu-latest
-    container:
-      image: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-      options: --init
+    runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
           name: ${{matrix.platform}}-build
-      - name: Run NP Tests
-        run: |
-          zstdcat build.tar.zst | tar x
-          cd build
-          ctest --output-on-failure -L "nonparallelizable_tests"
-      - name: Bundle logs from failed tests
-        if: failure()
-        run: tar -czf ${{matrix.platform}}-serial-logs.tar.gz build/var build/etc build/leap-ignition-wd
+      - name: Run tests in parallel containers
+        uses: ./.github/actions/parallel-ctest-containers
+        with:
+          container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          log-tarball-prefix: ${{matrix.platform}}
+          tests: ${{needs.Build.outputs.np-tests}}
       - name: Upload logs from failed tests  
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-serial-logs
-          path: ${{matrix.platform}}-serial-logs.tar.gz
+          name: ${{matrix.platform}}-np-logs
+          path: '*-logs.tar.gz'
 
   lr-tests:
     name: LR Tests
@@ -183,31 +181,26 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu20]
-        test-name: ${{fromJSON(needs.Build.outputs.lr-tests)}}
-    runs-on: ubuntu-latest
-    container:
-      image: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-      options: --init
+    runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
           name: ${{matrix.platform}}-build
-      - name: Run ${{matrix.test-name}} Test
-        run: |
-          zstdcat build.tar.zst | tar x
-          cd build
-          ctest --output-on-failure -R ${{matrix.test-name}}
-      - name: Bundle logs from failed tests
-        if: failure()
-        run: tar -czf ${{matrix.platform}}-${{matrix.test-name}}-logs.tar.gz build/var build/etc build/leap-ignition-wd
-      - name: Upload logs from failed tests  
+      - name: Run tests in parallel containers
+        uses: ./.github/actions/parallel-ctest-containers
+        with:
+          container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          log-tarball-prefix: ${{matrix.platform}}
+          tests: ${{needs.Build.outputs.lr-tests}}
+      - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-${{matrix.test-name}}-logs
-          path: ${{matrix.platform}}-${{matrix.test-name}}-logs.tar.gz
+          name: ${{matrix.platform}}-lr-logs
+          path: '*-logs.tar.gz'
 
   all-passing:
     name: All Required Tests Passed