Fixes #1058: Track system-tests coverage in Codecov.io

.github/codecov.yml

@@ -19,5 +19,17 @@
 
 # https://docs.codecov.com/docs/codecov-yaml
 
+comment:
+    layout: "header, diff, flags, components"  # show component info in the PR comment
+component_management:
+    individual_components:
+        - component_id: unittests
+          name: calculator
+          flag_regexes:
+              - ".*unittests"
+        - component_id: systemtests
+          name: systemtests
+          flag_regexes:
+              - ".*systemtests"
 github_checks:
     annotations: false

.github/workflows/build.yaml

@@ -325,6 +325,8 @@ jobs:
             containerTag: stream9
             cc: gcc
             cxx: g++
+            buildType: Coverage
+            covType: system
             runtimeCheck: OFF
             protonGitRef: ${{ github.event.inputs.protonBranch || 'main' }}
             shard: 1
@@ -334,6 +336,8 @@ jobs:
             containerTag: stream9
             cc: gcc
             cxx: g++
+            buildType: Coverage
+            covType: system
             runtimeCheck: OFF
             protonGitRef: ${{ github.event.inputs.protonBranch || 'main' }}
             shard: 2
@@ -375,13 +379,14 @@ jobs:
             protonGitRef: 0.38.0
             shard: 2
             shards: 2
-          # coverage
+          # unittest coverage
           - os: ubuntu-20.04
             container: 'fedora'
             containerTag: 37
             cc: gcc
             cxx: g++
             buildType: Coverage
+            covType: unit
             runtimeCheck: OFF
             protonGitRef: 0.38.0
             routerCTestExtraArgs: "-R 'unittests|unit_tests|threaded_timer_test|adaptor_buffer_test|router_engine_test|management_test|router_policy_test|test_command'"
@@ -574,11 +579,11 @@ jobs:
       - name: skupper-router cmake configure
         working-directory: ${{env.RouterBuildDir}}
         run: |
-          if [[ ${BuildType} == "Coverage" ]]; then CoverageOpts="--cov;--cov-report=xml;--cov-append"; fi
+          if [[ ${BuildType} == "Coverage" ]]; then CoverageOpts="-m;coverage;run"; fi
           cmake "${{github.workspace}}/skupper-router" \
             "-DCMAKE_INSTALL_PREFIX=${InstallPrefix}" \
             "-DCMAKE_BUILD_TYPE=${BuildType}" \
-            "-DPYTHON_TEST_COMMAND='-m;pytest;-vs;${CoverageOpts};--timeout=400;--junit-prefix=pytest.\${py_test_module};--junit-xml=junitxmls/\${py_test_module}.xml;--pyargs;\${py_test_module}'" \
+            "-DPYTHON_TEST_COMMAND='${CoverageOpts};-m;pytest;-vs;--timeout=400;--junit-prefix=pytest.\${py_test_module};--junit-xml=junitxmls/\${py_test_module}.xml;--pyargs;\${py_test_module}'" \
             ${RouterCMakeExtraArgs} -DQD_ENABLE_ASSERTIONS=${RouterCMakeAsserts} -DSANITIZE_PYTHON=${RouterCMakeSanitizePython:-ON}
 
       - name: skupper-router cmake build/install
@@ -619,30 +624,37 @@ jobs:
 
           ctest --timeout 1200 -V --output-junit=Testing/Test.xml --output-on-failure --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j${threads} ${{env.RouterCTestExtraArgs}}
 
-      - name: Process Coverage
-        if: ${{ matrix.buildType == 'Coverage' }}
+      - name: Process C Coverage
+        if: ${{ !cancelled() && matrix.buildType == 'Coverage' }}
         working-directory: ${{env.RouterBuildDir}}
         run: |
           dnf install -y lcov
           cmake --build . --target coverage
 
       # https://github.com/codecov/codecov-action
       - name: Upload C Coverage
-        if: ${{ matrix.buildType == 'Coverage' }}
+        if: ${{ !cancelled() && matrix.buildType == 'Coverage' }}
         uses: codecov/codecov-action@v3
         with:
-          flags: unittests
+          flags: ${{matrix.covType}}tests
           verbose: true
           gcov: true
           name: skupper-router
           root_dir: .
           working-directory: ${{github.workspace}}/skupper-router
 
+      - name: Process Python coverage
+        if: ${{ !cancelled() && matrix.buildType == 'Coverage' }}
+        run: |
+          coverage combine
+          coverage xml
+        working-directory: ${{env.RouterBuildDir}}/tests
+
       - name: Upload Python Coverage
-        if: ${{ matrix.buildType == 'Coverage' }}
+        if: ${{ !cancelled() && matrix.buildType == 'Coverage' }}
         uses: codecov/codecov-action@v3
         with:
-          flags: pyunittests
+          flags: py${{matrix.covType}}tests
           verbose: true
           directory: ${{env.RouterBuildDir}}/tests
           name: skupper-router

tests/.coveragerc

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[run]
+parallel = True

tests/CMakeLists.txt

@@ -192,6 +192,9 @@ foreach(py_test_module
   list(APPEND SYSTEM_TEST_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${py_test_module}.py)
 endforeach()
 
+# Place .coveragerc where coverage.py will find it when running system_tests
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${CMAKE_CURRENT_BINARY_DIR}/.coveragerc COPYONLY)
+
 # Location of tox.ini determines the testroot directory for Pytest
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tox.ini.in ${CMAKE_CURRENT_BINARY_DIR}/tox.ini)
 # Use tox to run the flake8 python linter tool on all the python