collect C++ code coverage (commaai#23619)

* collect cpp coverage

* install llvm

* compile with coverage

* cd to root

* remove broken gcda files

* remove newline

* delete more stuff

* script to collect coverage

* Update tools/collect_coverage.sh

* actually delete

.github/workflows/selfdrive_tests.yaml

@@ -209,7 +209,7 @@ jobs:
       run: eval "$BUILD"
     - name: Run valgrind
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) && \
+        ${{ env.RUN }} "scons -j$(nproc) --coverage && \
                         FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
     - name: Print logs
       if: always()
@@ -244,7 +244,7 @@ jobs:
       run: eval "$BUILD"
     - name: Run unit tests
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) --test && \
+        ${{ env.RUN }} "scons -j$(nproc) --test --coverage && \
                         coverage run selfdrive/test/test_fingerprints.py && \
                         $UNIT_TEST common && \
                         $UNIT_TEST opendbc/can && \
@@ -265,7 +265,7 @@ jobs:
                         ./selfdrive/proclogd/tests/test_proclog && \
                         ./selfdrive/ui/replay/tests/test_replay && \
                         ./selfdrive/camerad/test/ae_gray_test && \
-                        coverage xml"
+                        tools/collect_coverage.sh"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v2
 
@@ -299,9 +299,9 @@ jobs:
       run: eval "$BUILD"
     - name: Run replay
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) && \
+        ${{ env.RUN }} "scons -j$(nproc) --coverage && \
                         FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py && \
-                        coverage xml"
+                        tools/collect_coverage.sh"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v2
     - name: Print diff
@@ -354,10 +354,12 @@ jobs:
     - name: Test longitudinal
       run: |
         ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
-                        scons -j$(nproc) && \
+                        scons -j$(nproc) --coverage && \
                         cd selfdrive/test/longitudinal_maneuvers && \
                         coverage run ./test_longitudinal.py && \
-                        coverage xml"
+                        coverage xml && \
+                        cd ../../.. && \
+                        tools/collect_coverage.sh"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v2
     - uses: actions/upload-artifact@v2
@@ -401,9 +403,9 @@ jobs:
       run: eval "$BUILD"
     - name: Test car models
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) --test && \
+        ${{ env.RUN }} "scons -j$(nproc) --test --coverage && \
                         FILEREADER_CACHE=1 coverage run -m pytest selfdrive/test/test_models.py && \
-                        coverage xml && \
+                        tools/collect_coverage.sh && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 4

.gitignore

@@ -83,3 +83,7 @@ models/*.thneed
 build/
 
 !**/.gitkeep
+
+*.gcno
+*.gcda
+coverage.info

SConstruct

@@ -52,6 +52,11 @@ AddOption('--no-thneed',
           dest='no_thneed',
           help='avoid using thneed')
 
+AddOption('--coverage',
+          action='store_true',
+          dest='coverage',
+          help='outpute covarge information')
+
 real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
@@ -168,6 +173,10 @@ else:
   ccflags = []
   ldflags = []
 
+if GetOption('coverage'):
+  ccflags += ["-fprofile-arcs", "-ftest-coverage"]
+  ldflags += ["-fprofile-arcs", "-ftest-coverage"]
+
 # no --as-needed on mac linker
 if arch != "Darwin":
   ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
@@ -285,6 +294,10 @@ elif arch == "aarch64":
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
+if GetOption('coverage'):
+  envCython['CCFLAGS'] += ["-fprofile-arcs", "-ftest-coverage"]
+  envCython['LINKFLAGS'] += ["-fprofile-arcs", "-ftest-coverage"]
+
 Export('envCython')
 
 # Qt build environment

tools/collect_coverage.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+find opendbc/can/dbc_out -name "*.gcda" -delete
+find rednose/helpers -name "*.gcda" -delete
+find rednose_repo/rednose/helpers -name "*.gcda" -delete
+find cereal/gen -name "*.gcda" -delete
+
+lcov --capture --directory . --gcov-tool tools/gcov_for_clang.sh --output-file coverage.info
+lcov --remove coverage.info '*/third_party/*' --output-file coverage.info
+lcov --remove coverage.info '/usr/*' --output-file coverage.info
+
+coverage xml || true

tools/gcov_for_clang.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec llvm-cov gcov "$@"

tools/ubuntu_setup.sh

@@ -31,8 +31,9 @@ function install_ubuntu_common_requirements() {
     git \
     git-lfs \
     ffmpeg \
-    libavformat-dev \
+    lcov \
     libavcodec-dev \
+    libavformat-dev \
     libavdevice-dev \
     libavutil-dev \
     libavfilter-dev \
@@ -50,6 +51,7 @@ function install_ubuntu_common_requirements() {
     libusb-1.0-0-dev \
     libzmq3-dev \
     libsystemd-dev \
+    llvm \
     locales \
     opencl-headers \
     ocl-icd-libopencl1 \