From 48a8d01b05149757f69a6a65a22a280bf003cd24 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 26 Oct 2022 02:39:41 -0700 Subject: [PATCH] Allow C/C++ coverage collection for external targets Removes hardcoded filters that result in no coverage being reported for `cc_*` targets in external repositories even when matched by the `--instrumentation_filter`. Work towards #16228 Work towards #16208 Closes #16261. PiperOrigin-RevId: 483911568 Change-Id: Ibca6fb39a8e946e06beeb03414ad8ccbc42f53d6 --- .../shell/bazel/bazel_coverage_cc_test_gcc.sh | 145 +++++++++++ .../bazel/bazel_coverage_cc_test_llvm.sh | 229 +++++++++++++++++- tools/test/collect_cc_coverage.sh | 1 - tools/test/collect_coverage.sh | 1 - 4 files changed, 364 insertions(+), 12 deletions(-) diff --git a/src/test/shell/bazel/bazel_coverage_cc_test_gcc.sh b/src/test/shell/bazel/bazel_coverage_cc_test_gcc.sh index 3ad7498d50a531..ec3d9201b09440 100755 --- a/src/test/shell/bazel/bazel_coverage_cc_test_gcc.sh +++ b/src/test/shell/bazel/bazel_coverage_cc_test_gcc.sh @@ -924,4 +924,149 @@ EOF expect_log "WARNING: There was no coverage found." } +function setup_external_cc_target() { + cat > WORKSPACE <<'EOF' +local_repository( + name = "other_repo", + path = "other_repo", +) +EOF + + cat > BUILD <<'EOF' +cc_library( + name = "b", + srcs = ["b.cc"], + hdrs = ["b.h"], + visibility = ["//visibility:public"], +) +EOF + + cat > b.h <<'EOF' +int b(bool what); +EOF + + cat > b.cc <<'EOF' +int b(bool what) { + if (what) { + return 1; + } else { + return 2; + } +} +EOF + + mkdir -p other_repo + touch other_repo/WORKSPACE + + cat > other_repo/BUILD <<'EOF' +cc_library( + name = "a", + srcs = ["a.cc"], + hdrs = ["a.h"], + deps = ["@//:b"], +) + +cc_test( + name = "t", + srcs = ["t.cc"], + linkstatic = True, + deps = [":a"], +) +EOF + + cat > other_repo/a.h <<'EOF' +int a(bool what); +EOF + + cat > other_repo/a.cc <<'EOF' +#include "a.h" +#include "b.h" + +int a(bool what) { + if (what) { + return b(what); + } else { + return 1 + b(what); + } +} +EOF + + cat > other_repo/t.cc <<'EOF' +#include +#include "a.h" + +int main(void) { + a(true); +} +EOF +} + +function test_external_cc_target_can_collect_coverage() { + if is_gcov_missing_or_wrong_version; then + echo "Skipping test." && return + fi + + setup_external_cc_target + + bazel coverage --test_output=all --instrumentation_filter=// @other_repo//:t \ + &>"$TEST_log" || fail "Coverage for @other_repo//:t failed" + + local coverage_file_path="$(get_coverage_file_path_from_test_log)" + local expected_result_a_cc='SF:external/other_repo/a.cc +FN:4,_Z1ab +FNDA:1,_Z1ab +FNF:1 +FNH:1 +DA:4,1 +DA:5,1 +DA:6,1 +DA:8,0 +LH:3 +LF:4 +end_of_record' + local expected_result_b_cc='SF:b.cc +FN:1,_Z1bb +FNDA:1,_Z1bb +FNF:1 +FNH:1 +DA:1,1 +DA:2,1 +DA:3,1 +DA:5,0 +LH:3 +LF:4 +end_of_record' + + assert_coverage_result "$expected_result_a_cc" "$coverage_file_path" + assert_coverage_result "$expected_result_b_cc" "$coverage_file_path" +} + +function test_external_cc_target_coverage_not_collected_by_default() { + if is_gcov_missing_or_wrong_version; then + echo "Skipping test." && return + fi + + setup_external_cc_target + + bazel coverage --test_output=all @other_repo//:t \ + &>"$TEST_log" || fail "Coverage for @other_repo//:t failed" + + local coverage_file_path="$(get_coverage_file_path_from_test_log)" + local expected_result_b_cc='SF:b.cc +FN:1,_Z1bb +FNDA:1,_Z1bb +FNF:1 +FNH:1 +DA:1,1 +DA:2,1 +DA:3,1 +DA:5,0 +LH:3 +LF:4 +end_of_record' + + assert_coverage_result "$expected_result_b_cc" "$coverage_file_path" + assert_not_contains "SF:external/other_repo/a.cc" "$coverage_file_path" +} + run_suite "test tests" diff --git a/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh b/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh index be3b1ed0938e6f..e99c0b5bb98efa 100755 --- a/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh +++ b/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh @@ -109,17 +109,22 @@ function test_cc_test_llvm_coverage_doesnt_fail() { } function test_cc_test_llvm_coverage_produces_lcov_report() { - local -r llvm_profdata="/usr/bin/llvm-profdata-9" - if [[ ! -x ${llvm_profdata} ]]; then + local -r clang="/usr/bin/clang" + if [[ ! -x ${clang} ]]; then return fi + local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3) + if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then + # No lcov produced with <9.0, no branch coverage with 10.0 and 11.0. + echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return + fi - local -r clang="/usr/bin/clang-9" - if [[ ! -x ${clang} ]]; then + local -r llvm_profdata="/usr/bin/llvm-profdata" + if [[ ! -x ${llvm_profdata} ]]; then return fi - local -r llvm_cov="/usr/bin/llvm-cov-9" + local -r llvm_cov="/usr/bin/llvm-cov" if [[ ! -x ${llvm_cov} ]]; then return fi @@ -150,17 +155,22 @@ end_of_record" } function test_cc_test_with_runtime_objects_not_in_runfiles() { - local -r llvm_profdata="/usr/bin/llvm-profdata-9" - if [[ ! -x ${llvm_profdata} ]]; then + local -r clang="/usr/bin/clang" + if [[ ! -x ${clang} ]]; then return fi + local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3) + if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then + # No lcov produced with <9.0, no branch coverage with 10.0 and 11.0. + echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return + fi - local -r clang="/usr/bin/clang-9" - if [[ ! -x ${clang} ]]; then + local -r llvm_profdata="/usr/bin/llvm-profdata" + if [[ ! -x ${llvm_profdata} ]]; then return fi - local -r llvm_cov="/usr/bin/llvm-cov-9" + local -r llvm_cov="/usr/bin/llvm-cov" if [[ ! -x ${llvm_cov} ]]; then return fi @@ -218,5 +228,204 @@ end_of_record" assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))" } +function setup_external_cc_target() { + cat > WORKSPACE <<'EOF' +local_repository( + name = "other_repo", + path = "other_repo", +) +EOF + + cat > BUILD <<'EOF' +cc_library( + name = "b", + srcs = ["b.cc"], + hdrs = ["b.h"], + visibility = ["//visibility:public"], +) +EOF + + cat > b.h <<'EOF' +int b(bool what); +EOF + + cat > b.cc <<'EOF' +int b(bool what) { + if (what) { + return 1; + } else { + return 2; + } +} +EOF + + mkdir -p other_repo + touch other_repo/WORKSPACE + + cat > other_repo/BUILD <<'EOF' +cc_library( + name = "a", + srcs = ["a.cc"], + hdrs = ["a.h"], + deps = ["@//:b"], +) + +cc_test( + name = "t", + srcs = ["t.cc"], + linkstatic = True, + deps = [":a"], +) +EOF + + cat > other_repo/a.h <<'EOF' +int a(bool what); +EOF + + cat > other_repo/a.cc <<'EOF' +#include "a.h" +#include "b.h" + +int a(bool what) { + if (what) { + return b(what); + } else { + return 1 + b(what); + } +} +EOF + + cat > other_repo/t.cc <<'EOF' +#include +#include "a.h" + +int main(void) { + a(true); +} +EOF +} + +function test_external_cc_target_can_collect_coverage() { + local -r clang="/usr/bin/clang" + if [[ ! -x ${clang} ]]; then + return + fi + local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3) + if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then + # No lcov produced with <9.0, no branch coverage with 10.0 and 11.0. + echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return + fi + + local -r llvm_profdata="/usr/bin/llvm-profdata" + if [[ ! -x ${llvm_profdata} ]]; then + return + fi + + local -r llvm_cov="/usr/bin/llvm-cov" + if [[ ! -x ${llvm_cov} ]]; then + return + fi + + setup_external_cc_target + + BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \ + BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \ + --combined_report=lcov --test_output=all \ + @other_repo//:t --instrumentation_filter=// &>$TEST_log || fail "Coverage for @other_repo//:t failed" + + local expected_result='SF:b.cc +FN:1,_Z1bb +FNDA:1,_Z1bb +FNF:1 +FNH:1 +BRDA:2,0,0,1 +BRDA:2,0,1,0 +BRF:2 +BRH:1 +DA:1,1 +DA:2,1 +DA:3,1 +DA:4,1 +DA:5,0 +DA:6,0 +DA:7,1 +LH:5 +LF:7 +end_of_record +SF:external/other_repo/a.cc +FN:4,_Z1ab +FNDA:1,_Z1ab +FNF:1 +FNH:1 +BRDA:5,0,0,1 +BRDA:5,0,1,0 +BRF:2 +BRH:1 +DA:4,1 +DA:5,1 +DA:6,1 +DA:7,1 +DA:8,0 +DA:9,0 +DA:10,1 +LH:5 +LF:7 +end_of_record' + + assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))" + assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat)" +} + +function test_external_cc_target_coverage_not_collected_by_default() { + local -r clang="/usr/bin/clang" + if [[ ! -x ${clang} ]]; then + return + fi + local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3) + if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then + # No lcov produced with <9.0, no branch coverage with 10.0 and 11.0. + echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return + fi + + local -r llvm_profdata="/usr/bin/llvm-profdata" + if [[ ! -x ${llvm_profdata} ]]; then + return + fi + + local -r llvm_cov="/usr/bin/llvm-cov" + if [[ ! -x ${llvm_cov} ]]; then + return + fi + + setup_external_cc_target + + BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \ + BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \ + --combined_report=lcov --test_output=all \ + @other_repo//:t &>$TEST_log || fail "Coverage for @other_repo//:t failed" + + local expected_result='SF:b.cc +FN:1,_Z1bb +FNDA:1,_Z1bb +FNF:1 +FNH:1 +BRDA:2,0,0,1 +BRDA:2,0,1,0 +BRF:2 +BRH:1 +DA:1,1 +DA:2,1 +DA:3,1 +DA:4,1 +DA:5,0 +DA:6,0 +DA:7,1 +LH:5 +LF:7 +end_of_record' + + assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))" + assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat)" +} run_suite "test tests" \ No newline at end of file diff --git a/tools/test/collect_cc_coverage.sh b/tools/test/collect_cc_coverage.sh index eafa7b8fe8f1ce..54d9ba3e18f162 100755 --- a/tools/test/collect_cc_coverage.sh +++ b/tools/test/collect_cc_coverage.sh @@ -93,7 +93,6 @@ function llvm_coverage_lcov() { done < "${COVERAGE_MANIFEST}" "${LLVM_COV}" export -instr-profile "${output_file}.data" -format=lcov \ - -ignore-filename-regex='.*external/.+' \ -ignore-filename-regex='/tmp/.+' \ ${object_param} | sed 's#/proc/self/cwd/##' > "${output_file}" } diff --git a/tools/test/collect_coverage.sh b/tools/test/collect_coverage.sh index f1690d88a589e7..657e790b880796 100755 --- a/tools/test/collect_coverage.sh +++ b/tools/test/collect_coverage.sh @@ -235,7 +235,6 @@ LCOV_MERGER_CMD="${LCOV_MERGER} --coverage_dir=${COVERAGE_DIR} \ --filter_sources=/usr/lib/.+ \ --filter_sources=/usr/include.+ \ --filter_sources=/Applications/.+ \ - --filter_sources=.*external/.+ \ --source_file_manifest=${COVERAGE_MANIFEST}" if [[ $COVERAGE_REPORTED_TO_ACTUAL_SOURCES_FILE ]]; then