From a3640d4fdf3c85e8d3cc94bc35bab26809c26697 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 28 Sep 2022 11:57:15 +0200 Subject: [PATCH] Add coverage support to C++ regression tests Co-authored-by: Zhen Yu Ding --- .bazelrc | 12 ++++++++++++ .github/workflows/bazel_test.yml | 19 +++++++++++++++++++ BUILD | 1 + fuzzing/private/binary.bzl | 4 ++++ fuzzing/private/regression.bzl | 27 ++++++++++++++++++++++++++- fuzzing/tools/BUILD | 19 +++++++++++++++++++ fuzzing/tools/noop_lcov_merger.sh | 6 ++++++ 7 files changed, 87 insertions(+), 1 deletion(-) create mode 100755 fuzzing/tools/noop_lcov_merger.sh diff --git a/.bazelrc b/.bazelrc index 10430bd4..ebe78edf 100644 --- a/.bazelrc +++ b/.bazelrc @@ -87,3 +87,15 @@ build:ubsan-jazzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=jazzer build:ubsan-jazzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=ubsan # Workaround for https://github.com/bazelbuild/bazel/issues/11128 build:ubsan-jazzer --//fuzzing:cc_engine_sanitizer=ubsan + +# Coverage with Replay (C/C++ only) +coverage --//fuzzing:cc_engine=//fuzzing/engines:replay +coverage --@rules_fuzzing//fuzzing:cc_engine_instrumentation=none +coverage --@rules_fuzzing//fuzzing:cc_engine_sanitizer=none +coverage --instrument_test_targets +coverage --action_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1 +coverage --action_env=GCOV=llvm-profdata-10 +coverage --action_env=BAZEL_LLVM_COV=llvm-cov-10 +coverage --combined_report=lcov +coverage --experimental_use_llvm_covmap +coverage --experimental_generate_llvm_lcov diff --git a/.github/workflows/bazel_test.yml b/.github/workflows/bazel_test.yml index 85841915..3f694e02 100644 --- a/.github/workflows/bazel_test.yml +++ b/.github/workflows/bazel_test.yml @@ -94,6 +94,25 @@ jobs: bazel test --verbose_failures --test_output=all \ --build_tag_filters=fuzz-test --config=${{ matrix.config }} \ //examples:all + coverage: + name: Coverage gathering (C++) + runs-on: ubuntu-20.04 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update && sudo apt-get install -yq \ + clang-10 \ + libunwind-dev \ + libblocksruntime-dev + - name: Gather coverage + run: | + bazel coverage //examples:re2_fuzz_test + - name: Check coverage report + run: | + grep "SF:examples/re2_fuzz_test.cc" bazel-out/_coverage/_coverage_report.dat fuzzer_run_tests_java: name: Brief fuzz test runs (Java) runs-on: ubuntu-20.04 diff --git a/BUILD b/BUILD index 970c9bc2..3999f717 100644 --- a/BUILD +++ b/BUILD @@ -15,4 +15,5 @@ config_setting( name = "clang", flag_values = {"@bazel_tools//tools/cpp:compiler": "clang"}, + visibility = ["//:__subpackages__"], ) diff --git a/fuzzing/private/binary.bzl b/fuzzing/private/binary.bzl index ba44df5d..0572a0f6 100644 --- a/fuzzing/private/binary.bzl +++ b/fuzzing/private/binary.bzl @@ -131,6 +131,10 @@ def _fuzzing_binary_impl(ctx): engine_info = ctx.attr.engine[FuzzingEngineInfo], options_file = ctx.file.options, ), + coverage_common.instrumented_files_info( + ctx, + dependency_attributes = ["binary"], + ), ] _common_fuzzing_binary_attrs = { diff --git a/fuzzing/private/regression.bzl b/fuzzing/private/regression.bzl index 104993d3..6894267f 100644 --- a/fuzzing/private/regression.bzl +++ b/fuzzing/private/regression.bzl @@ -46,7 +46,14 @@ exec '{engine_launcher}' runfiles = ctx.runfiles() runfiles = runfiles.merge(ctx.attr.binary[DefaultInfo].default_runfiles) runfiles = runfiles.merge(binary_info.engine_info.launcher_runfiles) - return [DefaultInfo(executable = script, runfiles = runfiles)] + + return [ + DefaultInfo(executable = script, runfiles = runfiles), + coverage_common.instrumented_files_info( + ctx, + dependency_attributes = ["binary"], + ), + ] fuzzing_regression_test = rule( implementation = _fuzzing_regression_test_impl, @@ -61,6 +68,24 @@ Executes a fuzz test on its seed corpus. cfg = "target", mandatory = True, ), + "_lcov_merger": attr.label( + # As of Bazel 5.1.0, the following would work instead of the alias used below: + # default = configuration_field(fragment = "coverage", name = "output_generator") + default = "//fuzzing/tools:lcov_merger", + executable = True, + # This needs to be built in the target configuration so that the alias it points to can + # select on the value of --collect_code_coverage, which is disabled in the exec + # configuration. Since target and exec platform usually coincide for test execution, + # this should not cause any problems. + cfg = "target", + ), + "_collect_cc_coverage": attr.label( + # This target is just a shell script and can thus be depended on unconditionally + # without any effect on build times. + default = "@bazel_tools//tools/test:collect_cc_coverage", + executable = True, + cfg = "target", + ), }, test = True, ) diff --git a/fuzzing/tools/BUILD b/fuzzing/tools/BUILD index 21691a6f..6797bc2d 100644 --- a/fuzzing/tools/BUILD +++ b/fuzzing/tools/BUILD @@ -47,6 +47,25 @@ py_binary( ], ) +sh_binary( + name = "noop_lcov_merger", + srcs = ["noop_lcov_merger.sh"], +) + +config_setting( + name = "is_collecting_code_coverage", + values = {"collect_code_coverage": "true"}, +) + +alias( + name = "lcov_merger", + actual = select({ + ":is_collecting_code_coverage": "@bazel_tools//tools/test:lcov_merger", + "//conditions:default": ":noop_lcov_merger", + }), + visibility = ["//visibility:public"], +) + # Libraries. ############ diff --git a/fuzzing/tools/noop_lcov_merger.sh b/fuzzing/tools/noop_lcov_merger.sh new file mode 100755 index 00000000..b35b9c00 --- /dev/null +++ b/fuzzing/tools/noop_lcov_merger.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +# This script is a trivial to build replacement for the LCOV coverage merge tool +# shipped with Bazel in situations where code coverage is not being collected. +# It prevents the build time overhead and Java toolchain requirement incurred by +# the real tool when it is not needed. +exit 0