From 305174a9a384da14865b421b6bfa7264874ba9f4 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 7 Jun 2024 12:25:57 +0200 Subject: [PATCH] add DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE env variable to switch to single threaded code coverage mode if needed --- lib/datadog/ci/configuration/components.rb | 3 +- lib/datadog/ci/configuration/settings.rb | 6 ++ lib/datadog/ci/ext/settings.rb | 1 + lib/datadog/ci/itr/runner.rb | 13 +++- sig/datadog/ci/ext/settings.rbs | 1 + sig/datadog/ci/itr/runner.rbs | 5 +- .../minitest/helpers/addition_helper.rb | 5 ++ .../contrib/minitest/instrumentation_spec.rb | 60 ++++++++++++++++++- spec/support/contexts/ci_mode.rb | 2 + 9 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 spec/datadog/ci/contrib/minitest/helpers/addition_helper.rb diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index a9f97891..4bc5579a 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -129,7 +129,8 @@ def activate_ci!(settings) config_tags: custom_configuration_tags, coverage_writer: coverage_writer, enabled: settings.ci.enabled && settings.ci.itr_enabled, - bundle_location: settings.ci.itr_code_coverage_excluded_bundle_path + bundle_location: settings.ci.itr_code_coverage_excluded_bundle_path, + use_single_threaded_coverage: settings.ci.itr_code_coverage_use_single_threaded_mode ) git_tree_uploader = Git::TreeUploader.new(api: test_visibility_api) diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index 27be078b..5caa6f41 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -76,6 +76,12 @@ def self.add_settings!(base) end end + option :itr_code_coverage_use_single_threaded_mode do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE + o.default false + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index 917fdb33..37b1e653 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -13,6 +13,7 @@ module Settings ENV_ITR_ENABLED = "DD_CIVISIBILITY_ITR_ENABLED" ENV_GIT_METADATA_UPLOAD_ENABLED = "DD_CIVISIBILITY_GIT_METADATA_UPLOAD_ENABLED" ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH" + ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE" # Source: https://docs.datadoghq.com/getting_started/site/ DD_SITE_ALLOWLIST = %w[ diff --git a/lib/datadog/ci/itr/runner.rb b/lib/datadog/ci/itr/runner.rb index e35c857d..b1ccfa31 100644 --- a/lib/datadog/ci/itr/runner.rb +++ b/lib/datadog/ci/itr/runner.rb @@ -31,7 +31,8 @@ def initialize( api: nil, coverage_writer: nil, enabled: false, - bundle_location: nil + bundle_location: nil, + use_single_threaded_coverage: false ) @enabled = enabled @api = api @@ -43,6 +44,7 @@ def initialize( else bundle_location end + @use_single_threaded_coverage = use_single_threaded_coverage @test_skipping_enabled = false @code_coverage_enabled = false @@ -186,12 +188,15 @@ def write(event) def coverage_collector Thread.current[:dd_coverage_collector] ||= Coverage::DDCov.new( root: Git::LocalRepository.root, - ignored_path: @bundle_location + ignored_path: @bundle_location, + threading_mode: code_coverage_mode ) end def load_datadog_cov! require "datadog_cov.#{RUBY_VERSION}_#{RUBY_PLATFORM}" + + Datadog.logger.debug("Loaded Datadog code coverage collector, using coverage mode: #{code_coverage_mode}") rescue LoadError => e Datadog.logger.error("Failed to load coverage collector: #{e}. Code coverage will not be collected.") @@ -222,6 +227,10 @@ def fetch_skippable_tests(test_session:, git_tree_upload_worker:) Datadog.logger.debug { "Found #{@skippable_tests.count} skippable tests." } Datadog.logger.debug { "ITR correlation ID: #{@correlation_id}" } end + + def code_coverage_mode + @use_single_threaded_coverage ? :single : :multi + end end end end diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index 83ed4fad..841134d2 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -10,6 +10,7 @@ module Datadog ENV_ITR_ENABLED: String ENV_GIT_METADATA_UPLOAD_ENABLED: String ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH: String + ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE: String DD_SITE_ALLOWLIST: Array[String] end diff --git a/sig/datadog/ci/itr/runner.rbs b/sig/datadog/ci/itr/runner.rbs index 1ce20b44..7c4daae4 100644 --- a/sig/datadog/ci/itr/runner.rbs +++ b/sig/datadog/ci/itr/runner.rbs @@ -15,6 +15,7 @@ module Datadog @dd_env: String? @config_tags: Hash[String, String] @bundle_location: String? + @use_single_threaded_coverage: bool @skipped_tests_count: Integer @mutex: Thread::Mutex @@ -23,7 +24,7 @@ module Datadog attr_reader skipped_tests_count: Integer attr_reader correlation_id: String? - def initialize: (dd_env: String?, ?enabled: bool, ?coverage_writer: Datadog::CI::ITR::Coverage::Writer?, ?api: Datadog::CI::Transport::Api::Base?, ?config_tags: Hash[String, String]?, ?bundle_location: String?) -> void + def initialize: (dd_env: String?, ?enabled: bool, ?coverage_writer: Datadog::CI::ITR::Coverage::Writer?, ?api: Datadog::CI::Transport::Api::Base?, ?config_tags: Hash[String, String]?, ?bundle_location: String?, ?use_single_threaded_coverage: bool) -> void def configure: (Hash[String, untyped] remote_configuration, test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void @@ -58,6 +59,8 @@ module Datadog def fetch_skippable_tests: (test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void def increment_skipped_tests_counter: () -> void + + def code_coverage_mode: () -> Datadog::CI::ITR::Coverage::DDCov::threading_mode end end end diff --git a/spec/datadog/ci/contrib/minitest/helpers/addition_helper.rb b/spec/datadog/ci/contrib/minitest/helpers/addition_helper.rb new file mode 100644 index 00000000..f56753fe --- /dev/null +++ b/spec/datadog/ci/contrib/minitest/helpers/addition_helper.rb @@ -0,0 +1,5 @@ +module AdditionHelper + def self.add(a, b) + a + b + end +end diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb index a1196b81..e5ee8fb9 100644 --- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb @@ -27,7 +27,7 @@ def test_foo end end - context "with service name configured" do + context "with service name configured and code coverage enabled" do include_context "CI mode activated" do let(:integration_name) { :minitest } let(:integration_options) { {service_name: "ltest"} } @@ -402,12 +402,18 @@ def test_foo before(:context) do Minitest::Runnable.reset + require_relative "helpers/addition_helper" class SomeTest < Minitest::Test def test_pass assert true end def test_pass_other + # add thread to test that code coverage is collected + t = Thread.new do + AdditionHelper.add(1, 2) + end + t.join assert true end end @@ -494,6 +500,13 @@ def test_pass_other expect_coverage_events_belong_to_suite(first_test_suite_span) expect_coverage_events_belong_to_tests(test_spans) expect_non_empty_coverages + + # expect that background thread is covered + test_span = test_spans.find { |span| span.get_tag("test.name") == "test_pass_other" } + cov_event = find_coverage_for_test(test_span) + expect(cov_event.coverage.keys).to include( + File.expand_path(File.join(__dir__, "helpers/addition_helper.rb")) + ) end context "when ITR skips tests" do @@ -881,4 +894,49 @@ class SomeUnskippableSpec < Minitest::Spec end end end + + context "when using single threaded code coverage" do + include_context "CI mode activated" do + let(:integration_name) { :minitest } + + let(:itr_enabled) { true } + let(:code_coverage_enabled) { true } + let(:use_single_threaded_coverage) { true } + end + + before do + Minitest.run([]) + end + + before(:context) do + Thread.current[:dd_coverage_collector] = nil + + Minitest::Runnable.reset + + require_relative "helpers/addition_helper" + class SomeTestWithThreads < Minitest::Test + def test_with_background_thread + # add thread to test that code coverage is collected + t = Thread.new do + AdditionHelper.add(1, 2) + end + t.join + assert true + end + end + end + + it "does not cover the background thread" do + skip if PlatformHelpers.jruby? + + expect(test_spans).to have(1).item + expect(coverage_events).to have(1).item + + # expect that background thread is not covered + cov_event = find_coverage_for_test(first_test_span) + expect(cov_event.coverage.keys).not_to include( + File.expand_path(File.join(__dir__, "helpers/addition_helper.rb")) + ) + end + end end diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index f266c5c9..2b830136 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -23,6 +23,7 @@ let(:git_metadata_upload_enabled) { false } let(:require_git) { false } let(:bundle_path) { nil } + let(:use_single_threaded_coverage) { false } let(:itr_correlation_id) { "itr_correlation_id" } let(:itr_skippable_tests) { [] } @@ -77,6 +78,7 @@ c.ci.itr_enabled = itr_enabled c.ci.git_metadata_upload_enabled = git_metadata_upload_enabled c.ci.itr_code_coverage_excluded_bundle_path = bundle_path + c.ci.itr_code_coverage_use_single_threaded_mode = use_single_threaded_coverage unless integration_name == :no_instrument c.ci.instrument integration_name, integration_options end