Merge pull request #240 from DataDog/anmarchenko/total_code_coverage_…

…reporting

[SDTEST-60] Report total lines coverage percentage to Datadog

Steepfile

@@ -31,4 +31,5 @@ target :lib do
   library "capybara"
   library "timecop"
   library "webmock"
+  library "simplecov"
 end

lib/datadog/ci.rb

@@ -410,6 +410,7 @@ def test_optimisation
 require_relative "ci/contrib/rspec/integration"
 require_relative "ci/contrib/minitest/integration"
 require_relative "ci/contrib/selenium/integration"
+require_relative "ci/contrib/simplecov/integration"
 
 # Configuration extensions
 require_relative "ci/configuration/extensions"

lib/datadog/ci/contrib/simplecov/configuration/settings.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "datadog/core"
+
+require_relative "../ext"
+require_relative "../../settings"
+
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        module Configuration
+          # Custom settings for the Simplecov integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/simplecov/ext.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        # Simplecov integration constants
+        # @public_api
+        module Ext
+          ENV_ENABLED = "DD_CIVISIBILITY_SIMPLECOV_INSTRUMENTATION_ENABLED"
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/simplecov/integration.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        # Description of Simplecov integration
+        class Integration
+          include Datadog::CI::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new("0.18.0")
+
+          register_as :simplecov
+
+          def self.version
+            Gem.loaded_specs["simplecov"]&.version
+          end
+
+          def self.loaded?
+            !defined?(::SimpleCov).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          # additional instrumentations for test helpers are auto instrumented on test session start
+          def auto_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/simplecov/patcher.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "datadog/tracing/contrib/patcher"
+
+require_relative "result_extractor"
+
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        # Patcher enables patching of 'SimpleCov' module.
+        module Patcher
+          include Datadog::Tracing::Contrib::Patcher
+
+          module_function
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            ::SimpleCov.include(ResultExtractor)
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/simplecov/result_extractor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "coverage"
+
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        module ResultExtractor
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def __dd_peek_result
+              unless datadog_configuration[:enabled]
+                Datadog.logger.debug("SimpleCov instrumentation is disabled")
+                return nil
+              end
+
+              result = ::SimpleCov::UselessResultsRemover.call(
+                ::SimpleCov::ResultAdapter.call(::Coverage.peek_result)
+              )
+
+              ::SimpleCov::Result.new(add_not_loaded_files(result))
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:simplecov]
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/ext/test.rb

@@ -64,6 +64,9 @@ module Test
         TAG_EARLY_FLAKE_ENABLED = "test.early_flake.enabled" # true if early flake detection is enabled
         TAG_EARLY_FLAKE_ABORT_REASON = "test.early_flake.abort_reason" # reason why early flake detection was aborted
 
+        # Tags for total code coverage
+        TAG_CODE_COVERAGE_LINES_PCT = "test.code_coverage.lines_pct"
+
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
         SPAN_KIND_TEST = "test"

lib/datadog/ci/test_visibility/component.rb

@@ -4,6 +4,7 @@
 
 require_relative "context"
 require_relative "telemetry"
+require_relative "total_coverage"
 
 require_relative "../codeowners/parser"
 require_relative "../contrib/contrib"
@@ -188,6 +189,8 @@ def on_test_started(test)
         def on_test_session_finished(test_session)
           test_optimisation.write_test_session_tags(test_session)
 
+          TotalCoverage.extract_lines_pct(test_session)
+
           Telemetry.event_finished(test_session)
         end
 

lib/datadog/ci/test_visibility/total_coverage.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module TotalCoverage
+        def self.extract_lines_pct(test_session)
+          unless defined?(::SimpleCov)
+            Datadog.logger.debug("SimpleCov is not loaded, skipping code coverage extraction")
+            return
+          end
+
+          unless ::SimpleCov.running
+            Datadog.logger.debug("SimpleCov is not running, skipping code coverage extraction")
+            return
+          end
+
+          unless ::SimpleCov.respond_to?(:__dd_peek_result)
+            Datadog.logger.debug("SimpleCov is not patched, skipping code coverage extraction")
+            return
+          end
+
+          result = ::SimpleCov.__dd_peek_result
+          unless result
+            Datadog.logger.debug("SimpleCov result is nil, skipping code coverage extraction")
+            return
+          end
+
+          test_session.set_tag(Ext::Test::TAG_CODE_COVERAGE_LINES_PCT, result.covered_percent)
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/simplecov/configuration/settings.rbs

@@ -0,0 +1,12 @@
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        module Configuration
+          class Settings < Datadog::CI::Contrib::Settings
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/simplecov/ext.rbs

@@ -0,0 +1,11 @@
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        module Ext
+          ENV_ENABLED: "DD_CIVISIBILITY_SIMPLECOV_INSTRUMENTATION_ENABLED"
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/simplecov/integration.rbs

@@ -0,0 +1,26 @@
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        class Integration
+          extend Datadog::CI::Contrib::Integration::ClassMethods
+          include Datadog::CI::Contrib::Integration::InstanceMethods
+
+          MINIMUM_VERSION: Gem::Version
+
+          def self.version: () -> untyped
+
+          def self.loaded?: () -> bool
+
+          def self.compatible?: () -> bool
+
+          def auto_instrument?: () -> bool
+
+          def new_configuration: () -> untyped
+
+          def patcher: () -> untyped
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/simplecov/patcher.rbs

@@ -0,0 +1,15 @@
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        module Patcher
+          include Datadog::Tracing::Contrib::Patcher
+
+          def self?.target_version: () -> untyped
+
+          def self?.patch: () -> untyped
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/simplecov/result_extractor.rbs

@@ -0,0 +1,23 @@
+module Coverage
+  def self.peek_result: () -> untyped
+end
+
+module Datadog
+  module CI
+    module Contrib
+      module Simplecov
+        module ResultExtractor
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods
+            include ::SimpleCov
+
+            def __dd_peek_result: () -> ::SimpleCov::Result?
+
+            def datadog_configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/ext/test.rbs

@@ -100,6 +100,8 @@ module Datadog
 
         METRIC_CPU_COUNT: "_dd.host.vcpu_count"
 
+        TAG_CODE_COVERAGE_LINES_PCT: "test.code_coverage.lines_pct"
+
         module Status
           PASS: "pass"
 

sig/datadog/ci/test_visibility/total_coverage.rbs

@@ -0,0 +1,9 @@
+module Datadog
+  module CI
+    module TestVisibility
+      module TotalCoverage
+        def self.extract_lines_pct: (Datadog::CI::TestSession test_session) -> void
+      end
+    end
+  end
+end

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -180,10 +180,14 @@
       )
       expect(test_session_span).to have_pass_status
 
+      # ITR
       expect(test_session_span).to have_test_tag(:itr_test_skipping_enabled, "true")
       expect(test_session_span).to have_test_tag(:itr_test_skipping_type, "test")
       expect(test_session_span).to have_test_tag(:itr_tests_skipped, "false")
       expect(test_session_span).to have_test_tag(:itr_test_skipping_count, 0)
+
+      # Total code coverage
+      expect(test_session_span).to have_test_tag(:code_coverage_lines_pct)
     end
 
     it "creates test module span" do

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -440,11 +440,15 @@ def test_pass_other
             Datadog::CI::Contrib::Minitest::Integration.version.to_s
           )
 
+          # ITR
           expect(test_session_span).to have_test_tag(:itr_test_skipping_enabled, "true")
           expect(test_session_span).to have_test_tag(:itr_test_skipping_type, "test")
           expect(test_session_span).to have_test_tag(:itr_tests_skipped, "false")
           expect(test_session_span).to have_test_tag(:itr_test_skipping_count, 0)
 
+          # Total code coverage
+          expect(test_session_span).to have_test_tag(:code_coverage_lines_pct)
+
           expect(test_session_span).to have_pass_status
         end
 

spec/datadog/ci/contrib/rspec/instrumentation_spec.rb

@@ -554,6 +554,9 @@ def expect_failure
       expect(test_session_span).not_to have_test_tag(:itr_tests_skipped)
       expect(test_session_span).not_to have_test_tag(:itr_test_skipping_count)
 
+      # Total code coverage
+      expect(test_session_span).to have_test_tag(:code_coverage_lines_pct)
+
       expect(test_session_span).to have_pass_status
     end