Merge pull request #219 from DataDog/anmarchenko/auto_test_retries_rfc

[SDTEST-437] implement auto test retries RFC

lib/datadog/ci/configuration/components.rb

@@ -114,6 +114,7 @@ def activate_ci!(settings)
             library_settings_client: build_library_settings_client(settings, test_visibility_api)
           )
           @test_retries = TestRetries::Component.new(
+            retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled,
             retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
             retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
           )

lib/datadog/ci/configuration/settings.rb

@@ -88,6 +88,12 @@ def self.add_settings!(base)
                 o.default true
               end
 
+              option :retry_failed_tests_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED
+                o.default true
+              end
+
               option :retry_failed_tests_max_attempts do |o|
                 o.type :int
                 o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS
@@ -97,7 +103,7 @@ def self.add_settings!(base)
               option :retry_failed_tests_total_limit do |o|
                 o.type :int
                 o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT
-                o.default 100
+                o.default 1000
               end
 
               define_method(:instrument) do |integration_name, options = {}, &block|

lib/datadog/ci/ext/settings.rb

@@ -15,8 +15,9 @@ module Settings
         ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH"
         ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE"
         ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING = "DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING"
-        ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_MAX_ATTEMPTS"
-        ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_TOTAL_LIMIT"
+        ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"
+        ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT"
+        ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

lib/datadog/ci/ext/telemetry.rb

@@ -62,6 +62,7 @@ module Telemetry
         TAG_IS_UNSUPPORTED_CI = "is_unsupported_ci"
         TAG_BROWSER_DRIVER = "browser_driver"
         TAG_IS_RUM = "is_rum"
+        TAG_IS_RETRY = "is_retry"
         TAG_LIBRARY = "library"
         TAG_ENDPOINT = "endpoint"
         TAG_ERROR_TYPE = "error_type"

lib/datadog/ci/test_retries/component.rb

@@ -14,11 +14,11 @@ class Component
           :retry_failed_tests_total_limit, :retry_failed_tests_count
 
         def initialize(
+          retry_failed_tests_enabled:,
           retry_failed_tests_max_attempts:,
           retry_failed_tests_total_limit:
         )
-          # enabled only by remote settings
-          @retry_failed_tests_enabled = false
+          @retry_failed_tests_enabled = retry_failed_tests_enabled
           @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
           @retry_failed_tests_total_limit = retry_failed_tests_total_limit
           # counter that store the current number of failed tests retried
@@ -28,7 +28,7 @@ def initialize(
         end
 
         def configure(library_settings)
-          @retry_failed_tests_enabled = library_settings.flaky_test_retries_enabled?
+          @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled?
         end
 
         def with_retries(&block)

lib/datadog/ci/test_visibility/telemetry.rb

@@ -55,6 +55,9 @@ def self.event_tags_from_span(span)
           # codeowner tag
           tags[Ext::Telemetry::TAG_HAS_CODEOWNER] = "true" if span.get_tag(Ext::Test::TAG_CODEOWNERS)
 
+          # set is_retry tag if span represents a retried test
+          tags[Ext::Telemetry::TAG_IS_RETRY] = "true" if span.get_tag(Ext::Test::TAG_IS_RETRY)
+
           tags
         end
 

sig/datadog/ci/ext/settings.rbs

@@ -12,6 +12,7 @@ module Datadog
         ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH: String
         ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE: String
         ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING: String
+        ENV_RETRY_FAILED_TESTS_ENABLED: String
         ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS: String
         ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT: String
 

sig/datadog/ci/ext/telemetry.rbs

@@ -94,6 +94,8 @@ module Datadog
 
         TAG_IS_RUM: "is_rum"
 
+        TAG_IS_RETRY: "is_retry"
+
         TAG_LIBRARY: "library"
 
         TAG_ENDPOINT: "endpoint"

sig/datadog/ci/test_retries/component.rbs

@@ -12,7 +12,7 @@ module Datadog
 
         @mutex: Thread::Mutex
 
-        def initialize: (retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void
+        def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void
 
         def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void
 

spec/datadog/ci/configuration/settings_spec.rb

@@ -390,6 +390,117 @@ def patcher
         end
       end
 
+      describe "#retry_failed_tests_enabled" do
+        subject(:retry_failed_tests_enabled) { settings.ci.retry_failed_tests_enabled }
+
+        it { is_expected.to be true }
+
+        context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED}" do
+          around do |example|
+            ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED => enable) do
+              example.run
+            end
+          end
+
+          context "is not defined" do
+            let(:enable) { nil }
+
+            it { is_expected.to be true }
+          end
+
+          context "is set to true" do
+            let(:enable) { "true" }
+
+            it { is_expected.to be true }
+          end
+
+          context "is set to false" do
+            let(:enable) { "false" }
+
+            it { is_expected.to be false }
+          end
+        end
+      end
+
+      describe "#retry_failed_tests_enabled=" do
+        it "updates the #retry_failed_tests_enabled setting" do
+          expect { settings.ci.retry_failed_tests_enabled = false }
+            .to change { settings.ci.retry_failed_tests_enabled }
+            .from(true)
+            .to(false)
+        end
+      end
+
+      describe "#retry_failed_tests_max_attempts" do
+        subject(:retry_failed_tests_max_attempts) { settings.ci.retry_failed_tests_max_attempts }
+
+        it { is_expected.to eq 5 }
+
+        context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS}" do
+          around do |example|
+            ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS => attempts) do
+              example.run
+            end
+          end
+
+          context "is not defined" do
+            let(:attempts) { nil }
+
+            it { is_expected.to eq 5 }
+          end
+
+          context "is set to value" do
+            let(:attempts) { "10" }
+
+            it { is_expected.to eq attempts.to_i }
+          end
+        end
+      end
+
+      describe "#retry_failed_tests_max_attempts=" do
+        it "updates the #retry_failed_tests_max_attempts setting" do
+          expect { settings.ci.retry_failed_tests_max_attempts = 7 }
+            .to change { settings.ci.retry_failed_tests_max_attempts }
+            .from(5)
+            .to(7)
+        end
+      end
+
+      describe "#retry_failed_tests_total_limit" do
+        subject(:retry_failed_tests_total_limit) { settings.ci.retry_failed_tests_total_limit }
+
+        it { is_expected.to eq 1000 }
+
+        context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT}" do
+          around do |example|
+            ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT => attempts) do
+              example.run
+            end
+          end
+
+          context "is not defined" do
+            let(:attempts) { nil }
+
+            it { is_expected.to eq 1000 }
+          end
+
+          context "is set to value" do
+            let(:attempts) { "10" }
+
+            it { is_expected.to eq attempts.to_i }
+          end
+        end
+      end
+
+      describe "#retry_failed_tests_total_limit=" do
+        it "updates the #retry_failed_tests_total_limit setting" do
+          expect { settings.ci.retry_failed_tests_total_limit = 42 }
+            .to change { settings.ci.retry_failed_tests_total_limit }
+            .from(1000)
+            .to(42)
+        end
+      end
+
       describe "#instrument" do
         let(:integration_name) { :fake }
 

spec/datadog/ci/test_retries/component_spec.rb

@@ -2,11 +2,14 @@
 
 RSpec.describe Datadog::CI::TestRetries::Component do
   let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings) }
+
+  let(:retry_failed_tests_enabled) { true }
   let(:retry_failed_tests_max_attempts) { 1 }
   let(:retry_failed_tests_total_limit) { 12 }
 
   subject(:component) do
     described_class.new(
+      retry_failed_tests_enabled: retry_failed_tests_enabled,
       retry_failed_tests_max_attempts: retry_failed_tests_max_attempts,
       retry_failed_tests_total_limit: retry_failed_tests_total_limit
     )
@@ -38,6 +41,20 @@
         expect(component.retry_failed_tests_enabled).to be false
       end
     end
+
+    context "when flaky test retries are disabled in local settings" do
+      let(:retry_failed_tests_enabled) { false }
+
+      before do
+        allow(library_settings).to receive(:flaky_test_retries_enabled?).and_return(true)
+      end
+
+      it "disables retrying failed tests even if it's enabled remotely" do
+        subject
+
+        expect(component.retry_failed_tests_enabled).to be false
+      end
+    end
   end
 
   describe "#retry_failed_tests_max_attempts" do

spec/datadog/ci/test_visibility/telemetry_spec.rb

@@ -205,6 +205,36 @@
 
       it { event_finished }
     end
+
+    context "test span with retry" do
+      let(:span) do
+        Datadog::Tracing::SpanOperation.new(
+          "test_session",
+          type: Datadog::CI::Ext::AppTypes::TYPE_TEST,
+          tags: {
+            Datadog::CI::Ext::Test::TAG_FRAMEWORK => "rspec",
+            Datadog::CI::Ext::Environment::TAG_PROVIDER_NAME => "gha",
+            Datadog::CI::Ext::Test::TAG_CODEOWNERS => "@owner",
+            Datadog::CI::Ext::Test::TAG_IS_RUM_ACTIVE => "true",
+            Datadog::CI::Ext::Test::TAG_BROWSER_DRIVER => "selenium",
+            Datadog::CI::Ext::Test::TAG_IS_RETRY => "true"
+          }
+        )
+      end
+
+      let(:expected_tags) do
+        {
+          Datadog::CI::Ext::Telemetry::TAG_EVENT_TYPE => Datadog::CI::Ext::Telemetry::EventType::TEST,
+          Datadog::CI::Ext::Telemetry::TAG_TEST_FRAMEWORK => "rspec",
+          Datadog::CI::Ext::Telemetry::TAG_HAS_CODEOWNER => "true",
+          Datadog::CI::Ext::Telemetry::TAG_IS_RUM => "true",
+          Datadog::CI::Ext::Telemetry::TAG_BROWSER_DRIVER => "selenium",
+          Datadog::CI::Ext::Telemetry::TAG_IS_RETRY => "true"
+        }
+      end
+
+      it { event_finished }
+    end
   end
 
   describe ".test_session_started" do