fetch and store unique tests set when configuring TestRetries::Component

lib/datadog/ci/remote/component.rb

@@ -28,7 +28,7 @@ def configure(test_session)
           end
 
           test_optimisation.configure(library_configuration, test_session)
-          test_retries.configure(library_configuration)
+          test_retries.configure(library_configuration, test_session)
         end
 
         private

lib/datadog/ci/test_retries/component.rb

@@ -12,7 +12,8 @@ module TestRetries
       class Component
         attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts,
           :retry_failed_tests_total_limit, :retry_failed_tests_count,
-          :retry_new_tests_enabled, :retry_new_tests_duration_thresholds, :retry_new_tests_percentage_limit
+          :retry_new_tests_enabled, :retry_new_tests_duration_thresholds, :retry_new_tests_percentage_limit,
+          :retry_new_tests_unique_tests_set, :retry_new_tests_fault_reason
 
         def initialize(
           retry_failed_tests_enabled:,
@@ -30,13 +31,16 @@ def initialize(
           @retry_new_tests_enabled = retry_new_tests_enabled
           @retry_new_tests_duration_thresholds = nil
           @retry_new_tests_percentage_limit = 0
+          @retry_new_tests_unique_tests_set = Set.new
+          # indicates that retrying new tests failed and was disabled
+          @retry_new_tests_fault_reason = nil
 
           @unique_tests_client = unique_tests_client
 
           @mutex = Mutex.new
         end
 
-        def configure(library_settings)
+        def configure(library_settings, test_session)
           @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled?
           @retry_new_tests_enabled &&= library_settings.early_flake_detection_enabled?
 
@@ -45,6 +49,12 @@ def configure(library_settings)
           # configure retrying new tests
           @retry_new_tests_duration_thresholds = library_settings.slow_test_retries
           @retry_new_tests_percentage_limit = library_settings.faulty_session_threshold
+          @retry_new_tests_unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
+
+          if @retry_new_tests_unique_tests_set.empty?
+            @retry_new_tests_enabled = false
+            @retry_new_tests_fault_reason = "unique tests set is empty"
+          end
         end
 
         def with_retries(&block)

lib/datadog/ci/test_retries/unique_tests_client.rb

@@ -68,7 +68,7 @@ def initialize(dd_env:, api: nil, config_tags: {})
 
         def fetch_unique_tests(test_session)
           api = @api
-          return Response.new(nil) unless api
+          return Set.new unless api
 
           request_payload = payload(test_session)
           Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}")
@@ -99,7 +99,7 @@ def fetch_unique_tests(test_session)
             )
           end
 
-          Response.new(http_response)
+          Response.new(http_response).tests
         end
 
         private

sig/datadog/ci/test_retries/component.rbs

@@ -16,13 +16,17 @@ module Datadog
 
         attr_reader retry_new_tests_percentage_limit: Integer
 
+        attr_reader retry_new_tests_unique_tests_set: Set[String]
+
+        attr_reader retry_new_tests_fault_reason: String?
+
         @mutex: Thread::Mutex
 
         @unique_tests_client: Datadog::CI::TestRetries::UniqueTestsClient
 
         def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool, unique_tests_client: TestRetries::UniqueTestsClient) -> void
 
-        def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void
+        def configure: (Datadog::CI::Remote::LibrarySettings library_settings, Datadog::CI::TestSession test_session) -> void
 
         def with_retries: () { () -> void } -> void
 

sig/datadog/ci/test_retries/unique_tests_client.rbs

@@ -25,7 +25,7 @@ module Datadog
 
         def initialize: (?api: Datadog::CI::Transport::Api::Base?, dd_env: String?, ?config_tags: Hash[String, String]) -> void
 
-        def fetch_unique_tests: (Datadog::CI::TestSession test_session) -> Response
+        def fetch_unique_tests: (Datadog::CI::TestSession test_session) -> Set[String]
 
         private
 

spec/datadog/ci/remote/component_spec.rb

@@ -34,7 +34,7 @@
 
       before do
         expect(test_optimisation).to receive(:configure).with(library_configuration, test_session)
-        expect(test_retries).to receive(:configure).with(library_configuration)
+        expect(test_retries).to receive(:configure).with(library_configuration, test_session)
       end
 
       it { subject }
@@ -49,7 +49,7 @@
           .with(test_session).and_return(library_configuration)
 
         expect(test_optimisation).to receive(:configure).with(library_configuration, test_session)
-        expect(test_retries).to receive(:configure).with(library_configuration)
+        expect(test_retries).to receive(:configure).with(library_configuration, test_session)
       end
 
       it { subject }

spec/datadog/ci/test_retries/component_spec.rb

@@ -23,7 +23,13 @@
   let(:remote_flaky_test_retries_enabled) { false }
   let(:remote_early_flake_detection_enabled) { false }
 
-  let(:unique_tests_client) { instance_double(Datadog::CI::TestRetries::UniqueTestsClient) }
+  let(:unique_tests_set) { Set.new(["test1", "test2"]) }
+  let(:unique_tests_client) do
+    instance_double(
+      Datadog::CI::TestRetries::UniqueTestsClient,
+      fetch_unique_tests: unique_tests_set
+    )
+  end
 
   let(:slow_test_retries) do
     instance_double(
@@ -32,6 +38,9 @@
     )
   end
 
+  let(:tracer_span) { Datadog::Tracing::SpanOperation.new("session") }
+  let(:test_session) { Datadog::CI::TestSession.new(tracer_span) }
+
   subject(:component) do
     described_class.new(
       retry_failed_tests_enabled: retry_failed_tests_enabled,
@@ -43,7 +52,7 @@
   end
 
   describe "#configure" do
-    subject { component.configure(library_settings) }
+    subject { component.configure(library_settings, test_session) }
 
     context "when flaky test retries are enabled" do
       let(:remote_flaky_test_retries_enabled) { true }
@@ -79,12 +88,25 @@
     context "when early flake detection is enabled" do
       let(:remote_early_flake_detection_enabled) { true }
 
-      it "enables retrying new tests" do
-        subject
+      context "when unique tests set is empty" do
+        let(:unique_tests_set) { Set.new }
+
+        it "disables retrying new tests and adds fault reason" do
+          subject
 
-        expect(component.retry_new_tests_enabled).to be true
-        expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10)
-        expect(component.retry_new_tests_percentage_limit).to eq(retry_new_tests_percentage_limit)
+          expect(component.retry_new_tests_enabled).to be false
+          expect(component.retry_new_tests_fault_reason).to eq("unique tests set is empty")
+        end
+      end
+
+      context "when unique tests set is not empty" do
+        it "enables retrying new tests" do
+          subject
+
+          expect(component.retry_new_tests_enabled).to be true
+          expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10)
+          expect(component.retry_new_tests_percentage_limit).to eq(retry_new_tests_percentage_limit)
+        end
       end
     end
 
@@ -129,7 +151,7 @@
     let(:test_span) { instance_double(Datadog::CI::Test, failed?: test_failed) }
 
     before do
-      component.configure(library_settings)
+      component.configure(library_settings, test_session)
     end
 
     context "when retry failed tests is enabled" do
@@ -218,7 +240,7 @@
     end
 
     before do
-      component.configure(library_settings)
+      component.configure(library_settings, test_session)
     end
 
     context "when no retries strategy is used" do

spec/datadog/ci/test_retries/unique_tests_client_spec.rb

@@ -96,8 +96,7 @@
           end
 
           it "parses the response" do
-            expect(response.ok?).to be true
-            expect(response.tests).to eq(Set.new(["AdminControllerTest.test_new.", "AdminControllerTest.test_index.", "AdminControllerTest.test_create."]))
+            expect(response).to eq(Set.new(["AdminControllerTest.test_new.", "AdminControllerTest.test_index.", "AdminControllerTest.test_create."]))
           end
 
           it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request", 1
@@ -121,8 +120,7 @@
           end
 
           it "parses the response" do
-            expect(response.ok?).to be false
-            expect(response.tests).to be_empty
+            expect(response).to be_empty
           end
 
           it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request_errors", 1
@@ -146,8 +144,7 @@
           end
 
           it "parses the response" do
-            expect(response.ok?).to be true
-            expect(response.tests).to be_empty
+            expect(response).to be_empty
           end
         end
 
@@ -178,8 +175,7 @@
           end
 
           it "parses the response" do
-            expect(response.ok?).to be true
-            expect(response.tests).to be_empty
+            expect(response).to be_empty
           end
         end
       end
@@ -188,8 +184,7 @@
         let(:api) { nil }
 
         it "returns an empty response" do
-          expect(response.ok?).to be false
-          expect(response.tests).to be_empty
+          expect(response).to be_empty
         end
       end
     end

spec/support/contexts/ci_mode.rb

@@ -45,7 +45,6 @@
 
   let(:itr_correlation_id) { "itr_correlation_id" }
   let(:itr_skippable_tests) { [] }
-
   let(:skippable_tests_response) do
     instance_double(
       Datadog::CI::TestOptimisation::Skippable::Response,
@@ -55,6 +54,8 @@
     )
   end
 
+  let(:unique_tests_set) { Set.new }
+
   let(:test_visibility) { Datadog.send(:components).test_visibility }
 
   before do
@@ -104,6 +105,8 @@
     allow_any_instance_of(Datadog::CI::TestOptimisation::Skippable).to receive(:fetch_skippable_tests).and_return(skippable_tests_response)
     allow_any_instance_of(Datadog::CI::TestOptimisation::Coverage::Transport).to receive(:send_events).and_return([])
 
+    allow_any_instance_of(Datadog::CI::TestRetries::UniqueTestsClient).to receive(:fetch_unique_tests).and_return(unique_tests_set)
+
     Datadog.configure do |c|
       # library switch
       c.ci.enabled = ci_enabled