more tests for minitest retries, fix typing

sig/datadog/ci/contrib/minitest/runner.rbs

@@ -10,11 +10,15 @@ module Datadog
 
             def init_plugins: (*untyped) -> (nil | untyped)
 
+            def run_one_method: (untyped klass, String method_name) -> untyped
+
             private
 
             def datadog_configuration: () -> untyped
 
             def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
+
+            def test_retries_component: () -> Datadog::CI::TestRetries::Component
           end
         end
       end

sig/datadog/ci/contrib/minitest/test.rbs

@@ -13,6 +13,7 @@ module Datadog
 
           module InstanceMethods : ::Minitest::Test
             include ::Minitest::Test::LifecycleHooks
+
             extend ClassMethods
 
             def before_setup: () -> (nil | untyped)

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -961,8 +961,6 @@ def test_with_background_thread
     end
 
     before(:context) do
-      Thread.current[:dd_coverage_collector] = nil
-
       Minitest::Runnable.reset
 
       class FlakyTestSuite < Minitest::Test
@@ -989,7 +987,7 @@ def test_flaky
       expect(test_spans).to have(6).items
 
       failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
-      expect(failed_spans).to have(4).items # see steps.rb
+      expect(failed_spans).to have(4).items
       expect(passed_spans).to have(2).items
 
       test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
@@ -1021,8 +1019,6 @@ def test_flaky
     end
 
     before(:context) do
-      Thread.current[:dd_coverage_collector] = nil
-
       Minitest::Runnable.reset
 
       class FlakyTestSuite2 < Minitest::Test
@@ -1044,12 +1040,12 @@ def test_flaky
       end
     end
 
-    it "retries flaky test" do
+    it "retries flaky test without success" do
       # 1 initial run of flaky test + 3 retries without success + 1 passing test = 5 spans
       expect(test_spans).to have(5).items
 
       failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
-      expect(failed_spans).to have(4).items # see steps.rb
+      expect(failed_spans).to have(4).items
       expect(passed_spans).to have(1).items
 
       test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
@@ -1067,4 +1063,69 @@ def test_flaky
       expect(test_session_span).to have_fail_status
     end
   end
+
+  context "with failed test, flaky test, test retries enabled, and low overall failed tests retry limit" do
+    include_context "CI mode activated" do
+      let(:integration_name) { :minitest }
+
+      let(:flaky_test_retries_enabled) { true }
+      let(:retry_failed_tests_total_limit) { 1 }
+    end
+
+    before do
+      Minitest.run([])
+    end
+
+    before(:context) do
+      Minitest::Runnable.reset
+
+      class FailedAndFlakyTestSuite < Minitest::Test
+        # yep, this test is indeed order dependent!
+        i_suck_and_my_tests_are_order_dependent!
+
+        @@max_flaky_test_failures = 4
+        @@flaky_test_failures = 0
+
+        def test_failed
+          assert 1 + 1 == 4
+        end
+
+        def test_flaky
+          if @@flaky_test_failures < @@max_flaky_test_failures
+            @@flaky_test_failures += 1
+            assert 1 + 1 == 3
+          else
+            assert 1 + 1 == 2
+          end
+        end
+
+        def test_passed
+          assert true
+        end
+      end
+    end
+
+    it "retries flaky test without success" do
+      # 1 initial run of failed test + 5 retries without success + 1 run of flaky test without retries + 1 passing test = 8 spans
+      expect(test_spans).to have(8).items
+
+      failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
+      expect(failed_spans).to have(7).items
+      expect(passed_spans).to have(1).items
+
+      test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
+      expect(test_spans_by_test_name["test_flaky"]).to have(1).item
+      expect(test_spans_by_test_name["test_failed"]).to have(6).items
+      expect(test_spans_by_test_name["test_passed"]).to have(1).item
+
+      # count how many spans were marked as retries
+      retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+      expect(retries_count).to eq(5)
+
+      expect(test_suite_spans).to have(1).item
+      expect(test_suite_spans.first).to have_fail_status
+
+      expect(test_session_span).to have_fail_status
+    end
+  end
 end