add tag test.is_new

lib/datadog/ci/ext/test.rb

@@ -60,6 +60,7 @@ module Test
 
         # Tags for test retries
         TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
+        TAG_IS_NEW = "test.is_new" # true if test was marked as new by new test retries (early flake detection)
 
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"

lib/datadog/ci/test_retries/component.rb

@@ -88,7 +88,7 @@ def build_strategy(test_span)
             if should_retry_new_test?(test_span)
               Datadog.logger.debug("New test retry starts")
 
-              Strategy::RetryNew.new(duration_thresholds: @retry_new_tests_duration_thresholds)
+              Strategy::RetryNew.new(test_span, duration_thresholds: @retry_new_tests_duration_thresholds)
             elsif should_retry_failed_test?(test_span)
               Datadog.logger.debug("Failed test retry starts")
               @retry_failed_tests_count += 1

lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -10,11 +10,13 @@ module TestRetries
       module Strategy
         # retry every new test up to 10 times (early flake detection)
         class RetryNew < Base
-          def initialize(duration_thresholds:)
+          def initialize(test_span, duration_thresholds:)
             @duration_thresholds = duration_thresholds
             @attempts = 0
             # will be changed based on test span duration
             @max_attempts = 10
+
+            mark_new_test(test_span)
           end
 
           def should_retry?
@@ -25,6 +27,7 @@ def record_retry(test_span)
             super
 
             @attempts += 1
+            mark_new_test(test_span)
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
           end
@@ -34,6 +37,12 @@ def record_duration(duration)
 
             Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
           end
+
+          private
+
+          def mark_new_test(test_span)
+            test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+          end
         end
       end
     end

sig/datadog/ci/ext/test.rbs

@@ -88,6 +88,8 @@ module Datadog
 
         TAG_IS_RETRY: "test.is_retry"
 
+        TAG_IS_NEW: "test.is_new"
+
         module Status
           PASS: "pass"
 

sig/datadog/ci/test_retries/strategy/retry_new.rbs

@@ -8,7 +8,11 @@ module Datadog
           @attempts: Integer
           @max_attempts: Integer
 
-          def initialize: (duration_thresholds: Datadog::CI::Remote::SlowTestRetries) -> void
+          def initialize: (Datadog::CI::Test test_span, duration_thresholds: Datadog::CI::Remote::SlowTestRetries) -> void
+
+          private
+
+          def mark_new_test: (Datadog::CI::Test test_span) -> void
         end
       end
     end

spec/datadog/ci/contrib/rspec/instrumentation_spec.rb

@@ -982,10 +982,14 @@ def rspec_skipped_session_run
       # it retried the new test 10 times
       expect(test_spans_by_test_name["nested foo"]).to have(11).item
 
-      # count how many spans were marked as retries
+      # count how many tests were marked as retries
       retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
       expect(retries_count).to eq(10)
 
+      # count how many tests were marked as new
+      new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+      expect(new_tests_count).to eq(11)
+
       expect(test_suite_spans).to have(1).item
       expect(test_suite_spans.first).to have_fail_status
 
@@ -1011,6 +1015,10 @@ def rspec_skipped_session_run
         retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
         expect(retries_count).to eq(5)
 
+        # count how many tests were marked as new
+        new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+        expect(new_tests_count).to eq(6)
+
         expect(test_suite_spans).to have(1).item
         expect(test_session_span).to have_fail_status
       end
@@ -1045,6 +1053,10 @@ def rspec_skipped_session_run
       retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
       expect(retries_count).to eq(14)
 
+      # count how many tests were marked as new
+      new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+      expect(new_tests_count).to eq(11)
+
       expect(test_suite_spans).to have(1).item
       expect(test_suite_spans.first).to have_pass_status
 
@@ -1087,6 +1099,10 @@ def rspec_skipped_session_run
       retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
       expect(retries_count).to eq(20)
 
+      # count how many tests were marked as new
+      new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+      expect(new_tests_count).to eq(22)
+
       expect(test_suite_spans).to have(1).item
       expect(test_suite_spans.first).to have_pass_status