diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb index f9f4e554..454df778 100644 --- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb @@ -965,30 +965,106 @@ def test_with_background_thread Minitest::Runnable.reset - require_relative "helpers/addition_helper" - class SomeTestWithThreads < Minitest::Test - def test_with_background_thread - # add thread to test that code coverage is collected - t = Thread.new do - AdditionHelper.add(1, 2) + class FlakyTestSuite < Minitest::Test + @@max_flaky_test_failures = 4 + @@flaky_test_failures = 0 + + def test_passed + assert true + end + + def test_flaky + if @@flaky_test_failures < @@max_flaky_test_failures + @@flaky_test_failures += 1 + assert 1 + 1 == 3 + else + assert 1 + 1 == 2 end - t.join + end + end + end + + it "retries flaky test" do + # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans + expect(test_spans).to have(6).items + + failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } + expect(failed_spans).to have(4).items # see steps.rb + expect(passed_spans).to have(2).items + + test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } + expect(test_spans_by_test_name["test_flaky"]).to have(5).items + + # count how many spans were marked as retries + retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } + expect(retries_count).to eq(4) + + expect(test_spans_by_test_name["test_passed"]).to have(1).item + + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_pass_status + + expect(test_session_span).to have_pass_status + end + end + + context "with flaky test and test retries enabled with insufficient max retries" do + include_context "CI mode activated" do + let(:integration_name) { :minitest } + + let(:flaky_test_retries_enabled) { true } + let(:retry_failed_tests_max_attempts) { 3 } + end + + before do + Minitest.run([]) + end + + before(:context) do + Thread.current[:dd_coverage_collector] = nil + + Minitest::Runnable.reset + + class FlakyTestSuite2 < Minitest::Test + @@max_flaky_test_failures = 4 + @@flaky_test_failures = 0 + + def test_passed assert true end + + def test_flaky + if @@flaky_test_failures < @@max_flaky_test_failures + @@flaky_test_failures += 1 + assert 1 + 1 == 3 + else + assert 1 + 1 == 2 + end + end end end - it "does not cover the background thread" do - skip if PlatformHelpers.jruby? + it "retries flaky test" do + # 1 initial run of flaky test + 3 retries without success + 1 passing test = 5 spans + expect(test_spans).to have(5).items - expect(test_spans).to have(1).item - expect(coverage_events).to have(1).item + failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } + expect(failed_spans).to have(4).items # see steps.rb + expect(passed_spans).to have(1).items - # expect that background thread is not covered - cov_event = find_coverage_for_test(first_test_span) - expect(cov_event.coverage.keys).not_to include( - absolute_path("helpers/addition_helper.rb") - ) + test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } + expect(test_spans_by_test_name["test_flaky"]).to have(4).items + + # count how many spans were marked as retries + retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } + expect(retries_count).to eq(3) + + expect(test_spans_by_test_name["test_passed"]).to have(1).item + + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_fail_status + + expect(test_session_span).to have_fail_status end end end