diff --git a/Rakefile b/Rakefile index 79563a18b..bcf0c33ae 100644 --- a/Rakefile +++ b/Rakefile @@ -60,7 +60,10 @@ TEST_METADATA = { "minitest-5-shoulda-context-2-shoulda-matchers-6" => "❌ 2.7 / ❌ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby" }, "knapsack_rspec" => { - "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby" + "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ❌ jruby" + }, + "knapsack_rspec_go" => { + "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ❌ jruby" } } @@ -132,7 +135,8 @@ namespace :spec do :activesupport, :ci_queue_minitest, :ci_queue_rspec, - :knapsack_rspec + :knapsack_rspec, + :knapsack_rspec_go ].each do |contrib| desc "" # "Explicitly hiding from `rake -T`" RSpec::Core::RakeTask.new(contrib) do |t, args| diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb index fa6d3c901..53a2a624f 100644 --- a/lib/datadog/ci/contrib/rspec/example.rb +++ b/lib/datadog/ci/contrib/rspec/example.rb @@ -17,6 +17,7 @@ def self.included(base) module InstanceMethods def run(*) + return super if ::RSpec.configuration.dry_run? return super unless datadog_configuration[:enabled] test_name = full_description.strip diff --git a/lib/datadog/ci/contrib/rspec/example_group.rb b/lib/datadog/ci/contrib/rspec/example_group.rb index b13f21732..0a2680fb9 100644 --- a/lib/datadog/ci/contrib/rspec/example_group.rb +++ b/lib/datadog/ci/contrib/rspec/example_group.rb @@ -16,6 +16,7 @@ def self.included(base) # Instance methods for configuration module ClassMethods def run(reporter = ::RSpec::Core::NullReporter) + return super if ::RSpec.configuration.dry_run? return super unless datadog_configuration[:enabled] return super unless top_level? diff --git a/lib/datadog/ci/contrib/rspec/knapsack_pro/extension.rb b/lib/datadog/ci/contrib/rspec/knapsack_pro/extension.rb index bfe754531..f7fe2dcdc 100644 --- a/lib/datadog/ci/contrib/rspec/knapsack_pro/extension.rb +++ b/lib/datadog/ci/contrib/rspec/knapsack_pro/extension.rb @@ -8,7 +8,6 @@ module Datadog module CI module Contrib module RSpec - # Instrument RSpec::Core::Example module KnapsackPro module Extension def self.included(base) diff --git a/lib/datadog/ci/contrib/rspec/knapsack_pro/patcher.rb b/lib/datadog/ci/contrib/rspec/knapsack_pro/patcher.rb new file mode 100644 index 000000000..3346f6da6 --- /dev/null +++ b/lib/datadog/ci/contrib/rspec/knapsack_pro/patcher.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Contrib + module RSpec + module KnapsackPro + module Patcher + def self.patch + if defined?(::KnapsackPro::Extensions::RSpecExtension::Runner) && + ::RSpec::Core::Runner.ancestors.include?(::KnapsackPro::Extensions::RSpecExtension::Runner) + # knapsack already patched rspec runner + require_relative "runner" + ::RSpec::Core::Runner.include(KnapsackPro::Runner) + else + # knapsack didn't patch rspec runner yet + require_relative "extension" + ::KnapsackPro::Extensions::RSpecExtension.include(KnapsackPro::Extension) + end + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb b/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb index 9a5ec11af..59cbe98f7 100644 --- a/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb +++ b/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb @@ -15,6 +15,7 @@ def self.included(base) module InstanceMethods def knapsack__run_specs(*) + return super if ::RSpec.configuration.dry_run? return super unless datadog_configuration[:enabled] test_session = CI.start_test_session( diff --git a/lib/datadog/ci/contrib/rspec/patcher.rb b/lib/datadog/ci/contrib/rspec/patcher.rb index 41e6935da..8fec2cb1d 100644 --- a/lib/datadog/ci/contrib/rspec/patcher.rb +++ b/lib/datadog/ci/contrib/rspec/patcher.rb @@ -27,14 +27,16 @@ def patch ::RSpec::Queue::Runner.include(Runner) end - # Knapsack Pro test runner instrumentation - # https://github.com/KnapsackPro/knapsack_pro-ruby if knapsack_pro? - require_relative "knapsack_pro/extension" - ::KnapsackPro::Extensions::RSpecExtension.include(KnapsackPro::Extension) + # Knapsack Pro test runner instrumentation + # https://github.com/KnapsackPro/knapsack_pro-ruby + require_relative "knapsack_pro/patcher" + Datadog::CI::Contrib::RSpec::KnapsackPro::Patcher.patch end + # default rspec test runner instrumentation ::RSpec::Core::Runner.include(Runner) + ::RSpec::Core::Example.include(Example) ::RSpec::Core::ExampleGroup.include(ExampleGroup) end diff --git a/lib/datadog/ci/contrib/rspec/runner.rb b/lib/datadog/ci/contrib/rspec/runner.rb index 29d4c6923..2a52fac4b 100644 --- a/lib/datadog/ci/contrib/rspec/runner.rb +++ b/lib/datadog/ci/contrib/rspec/runner.rb @@ -15,6 +15,7 @@ def self.included(base) module InstanceMethods def run_specs(*) + return super if ::RSpec.configuration.dry_run? return super unless datadog_configuration[:enabled] test_session = CI.start_test_session( diff --git a/sig/datadog/ci/contrib/rspec/knapsack_pro/patcher.rbs b/sig/datadog/ci/contrib/rspec/knapsack_pro/patcher.rbs new file mode 100644 index 000000000..11d34d86b --- /dev/null +++ b/sig/datadog/ci/contrib/rspec/knapsack_pro/patcher.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module Contrib + module RSpec + module KnapsackPro + module Patcher + def self.patch: () -> void + end + end + end + end + end +end diff --git a/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb index 01dea690e..549b6d0f2 100644 --- a/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb @@ -2,8 +2,6 @@ require "fileutils" RSpec.describe "RSpec instrumentation with Knapsack Pro runner in queue mode" do - before { skip("jruby fails on file with emojis in name") if PlatformHelpers.jruby? } - include_context "CI mode activated" do let(:integration_name) { :rspec } end diff --git a/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb b/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb new file mode 100644 index 000000000..db44b0228 --- /dev/null +++ b/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb @@ -0,0 +1,79 @@ +require "knapsack_pro" +require "fileutils" + +RSpec.describe "Knapsack Pro runner when Datadog::CI is configured during the knapsack run like in rspec_go rake task" do + # Yields to a block in a new RSpec global context. All RSpec + # test configuration and execution should be wrapped in this method. + def with_new_rspec_environment + old_configuration = ::RSpec.configuration + old_world = ::RSpec.world + ::RSpec.configuration = ::RSpec::Core::Configuration.new + ::RSpec.world = ::RSpec::Core::World.new + + yield + ensure + ::RSpec.configuration = old_configuration + ::RSpec.world = old_world + end + + def devnull + File.new("/dev/null", "w") + end + + before do + allow_any_instance_of(Datadog::Core::Remote::Negotiation).to( + receive(:endpoint?).with("/evp_proxy/v4/").and_return(true) + ) + + allow(Datadog::CI::Utils::TestRun).to receive(:command).and_return("knapsack:queue:rspec") + + allow_any_instance_of(KnapsackPro::Runners::Queue::RSpecRunner).to receive(:test_file_paths).and_return( + ["./spec/datadog/ci/contrib/knapsack_rspec_go/suite_under_test/some_test_rspec.rb"], + [] + ) + end + + it "instruments this rspec session" do + with_new_rspec_environment do + ClimateControl.modify( + "KNAPSACK_PRO_CI_NODE_BUILD_ID" => "144", + "KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC" => "example_token", + "KNAPSACK_PRO_FIXED_QUEUE_SPLIT" => "true", + "KNAPSACK_PRO_QUEUE_ID" => nil + ) do + KnapsackPro::Adapters::RSpecAdapter.bind + KnapsackPro::Runners::Queue::RSpecRunner.run("--require knapsack_helper", devnull, devnull) + rescue ArgumentError + # suppress invalid API key error + end + end + + # test session and module traced + expect(test_session_span).not_to be_nil + expect(test_module_span).not_to be_nil + + # test session and module are failed + expect([test_session_span, test_module_span]).to all have_fail_status + + # single test suite span + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_test_tag(:status, Datadog::CI::Ext::Test::Status::FAIL) + expect(test_suite_spans.first).to have_test_tag( + :suite, + "SomeTest at ./spec/datadog/ci/contrib/knapsack_rspec_go/suite_under_test/some_test_rspec.rb" + ) + + # there is test span for every test case + expect(test_spans).to have(2).items + # test spans belong to a single test suite + expect(test_spans).to have_unique_tag_values_count(:test_suite_id, 1) + expect(test_spans).to have_tag_values_no_order( + :status, + [Datadog::CI::Ext::Test::Status::FAIL, Datadog::CI::Ext::Test::Status::PASS] + ) + + # every test span is connected to test module and test session + expect(test_spans).to all have_test_tag(:test_module_id) + expect(test_spans).to all have_test_tag(:test_session_id) + end +end diff --git a/spec/datadog/ci/contrib/knapsack_rspec_go/suite_under_test/some_test_rspec.rb b/spec/datadog/ci/contrib/knapsack_rspec_go/suite_under_test/some_test_rspec.rb new file mode 100644 index 000000000..7b775cb97 --- /dev/null +++ b/spec/datadog/ci/contrib/knapsack_rspec_go/suite_under_test/some_test_rspec.rb @@ -0,0 +1,13 @@ +require "rspec" + +RSpec.describe "SomeTest" do + context "nested" do + it "foo" do + # DO NOTHING + end + + it "fails" do + expect(1).to eq(2) + end + end +end diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb index d52c6bb17..e7cac9f58 100644 --- a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb @@ -27,7 +27,8 @@ def rspec_session_run( test: false, context: false, suite: false - } + }, + dry_run: false ) test_meta = unskippable[:test] ? {Datadog::CI::Ext::Test::ITR_UNSKIPPABLE_OPTION => true} : {} context_meta = unskippable[:context] ? {Datadog::CI::Ext::Test::ITR_UNSKIPPABLE_OPTION => true} : {} @@ -58,7 +59,11 @@ def rspec_session_run( end end - options = ::RSpec::Core::ConfigurationOptions.new(%w[--pattern none]) + options_array = %w[--pattern none] + if dry_run + options_array << "--dry-run" + end + options = ::RSpec::Core::ConfigurationOptions.new(options_array) ::RSpec::Core::Runner.new(options).run(devnull, devnull) spec @@ -104,7 +109,7 @@ def rspec_session_run( :source_file, "spec/datadog/ci/contrib/rspec/instrumentation_spec.rb" ) - expect(first_test_span).to have_test_tag(:source_start, "77") + expect(first_test_span).to have_test_tag(:source_start, "82") expect(first_test_span).to have_test_tag( :codeowners, "[\"@DataDog/ruby-guild\", \"@DataDog/ci-app-libraries\"]" @@ -773,4 +778,18 @@ def rspec_skipped_session_run end end end + + context "with dry run" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:integration_options) { {service_name: "lspec"} } + end + + it "does not instrument test session" do + rspec_session_run(dry_run: true) + + expect(test_session_span).to be_nil + expect(test_spans).to be_empty + end + end end diff --git a/spec/knapsack_helper.rb b/spec/knapsack_helper.rb new file mode 100644 index 000000000..dadf2e05b --- /dev/null +++ b/spec/knapsack_helper.rb @@ -0,0 +1,8 @@ +# this file is required for knapsack unit tests + +Datadog.configure do |c| + c.service = "knapsack_rspec_example" + c.ci.enabled = true + c.ci.git_metadata_upload_enabled = false + c.ci.instrument :rspec +end diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index 2b7c59d0e..f266c5c9a 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -88,5 +88,9 @@ Datadog::CI.send(:itr_runner)&.shutdown! Datadog::CI.send(:recorder)&.shutdown! + + Datadog.configure do |c| + c.ci.enabled = false + end end end diff --git a/vendor/rbs/knapsack_pro/0/knapsack_pro.rbs b/vendor/rbs/knapsack_pro/0/knapsack_pro.rbs index 58696d5ab..d44f12b65 100644 --- a/vendor/rbs/knapsack_pro/0/knapsack_pro.rbs +++ b/vendor/rbs/knapsack_pro/0/knapsack_pro.rbs @@ -12,6 +12,9 @@ module KnapsackPro module Extensions module RSpecExtension def setup!: () -> void + + module Runner + end end end end \ No newline at end of file diff --git a/vendor/rbs/rspec/0/rspec.rbs b/vendor/rbs/rspec/0/rspec.rbs index 5ccb0357e..d08273cc3 100644 --- a/vendor/rbs/rspec/0/rspec.rbs +++ b/vendor/rbs/rspec/0/rspec.rbs @@ -1,4 +1,5 @@ module RSpec + def self.configuration: () -> RSpec::Core::Configuration end module RSpec::Core @@ -36,3 +37,7 @@ end class RSpec::Core::NullReporter end + +class RSpec::Core::Configuration + def dry_run?: () -> bool +end