diff --git a/Appraisals b/Appraisals index 034acc21..77477d68 100644 --- a/Appraisals +++ b/Appraisals @@ -14,7 +14,7 @@ end alias original_appraise appraise REMOVED_GEMS = { - check: %w[rbs steep], + check: %w[rbs steep ruby_memcheck], development: %w[ruby-lsp ruby-lsp-rspec debug irb] } RUBY_VERSION = Gem::Version.new(RUBY_ENGINE_VERSION) diff --git a/Steepfile b/Steepfile index bc484a8a..7a3d755f 100644 --- a/Steepfile +++ b/Steepfile @@ -16,6 +16,7 @@ target :lib do library "tmpdir" library "fileutils" library "socket" + library "optparse" repo_path "vendor/rbs" library "ddtrace" diff --git a/datadog-ci.gemspec b/datadog-ci.gemspec index bdaa2d95..6b61d8f8 100644 --- a/datadog-ci.gemspec +++ b/datadog-ci.gemspec @@ -23,6 +23,9 @@ Gem::Specification.new do |spec| spec.homepage = "https://github.com/DataDog/datadog-ci-rb" spec.license = "BSD-3-Clause" + spec.bindir = "exe" + spec.executables = ["ddcirb"] + spec.metadata["allowed_push_host"] = "https://rubygems.org" spec.metadata["changelog_uri"] = "https://github.com/DataDog/datadog-ci-rb/blob/main/CHANGELOG.md" spec.metadata["homepage_uri"] = spec.homepage @@ -36,6 +39,7 @@ Gem::Specification.new do |spec| README.md ext/**/* lib/**/* + exe/**/* ]].select { |fn| File.file?(fn) } # We don't want directories, only files .reject { |fn| fn.end_with?(".so", ".bundle") } # Exclude local native binary artifacts diff --git a/docs/CommandLineInterface.md b/docs/CommandLineInterface.md new file mode 100644 index 00000000..67af6284 --- /dev/null +++ b/docs/CommandLineInterface.md @@ -0,0 +1,64 @@ +# Command line interface + +This library provides experimental command line interface `ddcirb` to get the percentage of the tests +that will be skipped for the current test run (only when using RSpec). + +## Usage + +This tool must be used on the same runner as your tests are running on with the same ENV variables. +Run the command in the same folder where you usually run your tests. Gem datadog-ci must be installed. + +Available commands: + +- `bundle exec ddcirb skipped-tests` - outputs the percentage of skipped tests to stdout. Note that it runs your specs +in dry run mode, don't forget to set RAILS_ENV=test environment variable. +- `bundle exec ddcirb skipped-tests-estimate` - estimates the percentage of skipped tests and outputs to stdout without loading +your test suite and running it in dry run mode. ATTENTION: this is considerably faster but could be very inaccurate. + +Example usage: + +```bash +$ RAILS_ENV=test bundle exec ddcirb skipped-tests +0.45 +``` + +Available arguments: + +- `-f, --file` - output to a file (example: `bundle exec ddcirb skipped-tests -f out`) +- `--verbose` - enable verbose output for debugging purposes (example: `bundle exec ddcirb skipped-tests --verbose`) +- `--spec-path` - path to the folder with RSpec tests (default: `spec`, example: `bundle exec ddcirb skipped-tests --spec-path="myapp/spec"`) +- `--rspec-opts` - additional options to pass to the RSpec when running it in dry run mode (example: `bundle exec ddcirb skipped-tests --rspec-opts="--require rails_helper"`) + +## Example usage in Circle CI + +This tool could be used to determine [Circle CI parallelism](https://support.circleci.com/hc/en-us/articles/14928385117851-How-to-dynamically-set-job-parallelism) dynamically: + +```yaml +version: 2.1 + +setup: true + +orbs: + continuation: circleci/continuation@0.2.0 + +jobs: + determine-parallelism: + docker: + - image: cimg/base:edge + resource_class: medium + steps: + - checkout + - run: + name: Determine parallelism + command: | + PARALLELISM=$(RAILS_ENV=test bundle exec ddcirb skipped-tests) + echo "{\"parallelism\": $PARALLELISM}" > pipeline-parameters.json + - continuation/continue: + configuration_path: .circleci/continue_config.yml + parameters: pipeline-parameters.json + +workflows: + build-setup: + jobs: + - determine-parallelism +``` diff --git a/exe/ddcirb b/exe/ddcirb new file mode 100755 index 00000000..2575638e --- /dev/null +++ b/exe/ddcirb @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require "datadog/ci/cli/cli" + +Datadog::CI::CLI.exec(ARGV.first) diff --git a/lib/datadog/ci/cli/cli.rb b/lib/datadog/ci/cli/cli.rb new file mode 100644 index 00000000..99df9df9 --- /dev/null +++ b/lib/datadog/ci/cli/cli.rb @@ -0,0 +1,24 @@ +require "datadog" +require "datadog/ci" + +require_relative "command/skippable_tests_percentage" +require_relative "command/skippable_tests_percentage_estimate" + +module Datadog + module CI + module CLI + def self.exec(action) + case action + when "skipped-tests", "skippable-tests" + Command::SkippableTestsPercentage.new.exec + when "skipped-tests-estimate", "skippable-tests-estimate" + Command::SkippableTestsPercentageEstimate.new.exec + else + puts("Usage: bundle exec ddcirb [command] [options]. Available commands:") + puts(" skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file") + puts(" skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file") + end + end + end + end +end diff --git a/lib/datadog/ci/cli/command/base.rb b/lib/datadog/ci/cli/command/base.rb new file mode 100644 index 00000000..4d88d1a7 --- /dev/null +++ b/lib/datadog/ci/cli/command/base.rb @@ -0,0 +1,58 @@ +require "optparse" + +module Datadog + module CI + module CLI + module Command + class Base + def exec + action = build_action + result = action&.call + + validate!(action) + output(result) + end + + private + + def build_action + end + + def options + return @options if defined?(@options) + + ddcirb_options = {} + OptionParser.new do |opts| + opts.banner = "Usage: bundle exec ddcirb [command] [options]\n Available commands: skippable-tests, skippable-tests-estimate" + + opts.on("-f", "--file FILENAME", "Output result to file FILENAME") + opts.on("--verbose", "Verbose output to stdout") + + command_options(opts) + end.parse!(into: ddcirb_options) + + @options = ddcirb_options + end + + def command_options(opts) + end + + def validate!(action) + if action.nil? || action.failed + Datadog.logger.error("ddcirb failed, exiting") + Kernel.exit(1) + end + end + + def output(result) + if options[:file] + File.write(options[:file], result) + else + print(result) + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/cli/command/skippable_tests_percentage.rb b/lib/datadog/ci/cli/command/skippable_tests_percentage.rb new file mode 100644 index 00000000..e1281616 --- /dev/null +++ b/lib/datadog/ci/cli/command/skippable_tests_percentage.rb @@ -0,0 +1,27 @@ +require_relative "base" +require_relative "../../test_optimisation/skippable_percentage/calculator" + +module Datadog + module CI + module CLI + module Command + class SkippableTestsPercentage < Base + private + + def build_action + ::Datadog::CI::TestOptimisation::SkippablePercentage::Calculator.new( + rspec_cli_options: (options[:"rspec-opts"] || "").split, + verbose: !options[:verbose].nil?, + spec_path: options[:"spec-path"] || "spec" + ) + end + + def command_options(opts) + opts.on("--rspec-opts=[OPTIONS]", "Command line options to pass to RSpec") + opts.on("--spec-path=[SPEC_PATH]", "Relative path to the spec directory, example: spec") + end + end + end + end + end +end diff --git a/lib/datadog/ci/cli/command/skippable_tests_percentage_estimate.rb b/lib/datadog/ci/cli/command/skippable_tests_percentage_estimate.rb new file mode 100644 index 00000000..4037c175 --- /dev/null +++ b/lib/datadog/ci/cli/command/skippable_tests_percentage_estimate.rb @@ -0,0 +1,21 @@ +require_relative "base" +require_relative "../../test_optimisation/skippable_percentage/estimator" + +module Datadog + module CI + module CLI + module Command + class SkippableTestsPercentageEstimate < Base + private + + def build_action + ::Datadog::CI::TestOptimisation::SkippablePercentage::Estimator.new( + verbose: !options[:verbose].nil?, + spec_path: options[:"spec-path"] || "spec" + ) + end + end + end + end + end +end diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 0fc277cb..8947464a 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -19,6 +19,7 @@ require_relative "../test_visibility/serializers/factories/test_suite_level" require_relative "../test_visibility/transport" require_relative "../transport/adapters/telemetry_webmock_safe_adapter" +require_relative "../test_visibility/null_transport" require_relative "../transport/api/builder" require_relative "../utils/parsing" require_relative "../utils/test_run" @@ -203,6 +204,9 @@ def build_test_visibility_api(settings) end def build_tracing_transport(settings, api) + # NullTransport ignores traces + return TestVisibility::NullTransport.new if settings.ci.discard_traces + # nil means that default legacy APM transport will be used (only for very old Datadog Agent versions) return nil if api.nil? TestVisibility::Transport.new( @@ -213,7 +217,8 @@ def build_tracing_transport(settings, api) end def build_coverage_writer(settings, api) - return nil if api.nil? + # nil means that coverage event will be ignored + return nil if api.nil? || settings.ci.discard_traces TestOptimisation::Coverage::Writer.new( transport: TestOptimisation::Coverage::Transport.new(api: api) diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index fc154923..8212913a 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -117,6 +117,12 @@ def self.add_settings!(base) o.default true end + # internal only + option :discard_traces do |o| + o.type :bool + o.default false + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/contrib/rspec/configuration/settings.rb b/lib/datadog/ci/contrib/rspec/configuration/settings.rb index 4e39211d..95d574b7 100644 --- a/lib/datadog/ci/contrib/rspec/configuration/settings.rb +++ b/lib/datadog/ci/contrib/rspec/configuration/settings.rb @@ -24,6 +24,12 @@ class Settings < Datadog::CI::Contrib::Settings Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME) end end + + # internal only + option :dry_run_enabled do |o| + o.type :bool + o.default false + end end end end diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb index bac1f27a..e7a2ac2e 100644 --- a/lib/datadog/ci/contrib/rspec/example.rb +++ b/lib/datadog/ci/contrib/rspec/example.rb @@ -17,7 +17,7 @@ def self.included(base) module InstanceMethods def run(*args) - return super if ::RSpec.configuration.dry_run? + return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled] return super unless datadog_configuration[:enabled] test_name = full_description.strip diff --git a/lib/datadog/ci/contrib/rspec/example_group.rb b/lib/datadog/ci/contrib/rspec/example_group.rb index a18a1900..35a38f74 100644 --- a/lib/datadog/ci/contrib/rspec/example_group.rb +++ b/lib/datadog/ci/contrib/rspec/example_group.rb @@ -16,7 +16,7 @@ def self.included(base) # Instance methods for configuration module ClassMethods def run(reporter = ::RSpec::Core::NullReporter) - return super if ::RSpec.configuration.dry_run? + return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled] return super unless datadog_configuration[:enabled] return super unless top_level? diff --git a/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb b/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb index d0988f5d..c4a386f1 100644 --- a/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb +++ b/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb @@ -15,7 +15,7 @@ def self.included(base) module InstanceMethods def knapsack__run_specs(*args) - return super if ::RSpec.configuration.dry_run? + return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled] return super unless datadog_configuration[:enabled] test_session = test_visibility_component.start_test_session( diff --git a/lib/datadog/ci/contrib/rspec/runner.rb b/lib/datadog/ci/contrib/rspec/runner.rb index eb27d402..49a88a10 100644 --- a/lib/datadog/ci/contrib/rspec/runner.rb +++ b/lib/datadog/ci/contrib/rspec/runner.rb @@ -15,7 +15,7 @@ def self.included(base) module InstanceMethods def run_specs(*args) - return super if ::RSpec.configuration.dry_run? + return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled] return super unless datadog_configuration[:enabled] test_session = test_visibility_component.start_test_session( diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index 0137ccf7..00912e43 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -76,7 +76,7 @@ def is_retry? # - tests that read files from disk # - tests that make network requests # - tests that call external processes - # - tests that use forking or threading + # - tests that use forking # # @return [void] def itr_unskippable! diff --git a/lib/datadog/ci/test_optimisation/component.rb b/lib/datadog/ci/test_optimisation/component.rb index 5f722d0d..562ca6c9 100644 --- a/lib/datadog/ci/test_optimisation/component.rb +++ b/lib/datadog/ci/test_optimisation/component.rb @@ -25,7 +25,9 @@ module TestOptimisation class Component include Core::Utils::Forking - attr_reader :correlation_id, :skippable_tests, :skipped_tests_count + attr_reader :correlation_id, :skippable_tests, :skippable_tests_fetch_error, + :skipped_tests_count, :total_tests_count, + :enabled, :test_skipping_enabled, :code_coverage_enabled def initialize( dd_env:, @@ -58,7 +60,9 @@ def initialize( @correlation_id = nil @skippable_tests = Set.new + @total_tests_count = 0 @skipped_tests_count = 0 + @mutex = Mutex.new Datadog.logger.debug("TestOptimisation initialized with enabled: #{@enabled}") @@ -159,14 +163,16 @@ def mark_if_skippable(test) end def count_skipped_test(test) - return if !test.skipped? || !test.skipped_by_itr? + @mutex.synchronize do + @total_tests_count += 1 - if forked? - Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" } - return - end + return if !test.skipped? || !test.skipped_by_itr? + + if forked? + Datadog.logger.warn { "ITR is not supported for forking test runners yet" } + return + end - @mutex.synchronize do Telemetry.itr_skipped @skipped_tests_count += 1 @@ -183,6 +189,10 @@ def write_test_session_tags(test_session) test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, @skipped_tests_count) end + def skippable_tests_count + skippable_tests.count + end + def shutdown! @coverage_writer&.stop end @@ -229,6 +239,7 @@ def fetch_skippable_tests(test_session) skippable_response = Skippable.new(api: @api, dd_env: @dd_env, config_tags: @config_tags) .fetch_skippable_tests(test_session) + @skippable_tests_fetch_error = skippable_response.error_message unless skippable_response.ok? @correlation_id = skippable_response.correlation_id @skippable_tests = skippable_response.tests diff --git a/lib/datadog/ci/test_optimisation/skippable.rb b/lib/datadog/ci/test_optimisation/skippable.rb index d65ae4f5..95a4dacb 100644 --- a/lib/datadog/ci/test_optimisation/skippable.rb +++ b/lib/datadog/ci/test_optimisation/skippable.rb @@ -42,6 +42,12 @@ def tests res end + def error_message + return nil if ok? + + "Status code: #{@http_response&.code}, response: #{@http_response&.payload}" + end + private def payload diff --git a/lib/datadog/ci/test_optimisation/skippable_percentage/base.rb b/lib/datadog/ci/test_optimisation/skippable_percentage/base.rb new file mode 100644 index 00000000..415a8949 --- /dev/null +++ b/lib/datadog/ci/test_optimisation/skippable_percentage/base.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Datadog + module CI + module TestOptimisation + module SkippablePercentage + class Base + attr_reader :failed + + def initialize(verbose: false, spec_path: "spec") + @verbose = verbose + @spec_path = spec_path + @failed = false + + log("Spec path: #{@spec_path}") + error!("Spec path is not a directory: #{@spec_path}") if !File.directory?(@spec_path) + end + + def call + 0.0 + end + + private + + def validate_test_optimisation_state! + unless test_optimisation.enabled + error!("ITR wasn't enabled, check the environment variables (DD_SERVICE, DD_ENV)") + end + + if test_optimisation.skippable_tests_fetch_error + error!("Skippable tests couldn't be fetched, error: #{test_optimisation.skippable_tests_fetch_error}") + end + end + + def log(message) + Datadog.logger.info(message) if @verbose + end + + def error!(message) + Datadog.logger.error(message) + @failed = true + end + + def test_optimisation + Datadog.send(:components).test_optimisation + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb b/lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb new file mode 100644 index 00000000..f2aa1b83 --- /dev/null +++ b/lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require_relative "base" + +module Datadog + module CI + module TestOptimisation + module SkippablePercentage + # This class calculates the percentage of tests that are going to be skipped in the next run + # without actually running the tests. + # + # It is useful to determine the number of parallel jobs that are required for the CI pipeline. + # + # NOTE: Only RSpec is supported at the moment. + class Calculator < Base + def initialize(rspec_cli_options: [], verbose: false, spec_path: "spec") + super(verbose: verbose, spec_path: spec_path) + + @rspec_cli_options = rspec_cli_options || [] + end + + def call + return 0.0 if @failed + + require_rspec! + return 0.0 if @failed + + configure_datadog + + exit_code = dry_run + if exit_code != 0 + Datadog.logger.error("RSpec dry-run failed with exit code #{exit_code}") + @failed = true + return 0.0 + end + + log("Total tests count: #{test_optimisation.total_tests_count}") + log("Skipped tests count: #{test_optimisation.skipped_tests_count}") + validate_test_optimisation_state! + + (test_optimisation.skipped_tests_count.to_f / test_optimisation.total_tests_count.to_f).floor(2) + end + + private + + def require_rspec! + require "rspec/core" + rescue LoadError + Datadog.logger.error("RSpec is not installed, currently this functionality is only supported for RSpec.") + @failed = true + end + + def configure_datadog + Datadog.configure do |c| + c.ci.enabled = true + c.ci.itr_enabled = true + c.ci.retry_failed_tests_enabled = false + c.ci.retry_new_tests_enabled = false + c.ci.discard_traces = true + c.ci.instrument :rspec, dry_run_enabled: true + c.tracing.enabled = true + end + end + + def dry_run + cli_options_array = @rspec_cli_options + ["--dry-run", @spec_path] + + rspec_config_options = ::RSpec::Core::ConfigurationOptions.new(cli_options_array) + devnull = File.new("/dev/null", "w") + out = @verbose ? $stdout : devnull + err = @verbose ? $stderr : devnull + + ::RSpec::Core::Runner.new(rspec_config_options).run(out, err) + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_optimisation/skippable_percentage/estimator.rb b/lib/datadog/ci/test_optimisation/skippable_percentage/estimator.rb new file mode 100644 index 00000000..045da6b7 --- /dev/null +++ b/lib/datadog/ci/test_optimisation/skippable_percentage/estimator.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require_relative "base" + +module Datadog + module CI + module TestOptimisation + module SkippablePercentage + # This class estimates the percentage of tests that are going to be skipped in the next run + # without actually running the tests. This estimate is very rough: + # + # - it counts the number of lines that start with "it" or "scenario" in the spec files, which could be inaccurate + # if you use shared examples + # - it only counts the number of tests that could be skipped, this does not mean that they will be actually skipped: + # if in this commit you replaced all the tests in your test suite with new ones, all the tests would be run (but + # this is highly unlikely) + # + # It is useful to determine the number of parallel jobs that are required for the CI pipeline. + # + # NOTE: Only RSpec is supported at the moment. + class Estimator < Base + def initialize(verbose: false, spec_path: "spec") + super + end + + def call + return 0.0 if @failed + + Datadog.configure do |c| + c.ci.enabled = true + c.ci.itr_enabled = true + c.ci.retry_failed_tests_enabled = false + c.ci.retry_new_tests_enabled = false + c.ci.discard_traces = true + c.tracing.enabled = true + end + + spec_files = Dir["#{@spec_path}/**/*_spec.rb"] + estimated_tests_count = spec_files.sum do |file| + content = File.read(file) + content.scan(/(^\s*it\s+)|(^\s*scenario\s+)/).size + end + + # starting and finishing a test session is required to get the skippable tests response + Datadog::CI.start_test_session(total_tests_count: estimated_tests_count)&.finish + skippable_tests_count = test_optimisation.skippable_tests_count + + log("Estimated tests count: #{estimated_tests_count}") + log("Skippable tests count: #{skippable_tests_count}") + validate_test_optimisation_state! + + [(skippable_tests_count.to_f / estimated_tests_count).floor(2), 0.99].min || 0.0 + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/null_transport.rb b/lib/datadog/ci/test_visibility/null_transport.rb new file mode 100644 index 00000000..6563ed0d --- /dev/null +++ b/lib/datadog/ci/test_visibility/null_transport.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Datadog + module CI + module TestVisibility + class NullTransport + def initialize + end + + def send_traces(traces) + [] + end + end + end + end +end diff --git a/sig/datadog/ci.rbs b/sig/datadog/ci.rbs index 6d85db27..553a37da 100644 --- a/sig/datadog/ci.rbs +++ b/sig/datadog/ci.rbs @@ -1,4 +1,6 @@ module Datadog + def self.configure: () ?{ (untyped config) -> untyped } -> untyped + module CI class ReservedTypeError < StandardError end diff --git a/sig/datadog/ci/cli/cli.rbs b/sig/datadog/ci/cli/cli.rbs new file mode 100644 index 00000000..45e48a8f --- /dev/null +++ b/sig/datadog/ci/cli/cli.rbs @@ -0,0 +1,7 @@ +module Datadog + module CI + module CLI + def self.exec: (String action) -> void + end + end +end diff --git a/sig/datadog/ci/cli/command/base.rbs b/sig/datadog/ci/cli/command/base.rbs new file mode 100644 index 00000000..5d08120c --- /dev/null +++ b/sig/datadog/ci/cli/command/base.rbs @@ -0,0 +1,25 @@ +module Datadog + module CI + module CLI + module Command + class Base + @options: Hash[Symbol, String] + + def exec: () -> void + + private + + def build_action: () -> untyped + + def options: () -> Hash[Symbol, String] + + def command_options: (untyped opts) -> void + + def validate!: (untyped action) -> void + + def output: (untyped result) -> void + end + end + end + end +end diff --git a/sig/datadog/ci/cli/command/skippable_tests_percentage.rbs b/sig/datadog/ci/cli/command/skippable_tests_percentage.rbs new file mode 100644 index 00000000..abb793bf --- /dev/null +++ b/sig/datadog/ci/cli/command/skippable_tests_percentage.rbs @@ -0,0 +1,15 @@ +module Datadog + module CI + module CLI + module Command + class SkippableTestsPercentage < Base + private + + def build_action: () -> untyped + + def command_options: (untyped opts) -> void + end + end + end + end +end diff --git a/sig/datadog/ci/cli/command/skippable_tests_percentage_estimate.rbs b/sig/datadog/ci/cli/command/skippable_tests_percentage_estimate.rbs new file mode 100644 index 00000000..bf4064c4 --- /dev/null +++ b/sig/datadog/ci/cli/command/skippable_tests_percentage_estimate.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module CLI + module Command + class SkippableTestsPercentageEstimate < Base + private + + def build_action: () -> untyped + end + end + end + end +end diff --git a/sig/datadog/ci/configuration/components.rbs b/sig/datadog/ci/configuration/components.rbs index 059e4d7e..835d5edf 100644 --- a/sig/datadog/ci/configuration/components.rbs +++ b/sig/datadog/ci/configuration/components.rbs @@ -26,7 +26,7 @@ module Datadog def check_dd_site: (untyped settings) -> void - def build_tracing_transport: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::TestVisibility::Transport? + def build_tracing_transport: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> (Datadog::CI::TestVisibility::Transport? | Datadog::CI::TestVisibility::NullTransport) def build_coverage_writer: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::TestOptimisation::Coverage::Writer? diff --git a/sig/datadog/ci/contrib/rspec/runner.rbs b/sig/datadog/ci/contrib/rspec/runner.rbs index 2b902879..b0b9c96a 100644 --- a/sig/datadog/ci/contrib/rspec/runner.rbs +++ b/sig/datadog/ci/contrib/rspec/runner.rbs @@ -5,9 +5,7 @@ module Datadog module Runner def self.included: (untyped base) -> untyped - module InstanceMethods - include ::RSpec::Core::Runner - + module InstanceMethods : ::RSpec::Core::Runner def run_specs: (untyped example_groups) -> untyped private diff --git a/sig/datadog/ci/test_optimisation/component.rbs b/sig/datadog/ci/test_optimisation/component.rbs index 0cf5ca5a..afbd7bf8 100644 --- a/sig/datadog/ci/test_optimisation/component.rbs +++ b/sig/datadog/ci/test_optimisation/component.rbs @@ -25,6 +25,11 @@ module Datadog attr_reader skippable_tests: Set[String] attr_reader skipped_tests_count: Integer attr_reader correlation_id: String? + attr_reader enabled: bool + attr_reader code_coverage_enabled: bool + attr_reader test_skipping_enabled: bool + attr_reader total_tests_count: Integer + attr_reader skippable_tests_fetch_error: String? def initialize: (dd_env: String?, ?enabled: bool, ?coverage_writer: Datadog::CI::TestOptimisation::Coverage::Writer?, ?api: Datadog::CI::Transport::Api::Base?, ?config_tags: Hash[String, String]?, ?bundle_location: String?, ?use_single_threaded_coverage: bool, ?use_allocation_tracing: bool) -> void @@ -48,6 +53,8 @@ module Datadog def shutdown!: () -> void + def skippable_tests_count: () -> Integer + private def coverage_collector: () -> Datadog::CI::TestOptimisation::Coverage::DDCov? diff --git a/sig/datadog/ci/test_optimisation/skippable.rbs b/sig/datadog/ci/test_optimisation/skippable.rbs index dc1333a6..17ac7f33 100644 --- a/sig/datadog/ci/test_optimisation/skippable.rbs +++ b/sig/datadog/ci/test_optimisation/skippable.rbs @@ -18,6 +18,8 @@ module Datadog def tests: () -> Set[String] + def error_message: () -> String? + private def payload: () -> Hash[String, untyped] diff --git a/sig/datadog/ci/test_optimisation/skippable_percentage/base.rbs b/sig/datadog/ci/test_optimisation/skippable_percentage/base.rbs new file mode 100644 index 00000000..241ad4b5 --- /dev/null +++ b/sig/datadog/ci/test_optimisation/skippable_percentage/base.rbs @@ -0,0 +1,28 @@ +module Datadog + module CI + module TestOptimisation + module SkippablePercentage + class Base + attr_reader failed: bool + + @verbose: bool + @spec_path: String + + def initialize: (?verbose: bool, ?spec_path: ::String) -> void + + def call: () -> Numeric + + private + + def validate_test_optimisation_state!: () -> void + + def log: (String message) -> void + + def error!: (String message) -> void + + def test_optimisation: () -> Datadog::CI::TestOptimisation::Component + end + end + end + end +end diff --git a/sig/datadog/ci/test_optimisation/skippable_percentage/calculator.rbs b/sig/datadog/ci/test_optimisation/skippable_percentage/calculator.rbs new file mode 100644 index 00000000..f025ec68 --- /dev/null +++ b/sig/datadog/ci/test_optimisation/skippable_percentage/calculator.rbs @@ -0,0 +1,23 @@ +module Datadog + module CI + module TestOptimisation + module SkippablePercentage + class Calculator < Base + @rspec_cli_options: Array[String] + + def initialize: (?rspec_cli_options: Array[String], ?verbose: bool, ?spec_path: ::String) -> void + + def call: () -> Numeric + + private + + def require_rspec!: () -> void + + def configure_datadog: () -> void + + def dry_run: () -> Integer + end + end + end + end +end diff --git a/sig/datadog/ci/test_optimisation/skippable_percentage/estimator.rbs b/sig/datadog/ci/test_optimisation/skippable_percentage/estimator.rbs new file mode 100644 index 00000000..8577ae84 --- /dev/null +++ b/sig/datadog/ci/test_optimisation/skippable_percentage/estimator.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module TestOptimisation + module SkippablePercentage + class Estimator < Base + def initialize: (?verbose: bool, ?spec_path: ::String) -> void + + def call: () -> Numeric + end + end + end + end +end diff --git a/sig/datadog/ci/test_visibility/null_transport.rbs b/sig/datadog/ci/test_visibility/null_transport.rbs new file mode 100644 index 00000000..e86080e0 --- /dev/null +++ b/sig/datadog/ci/test_visibility/null_transport.rbs @@ -0,0 +1,11 @@ +module Datadog + module CI + module TestVisibility + class NullTransport + def initialize: () -> void + + def send_traces: (untyped traces) -> ::Array[untyped] + end + end + end +end diff --git a/spec/datadog/ci/configuration/components_spec.rb b/spec/datadog/ci/configuration/components_spec.rb index aa120a9c..3a2980b8 100644 --- a/spec/datadog/ci/configuration/components_spec.rb +++ b/spec/datadog/ci/configuration/components_spec.rb @@ -45,6 +45,7 @@ settings.ci.itr_enabled = itr_enabled settings.ci.itr_code_coverage_use_single_threaded_mode = itr_code_coverage_use_single_threaded_mode settings.ci.itr_test_impact_analysis_use_allocation_tracing = itr_test_impact_analysis_use_allocation_tracing + settings.ci.discard_traces = discard_traces settings.site = dd_site settings.api_key = api_key @@ -116,6 +117,7 @@ let(:tracing_enabled) { true } let(:itr_code_coverage_use_single_threaded_mode) { false } let(:itr_test_impact_analysis_use_allocation_tracing) { true } + let(:discard_traces) { false } context "is enabled" do let(:enabled) { true } @@ -242,6 +244,16 @@ expect(components.test_visibility.itr_enabled?).to eq(false) end end + + context "and when discard_traces setting is enabled" do + let(:discard_traces) { true } + + it "sets tracing transport to TestVisibility::NullTransport" do + expect(settings.tracing.test_mode).to have_received(:writer_options=) do |options| + expect(options[:transport]).to be_kind_of(Datadog::CI::TestVisibility::NullTransport) + end + end + end end context "is enabled" do diff --git a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb index 3131c505..5fd90de6 100644 --- a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb @@ -41,24 +41,6 @@ FileUtils.rm_rf("log") end - def devnull - File.new("/dev/null", "w") - end - - # Yields to a block in a new RSpec global context. All RSpec - # test configuration and execution should be wrapped in this method. - def with_new_rspec_environment - old_configuration = ::RSpec.configuration - old_world = ::RSpec.world - ::RSpec.configuration = ::RSpec::Core::Configuration.new - ::RSpec.world = ::RSpec::Core::World.new - - yield - ensure - ::RSpec.configuration = old_configuration - ::RSpec.world = old_world - end - it "instruments this rspec session" do with_new_rspec_environment do ::RSpec::Queue::Runner.new(options).run(devnull, devnull) diff --git a/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb index acfaf0c4..65486f8d 100644 --- a/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/knapsack_rspec/instrumentation_spec.rb @@ -23,24 +23,6 @@ allow(KnapsackPro::Report).to receive(:save_node_queue_to_api).and_raise(ArgumentError) end - # Yields to a block in a new RSpec global context. All RSpec - # test configuration and execution should be wrapped in this method. - def with_new_rspec_environment - old_configuration = ::RSpec.configuration - old_world = ::RSpec.world - ::RSpec.configuration = ::RSpec::Core::Configuration.new - ::RSpec.world = ::RSpec::Core::World.new - - yield - ensure - ::RSpec.configuration = old_configuration - ::RSpec.world = old_world - end - - def devnull - File.new("/dev/null", "w") - end - it "instruments this rspec session" do with_new_rspec_environment do ClimateControl.modify( diff --git a/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb b/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb index ffb69f7e..43606d14 100644 --- a/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/knapsack_rspec_go/instrumentation_spec.rb @@ -14,24 +14,6 @@ let(:integration_name) { :rspec } end - # Yields to a block in a new RSpec global context. All RSpec - # test configuration and execution should be wrapped in this method. - def with_new_rspec_environment - old_configuration = ::RSpec.configuration - old_world = ::RSpec.world - ::RSpec.configuration = ::RSpec::Core::Configuration.new - ::RSpec.world = ::RSpec::Core::World.new - - yield - ensure - ::RSpec.configuration = old_configuration - ::RSpec.world = old_world - end - - def devnull - File.new("/dev/null", "w") - end - before do allow_any_instance_of(Datadog::Core::Remote::Negotiation).to( receive(:endpoint?).with("/evp_proxy/v4/").and_return(true) diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb index 22ed99d9..7d71a6b9 100644 --- a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb @@ -9,24 +9,6 @@ expect(Datadog::CI).to receive(:start_test).never end - # Yields to a block in a new RSpec global context. All RSpec - # test configuration and execution should be wrapped in this method. - def with_new_rspec_environment - old_configuration = ::RSpec.configuration - old_world = ::RSpec.world - ::RSpec.configuration = ::RSpec::Core::Configuration.new - ::RSpec.world = ::RSpec::Core::World.new - - yield - ensure - ::RSpec.configuration = old_configuration - ::RSpec.world = old_world - end - - def devnull - File.new("/dev/null", "w") - end - def rspec_session_run( with_failed_test: false, with_shared_test: false, @@ -166,7 +148,7 @@ def rspec_session_run( :source_file, "spec/datadog/ci/contrib/rspec/instrumentation_spec.rb" ) - expect(first_test_span).to have_test_tag(:source_start, "139") + expect(first_test_span).to have_test_tag(:source_start, "121") expect(first_test_span).to have_test_tag( :codeowners, "[\"@DataDog/ruby-guild\", \"@DataDog/ci-app-libraries\"]" @@ -596,7 +578,7 @@ def expect_failure :source_file, "spec/datadog/ci/contrib/rspec/instrumentation_spec.rb" ) - expect(first_test_suite_span).to have_test_tag(:source_start, "57") + expect(first_test_suite_span).to have_test_tag(:source_start, "39") expect(first_test_suite_span).to have_test_tag( :codeowners, "[\"@DataDog/ruby-guild\", \"@DataDog/ci-app-libraries\"]" @@ -851,16 +833,32 @@ def rspec_skipped_session_run end context "with dry run" do - include_context "CI mode activated" do - let(:integration_name) { :rspec } - let(:integration_options) { {service_name: "lspec"} } + context "normal instrumentation" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:integration_options) { {service_name: "lspec"} } + end + + it "does not instrument test session" do + rspec_session_run(dry_run: true) + + expect(test_session_span).to be_nil + expect(test_spans).to be_empty + end end - it "does not instrument test session" do - rspec_session_run(dry_run: true) + context "when dry run is enabled for rspec" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:integration_options) { {service_name: "lspec", dry_run_enabled: true} } + end + + it "instruments test session" do + rspec_session_run(dry_run: true) - expect(test_session_span).to be_nil - expect(test_spans).to be_empty + expect(test_session_span).not_to be_nil + expect(test_spans).not_to be_empty + end end end diff --git a/spec/datadog/ci/test_optimisation/component_spec.rb b/spec/datadog/ci/test_optimisation/component_spec.rb index 44dad042..3393ed96 100644 --- a/spec/datadog/ci/test_optimisation/component_spec.rb +++ b/spec/datadog/ci/test_optimisation/component_spec.rb @@ -77,7 +77,8 @@ fetch_skippable_tests: instance_double( Datadog::CI::TestOptimisation::Skippable::Response, correlation_id: "42", - tests: Set.new(["suite.test.", "suite.test2."]) + tests: Set.new(["suite.test.", "suite.test2."]), + ok?: true ) ) end @@ -274,7 +275,8 @@ fetch_skippable_tests: instance_double( Datadog::CI::TestOptimisation::Skippable::Response, correlation_id: "42", - tests: Set.new(["suite.test.", "suite2.test.", "suite.test3."]) + tests: Set.new(["suite.test.", "suite2.test.", "suite.test3."]), + ok?: true ) ) end @@ -349,6 +351,13 @@ .not_to change { component.skipped_tests_count } end + it "increments total tests count" do + expect { subject } + .to change { component.total_tests_count } + .from(0) + .to(1) + end + it_behaves_like "emits no metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_SKIPPED end @@ -365,6 +374,12 @@ .from(0) .to(1) end + it "increments total tests count" do + expect { subject } + .to change { component.total_tests_count } + .from(0) + .to(1) + end it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_SKIPPED, 1 end @@ -381,6 +396,13 @@ .not_to change { component.skipped_tests_count } end + it "increments total tests count" do + expect { subject } + .to change { component.total_tests_count } + .from(0) + .to(1) + end + it_behaves_like "emits no metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_SKIPPED end end diff --git a/spec/datadog/ci/test_optimisation/skippable_percentage/calculator_spec.rb b/spec/datadog/ci/test_optimisation/skippable_percentage/calculator_spec.rb new file mode 100644 index 00000000..60b60fee --- /dev/null +++ b/spec/datadog/ci/test_optimisation/skippable_percentage/calculator_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require_relative "../../../../../lib/datadog/ci/test_optimisation/skippable_percentage/calculator" + +RSpec.describe Datadog::CI::TestOptimisation::SkippablePercentage::Calculator do + let(:rspec_cli_options) { [] } + let(:verbose) { false } + let(:spec_path) { "spec/datadog/ci/test_optimisation/skippable_percentage/fixture_spec" } + + let(:calculator) { described_class.new(rspec_cli_options: rspec_cli_options, verbose: verbose, spec_path: spec_path) } + + before do + FileUtils.mkdir_p(spec_path) + + File.write(File.join(spec_path, "first_spec.rb"), <<~SPEC) + RSpec.describe 'FirstSpec' do + it 'test 1' do + end + + it 'test 2' do + end + + it 'test 3' do + end + end + SPEC + end + + after do + FileUtils.rm_rf(spec_path) + end + + describe "#call" do + subject(:call) { calculator.call } + + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:itr_enabled) { true } + let(:tests_skipping_enabled) { true } + + let(:itr_skippable_tests) do + Set.new([ + "FirstSpec at ./spec/datadog/ci/test_optimisation/skippable_percentage/fixture_spec/first_spec.rb.test 1.{\"arguments\":{},\"metadata\":{\"scoped_id\":\"1:1\"}}", + "FirstSpec at ./spec/datadog/ci/test_optimisation/skippable_percentage/fixture_spec/first_spec.rb.test 7.{\"arguments\":{},\"metadata\":{\"scoped_id\":\"1:1\"}}" + ]) + end + end + + it "returns the skippable percentage" do + with_new_rspec_environment do + expect(call).to be_within(0.01).of(0.33) + expect(calculator.failed).to be false + end + end + end +end diff --git a/spec/datadog/ci/test_optimisation/skippable_percentage/estimator_spec.rb b/spec/datadog/ci/test_optimisation/skippable_percentage/estimator_spec.rb new file mode 100644 index 00000000..d5a5a9c6 --- /dev/null +++ b/spec/datadog/ci/test_optimisation/skippable_percentage/estimator_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require_relative "../../../../../lib/datadog/ci/test_optimisation/skippable_percentage/estimator" + +RSpec.describe Datadog::CI::TestOptimisation::SkippablePercentage::Estimator do + let(:rspec_cli_options) { [] } + let(:verbose) { false } + let(:spec_path) { "spec/datadog/ci/test_optimisation/skippable_percentage/fixture_spec" } + + let(:calculator) { described_class.new(verbose: verbose, spec_path: spec_path) } + + before do + FileUtils.mkdir_p(spec_path) + + File.write(File.join(spec_path, "first_spec.rb"), <<~SPEC) + RSpec.describe 'FirstSpec' do + it 'test 1' do + end + + scenario 'test 2' do + end + + it 'test 3' do + end + + something_else 'test 4' do + end + end + SPEC + end + + after do + FileUtils.rm_rf(spec_path) + end + + describe "#call" do + subject(:call) { calculator.call } + + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:itr_enabled) { true } + let(:tests_skipping_enabled) { true } + + let(:itr_skippable_tests) do + Set.new([ + "FirstSpec at ./spec/datadog/ci/test_optimisation/skippable_percentage/fixture_spec/first_spec.rb.test 1.{\"arguments\":{},\"metadata\":{\"scoped_id\":\"1:1\"}}", + "FirstSpec at ./spec/datadog/ci/test_optimisation/skippable_percentage/fixture_spec/first_spec.rb.test 7.{\"arguments\":{},\"metadata\":{\"scoped_id\":\"1:1\"}}" + ]) + end + end + + it "returns the skippable percentage" do + with_new_rspec_environment do + expect(call).to be_within(0.01).of(0.66) + expect(calculator.failed).to be false + end + end + end +end diff --git a/spec/datadog/ci/test_optimisation/skippable_spec.rb b/spec/datadog/ci/test_optimisation/skippable_spec.rb index e00c9ce4..72af93db 100644 --- a/spec/datadog/ci/test_optimisation/skippable_spec.rb +++ b/spec/datadog/ci/test_optimisation/skippable_spec.rb @@ -106,6 +106,7 @@ expect(response.ok?).to be true expect(response.correlation_id).to eq("correlation_id_123") expect(response.tests).to eq(Set.new(["test_suite_name.test_name.string"])) + expect(response.error_message).to be_nil end it_behaves_like "emits telemetry metric", :inc, "itr_skippable_tests.request", 1 @@ -118,7 +119,7 @@ double( "http_response", ok?: false, - payload: "", + payload: "not authorized", request_compressed: false, duration_ms: 1.2, gzipped_content?: false, @@ -132,6 +133,7 @@ expect(response.ok?).to be false expect(response.correlation_id).to be_nil expect(response.tests).to be_empty + expect(response.error_message).to eq("Status code: 422, response: not authorized") end it_behaves_like "emits telemetry metric", :inc, "itr_skippable_tests.request_errors", 1 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7ff65b61..817e5c85 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -15,6 +15,7 @@ require_relative "support/platform_helpers" require_relative "support/synchronization_helpers" require_relative "support/file_helpers" +require_relative "support/rspec_helpers" # shared contexts require_relative "support/contexts/ci_mode" @@ -66,6 +67,7 @@ def self.load_plugins config.include SpanHelpers config.include SynchronizationHelpers config.include FileHelpers + config.include RSpecHelpers # Enable flags like --only-failures and --next-failure config.example_status_persistence_file_path = ".rspec_status" diff --git a/spec/support/rspec_helpers.rb b/spec/support/rspec_helpers.rb new file mode 100644 index 00000000..98e0e78d --- /dev/null +++ b/spec/support/rspec_helpers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module RSpecHelpers + # Yields to a block in a new RSpec global context. All RSpec + # test configuration and execution should be wrapped in this method. + def with_new_rspec_environment + old_configuration = ::RSpec.configuration + old_world = ::RSpec.world + ::RSpec.configuration = ::RSpec::Core::Configuration.new + ::RSpec.world = ::RSpec::Core::World.new + + yield + ensure + ::RSpec.configuration = old_configuration + ::RSpec.world = old_world + end + + def devnull + File.new("/dev/null", "w") + end +end diff --git a/vendor/rbs/rspec/0/rspec.rbs b/vendor/rbs/rspec/0/rspec.rbs index 0e82dd72..f4f228ca 100644 --- a/vendor/rbs/rspec/0/rspec.rbs +++ b/vendor/rbs/rspec/0/rspec.rbs @@ -28,7 +28,14 @@ module RSpec::Core::Example def finish: (untyped) -> void end -module RSpec::Core::Runner +class RSpec::Core::ConfigurationOptions + def initialize: (Array[String] args) -> void +end + +class RSpec::Core::Runner + def initialize: (RSpec::Core::ConfigurationOptions configuration) -> void + + def run: (untyped stdout, untyped stderr) -> Integer def run_specs: (untyped example_groups) -> untyped end