diff --git a/.github/workflows/datadog-sca.yml b/.github/workflows/datadog-sca.yml index 7e7b1d23..0a1d7bcb 100644 --- a/.github/workflows/datadog-sca.yml +++ b/.github/workflows/datadog-sca.yml @@ -20,6 +20,6 @@ jobs: with: dd_api_key: ${{ secrets.DD_API_KEY }} dd_app_key: ${{ secrets.DD_APP_KEY }} - dd_service: my-app + dd_service: datadog-ci-rb dd_env: ci dd_site: datadoghq.com diff --git a/ext/datadog_cov/datadog_cov.c b/ext/datadog_cov/datadog_cov.c index db76d28d..dab33db2 100644 --- a/ext/datadog_cov/datadog_cov.c +++ b/ext/datadog_cov/datadog_cov.c @@ -1,13 +1,26 @@ #include #include +#include + +#include + +// This is a native extension that collects a list of Ruby files that were executed during the test run. +// It is used to optimize the test suite by running only the tests that are affected by the changes. #define PROFILE_FRAMES_BUFFER_SIZE 1 // threading modes -#define SINGLE_THREADED_COVERAGE_MODE 0 -#define MULTI_THREADED_COVERAGE_MODE 1 +enum threading_mode +{ + single, + multi +}; + +// functions declarations +static void on_newobj_event(VALUE tracepoint_data, void *data); -char *ruby_strndup(const char *str, size_t size) +// utility functions +static char *ruby_strndup(const char *str, size_t size) { char *dup; @@ -18,30 +31,84 @@ char *ruby_strndup(const char *str, size_t size) return dup; } +// Equivalent to Ruby "begin/rescue nil" call, where we call a C function and +// swallow the exception if it occurs - const_source_location often fails with +// exceptions for classes that are defined in C or for anonymous classes. +static VALUE rescue_nil(VALUE (*function_to_call_safely)(VALUE), VALUE function_to_call_safely_arg) +{ + int exception_state; + // rb_protect sets exception_state to non-zero if an exception occurs + // see https://github.com/ruby/ruby/blob/3219ecf4f659908674f534491d8934ba54e1143d/include/ruby/internal/intern/proc.h#L349 + VALUE result = rb_protect(function_to_call_safely, function_to_call_safely_arg, &exception_state); + if (exception_state != 0) + { + return Qnil; + } + + return result; +} + +static int mark_key_for_gc_i(st_data_t key, st_data_t _value, st_data_t _data) +{ + VALUE klass = (VALUE)key; + // mark klass link for GC as non-movable to avoid changing hashtable's keys + rb_gc_mark(klass); + return ST_CONTINUE; +} + // Data structure struct dd_cov_data { + // Ruby hash with filenames impacted by the test. + VALUE impacted_files; + + // Root is the path to the root folder of the project under test. + // Files located outside of the root are ignored. char *root; long root_len; + // Ignored path contains path to the folder where bundled gems are located if + // gems are installed in the project folder. char *ignored_path; long ignored_path_len; - VALUE coverage; - + // Line tracepoint optimisation: cache last seen filename pointer to avoid + // unnecessary string comparison if we stay in the same file. uintptr_t last_filename_ptr; + // Line tracepoint can work in two modes: single threaded and multi threaded + // + // In single threaded mode line tracepoint will only cover the thread that started the coverage. + // This mode is useful for testing frameworks that run tests in multiple threads. + // Do not use single threaded mode for Rails applications unless you know that you + // don't run any background threads. + // + // In multi threaded mode line tracepoint will cover all threads. This mode is enabled by default + // and is recommended for most applications. + enum threading_mode threading_mode; // for single threaded mode: thread that is being covered VALUE th_covered; - int threading_mode; + // Allocation tracing is used to track test impact for objects that do not + // contain any methods that could be covered by line tracepoint. + // + // Allocation tracing works only in multi threaded mode. + VALUE object_allocation_tracepoint; + st_table *klasses_table; // { (VALUE) -> int } hashmap with class names that were covered by allocation during the test run }; static void dd_cov_mark(void *ptr) { struct dd_cov_data *dd_cov_data = ptr; - rb_gc_mark_movable(dd_cov_data->coverage); + rb_gc_mark_movable(dd_cov_data->impacted_files); rb_gc_mark_movable(dd_cov_data->th_covered); + rb_gc_mark_movable(dd_cov_data->object_allocation_tracepoint); + + // if GC starts withing dd_cov_allocate() call, klasses_table might not be initialized yet + if (dd_cov_data->klasses_table != NULL) + { + st_foreach(dd_cov_data->klasses_table, mark_key_for_gc_i, 0); + } } static void dd_cov_free(void *ptr) @@ -49,17 +116,20 @@ static void dd_cov_free(void *ptr) struct dd_cov_data *dd_cov_data = ptr; xfree(dd_cov_data->root); xfree(dd_cov_data->ignored_path); + st_free_table(dd_cov_data->klasses_table); xfree(dd_cov_data); } static void dd_cov_compact(void *ptr) { struct dd_cov_data *dd_cov_data = ptr; - dd_cov_data->coverage = rb_gc_location(dd_cov_data->coverage); + dd_cov_data->impacted_files = rb_gc_location(dd_cov_data->impacted_files); dd_cov_data->th_covered = rb_gc_location(dd_cov_data->th_covered); + dd_cov_data->object_allocation_tracepoint = rb_gc_location(dd_cov_data->object_allocation_tracepoint); + // keys for dd_cov_data->klasses_table are not moved by GC, so we don't need to update them } -const rb_data_type_t dd_cov_data_type = { +static const rb_data_type_t dd_cov_data_type = { .wrap_struct_name = "dd_cov", .function = { .dmark = dd_cov_mark, @@ -71,64 +141,48 @@ const rb_data_type_t dd_cov_data_type = { static VALUE dd_cov_allocate(VALUE klass) { struct dd_cov_data *dd_cov_data; - VALUE obj = TypedData_Make_Struct(klass, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); + VALUE dd_cov = TypedData_Make_Struct(klass, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); - dd_cov_data->coverage = rb_hash_new(); + dd_cov_data->impacted_files = rb_hash_new(); dd_cov_data->root = NULL; dd_cov_data->root_len = 0; dd_cov_data->ignored_path = NULL; dd_cov_data->ignored_path_len = 0; dd_cov_data->last_filename_ptr = 0; - dd_cov_data->threading_mode = MULTI_THREADED_COVERAGE_MODE; + dd_cov_data->threading_mode = multi; - return obj; -} + dd_cov_data->object_allocation_tracepoint = Qnil; + // numtable type is needed to store VALUE as a key + dd_cov_data->klasses_table = st_init_numtable(); -// DDCov methods -static VALUE dd_cov_initialize(int argc, VALUE *argv, VALUE self) -{ - VALUE opt; + return dd_cov; +} - rb_scan_args(argc, argv, "10", &opt); - VALUE rb_root = rb_hash_lookup(opt, ID2SYM(rb_intern("root"))); - if (!RTEST(rb_root)) - { - rb_raise(rb_eArgError, "root is required"); - } - VALUE rb_ignored_path = rb_hash_lookup(opt, ID2SYM(rb_intern("ignored_path"))); +// Helper functions (available in C only) - VALUE rb_threading_mode = rb_hash_lookup(opt, ID2SYM(rb_intern("threading_mode"))); - int threading_mode; - if (rb_threading_mode == ID2SYM(rb_intern("multi"))) - { - threading_mode = MULTI_THREADED_COVERAGE_MODE; - } - else if (rb_threading_mode == ID2SYM(rb_intern("single"))) - { - threading_mode = SINGLE_THREADED_COVERAGE_MODE; - } - else +// Checks if the filename is located under the root folder of the project (but not +// in the ignored folder) and adds it to the impacted_files hash. +static void record_impacted_file(struct dd_cov_data *dd_cov_data, VALUE filename) +{ + char *filename_ptr = RSTRING_PTR(filename); + // if the current filename is not located under the root, we skip it + if (strncmp(dd_cov_data->root, filename_ptr, dd_cov_data->root_len) != 0) { - rb_raise(rb_eArgError, "threading mode is invalid"); + return; } - struct dd_cov_data *dd_cov_data; - TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); - - dd_cov_data->threading_mode = threading_mode; - dd_cov_data->root_len = RSTRING_LEN(rb_root); - dd_cov_data->root = ruby_strndup(RSTRING_PTR(rb_root), dd_cov_data->root_len); - - if (RTEST(rb_ignored_path)) + // if ignored_path is provided and the current filename is located under the ignored_path, we skip it too + // this is useful for ignoring bundled gems location + if (dd_cov_data->ignored_path_len != 0 && strncmp(dd_cov_data->ignored_path, filename_ptr, dd_cov_data->ignored_path_len) == 0) { - dd_cov_data->ignored_path_len = RSTRING_LEN(rb_ignored_path); - dd_cov_data->ignored_path = ruby_strndup(RSTRING_PTR(rb_ignored_path), dd_cov_data->ignored_path_len); + return; } - return Qnil; + rb_hash_aset(dd_cov_data->impacted_files, filename, Qtrue); } -static void dd_cov_update_coverage(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) +// Executed on RUBY_EVENT_LINE event and captures the filename from rb_profile_frames. +static void on_line_event(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) { struct dd_cov_data *dd_cov_data; TypedData_Get_Struct(data, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); @@ -161,54 +215,182 @@ static void dd_cov_update_coverage(rb_event_flag_t event, VALUE data, VALUE self return; } - char *filename_ptr = RSTRING_PTR(filename); - // if the current filename is not located under the root, we skip it - if (strncmp(dd_cov_data->root, filename_ptr, dd_cov_data->root_len) != 0) + record_impacted_file(dd_cov_data, filename); +} + +// Get source location for a given class name +static VALUE get_source_location(VALUE klass_name) +{ + return rb_funcall(rb_cObject, rb_intern("const_source_location"), 1, klass_name); +} + +// Get source location for a given class name and swallow any exceptions +static VALUE safely_get_source_location(VALUE klass_name) +{ + return rescue_nil(get_source_location, klass_name); +} + +// This function is called for each class that was instantiated during the test run. +static int process_instantiated_klass(st_data_t key, st_data_t _value, st_data_t data) +{ + VALUE klass = (VALUE)key; + struct dd_cov_data *dd_cov_data = (struct dd_cov_data *)data; + + VALUE klass_name = rb_class_name(klass); + if (klass_name == Qnil) + { + return ST_CONTINUE; + } + + VALUE source_location = safely_get_source_location(klass_name); + if (source_location == Qnil || RARRAY_LEN(source_location) == 0) + { + return ST_CONTINUE; + } + + VALUE filename = RARRAY_AREF(source_location, 0); + if (filename == Qnil) + { + return ST_CONTINUE; + } + + record_impacted_file(dd_cov_data, filename); + return ST_CONTINUE; +} + +// Executed on RUBY_INTERNAL_EVENT_NEWOBJ event and captures the source file for the +// allocated object's class. +static void on_newobj_event(VALUE tracepoint_data, void *data) +{ + rb_trace_arg_t *tracearg = rb_tracearg_from_tracepoint(tracepoint_data); + VALUE new_object = rb_tracearg_object(tracearg); + + // To keep things fast and practical, we only care about objects that extend + // either Object or Struct. + enum ruby_value_type type = rb_type(new_object); + if (type != RUBY_T_OBJECT && type != RUBY_T_STRUCT) { return; } - // if ignored_path is provided and the current filename is located under the ignored_path, we skip it too - // this is useful for ignoring bundled gems location - if (dd_cov_data->ignored_path_len != 0 && strncmp(dd_cov_data->ignored_path, filename_ptr, dd_cov_data->ignored_path_len) == 0) + VALUE klass = rb_class_of(new_object); + if (klass == Qnil || klass == 0) + { + return; + } + // Skip anonymous classes starting with "#coverage, filename, Qtrue); + struct dd_cov_data *dd_cov_data = (struct dd_cov_data *)data; + + // We use VALUE directly as a key for the hashmap + // Ruby itself does it too: + // https://github.com/ruby/ruby/blob/94b87084a689a3bc732dcaee744508a708223d6c/ext/objspace/object_tracing.c#L113 + st_insert(dd_cov_data->klasses_table, (st_data_t)klass, 1); } -static VALUE dd_cov_start(VALUE self) +// DDCov instance methods available in Ruby +static VALUE dd_cov_initialize(int argc, VALUE *argv, VALUE self) { + VALUE opt; + + rb_scan_args(argc, argv, "10", &opt); + VALUE rb_root = rb_hash_lookup(opt, ID2SYM(rb_intern("root"))); + if (!RTEST(rb_root)) + { + rb_raise(rb_eArgError, "root is required"); + } + VALUE rb_ignored_path = rb_hash_lookup(opt, ID2SYM(rb_intern("ignored_path"))); + + VALUE rb_threading_mode = rb_hash_lookup(opt, ID2SYM(rb_intern("threading_mode"))); + enum threading_mode threading_mode; + if (rb_threading_mode == ID2SYM(rb_intern("multi"))) + { + threading_mode = multi; + } + else if (rb_threading_mode == ID2SYM(rb_intern("single"))) + { + threading_mode = single; + } + else + { + rb_raise(rb_eArgError, "threading mode is invalid"); + } + + VALUE rb_allocation_tracing_enabled = rb_hash_lookup(opt, ID2SYM(rb_intern("use_allocation_tracing"))); + if (rb_allocation_tracing_enabled == Qtrue && threading_mode == single) + { + rb_raise(rb_eArgError, "allocation tracing is not supported in single threaded mode"); + } struct dd_cov_data *dd_cov_data; TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); + dd_cov_data->threading_mode = threading_mode; + dd_cov_data->root_len = RSTRING_LEN(rb_root); + dd_cov_data->root = ruby_strndup(RSTRING_PTR(rb_root), dd_cov_data->root_len); + + if (RTEST(rb_ignored_path)) + { + dd_cov_data->ignored_path_len = RSTRING_LEN(rb_ignored_path); + dd_cov_data->ignored_path = ruby_strndup(RSTRING_PTR(rb_ignored_path), dd_cov_data->ignored_path_len); + } + + if (rb_allocation_tracing_enabled == Qtrue) + { + dd_cov_data->object_allocation_tracepoint = rb_tracepoint_new(Qnil, RUBY_INTERNAL_EVENT_NEWOBJ, on_newobj_event, (void *)dd_cov_data); + } + + return Qnil; +} + +// starts test impact collection, executed before the start of each test +static VALUE dd_cov_start(VALUE self) +{ + struct dd_cov_data *dd_cov_data; + TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); + if (dd_cov_data->root_len == 0) { rb_raise(rb_eRuntimeError, "root is required"); } - if (dd_cov_data->threading_mode == SINGLE_THREADED_COVERAGE_MODE) + // add line tracepoint + if (dd_cov_data->threading_mode == single) { VALUE thval = rb_thread_current(); - rb_thread_add_event_hook(thval, dd_cov_update_coverage, RUBY_EVENT_LINE, self); + rb_thread_add_event_hook(thval, on_line_event, RUBY_EVENT_LINE, self); dd_cov_data->th_covered = thval; } else { - rb_add_event_hook(dd_cov_update_coverage, RUBY_EVENT_LINE, self); + rb_add_event_hook(on_line_event, RUBY_EVENT_LINE, self); + } + + // add object allocation tracepoint + if (dd_cov_data->object_allocation_tracepoint != Qnil) + { + rb_tracepoint_enable(dd_cov_data->object_allocation_tracepoint); } return self; } +// stops test impact collection, executed after the end of each test +// returns the hash with impacted files and resets the internal state static VALUE dd_cov_stop(VALUE self) { struct dd_cov_data *dd_cov_data; TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); - if (dd_cov_data->threading_mode == SINGLE_THREADED_COVERAGE_MODE) + // stop line tracepoint + if (dd_cov_data->threading_mode == single) { VALUE thval = rb_thread_current(); if (!rb_equal(thval, dd_cov_data->th_covered)) @@ -216,17 +398,27 @@ static VALUE dd_cov_stop(VALUE self) rb_raise(rb_eRuntimeError, "Coverage was not started by this thread"); } - rb_thread_remove_event_hook(dd_cov_data->th_covered, dd_cov_update_coverage); + rb_thread_remove_event_hook(dd_cov_data->th_covered, on_line_event); dd_cov_data->th_covered = Qnil; } else { - rb_remove_event_hook(dd_cov_update_coverage); + rb_remove_event_hook(on_line_event); } - VALUE res = dd_cov_data->coverage; + // stop object allocation tracepoint + if (dd_cov_data->object_allocation_tracepoint != Qnil) + { + rb_tracepoint_disable(dd_cov_data->object_allocation_tracepoint); + } + + // process classes covered by allocation tracing + st_foreach(dd_cov_data->klasses_table, process_instantiated_klass, (st_data_t)dd_cov_data); + st_clear(dd_cov_data->klasses_table); + + VALUE res = dd_cov_data->impacted_files; - dd_cov_data->coverage = rb_hash_new(); + dd_cov_data->impacted_files = rb_hash_new(); dd_cov_data->last_filename_ptr = 0; return res; diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 26559b0f..c1ac872c 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -100,15 +100,7 @@ def activate_ci!(settings) settings.tracing.test_mode.writer_options = trace_writer_options # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component - @test_optimisation = TestOptimisation::Component.new( - api: test_visibility_api, - dd_env: settings.env, - config_tags: custom_configuration(settings), - coverage_writer: build_coverage_writer(settings, test_visibility_api), - enabled: settings.ci.enabled && settings.ci.itr_enabled, - bundle_location: settings.ci.itr_code_coverage_excluded_bundle_path, - use_single_threaded_coverage: settings.ci.itr_code_coverage_use_single_threaded_mode - ) + @test_optimisation = build_test_optimisation(settings, test_visibility_api) @test_visibility = TestVisibility::Component.new( test_optimisation: @test_optimisation, @@ -118,6 +110,42 @@ def activate_ci!(settings) ) end + def build_test_optimisation(settings, test_visibility_api) + if settings.ci.itr_code_coverage_use_single_threaded_mode && + settings.ci.itr_test_impact_analysis_use_allocation_tracing + Datadog.logger.warn( + "Intelligent test runner: Single threaded coverage mode is incompatible with allocation tracing. " \ + "Allocation tracing will be disabled. It means that test impact analysis will not be able to detect " \ + "instantiations of objects in your code, which is important for ActiveRecord models. " \ + "Please add your app/model folder to the list of tracked files or disable single threaded coverage mode." + ) + + settings.ci.itr_test_impact_analysis_use_allocation_tracing = false + end + + if RUBY_VERSION.start_with?("3.2.") && RUBY_VERSION < "3.2.3" && + settings.ci.itr_test_impact_analysis_use_allocation_tracing + Datadog.logger.warn( + "Intelligent test runner: Allocation tracing is not supported in Ruby versions 3.2.0, 3.2.1 and 3.2.2 and will be forcibly " \ + "disabled. This is due to a VM bug that can lead to crashes (https://bugs.ruby-lang.org/issues/19482). " \ + "Please update your Ruby version or add your app/model folder to the list of tracked files." \ + "Set env variable DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING to 0 to disable this warning." + ) + settings.ci.itr_test_impact_analysis_use_allocation_tracing = false + end + + TestOptimisation::Component.new( + api: test_visibility_api, + dd_env: settings.env, + config_tags: custom_configuration(settings), + coverage_writer: build_coverage_writer(settings, test_visibility_api), + enabled: settings.ci.enabled && settings.ci.itr_enabled, + bundle_location: settings.ci.itr_code_coverage_excluded_bundle_path, + use_single_threaded_coverage: settings.ci.itr_code_coverage_use_single_threaded_mode, + use_allocation_tracing: settings.ci.itr_test_impact_analysis_use_allocation_tracing + ) + end + def build_test_visibility_api(settings) if settings.ci.agentless_mode_enabled check_dd_site(settings) diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index 5caa6f41..d542d27d 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -82,6 +82,12 @@ def self.add_settings!(base) o.default false end + option :itr_test_impact_analysis_use_allocation_tracing do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING + o.default true + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index 37b1e653..9e9f4278 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -14,6 +14,7 @@ module Settings ENV_GIT_METADATA_UPLOAD_ENABLED = "DD_CIVISIBILITY_GIT_METADATA_UPLOAD_ENABLED" ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH" ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE" + ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING = "DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING" # Source: https://docs.datadoghq.com/getting_started/site/ DD_SITE_ALLOWLIST = %w[ diff --git a/lib/datadog/ci/test_optimisation/component.rb b/lib/datadog/ci/test_optimisation/component.rb index f70c1e47..ec39d93b 100644 --- a/lib/datadog/ci/test_optimisation/component.rb +++ b/lib/datadog/ci/test_optimisation/component.rb @@ -32,7 +32,8 @@ def initialize( coverage_writer: nil, enabled: false, bundle_location: nil, - use_single_threaded_coverage: false + use_single_threaded_coverage: false, + use_allocation_tracing: true ) @enabled = enabled @api = api @@ -45,6 +46,7 @@ def initialize( bundle_location end @use_single_threaded_coverage = use_single_threaded_coverage + @use_allocation_tracing = use_allocation_tracing @test_skipping_enabled = false @code_coverage_enabled = false @@ -189,7 +191,8 @@ def coverage_collector Thread.current[:dd_coverage_collector] ||= Coverage::DDCov.new( root: Git::LocalRepository.root, ignored_path: @bundle_location, - threading_mode: code_coverage_mode + threading_mode: code_coverage_mode, + use_allocation_tracing: @use_allocation_tracing ) end diff --git a/sig/datadog/ci/configuration/components.rbs b/sig/datadog/ci/configuration/components.rbs index 4b7d014d..52578e9d 100644 --- a/sig/datadog/ci/configuration/components.rbs +++ b/sig/datadog/ci/configuration/components.rbs @@ -13,6 +13,8 @@ module Datadog def activate_ci!: (untyped settings) -> untyped + def build_test_optimisation: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::TestOptimisation::Component + def build_test_visibility_api: (untyped settings) -> Datadog::CI::Transport::Api::Base? def serializers_factory: (untyped settings) -> (singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel)) diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index 841134d2..a8fa513e 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -11,6 +11,7 @@ module Datadog ENV_GIT_METADATA_UPLOAD_ENABLED: String ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH: String ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE: String + ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING: String DD_SITE_ALLOWLIST: Array[String] end diff --git a/sig/datadog/ci/test_optimisation/component.rbs b/sig/datadog/ci/test_optimisation/component.rbs index a9e016ab..f254edc5 100644 --- a/sig/datadog/ci/test_optimisation/component.rbs +++ b/sig/datadog/ci/test_optimisation/component.rbs @@ -14,8 +14,10 @@ module Datadog @api: Datadog::CI::Transport::Api::Base? @dd_env: String? @config_tags: Hash[String, String] + @bundle_location: String? @use_single_threaded_coverage: bool + @use_allocation_tracing: bool @skipped_tests_count: Integer @mutex: Thread::Mutex @@ -24,7 +26,7 @@ module Datadog attr_reader skipped_tests_count: Integer attr_reader correlation_id: String? - def initialize: (dd_env: String?, ?enabled: bool, ?coverage_writer: Datadog::CI::TestOptimisation::Coverage::Writer?, ?api: Datadog::CI::Transport::Api::Base?, ?config_tags: Hash[String, String]?, ?bundle_location: String?, ?use_single_threaded_coverage: bool) -> void + def initialize: (dd_env: String?, ?enabled: bool, ?coverage_writer: Datadog::CI::TestOptimisation::Coverage::Writer?, ?api: Datadog::CI::Transport::Api::Base?, ?config_tags: Hash[String, String]?, ?bundle_location: String?, ?use_single_threaded_coverage: bool, ?use_allocation_tracing: bool) -> void def configure: (Hash[String, untyped] remote_configuration, test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void diff --git a/sig/datadog/ci/test_optimisation/coverage/ddcov.rbs b/sig/datadog/ci/test_optimisation/coverage/ddcov.rbs index 0b5c9c56..59ee5219 100644 --- a/sig/datadog/ci/test_optimisation/coverage/ddcov.rbs +++ b/sig/datadog/ci/test_optimisation/coverage/ddcov.rbs @@ -5,7 +5,7 @@ module Datadog class DDCov type threading_mode = :multi | :single - def initialize: (root: String, ignored_path: String?, threading_mode: threading_mode) -> void + def initialize: (root: String, ignored_path: String?, threading_mode: threading_mode, use_allocation_tracing: bool) -> void def start: () -> void diff --git a/spec/datadog/ci/configuration/components_spec.rb b/spec/datadog/ci/configuration/components_spec.rb index 7b077a9b..d1677467 100644 --- a/spec/datadog/ci/configuration/components_spec.rb +++ b/spec/datadog/ci/configuration/components_spec.rb @@ -43,6 +43,8 @@ settings.ci.force_test_level_visibility = force_test_level_visibility settings.ci.agentless_url = agentless_url settings.ci.itr_enabled = itr_enabled + settings.ci.itr_code_coverage_use_single_threaded_mode = itr_code_coverage_use_single_threaded_mode + settings.ci.itr_test_impact_analysis_use_allocation_tracing = itr_test_impact_analysis_use_allocation_tracing settings.site = dd_site settings.api_key = api_key @@ -97,6 +99,8 @@ let(:evp_proxy_v4_supported) { false } let(:itr_enabled) { false } let(:tracing_enabled) { true } + let(:itr_code_coverage_use_single_threaded_mode) { false } + let(:itr_test_impact_analysis_use_allocation_tracing) { true } context "is enabled" do let(:enabled) { true } @@ -249,6 +253,17 @@ it "creates test visibility component with ITR enabled" do expect(components.test_visibility.itr_enabled?).to eq(true) + expect(settings.ci.itr_test_impact_analysis_use_allocation_tracing).to eq(true) + end + + context "when single threaded mode for line coverage is enabled" do + let(:itr_code_coverage_use_single_threaded_mode) { true } + + it "logs a warning and disables allocation tracing for ITR" do + expect(Datadog.logger).to have_received(:warn) + + expect(settings.ci.itr_test_impact_analysis_use_allocation_tracing).to eq(false) + end end end end diff --git a/spec/datadog/ci/configuration/settings_spec.rb b/spec/datadog/ci/configuration/settings_spec.rb index 8716cf35..960d2260 100644 --- a/spec/datadog/ci/configuration/settings_spec.rb +++ b/spec/datadog/ci/configuration/settings_spec.rb @@ -349,6 +349,47 @@ def patcher end end + describe "#itr_test_impact_analysis_use_allocation_tracing" do + subject(:itr_test_impact_analysis_use_allocation_tracing) { settings.ci.itr_test_impact_analysis_use_allocation_tracing } + + it { is_expected.to be true } + + context "when #{Datadog::CI::Ext::Settings::ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING}" do + around do |example| + ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING => enable) do + example.run + end + end + + context "is not defined" do + let(:enable) { nil } + + it { is_expected.to be true } + end + + context "is set to true" do + let(:enable) { "true" } + + it { is_expected.to be true } + end + + context "is set to false" do + let(:enable) { "false" } + + it { is_expected.to be false } + end + end + end + + describe "#itr_test_impact_analysis_use_allocation_tracing=" do + it "updates the #enabled setting" do + expect { settings.ci.itr_test_impact_analysis_use_allocation_tracing = false } + .to change { settings.ci.itr_test_impact_analysis_use_allocation_tracing } + .from(true) + .to(false) + end + end + describe "#instrument" do let(:integration_name) { :fake } diff --git a/spec/datadog/ci/contrib/minitest/helpers/simple_model.rb b/spec/datadog/ci/contrib/minitest/helpers/simple_model.rb new file mode 100644 index 00000000..b2b549ab --- /dev/null +++ b/spec/datadog/ci/contrib/minitest/helpers/simple_model.rb @@ -0,0 +1,2 @@ +class SimpleModel +end diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb index 1d79f697..76e8f89c 100644 --- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb @@ -403,12 +403,15 @@ def test_foo Minitest::Runnable.reset require_relative "helpers/addition_helper" + require_relative "helpers/simple_model" class SomeTest < Minitest::Test def test_pass assert true end def test_pass_other + # make sure that allocating objects is covered + SimpleModel.new # add thread to test that code coverage is collected t = Thread.new do AdditionHelper.add(1, 2) @@ -504,6 +507,7 @@ def test_pass_other test_span = test_spans.find { |span| span.get_tag("test.name") == "test_pass_other" } cov_event = find_coverage_for_test(test_span) expect(cov_event.coverage.keys).to include(absolute_path("helpers/addition_helper.rb")) + expect(cov_event.coverage.keys).to include(absolute_path("helpers/simple_model.rb")) end context "when test optimisation skips tests" do diff --git a/spec/datadog/ci/test_optimisation/runner_spec.rb b/spec/datadog/ci/test_optimisation/component_spec.rb similarity index 100% rename from spec/datadog/ci/test_optimisation/runner_spec.rb rename to spec/datadog/ci/test_optimisation/component_spec.rb diff --git a/spec/ddcov/app/model/measure.rb b/spec/ddcov/app/model/measure.rb new file mode 100644 index 00000000..f8b04210 --- /dev/null +++ b/spec/ddcov/app/model/measure.rb @@ -0,0 +1 @@ +Measure = Data.define(:amount, :unit) diff --git a/spec/ddcov/app/model/my_model.rb b/spec/ddcov/app/model/my_model.rb new file mode 100644 index 00000000..9757cb1b --- /dev/null +++ b/spec/ddcov/app/model/my_model.rb @@ -0,0 +1,2 @@ +class MyModel +end diff --git a/spec/ddcov/app/model/my_struct.rb b/spec/ddcov/app/model/my_struct.rb new file mode 100644 index 00000000..a22a8701 --- /dev/null +++ b/spec/ddcov/app/model/my_struct.rb @@ -0,0 +1 @@ +User = Struct.new(:name, :email) diff --git a/spec/ddcov/ddcov_spec.rb b/spec/ddcov/ddcov_spec.rb index 98b79313..98c45407 100644 --- a/spec/ddcov/ddcov_spec.rb +++ b/spec/ddcov/ddcov_spec.rb @@ -2,13 +2,24 @@ require "datadog_cov.#{RUBY_VERSION}_#{RUBY_PLATFORM}" +require_relative "app/model/my_model" +require_relative "app/model/my_struct" require_relative "calculator/calculator" require_relative "calculator/code_with_❤️" RSpec.describe Datadog::CI::TestOptimisation::Coverage::DDCov do let(:ignored_path) { nil } let(:threading_mode) { :multi } - subject { described_class.new(root: root, ignored_path: ignored_path, threading_mode: threading_mode) } + let(:use_allocation_tracing) { true } + + subject do + described_class.new( + root: root, + ignored_path: ignored_path, + threading_mode: threading_mode, + use_allocation_tracing: use_allocation_tracing + ) + end describe "code coverage collection" do let!(:calculator) { Calculator.new } @@ -155,11 +166,16 @@ context "multi threaded execution" do def thread_local_cov - Thread.current[:datadog_ci_cov] ||= described_class.new(root: root, threading_mode: threading_mode) + Thread.current[:datadog_ci_cov] ||= described_class.new( + root: root, + threading_mode: threading_mode, + use_allocation_tracing: use_allocation_tracing + ) end context "in single threaded coverage mode" do let(:threading_mode) { :single } + let(:use_allocation_tracing) { false } it "collects coverage for each thread separately" do t1_queue = Thread::Queue.new @@ -203,6 +219,16 @@ def thread_local_cov [t1, t2].each(&:join) end + + context "when allocation tracing is enabled" do + let(:use_allocation_tracing) { true } + + it "raises an error" do + expect { thread_local_cov }.to( + raise_error(ArgumentError, "allocation tracing is not supported in single threaded mode") + ) + end + end end context "in multi threaded code coverage mode" do @@ -278,5 +304,93 @@ def thread_local_cov end end end + + context "root in app folder" do + let(:root) { absolute_path("app") } + + context "allocation tracing is enabled" do + it "tracks coverage for empty model" do + subject.start + + MyModel.new + expect(calculator.add(1, 2)).to eq(3) + + coverage = subject.stop + expect(coverage.size).to eq(1) + expect(coverage.keys).to include(absolute_path("app/model/my_model.rb")) + + MyModel.new + + subject.start + coverage = subject.stop + expect(coverage.size).to eq(0) + end + + it "does not break when encountering anonymous class or internal Ruby classes implemented in C" do + subject.start + + MyModel.new + c = Class.new(Object) do + end + c.new + + # Trying to get non-existing constant could caise freezing of Ruby process when + # not safely getting source location of the constant in NEWOBJ tracepoint. + begin + Object.const_get(:fdsfdsfdsfds) + rescue + nil + end + + coverage = subject.stop + expect(coverage.size).to eq(1) + expect(coverage.keys).to include(absolute_path("app/model/my_model.rb")) + end + + it "tracks coverage for structs" do + subject.start + + User.new("john doe", "johndoe@mail.test") + + coverage = subject.stop + expect(coverage.size).to eq(1) + expect(coverage.keys).to include(absolute_path("app/model/my_struct.rb")) + end + + context "Data structs available since Ruby 3.2" do + before do + if RUBY_VERSION < "3.2" + skip + else + require_relative "app/model/measure" + end + end + + it "tracks coverage for Data structs" do + subject.start + + Measure.new(100, "km") + + coverage = subject.stop + expect(coverage.size).to eq(1) + expect(coverage.keys).to include(absolute_path("app/model/measure.rb")) + end + end + end + + context "allocation tracing is disabled" do + let(:use_allocation_tracing) { false } + + it "does not track coverage for empty model" do + subject.start + + MyModel.new + expect(calculator.add(1, 2)).to eq(3) + + coverage = subject.stop + expect(coverage.size).to eq(0) + end + end + end end end