diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index cb6fce18..4c8414f3 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "../ext/settings" +require_relative "../git/tree_uploader" require_relative "../itr/runner" require_relative "../itr/coverage/transport" require_relative "../itr/coverage/writer" @@ -12,6 +13,7 @@ require_relative "../test_visibility/transport" require_relative "../transport/api/builder" require_relative "../transport/remote_settings_api" +require_relative "../worker" module Datadog module CI @@ -35,6 +37,7 @@ def initialize(settings) def shutdown!(replacement = nil) super + @ci_recorder&.shutdown! @itr&.shutdown! end @@ -63,12 +66,12 @@ def activate_ci!(settings) if test_visibility_api # setup writer for code coverage payloads - coverage_writer = Datadog::CI::ITR::Coverage::Writer.new( - transport: Datadog::CI::ITR::Coverage::Transport.new(api: test_visibility_api) + coverage_writer = ITR::Coverage::Writer.new( + transport: ITR::Coverage::Transport.new(api: test_visibility_api) ) # configure tracing writer to send traces to CI visibility backend - writer_options[:transport] = Datadog::CI::TestVisibility::Transport.new( + writer_options[:transport] = TestVisibility::Transport.new( api: test_visibility_api, serializers_factory: serializers_factory(settings), dd_env: settings.env @@ -92,16 +95,26 @@ def activate_ci!(settings) dd_env: settings.env ) - itr = Datadog::CI::ITR::Runner.new( + itr = ITR::Runner.new( coverage_writer: coverage_writer, enabled: settings.ci.enabled && settings.ci.itr_enabled ) + git_tree_uploader = Git::TreeUploader.new(api: test_visibility_api) + git_tree_upload_worker = if settings.ci.git_metadata_upload_enabled + Worker.new do |repository_url| + git_tree_uploader.call(repository_url) + end + else + DummyWorker.new + end + # CI visibility recorder global instance @ci_recorder = TestVisibility::Recorder.new( test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility, itr: itr, - remote_settings_api: remote_settings_api + remote_settings_api: remote_settings_api, + git_tree_upload_worker: git_tree_upload_worker ) @itr = itr @@ -141,9 +154,9 @@ def build_test_visibility_api(settings) def serializers_factory(settings) if settings.ci.force_test_level_visibility - Datadog::CI::TestVisibility::Serializers::Factories::TestLevel + TestVisibility::Serializers::Factories::TestLevel else - Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel + TestVisibility::Serializers::Factories::TestSuiteLevel end end diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index be701747..64a8d0e2 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -61,6 +61,12 @@ def self.add_settings!(base) o.default false end + option :git_metadata_upload_enabled do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_GIT_METADATA_UPLOAD_ENABLED + o.default true + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index a6d16a30..9e2bfaf4 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -11,6 +11,7 @@ module Settings ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED" ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY" ENV_ITR_ENABLED = "DD_CIVISIBILITY_ITR_ENABLED" + ENV_GIT_METADATA_UPLOAD_ENABLED = "DD_CIVISIBILITY_GIT_METADATA_UPLOAD_ENABLED" # Source: https://docs.datadoghq.com/getting_started/site/ DD_SITE_ALLOWLIST = [ diff --git a/lib/datadog/ci/git/local_repository.rb b/lib/datadog/ci/git/local_repository.rb index 02dca051..4c0ac47e 100644 --- a/lib/datadog/ci/git/local_repository.rb +++ b/lib/datadog/ci/git/local_repository.rb @@ -9,6 +9,8 @@ module Datadog module CI module Git module LocalRepository + COMMAND_RETRY_COUNT = 3 + def self.root return @root if defined?(@root) @@ -195,7 +197,20 @@ def exec_git_command(cmd, stdin: nil) # no-dd-sa:ruby-security/shell-injection out, status = Open3.capture2e(cmd, stdin_data: stdin) - raise "Failed to run git command #{cmd}: #{out}" unless status.success? + if status.nil? + retry_count = COMMAND_RETRY_COUNT + Datadog.logger.debug { "Opening pipe failed, starting retries..." } + while status.nil? && retry_count.positive? + # no-dd-sa:ruby-security/shell-injection + out, status = Open3.capture2e(cmd, stdin_data: stdin) + Datadog.logger.debug { "After retry status is [#{status}]" } + retry_count -= 1 + end + end + + if status.nil? || !status.success? + raise "Failed to run git command [#{cmd}] with input [#{stdin}] and output [#{out}]" + end # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks # commit messages with UTF-8 characters like emojis @@ -213,7 +228,7 @@ def exec_git_command(cmd, stdin: nil) def log_failure(e, action) Datadog.logger.debug( - "Unable to read #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}" + "Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}" ) end end diff --git a/lib/datadog/ci/git/packfiles.rb b/lib/datadog/ci/git/packfiles.rb index 7b67877d..d4c48acf 100644 --- a/lib/datadog/ci/git/packfiles.rb +++ b/lib/datadog/ci/git/packfiles.rb @@ -60,7 +60,9 @@ def self.generate(included_commits:, excluded_commits:) rescue => e Datadog.logger.debug("Packfiles could not be generated, error: #{e}") ensure - FileUtils.remove_entry(current_process_tmp_folder) unless current_process_tmp_folder.nil? + if current_process_tmp_folder && File.exist?(current_process_tmp_folder) + FileUtils.remove_entry(current_process_tmp_folder) + end end end end diff --git a/lib/datadog/ci/git/tree_uploader.rb b/lib/datadog/ci/git/tree_uploader.rb index 36ea0142..43389090 100644 --- a/lib/datadog/ci/git/tree_uploader.rb +++ b/lib/datadog/ci/git/tree_uploader.rb @@ -69,6 +69,8 @@ def call(repository_url) Datadog.logger.debug("Packfile upload failed with #{e}") break end + ensure + Datadog.logger.debug("Git tree upload finished") end private diff --git a/lib/datadog/ci/test_visibility/null_recorder.rb b/lib/datadog/ci/test_visibility/null_recorder.rb index f43e6ee4..5a83df3f 100644 --- a/lib/datadog/ci/test_visibility/null_recorder.rb +++ b/lib/datadog/ci/test_visibility/null_recorder.rb @@ -42,6 +42,9 @@ def active_test_module def active_test_suite(test_suite_name) end + def shutdown! + end + private def skip_tracing(block = nil) diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index 8c2f8f38..6e3934c8 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -20,6 +20,7 @@ require_relative "../test_session" require_relative "../test_module" require_relative "../test_suite" +require_relative "../worker" module Datadog module CI @@ -30,7 +31,10 @@ class Recorder attr_reader :environment_tags, :test_suite_level_visibility_enabled def initialize( - itr:, remote_settings_api:, test_suite_level_visibility_enabled: false, + itr:, + remote_settings_api:, + git_tree_upload_worker: DummyWorker.new, + test_suite_level_visibility_enabled: false, codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse ) @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled @@ -43,6 +47,11 @@ def initialize( @itr = itr @remote_settings_api = remote_settings_api + @git_tree_upload_worker = git_tree_upload_worker + end + + def shutdown! + @git_tree_upload_worker.stop end def start_test_session(service: nil, tags: {}) @@ -56,6 +65,7 @@ def start_test_session(service: nil, tags: {}) test_session = build_test_session(tracer_span, tags) + @git_tree_upload_worker.perform(test_session.git_repository_url) configure_library(test_session) test_session diff --git a/lib/datadog/ci/worker.rb b/lib/datadog/ci/worker.rb new file mode 100644 index 00000000..ed879946 --- /dev/null +++ b/lib/datadog/ci/worker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require "datadog/core/worker" +require "datadog/core/workers/async" + +# general purpose async worker for CI +# executes given task once in separate thread +module Datadog + module CI + class Worker < Datadog::Core::Worker + include Datadog::Core::Workers::Async::Thread + + DEFAULT_SHUTDOWN_TIMEOUT = 60 + DEFAULT_WAIT_TIMEOUT = 60 + + def stop(timeout = DEFAULT_SHUTDOWN_TIMEOUT) + join(timeout) + end + + def wait_until_done(timeout = DEFAULT_WAIT_TIMEOUT) + join(timeout) + end + + def done? + started? && !running? + end + end + + class DummyWorker < Worker + def initialize + super { nil } + end + end + end +end diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index 2ead3eab..c8847c6c 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -8,6 +8,7 @@ module Datadog ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED: String ENV_FORCE_TEST_LEVEL_VISIBILITY: String ENV_ITR_ENABLED: String + ENV_GIT_METADATA_UPLOAD_ENABLED: String DD_SITE_ALLOWLIST: Array[String] end diff --git a/sig/datadog/ci/git/local_repository.rbs b/sig/datadog/ci/git/local_repository.rbs index 6354161c..25f4db7e 100644 --- a/sig/datadog/ci/git/local_repository.rbs +++ b/sig/datadog/ci/git/local_repository.rbs @@ -2,6 +2,8 @@ module Datadog module CI module Git module LocalRepository + COMMAND_RETRY_COUNT: 3 + @root: String? @repository_name: String? diff --git a/sig/datadog/ci/test_visibility/null_recorder.rbs b/sig/datadog/ci/test_visibility/null_recorder.rbs index 7b9b7f7b..a9132de0 100644 --- a/sig/datadog/ci/test_visibility/null_recorder.rbs +++ b/sig/datadog/ci/test_visibility/null_recorder.rbs @@ -24,6 +24,8 @@ module Datadog def active_span: () -> nil + def shutdown!: () -> nil + private def skip_tracing: (?untyped block) -> nil diff --git a/sig/datadog/ci/test_visibility/recorder.rbs b/sig/datadog/ci/test_visibility/recorder.rbs index 115b5325..0363922b 100644 --- a/sig/datadog/ci/test_visibility/recorder.rbs +++ b/sig/datadog/ci/test_visibility/recorder.rbs @@ -10,11 +10,12 @@ module Datadog @itr: Datadog::CI::ITR::Runner @remote_settings_api: Datadog::CI::Transport::RemoteSettingsApi @codeowners: Datadog::CI::Codeowners::Matcher + @git_tree_upload_worker: Datadog::CI::Worker attr_reader environment_tags: Hash[String, String] attr_reader test_suite_level_visibility_enabled: bool - def initialize: (?test_suite_level_visibility_enabled: bool, ?codeowners: Datadog::CI::Codeowners::Matcher, itr: Datadog::CI::ITR::Runner, remote_settings_api: Datadog::CI::Transport::RemoteSettingsApi) -> void + def initialize: (?test_suite_level_visibility_enabled: bool, ?codeowners: Datadog::CI::Codeowners::Matcher, itr: Datadog::CI::ITR::Runner, remote_settings_api: Datadog::CI::Transport::RemoteSettingsApi, ?git_tree_upload_worker: Datadog::CI::Worker) -> void def trace_test: (String span_name, String test_suite_name, ?service: String?, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Test span) -> untyped } -> untyped @@ -46,6 +47,8 @@ module Datadog def itr_enabled?: () -> bool + def shutdown!: () -> void + private def configure_library: (Datadog::CI::TestSession test_session) -> void diff --git a/sig/datadog/ci/worker.rbs b/sig/datadog/ci/worker.rbs new file mode 100644 index 00000000..21b86c17 --- /dev/null +++ b/sig/datadog/ci/worker.rbs @@ -0,0 +1,22 @@ +module Datadog + module CI + class Worker < Datadog::Core::Worker + include Datadog::Core::Workers::Async::Thread::PrependedMethods + include Datadog::Core::Workers::Async::Thread + + DEFAULT_SHUTDOWN_TIMEOUT: 60 + + DEFAULT_WAIT_TIMEOUT: 60 + + def stop: (?Integer timeout) -> void + + def wait_until_done: (?Integer timeout) -> void + + def done?: () -> bool + end + + class DummyWorker < Worker + def initialize: () -> void + end + end +end diff --git a/spec/datadog/ci/configuration/settings_spec.rb b/spec/datadog/ci/configuration/settings_spec.rb index 730f0aa4..f82dd250 100644 --- a/spec/datadog/ci/configuration/settings_spec.rb +++ b/spec/datadog/ci/configuration/settings_spec.rb @@ -258,6 +258,47 @@ def patcher end end + describe "#git_metadata_upload_enabled" do + subject(:git_metadata_upload_enabled) { settings.ci.git_metadata_upload_enabled } + + it { is_expected.to be true } + + context "when #{Datadog::CI::Ext::Settings::ENV_GIT_METADATA_UPLOAD_ENABLED}" do + around do |example| + ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_GIT_METADATA_UPLOAD_ENABLED => enable) do + example.run + end + end + + context "is not defined" do + let(:enable) { nil } + + it { is_expected.to be true } + end + + context "is set to true" do + let(:enable) { "true" } + + it { is_expected.to be true } + end + + context "is set to false" do + let(:enable) { "false" } + + it { is_expected.to be false } + end + end + end + + describe "#git_metadata_upload_enabled=" do + it "updates the #enabled setting" do + expect { settings.ci.git_metadata_upload_enabled = false } + .to change { settings.ci.git_metadata_upload_enabled } + .from(true) + .to(false) + end + end + describe "#instrument" do let(:integration_name) { :fake } diff --git a/spec/datadog/ci/git/local_repository_spec.rb b/spec/datadog/ci/git/local_repository_spec.rb index 9ad07089..2b53c092 100644 --- a/spec/datadog/ci/git/local_repository_spec.rb +++ b/spec/datadog/ci/git/local_repository_spec.rb @@ -371,4 +371,31 @@ def with_full_clone_git_dir it { is_expected.to be_falsey } end end + + context "with failing command" do + describe ".git_commits" do + subject { described_class.git_commits } + + context "succeeds on retries" do + before do + expect(Open3).to receive(:capture2e).and_return([nil, nil], [+"sha1\nsha2", double(success?: true)]) + end + + it { is_expected.to eq(%w[sha1 sha2]) } + end + + context "fails on retries" do + before do + expect(Open3).to( + receive(:capture2e) + .and_return([nil, nil]) + .at_most(described_class::COMMAND_RETRY_COUNT + 1) + .times + ) + end + + it { is_expected.to eq([]) } + end + end + end end diff --git a/spec/datadog/ci/test_visibility/recorder_spec.rb b/spec/datadog/ci/test_visibility/recorder_spec.rb index 34bdd013..fde3f1e1 100644 --- a/spec/datadog/ci/test_visibility/recorder_spec.rb +++ b/spec/datadog/ci/test_visibility/recorder_spec.rb @@ -402,6 +402,23 @@ expect(subject).to have_test_tag("my.tag", "my_value") end + context "with git upload enabled and gitdb api spy" do + let(:git_metadata_upload_enabled) { true } + let(:search_commits) { double("search_commits") } + let(:tags) { {"test.framework" => "my-framework"} } + + it "starts git metadata upload" do + expect(Datadog::CI::Git::SearchCommits).to receive(:new).and_return(search_commits) + expect(search_commits).to receive(:call) do |repo_url, commits| + expect(repo_url).to eq("git@github.com:DataDog/datadog-ci-rb.git") + + commits + end + + subject + end + end + it_behaves_like "span with environment tags" it_behaves_like "span with default tags" it_behaves_like "span with runtime tags" diff --git a/spec/datadog/ci/worker_spec.rb b/spec/datadog/ci/worker_spec.rb new file mode 100644 index 00000000..ce118761 --- /dev/null +++ b/spec/datadog/ci/worker_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require_relative "../../../lib/datadog/ci/worker" + +RSpec.describe Datadog::CI::Worker do + let(:task) { spy("task") } + let(:work) { task.call } + + subject(:worker) { described_class.new { work } } + + describe "#perform" do + it "executes the task" do + worker.perform + worker.stop + + expect(task).to have_received(:call) + end + end + + describe "#done?" do + context "when the worker has not started" do + it { is_expected.not_to be_done } + end + + context "when the worker has started" do + context "when the worker is running" do + let(:queue) { Queue.new } + subject(:worker) { described_class.new { queue.pop } } + + it do + worker.perform + is_expected.not_to be_done + + queue << :done + worker.stop + + is_expected.to be_done + end + end + + context "when the worker has stopped" do + it do + worker.perform + worker.stop + + is_expected.to be_done + end + end + end + end + + describe "#wait_until_done" do + it "waits until the worker is done" do + worker.perform + worker.wait_until_done + + expect(worker).to be_done + end + end +end diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index 6833e0d8..d44ad931 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -20,6 +20,7 @@ let(:itr_enabled) { false } let(:code_coverage_enabled) { false } let(:tests_skipping_enabled) { false } + let(:git_metadata_upload_enabled) { false } let(:recorder) { Datadog.send(:components).ci_recorder } @@ -49,6 +50,7 @@ c.ci.enabled = ci_enabled c.ci.force_test_level_visibility = force_test_level_visibility c.ci.itr_enabled = itr_enabled + c.ci.git_metadata_upload_enabled = git_metadata_upload_enabled unless integration_name == :no_instrument c.ci.instrument integration_name, integration_options end @@ -59,5 +61,6 @@ ::Datadog::Tracing.shutdown! Datadog::CI.send(:itr_runner)&.shutdown! + Datadog::CI.send(:recorder)&.shutdown! end end diff --git a/vendor/rbs/ddtrace/0/datadog/core/worker.rbs b/vendor/rbs/ddtrace/0/datadog/core/worker.rbs new file mode 100644 index 00000000..93eb887f --- /dev/null +++ b/vendor/rbs/ddtrace/0/datadog/core/worker.rbs @@ -0,0 +1,7 @@ +module Datadog + module Core + class Worker + def initialize: () { (untyped) -> untyped } -> void + end + end +end