[CIVIS-2938] Git upload async worker

lib/datadog/ci/configuration/components.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../ext/settings"
+require_relative "../git/tree_uploader"
 require_relative "../itr/runner"
 require_relative "../itr/coverage/transport"
 require_relative "../itr/coverage/writer"
@@ -12,6 +13,7 @@
 require_relative "../test_visibility/transport"
 require_relative "../transport/api/builder"
 require_relative "../transport/remote_settings_api"
+require_relative "../worker"
 
 module Datadog
   module CI
@@ -35,6 +37,7 @@ def initialize(settings)
         def shutdown!(replacement = nil)
           super
 
+          @ci_recorder&.shutdown!
           @itr&.shutdown!
         end
 
@@ -63,12 +66,12 @@ def activate_ci!(settings)
 
           if test_visibility_api
             # setup writer for code coverage payloads
-            coverage_writer = Datadog::CI::ITR::Coverage::Writer.new(
-              transport: Datadog::CI::ITR::Coverage::Transport.new(api: test_visibility_api)
+            coverage_writer = ITR::Coverage::Writer.new(
+              transport: ITR::Coverage::Transport.new(api: test_visibility_api)
             )
 
             # configure tracing writer to send traces to CI visibility backend
-            writer_options[:transport] = Datadog::CI::TestVisibility::Transport.new(
+            writer_options[:transport] = TestVisibility::Transport.new(
               api: test_visibility_api,
               serializers_factory: serializers_factory(settings),
               dd_env: settings.env
@@ -92,16 +95,26 @@ def activate_ci!(settings)
             dd_env: settings.env
           )
 
-          itr = Datadog::CI::ITR::Runner.new(
+          itr = ITR::Runner.new(
             coverage_writer: coverage_writer,
             enabled: settings.ci.enabled && settings.ci.itr_enabled
           )
 
+          git_tree_uploader = Git::TreeUploader.new(api: test_visibility_api)
+          git_tree_upload_worker = if settings.ci.git_metadata_upload_enabled
+            Worker.new do |repository_url|
+              git_tree_uploader.call(repository_url)
+            end
+          else
+            DummyWorker.new
+          end
+
           # CI visibility recorder global instance
           @ci_recorder = TestVisibility::Recorder.new(
             test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
             itr: itr,
-            remote_settings_api: remote_settings_api
+            remote_settings_api: remote_settings_api,
+            git_tree_upload_worker: git_tree_upload_worker
           )
 
           @itr = itr
@@ -141,9 +154,9 @@ def build_test_visibility_api(settings)
 
         def serializers_factory(settings)
           if settings.ci.force_test_level_visibility
-            Datadog::CI::TestVisibility::Serializers::Factories::TestLevel
+            TestVisibility::Serializers::Factories::TestLevel
           else
-            Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel
+            TestVisibility::Serializers::Factories::TestSuiteLevel
           end
         end
 

lib/datadog/ci/configuration/settings.rb

@@ -61,6 +61,12 @@ def self.add_settings!(base)
                 o.default false
               end
 
+              option :git_metadata_upload_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_GIT_METADATA_UPLOAD_ENABLED
+                o.default true
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

lib/datadog/ci/ext/settings.rb

@@ -11,6 +11,7 @@ module Settings
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
         ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
         ENV_ITR_ENABLED = "DD_CIVISIBILITY_ITR_ENABLED"
+        ENV_GIT_METADATA_UPLOAD_ENABLED = "DD_CIVISIBILITY_GIT_METADATA_UPLOAD_ENABLED"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = [

lib/datadog/ci/git/local_repository.rb

@@ -9,6 +9,8 @@ module Datadog
   module CI
     module Git
       module LocalRepository
+        COMMAND_RETRY_COUNT = 3
+
         def self.root
           return @root if defined?(@root)
 
@@ -195,7 +197,20 @@ def exec_git_command(cmd, stdin: nil)
             # no-dd-sa:ruby-security/shell-injection
             out, status = Open3.capture2e(cmd, stdin_data: stdin)
 
-            raise "Failed to run git command #{cmd}: #{out}" unless status.success?
+            if status.nil?
+              retry_count = COMMAND_RETRY_COUNT
+              Datadog.logger.debug { "Opening pipe failed, starting retries..." }
+              while status.nil? && retry_count.positive?
+                # no-dd-sa:ruby-security/shell-injection
+                out, status = Open3.capture2e(cmd, stdin_data: stdin)
+                Datadog.logger.debug { "After retry status is [#{status}]" }
+                retry_count -= 1
+              end
+            end
+
+            if status.nil? || !status.success?
+              raise "Failed to run git command [#{cmd}] with input [#{stdin}] and output [#{out}]"
+            end
 
             # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
             # commit messages with UTF-8 characters like emojis
@@ -213,7 +228,7 @@ def exec_git_command(cmd, stdin: nil)
 
           def log_failure(e, action)
             Datadog.logger.debug(
-              "Unable to read #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+              "Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
             )
           end
         end

lib/datadog/ci/git/packfiles.rb

@@ -60,7 +60,9 @@ def self.generate(included_commits:, excluded_commits:)
         rescue => e
           Datadog.logger.debug("Packfiles could not be generated, error: #{e}")
         ensure
-          FileUtils.remove_entry(current_process_tmp_folder) unless current_process_tmp_folder.nil?
+          if current_process_tmp_folder && File.exist?(current_process_tmp_folder)
+            FileUtils.remove_entry(current_process_tmp_folder)
+          end
         end
       end
     end

lib/datadog/ci/git/tree_uploader.rb

@@ -69,6 +69,8 @@ def call(repository_url)
             Datadog.logger.debug("Packfile upload failed with #{e}")
             break
           end
+        ensure
+          Datadog.logger.debug("Git tree upload finished")
         end
 
         private

lib/datadog/ci/test_visibility/null_recorder.rb

@@ -42,6 +42,9 @@ def active_test_module
         def active_test_suite(test_suite_name)
         end
 
+        def shutdown!
+        end
+
         private
 
         def skip_tracing(block = nil)

lib/datadog/ci/test_visibility/recorder.rb

@@ -20,6 +20,7 @@
 require_relative "../test_session"
 require_relative "../test_module"
 require_relative "../test_suite"
+require_relative "../worker"
 
 module Datadog
   module CI
@@ -30,7 +31,10 @@ class Recorder
         attr_reader :environment_tags, :test_suite_level_visibility_enabled
 
         def initialize(
-          itr:, remote_settings_api:, test_suite_level_visibility_enabled: false,
+          itr:,
+          remote_settings_api:,
+          git_tree_upload_worker: DummyWorker.new,
+          test_suite_level_visibility_enabled: false,
           codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse
         )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
@@ -43,6 +47,11 @@ def initialize(
 
           @itr = itr
           @remote_settings_api = remote_settings_api
+          @git_tree_upload_worker = git_tree_upload_worker
+        end
+
+        def shutdown!
+          @git_tree_upload_worker.stop
         end
 
         def start_test_session(service: nil, tags: {})
@@ -56,6 +65,7 @@ def start_test_session(service: nil, tags: {})
 
             test_session = build_test_session(tracer_span, tags)
 
+            @git_tree_upload_worker.perform(test_session.git_repository_url)
             configure_library(test_session)
 
             test_session

lib/datadog/ci/worker.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "datadog/core/worker"
+require "datadog/core/workers/async"
+
+# general purpose async worker for CI
+# executes given task once in separate thread
+module Datadog
+  module CI
+    class Worker < Datadog::Core::Worker
+      include Datadog::Core::Workers::Async::Thread
+
+      DEFAULT_SHUTDOWN_TIMEOUT = 60
+      DEFAULT_WAIT_TIMEOUT = 60
+
+      def stop(timeout = DEFAULT_SHUTDOWN_TIMEOUT)
+        join(timeout)
+      end
+
+      def wait_until_done(timeout = DEFAULT_WAIT_TIMEOUT)
+        join(timeout)
+      end
+
+      def done?
+        started? && !running?
+      end
+    end
+
+    class DummyWorker < Worker
+      def initialize
+        super { nil }
+      end
+    end
+  end
+end

sig/datadog/ci/ext/settings.rbs

@@ -8,6 +8,7 @@ module Datadog
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED: String
         ENV_FORCE_TEST_LEVEL_VISIBILITY: String
         ENV_ITR_ENABLED: String
+        ENV_GIT_METADATA_UPLOAD_ENABLED: String
 
         DD_SITE_ALLOWLIST: Array[String]
       end

sig/datadog/ci/git/local_repository.rbs

@@ -2,6 +2,8 @@ module Datadog
   module CI
     module Git
       module LocalRepository
+        COMMAND_RETRY_COUNT: 3
+
         @root: String?
         @repository_name: String?
 

sig/datadog/ci/test_visibility/null_recorder.rbs

@@ -24,6 +24,8 @@ module Datadog
 
         def active_span: () -> nil
 
+        def shutdown!: () -> nil
+
         private
 
         def skip_tracing: (?untyped block) -> nil

sig/datadog/ci/test_visibility/recorder.rbs

@@ -10,11 +10,12 @@ module Datadog
         @itr: Datadog::CI::ITR::Runner
         @remote_settings_api: Datadog::CI::Transport::RemoteSettingsApi
         @codeowners: Datadog::CI::Codeowners::Matcher
+        @git_tree_upload_worker: Datadog::CI::Worker
 
         attr_reader environment_tags: Hash[String, String]
         attr_reader test_suite_level_visibility_enabled: bool
 
-        def initialize: (?test_suite_level_visibility_enabled: bool, ?codeowners: Datadog::CI::Codeowners::Matcher, itr: Datadog::CI::ITR::Runner, remote_settings_api: Datadog::CI::Transport::RemoteSettingsApi) -> void
+        def initialize: (?test_suite_level_visibility_enabled: bool, ?codeowners: Datadog::CI::Codeowners::Matcher, itr: Datadog::CI::ITR::Runner, remote_settings_api: Datadog::CI::Transport::RemoteSettingsApi, ?git_tree_upload_worker: Datadog::CI::Worker) -> void
 
         def trace_test: (String span_name, String test_suite_name, ?service: String?, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Test span) -> untyped } -> untyped
 
@@ -46,6 +47,8 @@ module Datadog
 
         def itr_enabled?: () -> bool
 
+        def shutdown!: () -> void
+
         private
 
         def configure_library: (Datadog::CI::TestSession test_session) -> void

sig/datadog/ci/worker.rbs

@@ -0,0 +1,22 @@
+module Datadog
+  module CI
+    class Worker < Datadog::Core::Worker
+      include  Datadog::Core::Workers::Async::Thread::PrependedMethods
+      include Datadog::Core::Workers::Async::Thread
+
+      DEFAULT_SHUTDOWN_TIMEOUT: 60
+
+      DEFAULT_WAIT_TIMEOUT: 60
+
+      def stop: (?Integer timeout) -> void
+
+      def wait_until_done: (?Integer timeout) -> void
+
+      def done?: () -> bool
+    end
+
+    class DummyWorker < Worker
+      def initialize: () -> void
+    end
+  end
+end

spec/datadog/ci/configuration/settings_spec.rb

@@ -258,6 +258,47 @@ def patcher
         end
       end
 
+      describe "#git_metadata_upload_enabled" do
+        subject(:git_metadata_upload_enabled) { settings.ci.git_metadata_upload_enabled }
+
+        it { is_expected.to be true }
+
+        context "when #{Datadog::CI::Ext::Settings::ENV_GIT_METADATA_UPLOAD_ENABLED}" do
+          around do |example|
+            ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_GIT_METADATA_UPLOAD_ENABLED => enable) do
+              example.run
+            end
+          end
+
+          context "is not defined" do
+            let(:enable) { nil }
+
+            it { is_expected.to be true }
+          end
+
+          context "is set to true" do
+            let(:enable) { "true" }
+
+            it { is_expected.to be true }
+          end
+
+          context "is set to false" do
+            let(:enable) { "false" }
+
+            it { is_expected.to be false }
+          end
+        end
+      end
+
+      describe "#git_metadata_upload_enabled=" do
+        it "updates the #enabled setting" do
+          expect { settings.ci.git_metadata_upload_enabled = false }
+            .to change { settings.ci.git_metadata_upload_enabled }
+            .from(true)
+            .to(false)
+        end
+      end
+
       describe "#instrument" do
         let(:integration_name) { :fake }
 

spec/datadog/ci/git/local_repository_spec.rb

@@ -371,4 +371,31 @@ def with_full_clone_git_dir
       it { is_expected.to be_falsey }
     end
   end
+
+  context "with failing command" do
+    describe ".git_commits" do
+      subject { described_class.git_commits }
+
+      context "succeeds on retries" do
+        before do
+          expect(Open3).to receive(:capture2e).and_return([nil, nil], [+"sha1\nsha2", double(success?: true)])
+        end
+
+        it { is_expected.to eq(%w[sha1 sha2]) }
+      end
+
+      context "fails on retries" do
+        before do
+          expect(Open3).to(
+            receive(:capture2e)
+              .and_return([nil, nil])
+              .at_most(described_class::COMMAND_RETRY_COUNT + 1)
+              .times
+          )
+        end
+
+        it { is_expected.to eq([]) }
+      end
+    end
+  end
 end