[CIVIS-2938] Git tree uploader

Steepfile

@@ -13,6 +13,8 @@ target :lib do
   library "net-http"
   library "zlib"
   library "securerandom"
+  library "tmpdir"
+  library "fileutils"
 
   repo_path "vendor/rbs"
   library "ddtrace"

lib/datadog/ci/git/local_repository.rb

@@ -152,6 +152,8 @@ def self.git_generate_packfiles(included_commits:, excluded_commits:, path:)
             "git pack-objects --compression=9 --max-pack-size=3m #{path}/#{basename}",
             stdin: commit_tree
           )
+
+          basename
         rescue => e
           log_failure(e, "git generate packfiles")
           nil

lib/datadog/ci/git/packfiles.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+
+require_relative "local_repository"
+
+module Datadog
+  module CI
+    module Git
+      module Packfiles
+        def self.generate(included_commits:, excluded_commits:)
+          # @type var current_process_tmp_folder: String?
+          current_process_tmp_folder = nil
+
+          Dir.mktmpdir do |tmpdir|
+            prefix = LocalRepository.git_generate_packfiles(
+              included_commits: included_commits,
+              excluded_commits: excluded_commits,
+              path: tmpdir
+            )
+
+            if prefix.nil?
+              # git pack-files command fails if tmpdir is mounted on
+              # a different device from the current process directory
+              #
+              # @type var current_process_tmp_folder: String
+              current_process_tmp_folder = File.join(Dir.pwd, "tmp", "packfiles")
+              FileUtils.mkdir_p(current_process_tmp_folder)
+
+              prefix = LocalRepository.git_generate_packfiles(
+                included_commits: included_commits,
+                excluded_commits: excluded_commits,
+                path: current_process_tmp_folder
+              )
+
+              if prefix.nil?
+                Datadog.logger.debug("Packfiles generation failed twice, aborting")
+                break
+              end
+
+              tmpdir = current_process_tmp_folder
+            end
+
+            packfiles = Dir.entries(tmpdir) - %w[. ..]
+            if packfiles.empty?
+              Datadog.logger.debug("Empty packfiles list, aborting process")
+              break
+            end
+
+            packfiles.each do |packfile_name|
+              next unless packfile_name.start_with?(prefix)
+              next unless packfile_name.end_with?(".pack")
+
+              packfile_path = File.join(tmpdir, packfile_name)
+
+              yield packfile_path
+            end
+          end
+        rescue => e
+          Datadog.logger.debug("Packfiles could not be generated, error: #{e}")
+        ensure
+          FileUtils.remove_entry(current_process_tmp_folder) unless current_process_tmp_folder.nil?
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/git/tree_uploader.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+
+require_relative "local_repository"
+require_relative "search_commits"
+require_relative "upload_packfile"
+require_relative "packfiles"
+
+module Datadog
+  module CI
+    module Git
+      class TreeUploader
+        attr_reader :api
+
+        def initialize(api:)
+          @api = api
+        end
+
+        def call(repository_url)
+          if api.nil?
+            Datadog.logger.debug("API is not configured, aborting git upload")
+            return
+          end
+
+          Datadog.logger.debug { "Uploading git tree for repository #{repository_url}" }
+
+          # 2. Check if the repository clone is shallow and unshallow if appropriate
+          # TO BE ADDED IN CIVIS-2863
+          latest_commits = LocalRepository.git_commits
+          head_commit = latest_commits&.first
+          if head_commit.nil?
+            Datadog.logger.debug("Got empty latest commits list, aborting git upload")
+            return
+          end
+
+          begin
+            excluded_commits, included_commits = split_known_commits(repository_url, latest_commits)
+            if included_commits.empty?
+              Datadog.logger.debug("No new commits to upload")
+              return
+            end
+          rescue SearchCommits::ApiError => e
+            Datadog.logger.debug("SearchCommits failed with #{e}, aborting git upload")
+            return
+          end
+
+          Datadog.logger.debug { "Uploading packfiles for commits: #{included_commits}" }
+          uploader = UploadPackfile.new(
+            api: api,
+            head_commit_sha: head_commit,
+            repository_url: repository_url
+          )
+          Packfiles.generate(included_commits: included_commits, excluded_commits: excluded_commits) do |filepath|
+            uploader.call(filepath: filepath)
+          rescue UploadPackfile::ApiError => e
+            Datadog.logger.debug("Packfile upload failed with #{e}")
+            break
+          end
+        end
+
+        private
+
+        def split_known_commits(repository_url, latest_commits)
+          Datadog.logger.debug { "Checking the latest commits list with backend: #{latest_commits}" }
+          backend_commits = SearchCommits.new(api: api).call(repository_url, latest_commits)
+          latest_commits.partition do |commit|
+            backend_commits.include?(commit)
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/git/local_repository.rbs

@@ -29,11 +29,13 @@ module Datadog
 
         def self.git_commits: () -> Array[String]
 
-        def self.git_commits_rev_list: (included_commits: Array[String], excluded_commits: Array[String]) -> String?
+        def self.git_commits_rev_list: (included_commits: Enumerable[String], excluded_commits: Enumerable[String]) -> String?
+
+        def self.git_generate_packfiles: (included_commits: Enumerable[String], excluded_commits: Enumerable[String], path: String) -> String?
 
         private
 
-        def self.filter_invalid_commits: (Array[String] commits) -> Array[String]
+        def self.filter_invalid_commits: (Enumerable[String] commits) -> Array[String]
 
         def self.exec_git_command: (String ref, ?stdin: String?) -> String?
 

sig/datadog/ci/git/packfiles.rbs

@@ -0,0 +1,9 @@
+module Datadog
+  module CI
+    module Git
+      module Packfiles
+        def self.generate: (included_commits: Enumerable[String], excluded_commits: Enumerable[String]) { (String) -> untyped } -> void
+      end
+    end
+  end
+end

sig/datadog/ci/git/tree_uploader.rbs

@@ -0,0 +1,18 @@
+module Datadog
+  module CI
+    module Git
+      class TreeUploader
+        @api: Datadog::CI::Transport::Api::Base?
+
+        attr_reader api: Datadog::CI::Transport::Api::Base?
+
+        def initialize: (api: Datadog::CI::Transport::Api::Base?) -> void
+        def call: (String repository_url) -> void
+
+        private
+
+        def split_known_commits: (String repository_url, Array[String] latest_commits) -> [Array[String], Array[String]]
+      end
+    end
+  end
+end

spec/datadog/ci/git/local_repository_spec.rb

@@ -165,7 +165,7 @@ def with_custom_git_environment
       end
 
       it "generates packfiles in temp directory" do
-        expect(subject).not_to be_nil
+        expect(subject).to match(/^\h{8}$/)
         packfiles = Dir.entries(tmpdir) - %w[. ..]
         expect(packfiles).not_to be_empty
         expect(packfiles).to all(match(/^\h{8}-\h{40}\.(pack|idx|rev)$/))

spec/datadog/ci/git/packfiles_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require_relative "../../../../lib/datadog/ci/git/packfiles"
+
+RSpec.describe ::Datadog::CI::Git::Packfiles do
+  # skip for jruby for now - old git version DD docker image
+  before { skip if PlatformHelpers.jruby? }
+
+  let(:commits) { Datadog::CI::Git::LocalRepository.git_commits }
+  let(:included_commits) { commits[0..1] }
+  let(:excluded_commits) { commits[2..] }
+
+  describe ".generate" do
+    it "yields packfile" do
+      expect do |b|
+        described_class.generate(included_commits: included_commits, excluded_commits: excluded_commits, &b)
+      end.to yield_with_args(/\/.+\h{8}-\h{40}\.pack$/)
+    end
+
+    context "empty packfiles folder" do
+      before do
+        expect(Datadog::CI::Git::LocalRepository).to receive(:git_generate_packfiles).with(
+          included_commits: included_commits,
+          excluded_commits: excluded_commits,
+          path: String
+        ) do
+          "pref"
+        end
+      end
+
+      it "does not yield anything" do
+        expect do |b|
+          described_class.generate(included_commits: included_commits, excluded_commits: excluded_commits, &b)
+        end.not_to yield_control
+      end
+    end
+
+    context "something goes wrong" do
+      before do
+        expect(Dir).to receive(:mktmpdir).and_raise("error")
+      end
+
+      it "does not yield anything" do
+        expect do |b|
+          described_class.generate(included_commits: included_commits, excluded_commits: excluded_commits, &b)
+        end.not_to yield_control
+      end
+    end
+
+    context "tmp folder fails" do
+      let(:current_process_tmp_folder) { File.join(Dir.pwd, "tmp", "packfiles") }
+      let(:prefix) { "pref" }
+
+      before do
+        expect(Datadog::CI::Git::LocalRepository).to receive(:git_generate_packfiles).with(
+          included_commits: included_commits,
+          excluded_commits: excluded_commits,
+          path: String
+        ).and_return(nil)
+
+        expect(Datadog::CI::Git::LocalRepository).to receive(:git_generate_packfiles).with(
+          included_commits: included_commits,
+          excluded_commits: excluded_commits,
+          path: current_process_tmp_folder
+        ) do
+          File.write(File.join(current_process_tmp_folder, "#{prefix}-sha.idx"), "hello world")
+          File.write(File.join(current_process_tmp_folder, "other-sha.pack"), "hello world")
+          File.write(File.join(current_process_tmp_folder, "#{prefix}-sha.pack"), "hello world")
+
+          prefix
+        end
+      end
+
+      it "creates temporary folder in the current directory" do
+        expect do |b|
+          described_class.generate(included_commits: included_commits, excluded_commits: excluded_commits, &b)
+        end.to yield_with_args(/^#{current_process_tmp_folder}\/pref-sha.pack$/)
+
+        expect(File.exist?(current_process_tmp_folder)).to be_falsey
+      end
+    end
+
+    context "packfile generation fails" do
+      before do
+        allow(Datadog::CI::Git::LocalRepository).to receive(:git_generate_packfiles).and_return(nil)
+      end
+
+      it "does not yield anything" do
+        expect do |b|
+          described_class.generate(included_commits: included_commits, excluded_commits: excluded_commits, &b)
+        end.not_to yield_control
+      end
+    end
+  end
+end