From 07111cdd53225210152d4cdd427f63553f976179 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 5 Apr 2024 16:20:55 +0200 Subject: [PATCH] add Git::SearchCommits callable that sends backend API request to check which commits are already known to the backend --- lib/datadog/ci/ext/environment.rb | 27 ++- lib/datadog/ci/ext/transport.rb | 3 + lib/datadog/ci/git/search_commits.rb | 77 +++++++++ lib/datadog/ci/utils/git.rb | 6 + sig/datadog/ci/ext/environment.rbs | 2 - sig/datadog/ci/ext/transport.rbs | 4 +- sig/datadog/ci/git/search_commits.rbs | 26 +++ sig/datadog/ci/utils/git.rbs | 2 + spec/datadog/ci/git/search_commits_spec.rb | 156 ++++++++++++++++++ spec/datadog/ci/utils/git_spec.rb | 40 +++++ .../0/datadog/core/transport/response.rbs | 2 +- 11 files changed, 325 insertions(+), 20 deletions(-) create mode 100644 lib/datadog/ci/git/search_commits.rb create mode 100644 sig/datadog/ci/git/search_commits.rbs create mode 100644 spec/datadog/ci/git/search_commits_spec.rb diff --git a/lib/datadog/ci/ext/environment.rb b/lib/datadog/ci/ext/environment.rb index 81208d52..5e3c0034 100644 --- a/lib/datadog/ci/ext/environment.rb +++ b/lib/datadog/ci/ext/environment.rb @@ -3,6 +3,8 @@ require_relative "git" require_relative "environment/extractor" +require_relative "../utils/git" + module Datadog module CI module Ext @@ -21,8 +23,6 @@ module Environment TAG_NODE_NAME = "ci.node.name" TAG_CI_ENV_VARS = "_dd.ci.env_vars" - HEX_NUMBER_REGEXP = /[0-9a-f]{40}/i.freeze - module_function def tags(env) @@ -57,24 +57,19 @@ def validate_repository_url(repo_url) end def validate_git_sha(git_sha) - message = "DD_GIT_COMMIT_SHA must be a full-length git SHA." + return if Utils::Git.valid_commit_sha?(git_sha) - if git_sha.nil? || git_sha.empty? - message += " No value was set and no SHA was automatically extracted." - Datadog.logger.error(message) - return - end + message = "DD_GIT_COMMIT_SHA must be a full-length git SHA." - if git_sha.length < Git::SHA_LENGTH - message += " Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}." - Datadog.logger.error(message) - return + message += if git_sha.nil? || git_sha.empty? + " No value was set and no SHA was automatically extracted." + elsif git_sha.length < Git::SHA_LENGTH + " Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}." + else + " Expected SHA to be a valid HEX number, got #{git_sha}." end - unless HEX_NUMBER_REGEXP.match?(git_sha) - message += " Expected SHA to be a valid HEX number, got #{git_sha}." - Datadog.logger.error(message) - end + Datadog.logger.error(message) end end end diff --git a/lib/datadog/ci/ext/transport.rb b/lib/datadog/ci/ext/transport.rb index 07943696..b26604e2 100644 --- a/lib/datadog/ci/ext/transport.rb +++ b/lib/datadog/ci/ext/transport.rb @@ -27,6 +27,7 @@ module Transport TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov" DD_API_HOST_PREFIX = "api" + DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting" DD_API_SETTINGS_TYPE = "ci_app_test_service_libraries_settings" DD_API_SETTINGS_RESPONSE_DIG_KEYS = %w[data attributes].freeze @@ -36,6 +37,8 @@ module Transport DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git" DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze + DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits" + CONTENT_TYPE_MESSAGEPACK = "application/msgpack" CONTENT_TYPE_JSON = "application/json" CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data" diff --git a/lib/datadog/ci/git/search_commits.rb b/lib/datadog/ci/git/search_commits.rb new file mode 100644 index 00000000..5c26cb12 --- /dev/null +++ b/lib/datadog/ci/git/search_commits.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +require "json" +require "set" + +require_relative "../ext/transport" +require_relative "../utils/git" + +module Datadog + module CI + module Git + class SearchCommits + class ApiError < StandardError; end + + attr_reader :api + + def initialize(api:) + @api = api + end + + def call(repository_url, commits) + raise ApiError, "test visibility API is not configured" if api.nil? + + http_response = api.api_request( + path: Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH, + payload: request_payload(repository_url, commits) + ) + raise ApiError, "Failed to search commits: #{http_response.body}" unless http_response.ok? + + response_payload = parse_json_response(http_response) + extract_commits(response_payload) + end + + private + + def request_payload(repository_url, commits) + { + meta: { + repository_url: repository_url + }, + data: commits.filter_map do |commit| + next unless Utils::Git.valid_commit_sha?(commit) + + { + id: commit, + type: "commit" + } + end + }.to_json + end + + def parse_json_response(http_response) + JSON.parse(http_response.payload) + rescue JSON::ParserError => e + raise ApiError, "Failed to parse search commits response: #{e}. Payload was: #{http_response.payload}" + end + + def extract_commits(response_payload) + result = Set.new + + response_payload.fetch("data").each do |commit_json| + raise ApiError, "Invalid commit type response #{commit_json}" unless commit_json["type"] == "commit" + + commit_sha = commit_json["id"] + raise ApiError, "Invalid commit SHA response #{commit_sha}" unless Utils::Git.valid_commit_sha?(commit_sha) + + result.add(commit_sha) + end + + result + rescue KeyError => e + raise ApiError, "Malformed search commits response: #{e}. Payload was: #{response_payload}" + end + end + end + end +end diff --git a/lib/datadog/ci/utils/git.rb b/lib/datadog/ci/utils/git.rb index 4819d1e7..c6118030 100644 --- a/lib/datadog/ci/utils/git.rb +++ b/lib/datadog/ci/utils/git.rb @@ -7,6 +7,12 @@ module Datadog module CI module Utils module Git + def self.valid_commit_sha?(sha) + return false if sha.nil? + + sha.match?(/\A[0-9a-f]{40}\Z/) || sha.match?(/\A[0-9a-f]{64}\Z/) + end + def self.normalize_ref(ref) return nil if ref.nil? diff --git a/sig/datadog/ci/ext/environment.rbs b/sig/datadog/ci/ext/environment.rbs index 73f8b132..9e875937 100644 --- a/sig/datadog/ci/ext/environment.rbs +++ b/sig/datadog/ci/ext/environment.rbs @@ -27,8 +27,6 @@ module Datadog TAG_CI_ENV_VARS: String - HEX_NUMBER_REGEXP: Regexp - PROVIDERS: ::Array[Array[String | Symbol]] def self?.tags: (untyped env) -> Hash[String, String] diff --git a/sig/datadog/ci/ext/transport.rbs b/sig/datadog/ci/ext/transport.rbs index d273d008..4f96bb20 100644 --- a/sig/datadog/ci/ext/transport.rbs +++ b/sig/datadog/ci/ext/transport.rbs @@ -32,7 +32,7 @@ module Datadog DD_API_HOST_PREFIX: "api" - DD_API_SETTINGS_PATH: "/api/v2/ci/libraries/tests/services/setting" + DD_API_SETTINGS_PATH: "/api/v2/libraries/tests/services/setting" DD_API_SETTINGS_TYPE: "ci_app_test_service_libraries_settings" @@ -48,6 +48,8 @@ module Datadog DD_API_SETTINGS_RESPONSE_DEFAULT: Hash[String, untyped] + DD_API_GIT_SEARCH_COMMITS_PATH: "/api/v2/git/repository/search_commits" + CONTENT_TYPE_MESSAGEPACK: "application/msgpack" CONTENT_TYPE_JSON: "application/json" diff --git a/sig/datadog/ci/git/search_commits.rbs b/sig/datadog/ci/git/search_commits.rbs new file mode 100644 index 00000000..e650957f --- /dev/null +++ b/sig/datadog/ci/git/search_commits.rbs @@ -0,0 +1,26 @@ +module Datadog + module CI + module Git + class SearchCommits + @api: Datadog::CI::Transport::Api::Base? + + attr_reader api: Datadog::CI::Transport::Api::Base? + + class ApiError < StandardError + end + + def initialize: (api: Datadog::CI::Transport::Api::Base?) -> void + + def call: (String repository_url, Array[String] commits) -> Set[String] + + private + + def request_payload: (String repository_url, Array[String] commits) -> String + + def parse_json_response: (Datadog::Core::Transport::Response response) -> Hash[String, untyped] + + def extract_commits: (Hash[String, untyped] response) -> Set[String] + end + end + end +end diff --git a/sig/datadog/ci/utils/git.rbs b/sig/datadog/ci/utils/git.rbs index f4f09c75..e4138129 100644 --- a/sig/datadog/ci/utils/git.rbs +++ b/sig/datadog/ci/utils/git.rbs @@ -2,6 +2,8 @@ module Datadog module CI module Utils module Git + def self.valid_commit_sha?: (String? sha) -> bool + def self.normalize_ref: (String? name) -> String? def self.is_git_tag?: (String? ref) -> bool diff --git a/spec/datadog/ci/git/search_commits_spec.rb b/spec/datadog/ci/git/search_commits_spec.rb new file mode 100644 index 00000000..b639b04e --- /dev/null +++ b/spec/datadog/ci/git/search_commits_spec.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +require_relative "../../../../lib/datadog/ci/git/search_commits" + +RSpec.describe Datadog::CI::Git::SearchCommits do + let(:api) { double("api") } + subject(:search_commits) { described_class.new(api: api) } + + describe "#call" do + let(:repository_url) { "https://datadoghq.com/git/test.git" } + let(:commits) { ["c7f893648f656339f62fb7b4d8a6ecdf7d063835"] } + + context "when the API is not configured" do + let(:api) { nil } + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error(Datadog::CI::Git::SearchCommits::ApiError, "test visibility API is not configured") + end + end + + context "when the API is configured" do + before do + allow(api).to receive(:api_request).and_return(http_response) + end + + context "when the API request fails" do + let(:http_response) { double("http_response", ok?: false, body: "error message") } + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error(Datadog::CI::Git::SearchCommits::ApiError, "Failed to search commits: error message") + end + end + + context "when the API request is successful" do + let(:http_response) { double("http_response", ok?: true, payload: response_payload) } + let(:response_payload) do + { + "data" => [ + { + "id" => "c7f893648f656339f62fb7b4d8a6ecdf7d063835", + "type" => "commit" + } + ] + }.to_json + end + + it "returns the list of commit SHAs" do + expect(api).to receive(:api_request).with( + path: Datadog::CI::Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH, + payload: "{\"meta\":{\"repository_url\":\"https://datadoghq.com/git/test.git\"},\"data\":[{\"id\":\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\",\"type\":\"commit\"}]}" + ).and_return(http_response) + + expect(search_commits.call(repository_url, commits)).to eq(Set.new(["c7f893648f656339f62fb7b4d8a6ecdf7d063835"])) + end + + context "when the request contains an invalid commit SHA" do + let(:commits) { ["INVALID_SHA", "c7f893648f656339f62fb7b4d8a6ecdf7d063835"] } + + it "does not include the invalid commit SHA in the request" do + expect(api).to receive(:api_request).with( + path: Datadog::CI::Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH, + payload: "{\"meta\":{\"repository_url\":\"https://datadoghq.com/git/test.git\"},\"data\":[{\"id\":\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\",\"type\":\"commit\"}]}" + ).and_return(http_response) + + expect(search_commits.call(repository_url, commits)).to eq(Set.new(["c7f893648f656339f62fb7b4d8a6ecdf7d063835"])) + end + end + + context "when the response contains an invalid commit type" do + let(:response_payload) do + { + "data" => [ + { + "id" => "c7f893648f656339f62fb7b4d8a6ecdf7d063835", + "type" => "invalid" + } + ] + }.to_json + end + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error( + Datadog::CI::Git::SearchCommits::ApiError, + "Invalid commit type response {\"id\"=>\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\", \"type\"=>\"invalid\"}" + ) + end + end + + context "when the response contains an invalid commit SHA" do + let(:response_payload) do + { + "data" => [ + { + "id" => "INVALID_SHA", + "type" => "commit" + } + ] + }.to_json + end + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error(Datadog::CI::Git::SearchCommits::ApiError, "Invalid commit SHA response INVALID_SHA") + end + end + + context "when the response is not a valid JSON" do + let(:response_payload) { "invalid json" } + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error( + Datadog::CI::Git::SearchCommits::ApiError, + "Failed to parse search commits response: unexpected token at 'invalid json'. Payload was: invalid json" + ) + end + end + + context "when the response is missing the data key" do + let(:response_payload) { {}.to_json } + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error( + Datadog::CI::Git::SearchCommits::ApiError, + "Malformed search commits response: key not found: \"data\". Payload was: {}" + ) + end + end + + context "when the response is missing the commit type" do + let(:response_payload) do + { + "data" => [ + { + "id" => "c7f893648f656339f62fb7b4d8a6ecdf7d063835" + } + ] + }.to_json + end + + it "raises an error" do + expect { search_commits.call(repository_url, commits) } + .to raise_error( + Datadog::CI::Git::SearchCommits::ApiError, + "Invalid commit type response {\"id\"=>\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\"}" + ) + end + end + end + end + end +end diff --git a/spec/datadog/ci/utils/git_spec.rb b/spec/datadog/ci/utils/git_spec.rb index c17700c5..95e5df73 100644 --- a/spec/datadog/ci/utils/git_spec.rb +++ b/spec/datadog/ci/utils/git_spec.rb @@ -1,4 +1,44 @@ RSpec.describe ::Datadog::CI::Utils::Git do + describe ".valid_commit_sha?" do + subject { described_class.valid_commit_sha?(sha) } + + context "when input is nil" do + let(:sha) { nil } + + it { is_expected.to be_falsey } + end + + context "when input is a valid sha" do + let(:sha) { "c7f893648f656339f62fb7b4d8a6ecdf7d063835" } + + it { is_expected.to be_truthy } + end + + context "when input is a valid sha256" do + let(:sha) { "1b9affbba072ba2e923797d3b2050b9b9c8baacf696f84ac9940282b5568c547" } + + it { is_expected.to be_truthy } + end + + context "when input is a several valid shas separated by newline" do + let(:sha) { "c7f893648f656339f62fb7b4d8a6ecdf7d063835\nc7f893648f656339f62fb7b4d8a6ecdf7d063835" } + + it { is_expected.to be_falsey } + end + + context "when input is a an invalid sha" do + let(:sha) { "c7f893648g656339f62fb7b4d8a6ecdf7d063835" } + + it { is_expected.to be_falsey } + end + + context "when input is too short to be valid" do + let(:sha) { "c7f893648f656339f62fb7b4d8a6ecdf7d06383" } + + it { is_expected.to be_falsey } + end + end + describe ".normalize_ref" do subject { described_class.normalize_ref(ref) } diff --git a/vendor/rbs/ddtrace/0/datadog/core/transport/response.rbs b/vendor/rbs/ddtrace/0/datadog/core/transport/response.rbs index 888e6756..a793675e 100644 --- a/vendor/rbs/ddtrace/0/datadog/core/transport/response.rbs +++ b/vendor/rbs/ddtrace/0/datadog/core/transport/response.rbs @@ -2,7 +2,7 @@ module Datadog module Core module Transport module Response - def payload: () -> nil + def payload: () -> String def ok?: () -> nil