From db34827f2d76e1303edd89d96591659eccf36efb Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 16:22:14 +0200 Subject: [PATCH] add UniqueTestsClient to fetch a set of unique tests from backend --- .../ci/test_retries/unique_tests_client.rb | 132 +++++++++++ .../ci/test_retries/unique_tests_client.rbs | 36 +++ .../test_retries/unique_tests_client_spec.rb | 216 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 lib/datadog/ci/test_retries/unique_tests_client.rb create mode 100644 sig/datadog/ci/test_retries/unique_tests_client.rbs create mode 100644 spec/datadog/ci/test_retries/unique_tests_client_spec.rb diff --git a/lib/datadog/ci/test_retries/unique_tests_client.rb b/lib/datadog/ci/test_retries/unique_tests_client.rb new file mode 100644 index 00000000..eadfba98 --- /dev/null +++ b/lib/datadog/ci/test_retries/unique_tests_client.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require "json" + +require_relative "../ext/telemetry" +require_relative "../ext/transport" +require_relative "../transport/telemetry" +require_relative "../utils/telemetry" +require_relative "../utils/test_run" + +module Datadog + module CI + module TestRetries + # fetch a list of unique known tests from the backend + class UniqueTestsClient + class Response + def initialize(http_response) + @http_response = http_response + @json = nil + end + + def ok? + resp = @http_response + !resp.nil? && resp.ok? + end + + def tests + res = Set.new + + payload + .fetch("data", {}) + .fetch("attributes", {}) + .fetch("tests", {}) + .each do |_test_module, suites_hash| + suites_hash.each do |test_suite, tests| + tests.each do |test_name| + res << Utils::TestRun.datadog_test_id(test_name, test_suite) + end + end + end + + res + end + + private + + def payload + cached = @json + return cached unless cached.nil? + + resp = @http_response + return @json = {} if resp.nil? || !ok? + + begin + @json = JSON.parse(resp.payload) + rescue JSON::ParserError => e + Datadog.logger.error("Failed to parse unique known tests response payload: #{e}. Payload was: #{resp.payload}") + @json = {} + end + end + end + + def initialize(dd_env:, api: nil, config_tags: {}) + @api = api + @dd_env = dd_env + @config_tags = config_tags + end + + def fetch_unique_tests(test_session) + api = @api + return Response.new(nil) unless api + + request_payload = payload(test_session) + Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}") + + http_response = api.api_request( + path: Ext::Transport::DD_API_UNIQUE_TESTS_PATH, + payload: request_payload + ) + + Transport::Telemetry.api_requests( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST, + 1, + compressed: http_response.request_compressed + ) + Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms) + Utils::Telemetry.distribution( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES, + http_response.response_size.to_f, + {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s} + ) + + unless http_response.ok? + Transport::Telemetry.api_requests_errors( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS, + 1, + error_type: http_response.telemetry_error_type, + status_code: http_response.code + ) + end + + Response.new(http_response) + end + + private + + def payload(test_session) + { + "data" => { + "id" => Datadog::Core::Environment::Identity.id, + "type" => Ext::Transport::DD_API_UNIQUE_TESTS_TYPE, + "attributes" => { + "repository_url" => test_session.git_repository_url, + "service" => test_session.service, + "env" => @dd_env, + "sha" => test_session.git_commit_sha, + "configurations" => { + Ext::Test::TAG_OS_PLATFORM => test_session.os_platform, + Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture, + Ext::Test::TAG_OS_VERSION => test_session.os_version, + Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name, + Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version, + "custom" => @config_tags + } + } + } + }.to_json + end + end + end + end +end diff --git a/sig/datadog/ci/test_retries/unique_tests_client.rbs b/sig/datadog/ci/test_retries/unique_tests_client.rbs new file mode 100644 index 00000000..14c0c084 --- /dev/null +++ b/sig/datadog/ci/test_retries/unique_tests_client.rbs @@ -0,0 +1,36 @@ +module Datadog + module CI + module TestRetries + class UniqueTestsClient + @api: Datadog::CI::Transport::Api::Base? + @dd_env: String? + @config_tags: Hash[String, String] + + class Response + @http_response: Datadog::CI::Transport::Adapters::Net::Response? + @json: Hash[String, untyped]? + + def initialize: (Datadog::CI::Transport::Adapters::Net::Response? http_response) -> void + + def ok?: () -> bool + + def correlation_id: () -> String? + + def tests: () -> Set[String] + + private + + def payload: () -> Hash[String, untyped] + end + + def initialize: (?api: Datadog::CI::Transport::Api::Base?, dd_env: String?, ?config_tags: Hash[String, String]) -> void + + def fetch_unique_tests: (Datadog::CI::TestSession test_session) -> Response + + private + + def payload: (Datadog::CI::TestSession test_session) -> String + end + end + end +end diff --git a/spec/datadog/ci/test_retries/unique_tests_client_spec.rb b/spec/datadog/ci/test_retries/unique_tests_client_spec.rb new file mode 100644 index 00000000..8b0d3cec --- /dev/null +++ b/spec/datadog/ci/test_retries/unique_tests_client_spec.rb @@ -0,0 +1,216 @@ +# frozen_string_literal: true + +require_relative "../../../../lib/datadog/ci/test_retries/unique_tests_client" + +RSpec.describe Datadog::CI::TestRetries::UniqueTestsClient do + include_context "Telemetry spy" + + let(:api) { spy("api") } + let(:dd_env) { "ci" } + let(:config_tags) { {} } + + subject(:client) { described_class.new(api: api, dd_env: dd_env, config_tags: config_tags) } + + describe "#fetch_unique_tests" do + subject { client.fetch_unique_tests(test_session) } + + let(:service) { "service" } + let(:tracer_span) do + Datadog::Tracing::SpanOperation.new("session", service: service).tap do |span| + span.set_tags({ + "git.repository_url" => "repository_url", + "git.branch" => "branch", + "git.commit.sha" => "commit_sha", + "os.platform" => "platform", + "os.architecture" => "arch", + "os.version" => "version", + "runtime.name" => "runtime_name", + "runtime.version" => "runtime_version" + }) + end + end + let(:test_session) { Datadog::CI::TestSession.new(tracer_span) } + + let(:path) { Datadog::CI::Ext::Transport::DD_API_UNIQUE_TESTS_PATH } + + it "requests the unique tests" do + subject + + expect(api).to have_received(:api_request) do |args| + expect(args[:path]).to eq(path) + + data = JSON.parse(args[:payload])["data"] + + expect(data["type"]).to eq(Datadog::CI::Ext::Transport::DD_API_UNIQUE_TESTS_TYPE) + + attributes = data["attributes"] + expect(attributes["service"]).to eq(service) + expect(attributes["env"]).to eq(dd_env) + expect(attributes["repository_url"]).to eq("repository_url") + expect(attributes["sha"]).to eq("commit_sha") + + configurations = attributes["configurations"] + expect(configurations["os.platform"]).to eq("platform") + expect(configurations["os.architecture"]).to eq("arch") + expect(configurations["os.version"]).to eq("version") + expect(configurations["runtime.name"]).to eq("runtime_name") + expect(configurations["runtime.version"]).to eq("runtime_version") + end + end + + context "parsing response" do + subject(:response) { client.fetch_unique_tests(test_session) } + + context "when api is present" do + before do + allow(api).to receive(:api_request).and_return(http_response) + end + + context "when response is OK" do + let(:http_response) do + double( + "http_response", + ok?: true, + payload: { + data: { + id: "wTGavjGXpUg", + type: "ci_app_libraries_tests", + attributes: { + tests: { + "rspec" => { + "AdminControllerTest" => [ + "test_new", + "test_index", + "test_create" + ] + } + } + } + } + }.to_json, + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100 + ) + end + + it "parses the response" do + expect(response.ok?).to be true + expect(response.tests).to eq(Set.new(["AdminControllerTest.test_new.", "AdminControllerTest.test_index.", "AdminControllerTest.test_create."])) + end + + it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request", 1 + it_behaves_like "emits telemetry metric", :distribution, "early_flake_detection.request_ms" + it_behaves_like "emits telemetry metric", :distribution, "early_flake_detection.response_bytes" + end + + context "when response is not OK" do + let(:http_response) do + double( + "http_response", + ok?: false, + payload: "", + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100, + telemetry_error_type: nil, + code: 422 + ) + end + + it "parses the response" do + expect(response.ok?).to be false + expect(response.tests).to be_empty + end + + it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request_errors", 1 + end + + context "when response is OK but JSON is malformed" do + let(:http_response) do + double( + "http_response", + ok?: true, + payload: "not json", + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100 + ) + end + + before do + expect(Datadog.logger).to receive(:error).with(/Failed to parse unique known tests response payload/) + end + + it "parses the response" do + expect(response.ok?).to be true + expect(response.tests).to be_empty + end + end + + context "when response is OK but JSON has different format" do + let(:http_response) do + double( + "http_response", + ok?: true, + payload: { + "attributes" => { + "suite" => "test_suite_name", + "name" => "test_name", + "parameters" => "string", + "configurations" => { + "os.platform" => "linux", + "os.version" => "bionic", + "os.architecture" => "amd64", + "runtime.vendor" => "string", + "runtime.architecture" => "amd64" + } + } + }.to_json, + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100 + ) + end + + it "parses the response" do + expect(response.ok?).to be true + expect(response.tests).to be_empty + end + end + end + + context "when there is no api" do + let(:api) { nil } + + it "returns an empty response" do + expect(response.ok?).to be false + expect(response.tests).to be_empty + end + end + end + + context "when there are custom configurations" do + let(:config_tags) do + { + "tag1" => "value1" + } + end + + it "requests the unique tests with custom configurations" do + subject + + expect(api).to have_received(:api_request) do |args| + data = JSON.parse(args[:payload])["data"] + configurations = data["attributes"]["configurations"] + + expect(configurations["custom"]).to eq("tag1" => "value1") + end + end + end + end +end