Merge pull request #51 from DataDog/anmarchenko/introduce_api_concept

[CIVIS-2467] Support test visibility protocol via Datadog Agent with EVP proxy

lib/datadog/ci/configuration/components.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
+require "datadog/core/configuration/agent_settings_resolver"
+require "datadog/core/remote/negotiation"
+
+require_relative "../ext/transport"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/transport"
+require_relative "../transport/api/builder"
 
 module Datadog
   module CI
@@ -17,25 +22,13 @@ def initialize(settings)
         end
 
         def activate_ci!(settings)
-          agentless_transport = nil
+          test_visibility_transport = nil
+          agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
 
           if settings.ci.agentless_mode_enabled
-            if settings.api_key.nil?
-              # agentless mode is requested but no API key is provided -
-              # we cannot continue and log an error
-              # Tests are running without CI visibility enabled
-
-              Datadog.logger.error(
-                "DATADOG CONFIGURATION - CI VISIBILITY - ATTENTION - " \
-                "Agentless mode was enabled but DD_API_KEY is not set: CI visibility is disabled. " \
-                "Please make sure to set valid api key in DD_API_KEY environment variable"
-              )
-
-              settings.ci.enabled = false
-              return
-            else
-              agentless_transport = build_agentless_transport(settings)
-            end
+            test_visibility_transport = build_agentless_transport(settings)
+          elsif can_use_evp_proxy?(settings, agent_settings)
+            test_visibility_transport = build_evp_proxy_transport(settings, agent_settings)
           end
 
           # Deactivate telemetry
@@ -51,8 +44,8 @@ def activate_ci!(settings)
           settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Finished.new
 
           writer_options = settings.ci.writer_options
-          if agentless_transport
-            writer_options[:transport] = agentless_transport
+          if test_visibility_transport
+            writer_options[:transport] = test_visibility_transport
             writer_options[:shutdown_timeout] = 60
 
             settings.tracing.test_mode.async = true
@@ -61,15 +54,43 @@ def activate_ci!(settings)
           settings.tracing.test_mode.writer_options = writer_options
         end
 
+        def can_use_evp_proxy?(settings, agent_settings)
+          Datadog::Core::Remote::Negotiation.new(settings, agent_settings).endpoint?(
+            Ext::Transport::EVP_PROXY_PATH_PREFIX
+          )
+        end
+
         def build_agentless_transport(settings)
-          dd_site = settings.site || "datadoghq.com"
-          agentless_url = settings.ci.agentless_url ||
-            "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+          if settings.api_key.nil?
+            # agentless mode is requested but no API key is provided -
+            # we cannot continue and log an error
+            # Tests are running without CI visibility enabled
+
+            Datadog.logger.error(
+              "DATADOG CONFIGURATION - CI VISIBILITY - ATTENTION - " \
+              "Agentless mode was enabled but DD_API_KEY is not set: CI visibility is disabled. " \
+              "Please make sure to set valid api key in DD_API_KEY environment variable"
+            )
+
+            settings.ci.enabled = false
+
+            nil
+          else
+            Datadog.logger.debug("CI visibility configured to use agentless transport")
+
+            Datadog::CI::TestVisibility::Transport.new(
+              api: Transport::Api::Builder.build_ci_test_cycle_api(settings),
+              dd_env: settings.env
+            )
+          end
+        end
+
+        def build_evp_proxy_transport(settings, agent_settings)
+          Datadog.logger.debug("CI visibility configured to use agent transport via EVP proxy")
 
           Datadog::CI::TestVisibility::Transport.new(
-            api_key: settings.api_key,
-            url: agentless_url,
-            env: settings.env
+            api: Transport::Api::Builder.build_evp_proxy_api(agent_settings),
+            dd_env: settings.env
           )
         end
       end

lib/datadog/ci/ext/transport.rb

@@ -4,10 +4,15 @@ module Datadog
   module CI
     module Ext
       module Transport
+        DEFAULT_DD_SITE = "datadoghq.com"
+
         HEADER_DD_API_KEY = "DD-API-KEY"
         HEADER_CONTENT_TYPE = "Content-Type"
         HEADER_CONTENT_ENCODING = "Content-Encoding"
+        HEADER_EVP_SUBDOMAIN = "X-Datadog-EVP-Subdomain"
+        HEADER_CONTAINER_ID = "Datadog-Container-ID"
 
+        EVP_PROXY_PATH_PREFIX = "/evp_proxy/v2/"
         TEST_VISIBILITY_INTAKE_HOST_PREFIX = "citestcycle-intake"
         TEST_VISIBILITY_INTAKE_PATH = "/api/v2/citestcycle"
 

lib/datadog/ci/test_visibility/transport.rb

@@ -9,7 +9,6 @@
 
 require_relative "serializers/factories/test_level"
 require_relative "../ext/transport"
-require_relative "../transport/http"
 
 module Datadog
   module CI
@@ -20,33 +19,20 @@ class Transport
         DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
 
         attr_reader :serializers_factory,
-          :api_key,
+          :api,
           :max_payload_size,
-          :http,
-          :env
+          :dd_env
 
         def initialize(
-          api_key:,
-          url:,
-          env: nil,
+          api:,
+          dd_env: nil,
           serializers_factory: Datadog::CI::TestVisibility::Serializers::Factories::TestLevel,
           max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE
         )
           @serializers_factory = serializers_factory
-          @api_key = api_key
           @max_payload_size = max_payload_size
-          @env = env
-
-          uri = URI.parse(url)
-
-          raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
-
-          @http = Datadog::CI::Transport::HTTP.new(
-            host: uri.host,
-            port: uri.port,
-            ssl: uri.scheme == "https" || uri.port == 443,
-            compress: true
-          )
+          @dd_env = dd_env
+          @api = api
         end
 
         def send_traces(traces)
@@ -82,13 +68,9 @@ def send_traces(traces)
         private
 
         def send_payload(encoded_payload)
-          http.request(
+          api.request(
             path: Datadog::CI::Ext::Transport::TEST_VISIBILITY_INTAKE_PATH,
-            payload: encoded_payload,
-            headers: {
-              Ext::Transport::HEADER_DD_API_KEY => api_key,
-              Ext::Transport::HEADER_CONTENT_TYPE => Ext::Transport::CONTENT_TYPE_MESSAGEPACK
-            }
+            payload: encoded_payload
           )
         end
 
@@ -141,12 +123,12 @@ def pack_events(encoded_events)
           packer.write_map_header(1)
 
           packer.write("*")
-          metadata_fields_count = env ? 4 : 3
+          metadata_fields_count = dd_env ? 4 : 3
           packer.write_map_header(metadata_fields_count)
 
-          if env
+          if dd_env
             packer.write("env")
-            packer.write(env)
+            packer.write(dd_env)
           end
 
           packer.write("runtime-id")

lib/datadog/ci/transport/api/base.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class Base
+          attr_reader :http
+
+          def initialize(http:)
+            @http = http
+          end
+
+          def request(path:, payload:, verb: "post")
+            http.request(
+              path: path,
+              payload: payload,
+              verb: verb,
+              headers: headers
+            )
+          end
+
+          private
+
+          def headers
+            {
+              Ext::Transport::HEADER_CONTENT_TYPE => Ext::Transport::CONTENT_TYPE_MESSAGEPACK
+            }
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/transport/api/builder.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "ci_test_cycle"
+require_relative "evp_proxy"
+require_relative "../http"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        module Builder
+          def self.build_ci_test_cycle_api(settings)
+            dd_site = settings.site || Ext::Transport::DEFAULT_DD_SITE
+            url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            http = Datadog::CI::Transport::HTTP.new(
+              host: uri.host,
+              port: uri.port,
+              ssl: uri.scheme == "https" || uri.port == 443,
+              compress: true
+            )
+
+            CiTestCycle.new(api_key: settings.api_key, http: http)
+          end
+
+          def self.build_evp_proxy_api(agent_settings)
+            http = Datadog::CI::Transport::HTTP.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              ssl: agent_settings.ssl,
+              timeout: agent_settings.timeout_seconds,
+              compress: false
+            )
+
+            EvpProxy.new(http: http)
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/transport/api/ci_test_cycle.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class CiTestCycle < Base
+          attr_reader :api_key
+
+          def initialize(api_key:, http:)
+            @api_key = api_key
+
+            super(http: http)
+          end
+
+          private
+
+          def headers
+            headers = super
+            headers[Ext::Transport::HEADER_DD_API_KEY] = api_key
+            headers
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/transport/api/evp_proxy.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "datadog/core/environment/container"
+
+require_relative "base"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class EvpProxy < Base
+          def request(path:, payload:, verb: "post")
+            path = "#{Ext::Transport::EVP_PROXY_PATH_PREFIX}#{path.sub(/^\//, "")}"
+
+            super(
+              path: path,
+              payload: payload,
+              verb: verb
+            )
+          end
+
+          private
+
+          def container_id
+            return @container_id if defined?(@container_id)
+
+            @container_id = Datadog::Core::Environment::Container.container_id
+          end
+
+          def headers
+            headers = super
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX
+
+            c_id = container_id
+            headers[Ext::Transport::HEADER_CONTAINER_ID] = c_id unless c_id.nil?
+
+            headers
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/configuration/components.rbs

@@ -6,7 +6,9 @@ module Datadog
 
         def activate_ci!: (untyped settings) -> untyped
 
-        def build_agentless_transport: (untyped settings) -> Datadog::CI::TestVisibility::Transport
+        def build_agentless_transport: (untyped settings) -> Datadog::CI::TestVisibility::Transport?
+        def build_evp_proxy_transport: (untyped settings, untyped agent_settings) -> Datadog::CI::TestVisibility::Transport
+        def can_use_evp_proxy?: (untyped settings, untyped agent_settings) -> bool
       end
     end
   end

sig/datadog/ci/ext/transport.rbs

@@ -2,12 +2,20 @@ module Datadog
   module CI
     module Ext
       module Transport
+        DEFAULT_DD_SITE: "datadoghq.com"
+
         HEADER_DD_API_KEY: "DD-API-KEY"
 
         HEADER_CONTENT_TYPE: "Content-Type"
 
         HEADER_CONTENT_ENCODING: "Content-Encoding"
 
+        HEADER_EVP_SUBDOMAIN: "X-Datadog-EVP-Subdomain"
+
+        HEADER_CONTAINER_ID: "Datadog-Container-ID"
+
+        EVP_PROXY_PATH_PREFIX: "/evp_proxy/v2/"
+
         TEST_VISIBILITY_INTAKE_HOST_PREFIX: "citestcycle-intake"
 
         TEST_VISIBILITY_INTAKE_PATH: "/api/v2/citestcycle"