Merge pull request #170 from DataDog/anmarchenko/accept_gzip_skippabl…

…e_tests

[CIVIS-9901] accept gzipped responses from API

lib/datadog/ci/ext/transport.rb

@@ -7,6 +7,7 @@ module Transport
         DEFAULT_DD_SITE = "datadoghq.com"
 
         HEADER_DD_API_KEY = "DD-API-KEY"
+        HEADER_ACCEPT_ENCODING = "Accept-Encoding"
         HEADER_CONTENT_TYPE = "Content-Type"
         HEADER_CONTENT_ENCODING = "Content-Encoding"
         HEADER_EVP_SUBDOMAIN = "X-Datadog-EVP-Subdomain"
@@ -48,6 +49,8 @@ module Transport
         CONTENT_TYPE_JSON = "application/json"
         CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"
         CONTENT_ENCODING_GZIP = "gzip"
+
+        GZIP_MAGIC_NUMBER = "\x1F\x8B".b
       end
     end
   end

lib/datadog/ci/transport/api/agentless.rb

@@ -26,7 +26,14 @@ def citestcycle_request(path:, payload:, headers: {}, verb: "post")
           def api_request(path:, payload:, headers: {}, verb: "post")
             super
 
-            perform_request(@api_http, path: path, payload: payload, headers: headers, verb: verb)
+            perform_request(
+              @api_http,
+              path: path,
+              payload: payload,
+              headers: headers,
+              verb: verb,
+              accept_compressed_response: true
+            )
           end
 
           def citestcov_request(path:, payload:, headers: {}, verb: "post")
@@ -37,12 +44,13 @@ def citestcov_request(path:, payload:, headers: {}, verb: "post")
 
           private
 
-          def perform_request(http_client, path:, payload:, headers:, verb:)
+          def perform_request(http_client, path:, payload:, headers:, verb:, accept_compressed_response: false)
             http_client.request(
               path: path,
               payload: payload,
               headers: headers_with_default(headers),
-              verb: verb
+              verb: verb,
+              accept_compressed_response: accept_compressed_response
             )
           end
 

lib/datadog/ci/transport/gzip.rb

@@ -16,6 +16,18 @@ def compress(input)
           gzip_writer.close
           sio.string
         end
+
+        def decompress(input)
+          sio = StringIO.new(input)
+          gzip_reader = Zlib::GzipReader.new(
+            sio,
+            external_encoding: Encoding::UTF_8,
+            internal_encoding: Encoding::UTF_8
+          )
+          gzip_reader.read || ""
+        ensure
+          gzip_reader&.close
+        end
       end
     end
   end

lib/datadog/ci/transport/http.rb

@@ -32,12 +32,24 @@ def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress:
           @compress = compress.nil? ? false : compress
         end
 
-        def request(path:, payload:, headers:, verb: "post", retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
+        def request(
+          path:,
+          payload:,
+          headers:,
+          verb: "post",
+          retries: MAX_RETRIES,
+          backoff: INITIAL_BACKOFF,
+          accept_compressed_response: false
+        )
           if compress
             headers[Ext::Transport::HEADER_CONTENT_ENCODING] = Ext::Transport::CONTENT_ENCODING_GZIP
             payload = Gzip.compress(payload)
           end
 
+          if accept_compressed_response
+            headers[Ext::Transport::HEADER_ACCEPT_ENCODING] = Ext::Transport::CONTENT_ENCODING_GZIP
+          end
+
           Datadog.logger.debug do
             "Sending #{verb} request: host=#{host}; port=#{port}; ssl_enabled=#{ssl}; " \
               "compression_enabled=#{compress}; path=#{path}; payload_size=#{payload.size}"
@@ -91,12 +103,32 @@ def adapter
           @adapter ||= Datadog::Core::Transport::HTTP::Adapters::Net.new(settings)
         end
 
-        # this is needed because Datadog::Tracing::Writer is not fully compatiple with Datadog::Core::Transport
-        # TODO: remove when CI implements its own worker
+        # adds compatibility with Datadog::Tracing transport and
+        # provides ungzipping capabilities
         class ResponseDecorator < ::SimpleDelegator
+          def payload
+            return @decompressed_payload if defined?(@decompressed_payload)
+
+            if gzipped?(__getobj__.payload)
+              Datadog.logger.debug("Decompressing gzipped response payload")
+              @decompressed_payload = Gzip.decompress(__getobj__.payload)
+            else
+              __getobj__.payload
+            end
+          end
+
           def trace_count
             0
           end
+
+          def gzipped?(payload)
+            return false if payload.nil? || payload.empty?
+
+            first_bytes = payload[0, 2]
+            return false if first_bytes.nil? || first_bytes.empty?
+
+            first_bytes.b == Datadog::CI::Ext::Transport::GZIP_MAGIC_NUMBER
+          end
         end
 
         class AdapterSettings

sig/datadog/ci/ext/transport.rbs

@@ -6,6 +6,8 @@ module Datadog
 
         HEADER_DD_API_KEY: "DD-API-KEY"
 
+        HEADER_ACCEPT_ENCODING: "Accept-Encoding"
+
         HEADER_CONTENT_TYPE: "Content-Type"
 
         HEADER_CONTENT_ENCODING: "Content-Encoding"
@@ -63,6 +65,8 @@ module Datadog
         CONTENT_TYPE_MULTIPART_FORM_DATA: "multipart/form-data"
 
         CONTENT_ENCODING_GZIP: "gzip"
+
+        GZIP_MAGIC_NUMBER: String
       end
     end
   end

sig/datadog/ci/transport/api/agentless.rbs

@@ -22,7 +22,7 @@ module Datadog
 
           private
 
-          def perform_request: (Datadog::CI::Transport::HTTP client, path: String, payload: String, headers: Hash[String, String], verb: ::String) -> Datadog::CI::Transport::HTTP::ResponseDecorator
+          def perform_request: (Datadog::CI::Transport::HTTP client, path: String, payload: String, headers: Hash[String, String], verb: ::String, ?accept_compressed_response: bool) -> Datadog::CI::Transport::HTTP::ResponseDecorator
 
           def build_http_client: (String url, compress: bool) -> Datadog::CI::Transport::HTTP
 

sig/datadog/ci/transport/gzip.rbs

@@ -3,6 +3,7 @@ module Datadog
     module Transport
       module Gzip
         def self?.compress: (String input) -> String
+        def self?.decompress: (String input) -> String
       end
     end
   end

sig/datadog/ci/transport/http.rbs

@@ -1,4 +1,9 @@
 class SimpleDelegator
+  @decompressed_payload: String
+
+  def __getobj__: () -> Datadog::Core::Transport::Response
+  def gzipped?: (String payload) -> bool
+  def payload: () -> String
 end
 
 module Datadog
@@ -19,7 +24,7 @@ module Datadog
 
         def initialize: (host: String, ?port: Integer?, ?ssl: bool, ?timeout: Integer, ?compress: bool) -> void
 
-        def request: (?verb: String, payload: String, headers: Hash[String, String], path: String, ?retries: Integer, ?backoff: Integer) -> ResponseDecorator
+        def request: (?verb: String, payload: String, headers: Hash[String, String], path: String, ?retries: Integer, ?backoff: Integer, ?accept_compressed_response: bool) -> ResponseDecorator
 
         private
 

spec/datadog/ci/transport/api/agentless_spec.rb

@@ -66,7 +66,8 @@
           path: "path",
           payload: "payload",
           verb: "post",
-          headers: expected_headers
+          headers: expected_headers,
+          accept_compressed_response: false
         )
 
         subject.citestcycle_request(path: "path", payload: "payload")
@@ -120,7 +121,8 @@
           path: "path",
           payload: "payload",
           verb: "post",
-          headers: expected_headers
+          headers: expected_headers,
+          accept_compressed_response: false
         )
 
         subject.citestcycle_request(path: "path", payload: "payload")
@@ -131,7 +133,8 @@
           path: "path",
           payload: "payload",
           verb: "post",
-          headers: expected_headers.merge({"Content-Type" => "application/json"})
+          headers: expected_headers.merge({"Content-Type" => "application/json"}),
+          accept_compressed_response: false
         )
 
         subject.citestcycle_request(path: "path", payload: "payload", headers: {"Content-Type" => "application/json"})
@@ -154,7 +157,8 @@
           headers: {
             "DD-API-KEY" => "api_key",
             "Content-Type" => "application/json"
-          }
+          },
+          accept_compressed_response: true
         )
 
         subject.api_request(path: "path", payload: "payload")
@@ -199,6 +203,7 @@
           expect(args[:verb]).to eq("post")
           expect(args[:headers]).to eq(expected_headers)
           expect(args[:payload]).to eq(expected_payload)
+          expect(args[:accept_compressed_response]).to eq(false)
         end
       end
     end

spec/datadog/ci/transport/gzip_spec.rb

@@ -1,12 +1,9 @@
 require_relative "../../../../lib/datadog/ci/transport/gzip"
 
 RSpec.describe Datadog::CI::Transport::Gzip do
-  subject { described_class.compress(input) }
-
-  describe ".compress" do
-    let(:input) do
-      <<-LOREM
-        Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc et est eu dui dignissim tempus. Aliquam
+  let(:input) do
+    <<-LOREM
+        ❤️ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc et est eu dui dignissim tempus. Aliquam
         scelerisque posuere odio id sollicitudin. Etiam dolor lorem, interdum sed mollis consectetur, sagittis a massa.
         Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec gravida, libero ac gravida ornare, leo elit
         facilisis nunc, in pharetra odio lectus sit amet augue. Cras fermentum interdum tortor, pulvinar laoreet massa
@@ -18,17 +15,40 @@
         commodo sapien vel, consequat felis. Aenean velit turpis, rhoncus in ipsum ut, tempor iaculis nisi. Fusce
         faucibus consequat blandit. Nam maximus augue quis tellus cursus eleifend. Suspendisse auctor, orci sit amet
         vehicula molestie, magna nibh rutrum metus, eget sagittis felis mauris eu quam. Vivamus ut vulputate est.
-      LOREM
-    end
+    LOREM
+  end
+
+  describe ".compress" do
+    subject { described_class.compress(input) }
 
     it "compresses" do
       expect(input.size).to be > subject.size
     end
 
     it "can be decompressed with gzip" do
-      Zlib::GzipReader.new(StringIO.new(subject)) do |gzip|
+      Zlib::GzipReader.new(
+        StringIO.new(subject),
+        external_encoding: Encoding::UTF_8,
+        internal_encoding: Encoding::UTF_8
+      ) do |gzip|
         expect(gzip.read).to eq(input)
       end
     end
   end
+
+  describe ".decompress" do
+    subject { described_class.decompress(compressed_input) }
+
+    let(:compressed_input) do
+      sio = StringIO.new
+      writer = Zlib::GzipWriter.new(sio)
+      writer << input
+      writer.close
+      sio.string
+    end
+
+    it "decompresses" do
+      expect(subject).to eq(input)
+    end
+  end
 end

spec/datadog/ci/transport/http_spec.rb

@@ -89,32 +89,60 @@
     let(:path) { "/api/v1/intake" }
     let(:payload) { '{ "key": "value" }' }
     let(:headers) { {"Content-Type" => "application/json"} }
-    let(:request_options) { {} }
+    let(:expected_headers) { headers }
+    let(:request_options) { {accept_compressed_response: false} }
 
-    let(:http_response) { double("http_response", code: 200) }
+    let(:response_payload) { "sample payload" }
+    let(:http_response) { double("http_response", code: 200, payload: response_payload) }
 
     subject(:response) { transport.request(path: path, payload: payload, headers: headers, **request_options) }
 
     context "when request is successful" do
-      let(:env) do
+      let(:expected_env) do
         env = Datadog::Core::Transport::HTTP::Env.new(
           Datadog::Core::Transport::Request.new
         )
         env.body = payload
         env.path = path
-        env.headers = headers
+        env.headers = expected_headers
         env.verb = "post"
         env
       end
 
       before do
-        expect(adapter).to receive(:call).with(env).and_return(http_response)
+        expect(adapter).to receive(:call).with(expected_env).and_return(http_response)
       end
 
       it "produces a response" do
         is_expected.to be_a_kind_of(described_class::ResponseDecorator)
 
         expect(response.code).to eq(200)
+        expect(response.payload).to eq("sample payload")
+      end
+
+      context "when accepting gzipped response" do
+        let(:expected_headers) { {"Content-Type" => "application/json", "Accept-Encoding" => "gzip"} }
+        let(:request_options) { {accept_compressed_response: true} }
+
+        it "adds Accept-Encoding header" do
+          is_expected.to be_a_kind_of(described_class::ResponseDecorator)
+
+          expect(response.code).to eq(200)
+          expect(response.payload).to eq("sample payload")
+        end
+
+        context "when response is gzipped" do
+          let(:response_payload) do
+            Datadog::CI::Transport::Gzip.compress("sample payload")
+          end
+
+          it "decompressed response payload" do
+            is_expected.to be_a_kind_of(described_class::ResponseDecorator)
+
+            expect(response.code).to eq(200)
+            expect(response.payload).to eq("sample payload")
+          end
+        end
       end
     end