Support the Automate /compliance/profiles/metasearch endpoint to send reduced reports

attributes/default.rb

@@ -88,3 +88,8 @@
 # <chef_cache_path>/cookbooks/audit/inspec-<YYYYMMDDHHMMSS>.json
 default['audit']['json_file']['location'] =
   File.expand_path(Time.now.utc.strftime('../../inspec-%Y%m%d%H%M%S.json'), __FILE__)
+
+# Control results that have a `run_time` below this limit will
+# be stripped of the `start_time` and `run_time` fields to
+# reduce the size of the reports being sent to Automate
+default['audit']['run_time_limit'] = 1.0

files/default/handler/audit_report.rb

@@ -230,6 +230,7 @@ def send_report(reporter, server, source_location, report)
         # Set `insecure` here to avoid passing 6 aruguments to `AuditReport#send_report`
         # See `cookstyle` Metrics/ParameterLists
         insecure = node['audit']['insecure']
+        run_time_limit = node['audit']['run_time_limit']
 
         # TODO: harmonize reporter interface
         if reporter == 'chef-automate'
@@ -240,6 +241,7 @@ def send_report(reporter, server, source_location, report)
             node_info: gather_nodeinfo,
             insecure: insecure,
             source_location: source_location,
+            run_time_limit: run_time_limit,
           }
           Reporter::ChefAutomate.new(opts).send_report(report)
         elsif reporter == 'chef-server-automate'
@@ -255,6 +257,7 @@ def send_report(reporter, server, source_location, report)
               insecure: insecure,
               url: url,
               source_location: source_location,
+              run_time_limit: run_time_limit,
             }
             Reporter::ChefServerAutomate.new(opts).send_report(report)
           else

libraries/helper.rb

@@ -150,6 +150,72 @@ def get_reporters(audit)
     end
     handle_reporters(audit['reporter'])
   end
+
+  # Extracts all the profile sha256 IDs from an inspec report
+  def report_profile_sha256s(report)
+    return [] unless report.is_a?(Hash) && report[:profiles].is_a?(Array)
+    report[:profiles].map { |p| p[:sha256] }
+  end
+
+  # Strips profile metadata (title, copyright, controls title, code block, descriptions, etc)
+  # from the `report` profiles that are not in the `missing_report_shas` array.
+  # Control results are also stripped of the `start_time` and `run_time` if they don't reach the `run_time_limit` threshold (seconds)
+  def strip_profiles_meta(report, missing_report_shas, run_time_limit)
+    return report unless report.is_a?(Hash) && report[:profiles].is_a?(Array)
+    report[:profiles].each do |p|
+      next if missing_report_shas.include?(p[:sha256])
+      # Profile 'name' is a required property. By not sending it in the report, we make it clear to the ingestion backend that the profile metadata has been stripped from this profile in the report.
+      # Profile 'title' and 'version' are still kept for troubleshooting purposes in the backend.
+      p.delete(:name)
+      p.delete(:groups)
+      p.delete(:copyright_email)
+      p.delete(:copyright)
+      p.delete(:summary)
+      p.delete(:supports)
+      p.delete(:license)
+      p.delete(:maintainer)
+      next unless p[:controls].is_a?(Array)
+      p[:controls].each do |c|
+        c.delete(:code)
+        c.delete(:desc)
+        c.delete(:descriptions)
+        c.delete(:impact)
+        c.delete(:refs)
+        c.delete(:tags)
+        c.delete(:title)
+        c.delete(:source_location)
+        c.delete(:waiver_data) if c[:waiver_data] == {}
+        next unless c[:results].is_a?(Array)
+        c[:results].each do |r|
+          if r[:run_time].is_a?(Float) && r[:run_time] < run_time_limit
+            r.delete(:start_time)
+            r.delete(:run_time)
+          end
+        end
+      end
+    end
+    report[:run_time_limit] = run_time_limit
+    report
+  end
+
+  # Contacts the metasearch Automate API to check which of the inspec profile sha256 ids
+  # passed in via `report_shas` are missing from the Automate profiles metadata database.
+  def missing_automate_profiles(automate_url, headers, report_shas)
+    Chef::Log.debug "Checking the Automate profiles metadata for: #{report_shas}"
+    meta_url = URI(automate_url)
+    meta_url.path = '/compliance/profiles/metasearch'
+    http = Chef::HTTP.new(meta_url.to_s)
+    response_str = http.post(nil, "{\"sha256\": #{report_shas}}", headers)
+    missing_shas = JSON.parse(response_str)['missing_sha256']
+    unless missing_shas.empty?
+      Chef::Log.info "Automate is missing metadata for the following profile ids: #{missing_shas}"
+    end
+    missing_shas
+  rescue => e
+    Chef::Log.error "missing_automate_profiles error: #{e.message}"
+    # If we get an error it's safer to assume none of the profile shas exist in Automate
+    report_shas
+  end
 end
 
 ::Chef::DSL::Recipe.include ReportHelpers

libraries/reporters/automate.rb

@@ -24,6 +24,7 @@ def initialize(opts)
       @organization_name = opts[:node_info][:organization_name]
       @ipaddress         = opts[:node_info][:ipaddress]
       @fqdn              = opts[:node_info][:fqdn]
+      @run_time_limit    = opts[:run_time_limit]
 
       if defined?(Chef) &&
          defined?(Chef::Config) &&
@@ -44,23 +45,7 @@ def send_report(report)
         return false
       end
 
-      json_report = enriched_report(report).to_json
-      report_size = json_report.bytesize
-      # Automate GRPC currently has a message limit of ~4MB
-      # https://github.com/chef/automate/issues/1417#issuecomment-541908157
-      if report_size > 4 * 1024 * 1024
-        Chef::Log.warn "Compliance report size is #{(report_size / (1024 * 1024.0)).round(2)} MB."
-        Chef::Log.warn 'Automate has an internal 4MB limit that is not currently configurable.'
-      end
-
-      unless json_report
-        Chef::Log.warn 'Something went wrong, report can\'t be nil'
-        return false
-      end
-
-      if defined?(Chef) &&
-         defined?(Chef::Config)
-
+      if defined?(Chef) && defined?(Chef::Config)
         headers = { 'Content-Type' => 'application/json' }
         unless @token.nil?
           headers['x-data-collector-token'] = @token
@@ -74,6 +59,28 @@ def send_report(report)
           Chef::Config[:ssl_verify_mode] = :verify_none
         end
 
+        all_report_shas = report_profile_sha256s(report)
+        missing_report_shas = missing_automate_profiles(@url, headers, all_report_shas)
+
+        full_report = enriched_report(report)
+
+        # If the Automate backend has the profile metadata for at least one profile, proceed with metadata stripping
+        full_report = strip_profiles_meta(full_report, missing_report_shas, 1) if missing_report_shas.length < all_report_shas.length
+        json_report = full_report.to_json
+
+        report_size = json_report.bytesize
+        # Automate GRPC currently has a message limit of ~4MB
+        # https://github.com/chef/automate/issues/1417#issuecomment-541908157
+        if report_size > 4 * 1024 * 1024
+          Chef::Log.warn "Compliance report size is #{(report_size / (1024 * 1024.0)).round(2)} MB."
+          Chef::Log.warn 'Automate has an internal 4MB limit that is not currently configurable.'
+        end
+
+        unless json_report
+          Chef::Log.warn 'Something went wrong, report can\'t be nil'
+          return false
+        end
+
         begin
           Chef::Log.info "Report to Chef Automate: #{@url}"
           Chef::Log.debug "Audit Report: #{json_report}"

spec/data/mock.rb

@@ -20,6 +20,7 @@ def self.inspec_results
       [{ "name": 'tmp_compliance_profile',
          "title": '/tmp Compliance Profile',
          "summary": 'An Example Compliance Profile',
+         "sha256": '7bd598e369970002fc6f2d16d5b988027d58b044ac3fa30ae5fc1b8492e215cd',
          "version": '0.1.1',
          "maintainer": 'Nathen Harvey <nharvey@chef.io>',
          "license": 'Apache 2.0 License',
@@ -34,17 +35,23 @@ def self.inspec_results
             "code": "control 'tmp-1.0' do\n  impact 0.3\n  title 'A /tmp directory must exist'\n  desc 'A /tmp directory must exist'\n  describe file '/tmp' do\n    it { should be_directory }\n  end\nend\n",
             "source_location": { "ref": '/Users/vjeffrey/code/delivery/insights/data_generator/chef-client/cache/cookbooks/test-cookbook/recipes/../files/default/compliance_profiles/tmp_compliance_profile/controls/tmp.rb', "line": 3 },
             "id": 'tmp-1.0',
-            "results": [{ "status": 'passed', "code_desc": 'File /tmp should be directory', "run_time": 0.002312, "start_time": '2016-10-19 11:09:43 -0400' }] },
+            "results": [
+              { "status": 'passed', "code_desc": 'File /tmp should be directory', "run_time": 0.002312, "start_time": '2016-10-19 11:09:43 -0400' },
+            ],
+          },
           { "title": '/tmp directory is owned by the root user',
             "desc": 'The /tmp directory must be owned by the root user',
             "impact": 0.3,
             "refs": [{ "url": 'https://pages.chef.io/rs/255-VFB-268/images/compliance-at-velocity2015.pdf', "ref": 'Compliance Whitepaper' }],
             "tags": { "production": nil, "development": nil, "identifier": 'value', "remediation": 'https://github.com/chef-cookbooks/audit' },
-            "code":
-            "control 'tmp-1.1' do\n  impact 0.3\n  title '/tmp directory is owned by the root user'\n  desc 'The /tmp directory must be owned by the root user'\n  tag 'production','development'\n  tag identifier: 'value'\n  tag remediation: 'https://github.com/chef-cookbooks/audit'\n  ref 'Compliance Whitepaper', url: 'https://pages.chef.io/rs/255-VFB-268/images/compliance-at-velocity2015.pdf'\n  describe file '/tmp' do\n    it { should be_owned_by 'root' }\n  end\nend\n",
+            "code": "control 'tmp-1.1' do\n  impact 0.3\n  title '/tmp directory is owned by the root user'\n  desc 'The /tmp directory must be owned by the root user'\n  tag 'production','development'\n  tag identifier: 'value'\n  tag remediation: 'https://github.com/chef-cookbooks/audit'\n  ref 'Compliance Whitepaper', url: 'https://pages.chef.io/rs/255-VFB-268/images/compliance-at-velocity2015.pdf'\n  describe file '/tmp' do\n    it { should be_owned_by 'root' }\n  end\nend\n",
             "source_location": { "ref": '/Users/vjeffrey/code/delivery/insights/data_generator/chef-client/cache/cookbooks/test-cookbook/recipes/../files/default/compliance_profiles/tmp_compliance_profile/controls/tmp.rb', "line": 12 },
             "id": 'tmp-1.1',
-            "results": [{ "status": 'passed', "code_desc": 'File /tmp should be owned by "root"', "run_time": 0.028845, "start_time": '2016-10-19 11:09:43 -0400' }] }],
+            "results": [
+              { "status": 'passed', "code_desc": 'File /tmp should be owned by "root"', "run_time": 1.228845, "start_time": '2016-10-19 11:09:43 -0400' },
+            ],
+          },
+        ],
          "groups": [{ "title": '/tmp Compliance Profile', "controls": ['tmp-1.0', 'tmp-1.1'], "id": 'controls/tmp.rb' }],
          "attributes": [{ "name": 'syslog_pkg', "options": { "default": 'rsyslog', "description": 'syslog package...' } }] }],
       "other_checks": [],