feat: add insights instrumentation - events and metrics

lib/honeybadger/agent.rb

@@ -8,7 +8,10 @@
 require 'honeybadger/logging'
 require 'honeybadger/worker'
 require 'honeybadger/events_worker'
+require 'honeybadger/metrics_worker'
 require 'honeybadger/breadcrumbs'
+require 'honeybadger/registry'
+require 'honeybadger/registry_execution'
 
 module Honeybadger
   # The Honeybadger agent contains all the methods for interacting with the
@@ -358,6 +361,7 @@ def flush
     ensure
       worker.flush
       events_worker&.flush
+      metrics_worker&.flush
     end
 
     # Stops the Honeybadger service.
@@ -367,6 +371,7 @@ def flush
     def stop(force = false)
       worker.shutdown(force)
       events_worker&.shutdown(force)
+      metrics_worker&.shutdown(force)
       true
     end
 
@@ -387,20 +392,46 @@ def stop(force = false)
     def event(event_type, payload = {})
       init_events_worker
 
-      ts = DateTime.now.new_offset(0).rfc3339
+      ts = Time.now.utc.strftime("%FT%T.%LZ")
       merged = {ts: ts}
 
       if event_type.is_a?(String)
-        merged.merge!(event_type: event_type)
+        merged[:event_type] = event_type
       else
         merged.merge!(Hash(event_type))
       end
 
+      if (request_id = context_manager.get_request_id)
+        merged[:request_id] = request_id
+      end
+
+      if config[:'events.attach_hostname']
+        merged[:hostname] = config[:hostname].to_s
+      end
+
       merged.merge!(Hash(payload))
 
+      return if config.ignored_events.any? { |check| merged[:event_type]&.match?(check) }
+
       events_worker.push(merged)
     end
 
+    # @api private
+    def collect(collector)
+      return unless config.insights_enabled?
+
+      init_metrics_worker
+      metrics_worker.push(collector)
+    end
+
+    # @api private
+    def registry
+      return @registry if defined?(@registry)
+      @registry = Honeybadger::Registry.new.tap do |r|
+        collect(Honeybadger::RegistryExecution.new(r, config, {}))
+      end
+    end
+
     # @api private
     attr_reader :config
 
@@ -467,13 +498,15 @@ def event(event_type, payload = {})
     # @api private
     def with_rack_env(rack_env, &block)
       context_manager.set_rack_env(rack_env)
+      context_manager.set_request_id(rack_env["action_dispatch.request_id"] || SecureRandom.uuid)
       yield
     ensure
       context_manager.set_rack_env(nil)
+      context_manager.set_request_id(nil)
     end
 
     # @api private
-    attr_reader :worker, :events_worker
+    attr_reader :worker, :events_worker, :metrics_worker
 
     # @api private
     # @!method init!(...)
@@ -485,6 +518,35 @@ def with_rack_env(rack_env, &block)
     # @see Config#backend
     def_delegators :config, :backend
 
+    # @api private
+    # @!method time
+    # @see Honeybadger::Instrumentation#time
+    def_delegator :instrumentation, :time
+
+    # @api private
+    # @!method histogram
+    # @see Honeybadger::Instrumentation#histogram
+    def_delegator :instrumentation, :histogram
+
+    # @api private
+    # @!method gauge
+    # @see Honeybadger::Instrumentation#gauge
+    def_delegator :instrumentation, :gauge
+
+    # @api private
+    # @!method increment_counter
+    # @see Honeybadger::Instrumentation#increment_counter
+    def_delegator :instrumentation, :increment_counter
+
+    # @api private
+    # @!method decrement_counter
+    # @see Honeybadger::Instrumentation#decrement_counter
+    def_delegator :instrumentation, :decrement_counter
+
+    def instrumentation
+      @instrumentation ||= Honeybadger::Instrumentation.new(self)
+    end
+
     private
 
     def validate_notify_opts!(opts)
@@ -520,6 +582,11 @@ def init_events_worker
       @events_worker = EventsWorker.new(config)
     end
 
+    def init_metrics_worker
+      return if @metrics_worker
+      @metrics_worker = MetricsWorker.new(config)
+    end
+
     def with_error_handling
       yield
     rescue => ex

lib/honeybadger/config.rb

@@ -188,6 +188,12 @@ def ignored_classes
       DEFAULTS[:'exceptions.ignore'] | Array(ignore)
     end
 
+    def ignored_events
+      self[:'events.ignore'].map do |check|
+        check.is_a?(String) ? /^#{check}$/ : check
+      end
+    end
+
     def ca_bundle_path
       if self[:'connection.system_ssl_cert_chain'] && File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE)
         OpenSSL::X509::DEFAULT_CERT_FILE
@@ -224,6 +230,10 @@ def max_queue_size
       self[:max_queue_size]
     end
 
+    def events_max_queue_size
+      self[:'events.max_queue_size']
+    end
+
     def events_batch_size
       self[:'events.batch_size']
     end
@@ -262,6 +272,28 @@ def load_plugin?(name)
       includes_token?(self[:plugins], name)
     end
 
+    def insights_enabled?
+      return false if defined?(::Rails.application) && ::Rails.const_defined?("Console")
+      !!self[:'insights.enabled']
+    end
+
+    def cluster_collection?(name)
+      return false unless insights_enabled?
+      return true if self[:"#{name}.insights.cluster_collection"].nil?
+      !!self[:"#{name}.insights.cluster_collection"]
+    end
+
+    def collection_interval(name)
+      return nil unless insights_enabled?
+      self[:"#{name}.insights.collection_interval"]
+    end
+
+    def load_plugin_insights?(name)
+      return false unless insights_enabled?
+      return true if self[:"#{name}.insights.enabled"].nil?
+      !!self[:"#{name}.insights.enabled"]
+    end
+
     def root_regexp
       return @root_regexp if @root_regexp
       return nil if @no_root

lib/honeybadger/config/defaults.rb

@@ -91,16 +91,31 @@ class Boolean; end
         default: 100,
         type: Integer
       },
+      :'events.max_queue_size' => {
+        description: 'Maximum number of event for the event worker queue.',
+        default: 10000,
+        type: Integer
+      },
       :'events.batch_size' => {
         description: 'Send events batch if n events have accumulated',
-        default: 100,
+        default: 1000,
         type: Integer
       },
       :'events.timeout' => {
         description: 'Timeout after which the events batch will be sent regardless (in milliseconds)',
         default: 30_000,
         type: Integer
       },
+      :'events.attach_hostname' => {
+        description: 'Add the hostname to all event paylaods.',
+        default: true,
+        type: Boolean
+      },
+      :'events.ignore' => {
+        description: 'A list of events to ignore. Use a string to specify exact matches, or regex for more flexibility.',
+        default: [],
+        type: Array
+      },
       plugins: {
         description: 'An optional list of plugins to load. Default is to load all plugins.',
         default: nil,
@@ -311,6 +326,36 @@ class Boolean; end
         default: true,
         type: Boolean
       },
+      :'sidekiq.insights.cluster_collection' => {
+        description: 'Collect cluster based metrics for Sidekiq.',
+        default: true,
+        type: Boolean
+      },
+      :'sidekiq.insights.collection_interval' => {
+        description: 'The frequency in which Sidekiq cluster metrics are sampled.',
+        default: 60,
+        type: Integer
+      },
+      :'solid_queue.insights.cluster_collection' => {
+        description: 'Collect cluster based metrics for SolidQueue.',
+        default: true,
+        type: Boolean
+      },
+      :'solid_queue.insights.collection_interval' => {
+        description: 'The frequency in which SolidQueue cluster metrics are sampled.',
+        default: 60,
+        type: Integer
+      },
+      :'net_http.insights.enabled' => {
+        description: 'Allow automatic instrumentation of Net::HTTP requests.',
+        default: true,
+        type: Boolean
+      },
+      :'net_http.insights.full_url' => {
+        description: 'Record the full request url during instrumentation.',
+        default: false,
+        type: Boolean
+      },
       :'sinatra.enabled' => {
         description: 'Enable Sinatra auto-initialization.',
         default: true,
@@ -343,6 +388,16 @@ class Boolean; end
         description: 'Enable/Disable automatic breadcrumbs from log messages.',
         default: true,
         type: Boolean
+      },
+      :'insights.enabled' => {
+        description: "Enable/Disable Honeybadger Insights built-in instrumentation.",
+        default: false,
+        type: Boolean
+      },
+      :'insights.registry_flush_interval' => {
+        description: "Number of seconds between registry flushes.",
+        default: 60,
+        type: Integer
       }
     }.freeze
 

lib/honeybadger/context_manager.rb

@@ -61,15 +61,24 @@ def get_rack_env
       @mutex.synchronize { @rack_env }
     end
 
+    def set_request_id(request_id)
+      @mutex.synchronize { @request_id = request_id }
+    end
+
+    def get_request_id
+      @mutex.synchronize { @request_id }
+    end
+
     private
 
-    attr_accessor :custom, :rack_env
+    attr_accessor :custom, :rack_env, :request_id
 
     def _initialize
       @mutex.synchronize do
         @global_context = nil
         @local_context = nil
         @rack_env = nil
+        @request_id = nil
       end
     end
   end

lib/honeybadger/counter.rb

@@ -0,0 +1,18 @@
+require 'honeybadger/metric'
+
+module Honeybadger
+  class Counter < Metric
+    def count(by=1)
+      return unless by
+
+      @samples += 1
+
+      @counter ||= 0
+      @counter = @counter + by
+    end
+
+    def payloads
+      [{ counter: @counter }]
+    end
+  end
+end

lib/honeybadger/events_worker.rb

@@ -44,7 +44,7 @@ def initialize(config)
     def push(msg)
       return false unless start
 
-      if queue.size >= config.max_queue_size
+      if queue.size >= config.events_max_queue_size
         warn { sprintf('Unable to send event; reached max queue size of %s.', queue.size) }
         return false
       end

lib/honeybadger/gauge.rb

@@ -0,0 +1,30 @@
+require 'honeybadger/metric'
+
+module Honeybadger
+  class Gauge < Metric
+    def record(value)
+      return unless value
+
+      @samples += 1
+
+      @total ||= 0
+      @total = @total + value
+
+      @min = value if @min.nil? || @min > value
+      @max = value if @max.nil? || @max < value
+      @avg = @total.to_f / @samples
+      @latest = value
+    end
+
+    def payloads
+      [
+        {
+          min: @min,
+          max: @max,
+          avg: @avg,
+          latest: @latest
+        }
+      ]
+    end
+  end
+end

lib/honeybadger/histogram.rb

@@ -0,0 +1,32 @@
+require 'honeybadger/metric'
+
+module Honeybadger
+  class Histogram < Metric
+    DEFAULT_BINS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
+    INFINITY = 1e20.to_f # not quite, but pretty much
+
+    def record(value)
+      return unless value
+
+      @samples += 1
+      @bin_counts ||= Hash.new(0)
+      @bin_counts[find_bin(value)] += 1
+    end
+
+    def find_bin(value)
+      bin = bins.find {|b| b >= value  }
+      bin = INFINITY if bin.nil?
+      bin
+    end
+
+    def bins
+      @attributes.fetch(:bins, DEFAULT_BINS).sort
+    end
+
+    def payloads
+      [{
+        bins: (bins + [INFINITY]).map { |bin| [bin.to_f, @bin_counts[bin]] }
+      }]
+    end
+  end
+end