Dynamic integration registration system (#278)

* add dynamic integration registration system

This commit adds a dynamic integration registration system similar to
(and mostly copied from) Prometheus' SD registration mechanism.

* fix lint error

* fix runtime errors

* refactor integrations

1. Integration interface moved to integrations
2. Integration Config interface now responsible for providing
   name/config

* marshal manager config with embedded integrations

cmd/agent/main.go

@@ -9,6 +9,7 @@ import (
 
 	// Adds version information
 	_ "github.com/grafana/agent/pkg/build"
+
 	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/loki"
 	"github.com/grafana/agent/pkg/tempo"
@@ -23,6 +24,9 @@ import (
 
 	// Register Prometheus SD components
 	_ "github.com/prometheus/prometheus/discovery/install"
+
+	// Register integrations
+	_ "github.com/grafana/agent/pkg/integrations/install"
 )
 
 func init() {

go.mod

@@ -48,6 +48,7 @@ require (
 	google.golang.org/grpc v1.33.2
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )
 
 // Needed for Cortex's dependencies to work properly.

pkg/config/config.go

@@ -19,11 +19,11 @@ import (
 
 // Config contains underlying configurations for the agent
 type Config struct {
-	Server       server.Config       `yaml:"server"`
-	Prometheus   prom.Config         `yaml:"prometheus,omitempty"`
-	Loki         loki.Config         `yaml:"loki,omitempty"`
-	Integrations integrations.Config `yaml:"integrations"`
-	Tempo        tempo.Config        `yaml:"tempo,omitempty"`
+	Server       server.Config              `yaml:"server"`
+	Prometheus   prom.Config                `yaml:"prometheus,omitempty"`
+	Loki         loki.Config                `yaml:"loki,omitempty"`
+	Integrations integrations.ManagerConfig `yaml:"integrations"`
+	Tempo        tempo.Config               `yaml:"tempo,omitempty"`
 }
 
 // ApplyDefaults sets default values in the config

pkg/integrations/agent/agent.go

@@ -6,35 +6,44 @@ package agent
 import (
 	"context"
 
+	"github.com/go-kit/kit/log"
 	"github.com/gorilla/mux"
+	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/integrations/config"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // Config controls the Agent integration.
 type Config struct {
-	CommonConfig config.Common `yaml:",inline"`
+	Common config.Common `yaml:",inline"`
+}
+
+func (c *Config) Name() string {
+	return "agent"
+}
+
+func (c *Config) CommonConfig() config.Common {
+	return c.Common
+}
 
-	// Enabled enables the Agent integration.
-	Enabled bool
+func (c *Config) NewIntegration(_ log.Logger) (integrations.Integration, error) {
+	return New(c), nil
+}
+
+func init() {
+	integrations.RegisterIntegration(&Config{})
 }
 
 // Integration is the Agent integration. The Agent integration scrapes the
 // Agent's own metrics.
 type Integration struct {
-	c Config
+	c *Config
 }
 
-func New(c Config) *Integration {
+func New(c *Config) *Integration {
 	return &Integration{c: c}
 }
 
-// CommonConfig satisfies Integration.CommonConfig.
-func (i *Integration) CommonConfig() config.Common { return i.c.CommonConfig }
-
-// Name satisfies Integration.Name.
-func (i *Integration) Name() string { return "agent" }
-
 // RegisterRoutes satisfies Integration.RegisterRoutes.
 func (i *Integration) RegisterRoutes(r *mux.Router) error {
 	// Note that if the weaveworks common server is set to not register
@@ -48,7 +57,7 @@ func (i *Integration) RegisterRoutes(r *mux.Router) error {
 // ScrapeConfigs satisfies Integration.ScrapeConfigs.
 func (i *Integration) ScrapeConfigs() []config.ScrapeConfig {
 	return []config.ScrapeConfig{{
-		JobName:     i.Name(),
+		JobName:     i.c.Name(),
 		MetricsPath: "/metrics",
 	}}
 }

pkg/integrations/common/collector_integration.go → pkg/integrations/collector_integration.go

@@ -1,6 +1,4 @@
-// Package common implements a bare-bones Integration that can be used by
-// exporters that have no logic associated with them.
-package common
+package integrations
 
 import (
 	"context"
@@ -18,28 +16,20 @@ import (
 // collector.
 type CollectorIntegration struct {
 	name                   string
-	cfg                    config.Common
 	c                      prometheus.Collector
 	includeExporterMetrics bool
 }
 
 // NewCollectorIntegration creates a basic integration that exposes metrics
 // from a prometheus.Collector.
-func NewCollectorIntegration(name string, cfg config.Common, c prometheus.Collector, includeExporterMetrics bool) *CollectorIntegration {
+func NewCollectorIntegration(name string, c prometheus.Collector, includeExporterMetrics bool) *CollectorIntegration {
 	return &CollectorIntegration{
 		name:                   name,
-		cfg:                    cfg,
 		c:                      c,
 		includeExporterMetrics: includeExporterMetrics,
 	}
 }
 
-// CommonConfig satisfies Integration.CommonConfig.
-func (i *CollectorIntegration) CommonConfig() config.Common { return i.cfg }
-
-// Name satisfies Integration.Name.
-func (i *CollectorIntegration) Name() string { return i.name }
-
 // RegisterRoutes satisfies Integration.RegisterRoutes. The mux.Router provided
 // here is expected to be a subrouter, where all registered paths will be
 // registered within that subroute.
@@ -84,7 +74,7 @@ func (i *CollectorIntegration) handler() (http.Handler, error) {
 // ScrapeConfigs satisfies Integration.ScrapeConfigs.
 func (i *CollectorIntegration) ScrapeConfigs() []config.ScrapeConfig {
 	return []config.ScrapeConfig{{
-		JobName:     i.Name(),
+		JobName:     i.name,
 		MetricsPath: "/metrics",
 	}}
 }

pkg/integrations/config/config.go

@@ -11,10 +11,11 @@ import (
 // Common is a set of common options shared by all integrations. It should be
 // utilised by an integration's config by inlining the common options:
 //
-// type IntegrationConfig struct {
-//   Common config.Common `yaml:",inline"`
-// }
+//   type IntegrationConfig struct {
+//     Common config.Common `yaml:",inline"`
+//   }
 type Common struct {
+	Enabled              bool              `yaml:"enabled"`
 	ScrapeIntegration    *bool             `yaml:"scrape_integration"`
 	ScrapeInterval       time.Duration     `yaml:"scrape_interval"`
 	ScrapeTimeout        time.Duration     `yaml:"scrape_timeout"`

pkg/integrations/consul_exporter/consul_exporter.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/go-kit/kit/log"
-	"github.com/grafana/agent/pkg/integrations/common"
+	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/integrations/config"
 	consul_api "github.com/hashicorp/consul/api"
 	"github.com/prometheus/consul_exporter/pkg/exporter"
@@ -21,10 +21,7 @@ var DefaultConfig = Config{
 
 // Config controls the consul_exporter integration.
 type Config struct {
-	// Enabled enables the integration.
-	Enabled bool `yaml:"enabled"`
-
-	CommonConfig config.Common `yaml:",inline"`
+	Common config.Common `yaml:",inline"`
 
 	Server             string        `yaml:"server"`
 	CAFile             string        `yaml:"ca_file"`
@@ -50,9 +47,25 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return unmarshal((*plain)(c))
 }
 
+func (c *Config) Name() string {
+	return "consul_exporter"
+}
+
+func (c *Config) CommonConfig() config.Common {
+	return c.Common
+}
+
+func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) {
+	return New(l, c)
+}
+
+func init() {
+	integrations.RegisterIntegration(&Config{})
+}
+
 // New creates a new consul_exporter integration. The integration scrapes
 // metrics from a consul process.
-func New(log log.Logger, c Config) (common.Integration, error) {
+func New(log log.Logger, c *Config) (integrations.Integration, error) {
 	var (
 		consulOpts = exporter.ConsulOpts{
 			CAFile:       c.CAFile,
@@ -75,10 +88,5 @@ func New(log log.Logger, c Config) (common.Integration, error) {
 		return nil, err
 	}
 
-	return common.NewCollectorIntegration(
-		"consul_exporter",
-		c.CommonConfig,
-		e,
-		false,
-	), nil
+	return integrations.NewCollectorIntegration(c.Name(), e, false), nil
 }

pkg/integrations/dnsmasq_exporter/dnsmasq_exporter.go

@@ -4,7 +4,7 @@ package dnsmasq_exporter //nolint:golint
 import (
 	"github.com/go-kit/kit/log"
 	"github.com/google/dnsmasq_exporter/collector"
-	"github.com/grafana/agent/pkg/integrations/common"
+	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/integrations/config"
 	"github.com/miekg/dns"
 )
@@ -17,10 +17,7 @@ var DefaultConfig Config = Config{
 
 // Config controls the dnsmasq_exporter integration.
 type Config struct {
-	// Enabled enables the integration.
-	Enabled bool `yaml:"enabled"`
-
-	CommonConfig config.Common `yaml:",inline"`
+	Common config.Common `yaml:",inline"`
 
 	// DnsmasqAddress is the address of the dnsmasq server (host:port).
 	DnsmasqAddress string `yaml:"dnsmasq_address"`
@@ -29,6 +26,18 @@ type Config struct {
 	LeasesPath string `yaml:"leases_path"`
 }
 
+func (c *Config) Name() string {
+	return "dnsmasq_exporter"
+}
+
+func (c *Config) CommonConfig() config.Common {
+	return c.Common
+}
+
+func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) {
+	return New(l, c)
+}
+
 // UnmarshalYAML implements yaml.Unmarshaler for Config.
 func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultConfig
@@ -37,17 +46,16 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return unmarshal((*plain)(c))
 }
 
+func init() {
+	integrations.RegisterIntegration(&Config{})
+}
+
 // New creates a new dnsmasq_exporter integration. The integration scrapes metrics
 // from a dnsmasq server.
-func New(log log.Logger, c Config) (common.Integration, error) {
+func New(log log.Logger, c *Config) (integrations.Integration, error) {
 	exporter := collector.New(&dns.Client{
 		SingleInflight: true,
 	}, c.DnsmasqAddress, c.LeasesPath)
 
-	return common.NewCollectorIntegration(
-		"dnsmasq_exporter",
-		c.CommonConfig,
-		exporter,
-		false,
-	), nil
+	return integrations.NewCollectorIntegration(c.Name(), exporter, false), nil
 }

pkg/integrations/install/install.go

@@ -0,0 +1,15 @@
+// Package install registers all in-source integrations for use.
+package install
+
+import (
+	_ "github.com/grafana/agent/pkg/integrations/agent"              // register agent
+	_ "github.com/grafana/agent/pkg/integrations/consul_exporter"    // register consul_exporter
+	_ "github.com/grafana/agent/pkg/integrations/dnsmasq_exporter"   // register dnsmasq_exporter
+	_ "github.com/grafana/agent/pkg/integrations/memcached_exporter" // register memcached_exporter
+	_ "github.com/grafana/agent/pkg/integrations/mysqld_exporter"    // register mysqld_exporter
+	_ "github.com/grafana/agent/pkg/integrations/node_exporter"      // register node_exporter
+	_ "github.com/grafana/agent/pkg/integrations/postgres_exporter"  // register postgres_exporter
+	_ "github.com/grafana/agent/pkg/integrations/process_exporter"   // register process_exporter
+	_ "github.com/grafana/agent/pkg/integrations/redis_exporter"     // register redis_exporter
+	_ "github.com/grafana/agent/pkg/integrations/statsd_exporter"    // register statsd_exporter
+)

pkg/integrations/integration.go

@@ -0,0 +1,43 @@
+package integrations
+
+import (
+	"context"
+
+	"github.com/go-kit/kit/log"
+	"github.com/gorilla/mux"
+	"github.com/grafana/agent/pkg/integrations/config"
+)
+
+// Config provides the configuration and constructor for an integration.
+type Config interface {
+	// Name returns the name of the integration and the key that will be used to
+	// pull the configuration from the Agent config YAML.
+	Name() string
+
+	// CommonConfig returns the set of common configuration values present across
+	// all integrations.
+	CommonConfig() config.Common
+
+	// NewIntegration returns an integration for the given with the given logger.
+	NewIntegration(l log.Logger) (Integration, error)
+}
+
+// An Integration is a process that integrates with some external system and
+// pulls telemetry data.
+type Integration interface {
+	// RegisterRoutes should register any HTTP handlers needed for the
+	// integrations. The mux router provided will be a subrouter for the path
+	// /integrations/<integration name>, where the integration name is retrieved
+	// by the config that created this integration.
+	RegisterRoutes(r *mux.Router) error
+
+	// ScrapeConfigs returns a set of scrape configs that determine where metrics
+	// can be scraped.
+	ScrapeConfigs() []config.ScrapeConfig
+
+	// Run should start the integration and do any required tasks, if necessary.
+	// For example, an Integration that requires a persistent connection to a
+	// database would establish that connection here. If the integration doesn't
+	// need to do anything, it should wait for the ctx to be canceled.
+	Run(ctx context.Context) error
+}