Github exporter integration

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Main (unreleased)
 
+- [FEATURE] Added [Github exporter](https://github.com/infinityworks/github-exporter) integration. (@rgeyer)
+
 - [FEATURE] Add TLS config options for tempo `remote_write`s. (@mapno)
 
 - [FEATURE] Add support for OTLP HTTP trace exporting. (@mapno)

docs/configuration/integrations/_index.md

@@ -83,6 +83,9 @@ windows_exporter: <windows_exporter_config>
 # Controls the kafka_exporter integration
 kafka_exporter: <kafka_exporter_config>
 
+# Controls the github_exporter integration
+github_exporter: <github_exporter_config>
+
 # Automatically collect metrics from enabled integrations. If disabled,
 # integrations will be run but not scraped and thus not remote_written. Metrics
 # for integrations will be exposed at /integrations/<integration_key>/metrics

docs/configuration/integrations/github-exporter-config.md

@@ -0,0 +1,72 @@
++++
+title = "github_exporter_config"
++++
+
+# github_exporter_config
+
+The `github_exporter_config` block configures the `github_exporter` integration,
+which is an embedded version of
+[`github_exporter`](https://github.com/infinityworks/github-exporter). This allows for the collection of metrics from the github api.
+
+Full reference of options:
+
+```yaml
+  # Enables the github_exporter integration, allowing the Agent to automatically
+  # collect metrics for the specified github objects.
+  [enabled: <boolean> | default = false]
+
+  # Automatically collect metrics from this integration. If disabled,
+  # the github_exporter integration will be run but not scraped and thus not
+  # remote-written. Metrics for the integration will be exposed at
+  # /integrations/github_exporter/metrics and can be scraped by an external
+  # process.
+  [scrape_integration: <boolean> | default = <integrations_config.scrape_integrations>]
+
+  # How often should the metrics be collected? Defaults to
+  # prometheus.global.scrape_interval.
+  [scrape_interval: <duration> | default = <global_config.scrape_interval>]
+
+  # The timeout before considering the scrape a failure. Defaults to
+  # prometheus.global.scrape_timeout.
+  [scrape_timeout: <duration> | default = <global_config.scrape_timeout>]
+
+  # Allows for relabeling labels on the target.
+  relabel_configs:
+    [- <relabel_config> ... ]
+
+  # Relabel metrics coming from the integration, allowing to drop series
+  # from the integration that you don't care about.
+  metric_relabel_configs:
+    [ - <relabel_config> ... ]
+
+  # How frequent to truncate the WAL for this integration.
+  [wal_truncate_frequency: <duration> | default = "60m"]
+
+  #
+  # Exporter-specific configuration options
+  #
+
+  # The full URI of the github API.
+  [api_url: <string> | default = "https://api.github.com"]
+
+  # A list of github repositories for which to collect metrics.
+  repositories:
+    [ - <string> ]
+
+  # A list of github organizations for which to collect metrics.
+  organizations:
+    [ - <string> ]
+
+  # A list of github users for which to collect metrics.
+  users:
+    [ - <string> ]
+
+  # A github authentication token that allows the API to be queried more often.
+  # Optional, but recommended.
+  [api_token: <string>]
+
+  # A path to a file containing a github authentication token that allows the
+  # API to be queried more often. If supplied, this supercedes `api_token`
+  # Optional, but recommended.
+  [api_token_file: <string>]
+```

example/docker-compose/agent/config/agent-local.yaml

@@ -72,4 +72,7 @@ integrations:
   kafka_exporter:
     enabled: true
     kafka_uris: [localhost:9093]
-
+  github_exporter:
+    enabled: true
+    repositories:
+      - grafana/agent

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/grafana/loki v1.6.2-0.20210429132126-d88f3996eaa2
 	github.com/hashicorp/consul/api v1.8.1
 	github.com/hashicorp/go-getter v1.5.3
+	github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034
 	github.com/jsternberg/zap-logfmt v1.2.0
 	github.com/justwatchcom/elasticsearch_exporter v1.1.0
 	github.com/klauspost/compress v1.13.1 // indirect
@@ -99,11 +100,11 @@ replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20200521142311-984
 // TODO(rfratto): remove forks when changes are merged upstream
 replace (
 	github.com/google/dnsmasq_exporter => github.com/grafana/dnsmasq_exporter v0.2.1-0.20201029182940-e5169b835a23
+	github.com/infinityworks/github-exporter => github.com/rgeyer/github-exporter v0.0.0-20210722215637-d0cec2ee0dc8
 	github.com/ncabatoff/process-exporter => github.com/grafana/process-exporter v0.7.3-0.20210106202358-831154072e2a
 	github.com/prometheus-community/postgres_exporter => github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520
 	github.com/prometheus-community/windows_exporter => github.com/grafana/windows_exporter v0.15.1-0.20210325142439-9e8f66d53433
 	github.com/prometheus/mysqld_exporter => github.com/grafana/mysqld_exporter v0.12.2-0.20201015182516-5ac885b2d38a
-
 )
 
 // Required for redis_exporter, which is incompatible with v2.0.0+incompatible.

go.sum

@@ -1021,6 +1021,8 @@ github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU=
+github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg=
 github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY=
 github.com/influxdata/go-syslog/v2 v2.0.1/go.mod h1:hjvie1UTaD5E1fTnDmxaCw8RRDrT4Ve+XHr5O2dKSCo=
 github.com/influxdata/go-syslog/v3 v3.0.1-0.20201128200927-a1889d947b48 h1:0WbZ+ZVg74wbyQoRx1TD4D1Xoz8MsXJSTwdP9F7RMeQ=
@@ -1546,6 +1548,8 @@ github.com/rfratto/go-yaml v0.0.0-20200521142311-984fc90c8a04 h1:1HAVwfHi7d7nGq7
 github.com/rfratto/go-yaml v0.0.0-20200521142311-984fc90c8a04/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 github.com/rfratto/weaveworks-common v0.0.0-20210326192855-c95210d58ba7 h1:pi0dqcmXlW/cEqOG3OH0yy3BLHwmshd0w+0JLmK4dX4=
 github.com/rfratto/weaveworks-common v0.0.0-20210326192855-c95210d58ba7/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs=
+github.com/rgeyer/github-exporter v0.0.0-20210722215637-d0cec2ee0dc8 h1:wNuNGrFzFmZlhrtz1Q8EiK1Ob6yWli8lX7D2AGmSGzE=
+github.com/rgeyer/github-exporter v0.0.0-20210722215637-d0cec2ee0dc8/go.mod h1:6XoOvFDTfk3aqGaOLHLxoWiZNx4zHobApOhKc3oHF/g=
 github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
 github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -1665,6 +1669,7 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
+github.com/steinfletcher/apitest v1.3.8/go.mod h1:LOVbGzWvWCiiVE4PZByfhRnA5L00l5uZQEx403xQ4K8=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/streadway/amqp v0.0.0-20180528204448-e5adc2ada8b8/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -1715,6 +1720,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
+github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:Q5IRRDY+cjIaiOjTAnXN5LKQV5MPqVx5ofQn85Jy5Yw=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber/jaeger-client-go v2.20.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=

pkg/integrations/github_exporter/github_exporter.go

@@ -0,0 +1,100 @@
+package github_exporter //nolint:golint
+
+import (
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/grafana/agent/pkg/integrations"
+	"github.com/grafana/agent/pkg/integrations/config"
+	gh_config "github.com/infinityworks/github-exporter/config"
+	"github.com/infinityworks/github-exporter/exporter"
+)
+
+// DefaultConfig holds the default settings for the github_exporter integration
+var DefaultConfig Config = Config{
+	APIURL: "https://api.github.com",
+}
+
+// Config controls github_exporter
+type Config struct {
+	Common config.Common `yaml:",inline"`
+
+	// URL for the github API
+	APIURL string `yaml:"api_url,omitempty"`
+
+	// A list of github repositories for which to collect metrics.
+	Repositories []string `yaml:"repositories,omitempty"`
+
+	// A list of github organizations for which to collect metrics.
+	Organizations []string `yaml:"organizations,omitempty"`
+
+	// A list of github users for which to collect metrics.
+	Users []string `yaml:"users,omitempty"`
+
+	// A github authentication token that allows the API to be queried more often.
+	APIToken string `yaml:"api_token,omitempty"`
+
+	// A path to a file containing a github authentication token that allows the API to be queried more often. If supplied, this supercedes `api_token`
+	APITokenFile string `yaml:"api_token_file,omitempty"`
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler for Config
+func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultConfig
+
+	type plain Config
+	return unmarshal((*plain)(c))
+}
+
+// Name returns the name of the integration that this config represents.
+func (c *Config) Name() string {
+	return "github_exporter"
+}
+
+// CommonConfig returns the common settings shared across all configs for
+// integrations.
+func (c *Config) CommonConfig() config.Common {
+	return c.Common
+}
+
+// NewIntegration creates a new github_exporter
+func (c *Config) NewIntegration(logger log.Logger) (integrations.Integration, error) {
+	return New(logger, c)
+}
+
+func init() {
+	integrations.RegisterIntegration(&Config{})
+}
+
+// New creates a new github_exporter integration.
+func New(logger log.Logger, c *Config) (integrations.Integration, error) {
+
+	conf := gh_config.Config{}
+	err := conf.SetAPIURL(c.APIURL)
+	if err != nil {
+		level.Error(logger).Log("msg", "api url is invalid", "err", err)
+		return nil, err
+	}
+	conf.SetRepositories(c.Repositories)
+	conf.SetOrganisations(c.Organizations)
+	conf.SetUsers(c.Users)
+	if c.APIToken != "" {
+		conf.SetAPIToken(c.APIToken)
+	}
+	if c.APITokenFile != "" {
+		err = conf.SetAPITokenFromFile(c.APITokenFile)
+		if err != nil {
+			level.Error(logger).Log("msg", "unable to load Github API token from file", "err", err)
+			return nil, err
+		}
+	}
+
+	ghExporter := exporter.Exporter{
+		APIMetrics: exporter.AddMetrics(),
+		Config:     conf,
+	}
+
+	return integrations.NewCollectorIntegration(
+		c.Name(),
+		integrations.WithCollectors(&ghExporter),
+	), nil
+}

pkg/integrations/install/install.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/grafana/agent/pkg/integrations/consul_exporter"        // register consul_exporter
 	_ "github.com/grafana/agent/pkg/integrations/dnsmasq_exporter"       // register dnsmasq_exporter
 	_ "github.com/grafana/agent/pkg/integrations/elasticsearch_exporter" // register elasticsearch_exporter
+	_ "github.com/grafana/agent/pkg/integrations/github_exporter"        // register github_exporter
 	_ "github.com/grafana/agent/pkg/integrations/kafka_exporter"         // register kafka_exporter
 	_ "github.com/grafana/agent/pkg/integrations/memcached_exporter"     // register memcached_exporter
 	_ "github.com/grafana/agent/pkg/integrations/mysqld_exporter"        // register mysqld_exporter