diff --git a/README.md b/README.md index 31e7170..74fe3cd 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ ENV Variable | Description |----------|-----| | DEBUG | If set to true also debug information will be logged, otherwise only info | | DIGITALOCEAN_TOKEN | Token for API access | -| HTTP_TIMEOUT | Timeout for the godo client, default: `5000`ms | +| HTTP_TIMEOUT | Timeout for the godo client, default: `5000`ms | | WEB_ADDR | Address for this exporter to run, default: `:9212` | | WEB_PATH | Path for metrics, default: `/metrics` | @@ -62,6 +62,8 @@ Read-only tokens are sufficient. | digitalocean_droplet_price_monthly | gauge | 4 | Price of the Droplet billed monthly in dollars | digitalocean_droplet_up | gauge | 4 | If 1 the droplet is up and running, 0 otherwise | digitalocean_floating_ipv4_active | gauge | 1 | If 1 the floating ip used by a droplet, 0 otherwise +| digitalocean_incidents | gauge | 1 | Number of active regional incidents associated with digitalocean services +| digitalocean_incidents_total | gauge | 0 | Number of active total incidents associated with digitalocean services | digitalocean_key | gauge | 1 | Information about keys in your digitalocean account | digitalocean_loadbalancer_droplets | gauge | 1 | The number of droplets this load balancer is proxying to | digitalocean_loadbalancer_status | gauge | 1 | The status of the load balancer, 1 if active diff --git a/collector/incidents.go b/collector/incidents.go new file mode 100644 index 0000000..14e9192 --- /dev/null +++ b/collector/incidents.go @@ -0,0 +1,138 @@ +package collector + +import ( + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" +) + +const doStatusAPIURL = "https://s2k7tnzlhrpw.statuspage.io/api/v2/summary.json" + +var regionRegex = regexp.MustCompile("[A-Z]{3}\\d{1}") + +// DOIncidentAPIResponse stores active digitalocean incidents with their Name(title) to extract the region name +type DOIncidentAPIResponse struct { + Incidents []struct { + Name string `json:"name"` + } `json:"incidents"` +} + +// IncidentCollector collects number of active incidents associated with digital ocean services +type IncidentCollector struct { + logger log.Logger + errors *prometheus.CounterVec + timeout time.Duration + + Incidents *prometheus.Desc + IncidentsTotal *prometheus.Desc +} + +// NewIncidentCollector returns a new IncidentCollector. +func NewIncidentCollector(logger log.Logger, errors *prometheus.CounterVec, timeout time.Duration) *IncidentCollector { + errors.WithLabelValues("incidents").Add(0) + + labels := []string{"region"} + return &IncidentCollector{ + logger: logger, + errors: errors, + timeout: timeout, + + Incidents: prometheus.NewDesc( + "digitalocean_incidents", + "Number of regional active incidents at digitalocean", + labels, nil, + ), + IncidentsTotal: prometheus.NewDesc( + "digitalocean_incidents_total", + "Number of total active incidents at digitalocean", + nil, nil, + ), + } +} + +// Describe sends the super-set of all possible descriptors of metrics +// collected by this Collector. +func (c *IncidentCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.Incidents +} + +// GetIncidents fetches active incidents associated with digital ocean services +func GetIncidents(client *http.Client) (DOIncidentAPIResponse, error) { + r, err := client.Get(doStatusAPIURL) + if err != nil { + return DOIncidentAPIResponse{}, err + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + return DOIncidentAPIResponse{}, fmt.Errorf("unable to retrieve incidents: %w", err) + } + + var doIncidents DOIncidentAPIResponse + if err := json.NewDecoder(r.Body).Decode(&doIncidents); err != nil { + return DOIncidentAPIResponse{}, err + } + + return doIncidents, nil +} + +// parseRegion extracts the region code for digitalocean datacenters in an incident titl(e.g. NYC1, SFO3) +func parseRegion(s string) string { + region := regionRegex.FindString(s) + // Not all incidents have regions reported + if region == "" { + return "unspecified" + } + return strings.ToLower(region) +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (c *IncidentCollector) Collect(ch chan<- prometheus.Metric) { + // Datastore to count all incidents per region + regionalIncidents := make(map[string]int) + client := http.Client{Timeout: c.timeout} + doStatus, err := GetIncidents(&client) + if err != nil { + c.errors.WithLabelValues("incidents").Add(1) + level.Warn(c.logger).Log( + "msg", "can't retrieve incidents", + "err", err, + ) + } + // Count all incidents per region + for _, incident := range doStatus.Incidents { + // Extract region name from incident title(if present) + region := parseRegion(incident.Name) + if _, ok := regionalIncidents[region]; ok { + // If key is present, increment + regionalIncidents[region]++ + } else { + // If key is not present, create with initial value of 1 + regionalIncidents[region] = 1 + } + } + + // Create metric per region + for region, incidentCount := range regionalIncidents { + ch <- prometheus.MustNewConstMetric( + c.Incidents, + prometheus.GaugeValue, + float64(incidentCount), + region, + ) + } + + // Create metric for all incidents + ch <- prometheus.MustNewConstMetric( + c.IncidentsTotal, + prometheus.GaugeValue, + float64(len(doStatus.Incidents)), + ) +} diff --git a/go.sum b/go.sum index 92b8570..be71bfa 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/digitalocean/godo v1.29.0 h1:KgNNU0k9SZqVgn7m8NN9iDsq0+nluHBe8HR9QE0QVmA= github.com/digitalocean/godo v1.29.0/go.mod h1:iJnN9rVu6K5LioLxLimlq0uRI+y/eAQjROUmeU/r0hY= @@ -23,12 +24,14 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -39,6 +42,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -49,6 +53,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -70,6 +75,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -89,8 +95,10 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 0716d19..e98947c 100644 --- a/main.go +++ b/main.go @@ -105,6 +105,7 @@ func main() { r.MustRegister(collector.NewSnapshotCollector(logger, errors, client, timeout)) r.MustRegister(collector.NewVolumeCollector(logger, errors, client, timeout)) r.MustRegister(collector.NewKubernetesCollector(logger, errors, client, timeout)) + r.MustRegister(collector.NewIncidentCollector(logger, errors, timeout)) http.Handle(c.WebPath, promhttp.HandlerFor(r, promhttp.HandlerOpts{}),