Merge v0.18.1 into main

CHANGELOG.md

@@ -35,6 +35,11 @@
   `loki_tag` is now `logs_instance_tag`, and `backend: loki` is now
   `backend: logs_instance`. (@rfratto)
 
+
+# v0.18.1 (2021-08-09)
+
+- [BUGFIX] Reduce number of consul calls when ran in scrape service mode (@mattdurham)
+
 # v0.18.0 (2021-07-29)
 
 - [FEATURE] Added [Github exporter](https://github.com/infinityworks/github-exporter) integration. (@rgeyer)

docs/configuration/integrations/process-exporter-config.md

@@ -35,7 +35,7 @@ metadata:
   name: agent
 spec:
   containers:
-  - image: grafana/agent:v0.16.1
+  - image: grafana/agent:v0.18.1
     name: agent
     args:
     - --config.file=/etc/agent-config/agent.yaml

docs/getting-started/_index.md

@@ -24,7 +24,7 @@ Currently, there are six ways to install the agent:
 docker run \
   -v /tmp/agent:/etc/agent/data \
   -v /path/to/config.yaml:/etc/agent/agent.yaml \
-  grafana/agent:v0.16.1
+  grafana/agent:v0.18.1
 ```
 
 Replace `/tmp/agent` with the folder you wish to store WAL data in. WAL data is

docs/operator/getting-started.md

@@ -62,7 +62,7 @@ spec:
       serviceAccountName: grafana-agent-operator
       containers:
       - name: operator
-        image: grafana/agent-operator:v0.16.1
+        image: grafana/agent-operator:v0.18.1
 ---
 
 apiVersion: v1

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/grafana/loki v1.6.2-0.20210429132126-d88f3996eaa2
 	github.com/hashicorp/consul/api v1.8.1
+	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-getter v1.5.3
 	github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034
 	github.com/jsternberg/zap-logfmt v1.2.0

pkg/operator/defaults.go

@@ -7,6 +7,9 @@ var (
 		"v0.15.0",
 		// "v0.16.0", // Pulled due to critical bug fixed in v0.16.1.
 		"v0.16.1",
+		"v0.17.0",
+		"v0.18.0",
+		"v0.18.1",
 
 		// NOTE(rfratto): when performing an upgrade, add the newest version above instead of changing the existing reference.
 	}

pkg/prom/instance/configstore/remote.go

@@ -2,10 +2,20 @@ package configstore
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 	"sync"
 
+	"github.com/weaveworks/common/instrument"
+
+	"github.com/cortexproject/cortex/pkg/ring/kv/consul"
+
+	"github.com/hashicorp/go-cleanhttp"
+
+	"github.com/hashicorp/consul/api"
+
 	"github.com/cortexproject/cortex/pkg/ring/kv"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
@@ -14,14 +24,28 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+/***********************************************************************************************************************
+The consul code skipping the cortex handler is due to performance issue with a large number of configs and overloading
+consul. See issue https://github.com/grafana/agent/issues/789. The long term method will be to refactor and extract
+the cortex code so other stores can also benefit from this. @mattdurham
+***********************************************************************************************************************/
+
+// This is copied from cortex code so that stats stay the same
+var consulRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	Namespace: "cortex",
+	Name:      "consul_request_duration_seconds",
+	Help:      "Time spent on consul requests.",
+	Buckets:   prometheus.DefBuckets,
+}, []string{"operation", "status_code"}))
+
 // Remote loads instance files from a remote KV store. The KV store
 // can be swapped out in real time.
 type Remote struct {
 	log log.Logger
 	reg *util.Unregisterer
 
 	kvMut    sync.RWMutex
-	kv       kv.Client
+	kv       *agentRemoteClient
 	reloadKV chan struct{}
 
 	cancelCtx  context.Context
@@ -31,6 +55,14 @@ type Remote struct {
 	configsCh  chan WatchEvent
 }
 
+// agentRemoteClient is a simple wrapper to allow the shortcircuit of consul, while being backwards compatible with non
+// consul kv stores
+type agentRemoteClient struct {
+	kv.Client
+	consul       *api.Client
+	consulConfig consul.Config
+}
+
 // NewRemote creates a new Remote store that uses a Key-Value client to store
 // and retrieve configs. If enable is true, the store will be immediately
 // connected to. Otherwise, it can be lazily loaded by enabling later through
@@ -70,23 +102,48 @@ func (r *Remote) ApplyConfig(cfg kv.Config, enable bool) error {
 	r.reg.UnregisterAll()
 
 	if !enable {
-		r.setClient(nil)
+		r.setClient(nil, nil)
 		return nil
 	}
 
 	cli, err := kv.NewClient(cfg, GetCodec(), kv.RegistererWithKVName(r.reg, "agent_configs"))
+	// This is a hack to get a consul client, the client above has it embedded but its not exposed
+	var consulClient *api.Client
+	if cfg.Store == "consul" {
+		consulClient, err = api.NewClient(&api.Config{
+			Address: cfg.Consul.Host,
+			Token:   cfg.Consul.ACLToken,
+			Scheme:  "http",
+			HttpClient: &http.Client{
+				Transport: cleanhttp.DefaultPooledTransport(),
+				// See https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
+				Timeout: cfg.Consul.HTTPClientTimeout,
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+	}
 	if err != nil {
 		return fmt.Errorf("failed to create kv client: %w", err)
 	}
 
-	r.setClient(cli)
+	r.setClient(cli, consulClient)
 	return nil
 }
 
 // setClient sets the active client and notifies run to restart the
 // kv watcher.
-func (r *Remote) setClient(client kv.Client) {
-	r.kv = client
+func (r *Remote) setClient(client kv.Client, consulClient *api.Client) {
+	if client == nil && consulClient == nil {
+		r.kv = nil
+	} else {
+		r.kv = &agentRemoteClient{
+			Client: client,
+			consul: consulClient,
+		}
+	}
 	r.reloadKV <- struct{}{}
 }
 
@@ -119,7 +176,7 @@ Outer:
 	}
 }
 
-func (r *Remote) watchKV(ctx context.Context, client kv.Client) {
+func (r *Remote) watchKV(ctx context.Context, client *agentRemoteClient) {
 	// Edge case: client was unset, nothing to do here.
 	if client == nil {
 		level.Info(r.log).Log("msg", "not watching the KV, none set")
@@ -162,6 +219,36 @@ func (r *Remote) List(ctx context.Context) ([]string, error) {
 	return r.kv.List(ctx, "")
 }
 
+// listConsul returns Key Value Pairs instead of []string
+func (r *Remote) listConsul(ctx context.Context) (api.KVPairs, error) {
+	if r.kv == nil {
+		return nil, ErrNotConnected
+	}
+
+	var pairs api.KVPairs
+	options := &api.QueryOptions{
+		AllowStale:        !r.kv.consulConfig.ConsistentReads,
+		RequireConsistent: r.kv.consulConfig.ConsistentReads,
+	}
+	// This is copied from cortex list so that stats stay the same
+	err := instrument.CollectedRequest(ctx, "List", consulRequestDuration, instrument.ErrorCode, func(ctx context.Context) error {
+		var err error
+		// This mirrors the default prefix in cortex
+		pairs, _, err = r.kv.consul.KV().List("configurations/", options.WithContext(ctx))
+		return err
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	// This mirrors the previous behavior of returning a blank array as opposed to nil.
+	if pairs == nil {
+		blankPairs := make(api.KVPairs, 0)
+		return blankPairs, nil
+	}
+	return pairs, nil
+}
+
 // Get retrieves an individual config from the KV store.
 func (r *Remote) Get(ctx context.Context, key string) (instance.Config, error) {
 	r.kvMut.RLock()
@@ -260,6 +347,69 @@ func (r *Remote) all(ctx context.Context, keep func(key string) bool) (<-chan in
 		return nil, ErrNotConnected
 	}
 
+	// If we are using a consul client then do the short circuit way, this is done so that we receive all the key value pairs
+	//	in one call then, operate on them in memory. Previously we retrieved the list (which stripped the values)
+	//	then ran a goroutine to get each individual value from consul. In situations with an extremely large number of
+	// 	configs this overloaded the consul instances. This reduces that to one call, that was being made anyways.
+	if r.kv.consul != nil {
+		return r.allConsul(ctx, keep)
+	}
+	return r.allOther(ctx, keep)
+
+}
+
+// allConsul is ONLY usable when consul is the keystore. This is a performance improvement in using the client directly
+//	instead of the cortex multi store kv interface. That interface returns the list then each value must be retrieved
+//	individually. This returns all the keys and values in one call and works on them in memory
+func (r *Remote) allConsul(ctx context.Context, keep func(key string) bool) (<-chan instance.Config, error) {
+	if r.kv.consul == nil {
+		level.Error(r.log).Log("err", "allConsul called but consul client nil")
+		return nil, errors.New("allConsul called but consul client nil")
+	}
+	var configs []*instance.Config
+	c := GetCodec()
+
+	pairs, err := r.listConsul(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+	for _, kvp := range pairs {
+		if keep != nil && !keep(kvp.Key) {
+			level.Debug(r.log).Log("msg", "skipping key that was filtered out", "key", kvp.Key)
+			continue
+		}
+		value, err := c.Decode(kvp.Value)
+		if err != nil {
+			level.Error(r.log).Log("msg", "failed to decode config from store", "key", kvp.Key, "err", err)
+			continue
+		}
+		if value == nil {
+			// Config was deleted since we called list, skip it.
+			level.Debug(r.log).Log("msg", "skipping key that was deleted after list was called", "key", kvp.Key)
+			continue
+		}
+
+		cfg, err := instance.UnmarshalConfig(strings.NewReader(value.(string)))
+		if err != nil {
+			level.Error(r.log).Log("msg", "failed to unmarshal config from store", "key", kvp.Key, "err", err)
+			continue
+		}
+		configs = append(configs, cfg)
+	}
+	ch := make(chan instance.Config, len(configs))
+	for _, cfg := range configs {
+		ch <- *cfg
+	}
+	close(ch)
+	return ch, nil
+}
+
+func (r *Remote) allOther(ctx context.Context, keep func(key string) bool) (<-chan instance.Config, error) {
+	if r.kv == nil {
+		return nil, ErrNotConnected
+	}
+
 	keys, err := r.kv.List(ctx, "")
 	if err != nil {
 		return nil, fmt.Errorf("failed to list configs: %w", err)

production/README.md

@@ -26,7 +26,7 @@ directory on your host that you want the agent to store its WAL.
 docker run \
   -v /tmp/agent:/etc/agent/data \
   -v /path/to/config.yaml:/etc/agent/agent.yaml \
-  grafana/agent:v0.16.1
+  grafana/agent:v0.18.1
 ```
 
 ## Running the Agent locally

production/grafanacloud-install.sh

@@ -47,7 +47,7 @@ PACKAGE_SYSTEM=${PACKAGE_SYSTEM:=}
 #
 # Global constants.
 #
-RELEASE_VERSION="0.18.0"
+RELEASE_VERSION="0.18.1"
 
 RELEASE_URL="https://github.com/grafana/agent/releases/download/v${RELEASE_VERSION}"
 DEB_URL="${RELEASE_URL}/grafana-agent-${RELEASE_VERSION}-1.${ARCH}.deb"

production/kubernetes/agent-bare.yaml

@@ -66,7 +66,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: grafana/agent:v0.16.1
+        image: grafana/agent:v0.18.1
         imagePullPolicy: IfNotPresent
         name: agent
         ports:

production/kubernetes/agent-loki.yaml

@@ -64,7 +64,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: grafana/agent:v0.16.1
+        image: grafana/agent:v0.18.1
         imagePullPolicy: IfNotPresent
         name: agent
         ports:

production/kubernetes/agent-tempo.yaml

@@ -109,7 +109,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: grafana/agent:v0.16.1
+        image: grafana/agent:v0.18.1
         imagePullPolicy: IfNotPresent
         name: agent
         ports:

production/kubernetes/build/lib/version.libsonnet

@@ -1 +1 @@
-'grafana/agent:v0.16.1'
+'grafana/agent:v0.18.1'

production/kubernetes/install-bare.sh

@@ -25,7 +25,7 @@ check_installed() {
 check_installed curl
 check_installed envsubst
 
-MANIFEST_BRANCH=v0.16.1
+MANIFEST_BRANCH=v0.18.1
 MANIFEST_URL=${MANIFEST_URL:-https://raw.githubusercontent.com/grafana/agent/${MANIFEST_BRANCH}/production/kubernetes/agent-bare.yaml}
 NAMESPACE=${NAMESPACE:-default}
 

production/tanka/grafana-agent/v1/main.libsonnet

@@ -15,8 +15,8 @@ local service = k.core.v1.service;
 (import './lib/tempo.libsonnet') +
 {
   _images:: {
-    agent: 'grafana/agent:v0.16.1',
-    agentctl: 'grafana/agentctl:v0.16.1',
+    agent: 'grafana/agent:v0.18.1',
+    agentctl: 'grafana/agentctl:v0.18.1',
   },
 
   // new creates a new DaemonSet deployment of the grafana-agent. By default,