Skip to content

Commit

Permalink
Grafana Agent Crow (#773)
Browse files Browse the repository at this point in the history
* basic working example

* increase default sample backoff to 1.25s

* increase check frequency

* tests for ready

* review feedback

* finish initial implementation

* sign drone

* fix lint issues

* Update pkg/crow/samples.go
  • Loading branch information
rfratto authored Aug 12, 2021
1 parent b5416fd commit cbde019
Show file tree
Hide file tree
Showing 15 changed files with 986 additions and 38 deletions.
6 changes: 3 additions & 3 deletions .drone/drone.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ steps:
- tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz
- rm go1.16.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator
- make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator cmd/grafana-agent-crow/grafana-agent-crow
- make DRONE=true BUILD_IN_CONTAINER=false test
depends_on:
- Lint
Expand Down Expand Up @@ -115,9 +115,9 @@ steps:
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-image
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agentctl-image
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-operator-image
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true grafana-agent-crow-image
- docker buildx rm multiarch


depends_on:
- Test

Expand Down Expand Up @@ -173,6 +173,6 @@ volumes:
path: /var/run/docker.sock
---
kind: signature
hmac: 46567e17e4575977c0b42ad45c3013f159b1d510df27656349d91a68853be485
hmac: fe10154b59ce9b9446f781eb801b147c689f1f6e3194c2d8d20447c4e0715b6e

...
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ vendor
cmd/agent/agent
cmd/agentctl/agentctl
cmd/agent-operator/agent-operator
cmd/grafana-agent-crow/grafana-agent-crow
dist/
packaging/windows/LICENSE
packaging/windows/agent-windows-amd64.exe
Expand Down
28 changes: 16 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -177,38 +177,40 @@ all: protos agent agentctl
agent: cmd/agent/agent
agentctl: cmd/agentctl/agentctl
agent-operator: cmd/agent-operator/agent-operator



grafana-agent-crow: cmd/grafana-agent-crow/grafana-agent-crow

# In general DRONE variable should overwrite any other options, if DRONE is not set then fallback to normal behavior

cmd/agent/agent: seego cmd/agent/main.go
cmd/agent/agent: seego cmd/agent/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)


cmd/agentctl/agentctl: seego cmd/agentctl/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)


cmd/agent-operator/agent-operator: cmd/agent-operator/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

cmd/grafana-agent-crow/grafana-agent-crow: cmd/grafana-agent-crow/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

agent-image:
$(docker-build) -t $(IMAGE_PREFIX)/agent:latest -t $(IMAGE_PREFIX)/agent:$(IMAGE_TAG) -f cmd/agent/$(DOCKERFILE) .
agentctl-image:
$(docker-build) -t $(IMAGE_PREFIX)/agentctl:latest -t $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG) -f cmd/agentctl/$(DOCKERFILE) .
agent-operator-image:
$(docker-build) -t $(IMAGE_PREFIX)/agent-operator:latest -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_TAG) -f cmd/agent-operator/$(DOCKERFILE) .

grafana-agent-crow-image:
$(docker-build) -t $(IMAGE_PREFIX)/agent-crow:latest -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f cmd/grafana-agent-crow/$(DOCKERFILE) .

install:
CGO_ENABLED=1 go install $(CGO_FLAGS) ./cmd/agent
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent-operator
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/grafana-agent-crow

#######################
# Development targets #
Expand All @@ -217,12 +219,14 @@ install:
lint:
GO111MODULE=on golangci-lint run -v --timeout=10m

# We have to run test twice: once for all packages with -race and then once more without -race
# for packages that have known race detection issues
# Run tests with -online flag set to true, to include tests requiring network connectivity in CI
# We have to run test twice: once for all packages with -race and then once
# more without -race for packages that have known race detection issues.
#
# Run tests with -tags=has_network to include tests that require network
# connectivity.
test:
CGO_ENABLED=1 go test $(CGO_FLAGS) -race -cover -coverprofile=cover.out -p=4 ./... -- -online
CGO_ENABLED=1 go test $(CGO_FLAGS) -cover -coverprofile=cover-norace.out -p=4 ./pkg/integrations/node_exporter ./pkg/logs -- -online
CGO_ENABLED=1 go test $(CGO_FLAGS) -tags=has_network -race -cover -coverprofile=cover.out -p=4 ./...
CGO_ENABLED=1 go test $(CGO_FLAGS) -tags=has_network -cover -coverprofile=cover-norace.out -p=4 ./pkg/integrations/node_exporter ./pkg/logs

clean:
rm -rf cmd/agent/agent
Expand Down
11 changes: 11 additions & 0 deletions cmd/grafana-agent-crow/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
FROM golang:1.16-buster as build
COPY . /src/agent
WORKDIR /src/agent
ARG RELEASE_BUILD=true
ARG IMAGE_TAG

RUN make clean && make IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false grafana-agent-crow

FROM debian:buster-slim
COPY --from=build /src/agent/cmd/grafana-agent-crow/grafana-agent-crow /bin/grafana-agent-crow
ENTRYPOINT ["/bin/grafana-agent-crow"]
24 changes: 24 additions & 0 deletions cmd/grafana-agent-crow/Dockerfile.buildx
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
FROM --platform=$BUILDPLATFORM rfratto/seego:latest as build
ARG TARGETPLATFORM
ARG BUILDPLATFORM

# Use custom Go version instead of one prepacked in seego
ENV GOLANG_VERSION 1.16
ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
ENV GOLANG_DOWNLOAD_SHA256 013a489ebb3e24ef3d915abe5b94c3286c070dfe0818d5bca8108f1d6e8440d2
RUN rm -rf /usr/local/go \
&& curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
&& echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \
&& tar -C /usr/local -xzf golang.tar.gz \
&& rm golang.tar.gz

COPY . /src/agent
WORKDIR /src/agent
ARG RELEASE_BUILD=true
ARG IMAGE_TAG

RUN make clean && IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false DRONE=true make grafana-agent-crow

FROM debian:buster-slim
COPY --from=build /src/agent/cmd/grafana-agent-crow/grafana-agent-crow /bin/grafana-agent-crow
ENTRYPOINT ["/bin/grafana-agent-crow"]
81 changes: 81 additions & 0 deletions cmd/grafana-agent-crow/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Comand grafana-agent-crow is a correctness checker tool that validates that
// scraped metrics are delivered to a remote_write endpoint. Inspired by Loki
// Canary and Cortex test-exporter.
package main

import (
"flag"
"fmt"
"os"

// Adds version information
_ "github.com/grafana/agent/pkg/build"

"github.com/go-kit/kit/log/level"
"github.com/grafana/agent/pkg/crow"
"github.com/grafana/agent/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/common/version"
"github.com/weaveworks/common/server"
)

func init() {
prometheus.MustRegister(version.NewCollector("grafana_agent_crow"))
}

func main() {
var (
fs = flag.NewFlagSet(os.Args[0], flag.ExitOnError)

serverCfg server.Config
crowCfg crow.Config
showVersion bool
)

serverCfg.RegisterFlags(fs)
crowCfg.RegisterFlagsWithPrefix(fs, "crow.")
fs.BoolVar(&showVersion, "version", false, "show version")

if err := fs.Parse(os.Args[1:]); err != nil {
fmt.Fprintln(os.Stderr, "failed to parse flags", err)
os.Exit(1)
}
if showVersion {
fmt.Println(version.Print(os.Args[0]))
os.Exit(0)
}

l := util.NewLogger(&serverCfg)
crowCfg.Log = l
serverCfg.Log = util.GoKitLogger(l)

s, err := server.New(serverCfg)
if err != nil {
level.Error(l).Log("msg", "failed to initialize server", "err", err)
os.Exit(1)
}

c, err := crow.New(crowCfg)
if err != nil {
level.Error(l).Log("msg", "failed to initialize crow", "err", err)
os.Exit(1)
}
defer c.Stop()

// The server comes with a /metrics endpoint by default using s.Registerer.
// Create a /validate endpoint to handle our validation metrics.
validator := prometheus.NewRegistry()
s.HTTP.Handle("/validate", promhttp.HandlerFor(validator, promhttp.HandlerOpts{
EnableOpenMetrics: true,
}))

// Register crow's metrics to /metrics and /valiate respectively.
s.Registerer.MustRegister(c.StateMetrics())
validator.MustRegister(c.TestMetrics())

if err := s.Run(); err != nil {
level.Error(l).Log("msg", "server exited with error", "err", err)
os.Exit(1)
}
}
23 changes: 14 additions & 9 deletions example/docker-compose/agent/config/agent.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,23 @@ server:

prometheus:
global:
scrape_interval: 5s
scrape_interval: 30s
configs:
- name: test
host_filter: false
scrape_configs:
- job_name: crow
static_configs:
- targets: ['crow:8080']
labels:
cluster: 'docker_compose'
container: 'crow'
pod: 'crow'
- job_name: crow_validate
metrics_path: /validate
static_configs:
- targets: ['crow:8080']

- job_name: local_scrape
static_configs:
- targets: ['127.0.0.1:12345']
Expand Down Expand Up @@ -43,10 +55,6 @@ prometheus:
container: 'avalanche-3'
remote_write:
- url: http://cortex:9009/api/prom/push
- name: tempo
host_filter: false
remote_write:
- url: http://cortex:9009/api/prom/push

loki:
configs:
Expand Down Expand Up @@ -88,8 +96,5 @@ tempo:
spans: true
processes: true
roots: true
labels:
- svc
- status
spanmetrics:
prom_instance: tempo
prom_instance: test
7 changes: 7 additions & 0 deletions example/docker-compose/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ services:
depends_on:
- tempo

crow:
image: grafana/agent-crow:latest
command:
- -log.level=debug
- -server.http-listen-port=8080
- -crow.prometheus-addr=http://cortex:9009/api/prom

loki:
image: grafana/loki:2.2.1
ports:
Expand Down
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@ go 1.16

require (
contrib.go.opencensus.io/exporter/prometheus v0.3.0
github.com/Microsoft/hcsshim v0.8.16 // indirect
github.com/Shopify/sarama v1.29.1
github.com/cortexproject/cortex v1.8.2-0.20210428155238-d382e1d80eaf
github.com/davidmparrott/kafka_exporter/v2 v2.0.1
github.com/docker/docker v20.10.7+incompatible // indirect
github.com/drone/envsubst v1.0.2
github.com/fatih/structs v1.1.0
github.com/go-kit/kit v0.11.0
github.com/go-logfmt/logfmt v0.5.0
github.com/go-logr/logr v0.4.0
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.2
github.com/google/dnsmasq_exporter v0.0.0-00010101000000-000000000000
Expand All @@ -21,6 +24,7 @@ require (
github.com/hashicorp/consul/api v1.8.1
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-getter v1.5.3
github.com/imdario/mergo v0.3.12 // indirect
github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034
github.com/jsternberg/zap-logfmt v1.2.0
github.com/justwatchcom/elasticsearch_exporter v1.1.0
Expand All @@ -35,6 +39,7 @@ require (
github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor v0.30.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.30.0
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e
github.com/opentracing-contrib/go-stdlib v1.0.0
github.com/opentracing/opentracing-go v1.2.0
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/errors v0.9.1
Expand Down
Loading

0 comments on commit cbde019

Please sign in to comment.