Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Grafana Agent Crow #773

Merged
merged 9 commits into from
Aug 12, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .drone/drone.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ steps:
- tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz
- rm go1.16.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator
- make cmd/agent/agent cmd/agentctl/agentctl cmd/agent-operator/agent-operator cmd/grafana-agent-crow/grafana-agent-crow
- make DRONE=true BUILD_IN_CONTAINER=false test
depends_on:
- Lint
Expand Down Expand Up @@ -115,9 +115,9 @@ steps:
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-image
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agentctl-image
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true agent-operator-image
- make DRONE=true CROSS_BUILD=true BUILD_IN_CONTAINER=true RELEASE_BUILD=true grafana-agent-crow-image
- docker buildx rm multiarch


depends_on:
- Test

Expand Down Expand Up @@ -173,6 +173,6 @@ volumes:
path: /var/run/docker.sock
---
kind: signature
hmac: 46567e17e4575977c0b42ad45c3013f159b1d510df27656349d91a68853be485
hmac: fe10154b59ce9b9446f781eb801b147c689f1f6e3194c2d8d20447c4e0715b6e

...
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ vendor
cmd/agent/agent
cmd/agentctl/agentctl
cmd/agent-operator/agent-operator
cmd/grafana-agent-crow/grafana-agent-crow
dist/
packaging/windows/LICENSE
packaging/windows/agent-windows-amd64.exe
Expand Down
28 changes: 16 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -177,38 +177,40 @@ all: protos agent agentctl
agent: cmd/agent/agent
agentctl: cmd/agentctl/agentctl
agent-operator: cmd/agent-operator/agent-operator



grafana-agent-crow: cmd/grafana-agent-crow/grafana-agent-crow

# In general DRONE variable should overwrite any other options, if DRONE is not set then fallback to normal behavior

cmd/agent/agent: seego cmd/agent/main.go
cmd/agent/agent: seego cmd/agent/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)


cmd/agentctl/agentctl: seego cmd/agentctl/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)


cmd/agent-operator/agent-operator: cmd/agent-operator/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

cmd/grafana-agent-crow/grafana-agent-crow: cmd/grafana-agent-crow/main.go
$(ALL_CGO_BUILD_FLAGS) ; $(seego) build $(CGO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

agent-image:
$(docker-build) -t $(IMAGE_PREFIX)/agent:latest -t $(IMAGE_PREFIX)/agent:$(IMAGE_TAG) -f cmd/agent/$(DOCKERFILE) .
agentctl-image:
$(docker-build) -t $(IMAGE_PREFIX)/agentctl:latest -t $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG) -f cmd/agentctl/$(DOCKERFILE) .
agent-operator-image:
$(docker-build) -t $(IMAGE_PREFIX)/agent-operator:latest -t $(IMAGE_PREFIX)/agent-operator:$(IMAGE_TAG) -f cmd/agent-operator/$(DOCKERFILE) .

grafana-agent-crow-image:
$(docker-build) -t $(IMAGE_PREFIX)/agent-crow:latest -t $(IMAGE_PREFIX)/agent-crow:$(IMAGE_TAG) -f cmd/grafana-agent-crow/$(DOCKERFILE) .

install:
CGO_ENABLED=1 go install $(CGO_FLAGS) ./cmd/agent
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent-operator
CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/grafana-agent-crow

#######################
# Development targets #
Expand All @@ -217,12 +219,14 @@ install:
lint:
GO111MODULE=on golangci-lint run -v --timeout=10m

# We have to run test twice: once for all packages with -race and then once more without -race
# for packages that have known race detection issues
# Run tests with -online flag set to true, to include tests requiring network connectivity in CI
# We have to run test twice: once for all packages with -race and then once
# more without -race for packages that have known race detection issues.
#
# Run tests with -tags=has_network to include tests that require network
# connectivity.
test:
CGO_ENABLED=1 go test $(CGO_FLAGS) -race -cover -coverprofile=cover.out -p=4 ./... -- -online
CGO_ENABLED=1 go test $(CGO_FLAGS) -cover -coverprofile=cover-norace.out -p=4 ./pkg/integrations/node_exporter ./pkg/logs -- -online
Comment on lines -224 to -225
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Turns out flags after -- aren't read and you can't omit the -- otherwise -online will be passed to every package, even packages that don't have them defined. Build tags are the best choice here.

CGO_ENABLED=1 go test $(CGO_FLAGS) -tags=has_network -race -cover -coverprofile=cover.out -p=4 ./...
CGO_ENABLED=1 go test $(CGO_FLAGS) -tags=has_network -cover -coverprofile=cover-norace.out -p=4 ./pkg/integrations/node_exporter ./pkg/logs

clean:
rm -rf cmd/agent/agent
Expand Down
11 changes: 11 additions & 0 deletions cmd/grafana-agent-crow/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
FROM golang:1.16-buster as build
COPY . /src/agent
WORKDIR /src/agent
ARG RELEASE_BUILD=true
ARG IMAGE_TAG

RUN make clean && make IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false grafana-agent-crow

FROM debian:buster-slim
COPY --from=build /src/agent/cmd/grafana-agent-crow/grafana-agent-crow /bin/grafana-agent-crow
ENTRYPOINT ["/bin/grafana-agent-crow"]
24 changes: 24 additions & 0 deletions cmd/grafana-agent-crow/Dockerfile.buildx
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
FROM --platform=$BUILDPLATFORM rfratto/seego:latest as build
ARG TARGETPLATFORM
ARG BUILDPLATFORM

# Use custom Go version instead of one prepacked in seego
ENV GOLANG_VERSION 1.16
ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
ENV GOLANG_DOWNLOAD_SHA256 013a489ebb3e24ef3d915abe5b94c3286c070dfe0818d5bca8108f1d6e8440d2
RUN rm -rf /usr/local/go \
&& curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
&& echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \
&& tar -C /usr/local -xzf golang.tar.gz \
&& rm golang.tar.gz

COPY . /src/agent
WORKDIR /src/agent
ARG RELEASE_BUILD=true
ARG IMAGE_TAG

RUN make clean && IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false DRONE=true make grafana-agent-crow

FROM debian:buster-slim
COPY --from=build /src/agent/cmd/grafana-agent-crow/grafana-agent-crow /bin/grafana-agent-crow
ENTRYPOINT ["/bin/grafana-agent-crow"]
81 changes: 81 additions & 0 deletions cmd/grafana-agent-crow/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Comand grafana-agent-crow is a correctness checker tool that validates that
// scraped metrics are delivered to a remote_write endpoint. Inspired by Loki
// Canary and Cortex test-exporter.
package main

import (
"flag"
"fmt"
"os"

// Adds version information
_ "github.com/grafana/agent/pkg/build"

"github.com/go-kit/kit/log/level"
"github.com/grafana/agent/pkg/crow"
"github.com/grafana/agent/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/common/version"
"github.com/weaveworks/common/server"
)

func init() {
prometheus.MustRegister(version.NewCollector("grafana_agent_crow"))
}

func main() {
var (
fs = flag.NewFlagSet(os.Args[0], flag.ExitOnError)

serverCfg server.Config
crowCfg crow.Config
showVersion bool
)

serverCfg.RegisterFlags(fs)
crowCfg.RegisterFlagsWithPrefix(fs, "crow.")
fs.BoolVar(&showVersion, "version", false, "show version")

if err := fs.Parse(os.Args[1:]); err != nil {
fmt.Fprintln(os.Stderr, "failed to parse flags", err)
os.Exit(1)
}
if showVersion {
fmt.Println(version.Print(os.Args[0]))
os.Exit(0)
}

l := util.NewLogger(&serverCfg)
crowCfg.Log = l
serverCfg.Log = util.GoKitLogger(l)

s, err := server.New(serverCfg)
if err != nil {
level.Error(l).Log("msg", "failed to initialize server", "err", err)
os.Exit(1)
}

c, err := crow.New(crowCfg)
if err != nil {
level.Error(l).Log("msg", "failed to initialize crow", "err", err)
os.Exit(1)
}
defer c.Stop()

// The server comes with a /metrics endpoint by default using s.Registerer.
// Create a /validate endpoint to handle our validation metrics.
validator := prometheus.NewRegistry()
s.HTTP.Handle("/validate", promhttp.HandlerFor(validator, promhttp.HandlerOpts{
EnableOpenMetrics: true,
}))

// Register crow's metrics to /metrics and /valiate respectively.
s.Registerer.MustRegister(c.StateMetrics())
validator.MustRegister(c.TestMetrics())

if err := s.Run(); err != nil {
level.Error(l).Log("msg", "server exited with error", "err", err)
os.Exit(1)
}
}
23 changes: 14 additions & 9 deletions example/docker-compose/agent/config/agent.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,23 @@ server:

prometheus:
global:
scrape_interval: 5s
scrape_interval: 30s
configs:
- name: test
host_filter: false
scrape_configs:
- job_name: crow
static_configs:
- targets: ['crow:8080']
labels:
cluster: 'docker_compose'
container: 'crow'
pod: 'crow'
- job_name: crow_validate
metrics_path: /validate
static_configs:
- targets: ['crow:8080']

- job_name: local_scrape
static_configs:
- targets: ['127.0.0.1:12345']
Expand Down Expand Up @@ -43,10 +55,6 @@ prometheus:
container: 'avalanche-3'
remote_write:
- url: http://cortex:9009/api/prom/push
- name: tempo
host_filter: false
remote_write:
- url: http://cortex:9009/api/prom/push

loki:
configs:
Expand Down Expand Up @@ -88,8 +96,5 @@ tempo:
spans: true
processes: true
roots: true
labels:
- svc
- status
Comment on lines -91 to -93
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The labels field was removed at some point.

spanmetrics:
prom_instance: tempo
prom_instance: test
7 changes: 7 additions & 0 deletions example/docker-compose/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ services:
depends_on:
- tempo

crow:
image: grafana/agent-crow:latest
command:
- -log.level=debug
- -server.http-listen-port=8080
- -crow.prometheus-addr=http://cortex:9009/api/prom

loki:
image: grafana/loki:2.2.1
ports:
Expand Down
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@ go 1.16

require (
contrib.go.opencensus.io/exporter/prometheus v0.3.0
github.com/Microsoft/hcsshim v0.8.16 // indirect
github.com/Shopify/sarama v1.29.1
github.com/cortexproject/cortex v1.8.2-0.20210428155238-d382e1d80eaf
github.com/davidmparrott/kafka_exporter/v2 v2.0.1
github.com/docker/docker v20.10.7+incompatible // indirect
github.com/drone/envsubst v1.0.2
github.com/fatih/structs v1.1.0
github.com/go-kit/kit v0.11.0
github.com/go-logfmt/logfmt v0.5.0
github.com/go-logr/logr v0.4.0
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.2
github.com/google/dnsmasq_exporter v0.0.0-00010101000000-000000000000
Expand All @@ -21,6 +24,7 @@ require (
github.com/hashicorp/consul/api v1.8.1
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-getter v1.5.3
github.com/imdario/mergo v0.3.12 // indirect
github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034
github.com/jsternberg/zap-logfmt v1.2.0
github.com/justwatchcom/elasticsearch_exporter v1.1.0
Expand All @@ -35,6 +39,7 @@ require (
github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor v0.30.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.30.0
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e
github.com/opentracing-contrib/go-stdlib v1.0.0
github.com/opentracing/opentracing-go v1.2.0
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/errors v0.9.1
Expand Down
Loading