Switch to using buf to manage protobuf code (#641)

This required some rework around code generation (e.g. we use enumer to
generate strings out of enums, enumer is part of grafana-build-tools, so
that required some adjusting to work correctly).

Because of the above, this is adding a `scripts/docker-run` script that
is used to run tools from grafana-build-tools consistenly.

Signed-off-by: Marcelo E. Magallon <marcelo.magallon@grafana.com>

.gbt.mk

@@ -0,0 +1,7 @@
+# This file is used by the makefile in order to obtain the version of the
+# Grafana Build Tools image to use. This is *also* used by scripts/docker-run
+# to obtain the same information. That means this file must be both a Makefile
+# and a shell script. This is achieved by using the `VAR=value` syntax, which
+# is valid in both Makefile and shell.
+
+GBT_IMAGE=ghcr.io/grafana/grafana-build-tools:v0.10.0

Makefile

@@ -11,7 +11,9 @@ all: deps build
 
 include $(ROOTDIR)/scripts/make/vars.mk
 
-include config.mk
+include $(ROOTDIR)/config.mk
+
+include $(ROOTDIR)/.gbt.mk
 
 -include $(ROOTDIR)/scripts/make/local.mk
 

config.mk

@@ -3,6 +3,4 @@
 
 DOCKER_TAG = grafana/synthetic-monitoring-agent
 
-GO_TOOLS_IMAGE := ghcr.io/grafana/grafana-build-tools:v0.10.0
-
 PLATFORMS := $(sort $(HOST_OS)/$(HOST_ARCH) linux/amd64 linux/arm64)

pkg/pb/synthetic_monitoring/buf.gen.yaml

@@ -0,0 +1,7 @@
+version: v1
+plugins:
+  - name: gogofast
+    out: .
+    opt:
+      - plugins=grpc
+      - paths=source_relative

pkg/pb/synthetic_monitoring/buf.lock

@@ -0,0 +1,8 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: gogo
+    repository: protobuf
+    commit: 5461a3dfa9d941da82028ab185dc2a0e
+    digest: shake256:37c7c75224982038cb1abf45b481ef06716c1f806ffaa162018d0df092bd11a2a9b62c2d0dc0a2ae43beff86b6014fc0eb8c594ffd84d52ade4b08fca901eadc

pkg/pb/synthetic_monitoring/buf.yaml

@@ -0,0 +1,11 @@
+version: v1
+name: buf.build/grafana/synthetic-monitoring-agent
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - DEFAULT
+deps:
+  - buf.build/gogo/protobuf
+

pkg/pb/synthetic_monitoring/checks_extra.go

@@ -16,8 +16,8 @@
 // messages used to communicate with synthetic-monitoring-api.
 package synthetic_monitoring
 
-//go:generate go run github.com/dmarkham/enumer -type=CheckType,CheckClass -trimprefix=CheckType,CheckClass -transform=lower -output=string.go
-//go:generate go run github.com/dmarkham/enumer -type=MultiHttpEntryAssertionType,MultiHttpEntryAssertionSubjectVariant,MultiHttpEntryAssertionConditionVariant,MultiHttpEntryVariableType -trimprefix=MultiHttpEntryAssertionType_,MultiHttpEntryAssertionSubjectVariant_,MultiHttpEntryAssertionConditionVariant_,MultiHttpEntryVariableType_ -transform=upper -output=multihttp_string.go
+//go:generate ../../../scripts/enumer -type=CheckType -trimprefix=CheckType -transform=lower -output=string.go
+//go:generate ../../../scripts/enumer -type=MultiHttpEntryAssertionType,MultiHttpEntryAssertionSubjectVariant,MultiHttpEntryAssertionConditionVariant,MultiHttpEntryVariableType -trimprefix=MultiHttpEntryAssertionType_,MultiHttpEntryAssertionSubjectVariant_,MultiHttpEntryAssertionConditionVariant_,MultiHttpEntryVariableType_ -transform=upper -output=multihttp_string.go
 
 import (
 	"errors"

scripts/docker-run

@@ -0,0 +1,51 @@
+#!/usr/bin/env sh
+
+set -e
+
+# This is a fallback value; the actual value should be passed in the environment.
+CGO_ENABLED=${CGO_ENABLED:-0}
+
+if test -z "${ROOTDIR}" ; then
+	ROOTDIR=$(dirname "$(go env GOMOD)")
+	if test -z "${ROOTDIR}" ; then
+		ROOTDIR=$(git rev-parse --show-toplevel)
+	fi
+	ROOTDIR=$(realpath -m "${ROOTDIR}")
+fi
+
+. "${ROOTDIR}/.gbt.mk"
+
+if test -z "${GOPATH}" ; then
+	GOPATH=$(go env GOPATH)
+fi
+
+if test -z "${GOMODCACHE}" ; then
+	GOMODCACHE=$(go env GOMODCACHE)
+fi
+
+if test -z "${CI}" ; then
+	CI=false
+fi
+
+if test -z "${GOOS}" ; then
+	GOOS=$(go env GOOS)
+fi
+
+if test -z "${GOARCH}" ; then
+	GOARCH=$(go env GOARCH)
+fi
+
+exec docker run                                  \
+	--rm                                     \
+	--user "$(id -u):$(id -g)"               \
+	--volume "${ROOTDIR}:${ROOTDIR}"         \
+	--volume "${HOME}/.cache:/.cache"        \
+	--volume "${GOPATH}:/go"                 \
+	--volume "${GOMODCACHE}:/go/pkg/mod"     \
+	--workdir "${PWD}"                       \
+	--env CI="${CI}"                         \
+	--env CGO_ENABLED="${CGO_ENABLED}"       \
+	--env GOOS="${GOOS}"                     \
+	--env GOARCH="${GOARCH}"                 \
+	"${GBT_IMAGE}"                           \
+	"$@"

scripts/enumer

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+if command -v enumer > /dev/null 2>&1 ; then
+	exec enumer "$@"
+fi
+
+if test -z "${ROOTDIR}" ; then
+	ROOTDIR=$(dirname "$(go env GOMOD)")
+	if test -z "${ROOTDIR}" ; then
+		ROOTDIR=$(git rev-parse --show-toplevel)
+	fi
+	ROOTDIR=$(realpath -m "${ROOTDIR}")
+fi
+
+"${ROOTDIR}/scripts/docker-run" enumer "$@"
+

scripts/genproto.sh

@@ -11,45 +11,20 @@ if test ! -e "scripts/genproto.sh" ; then
 	exit 255
 fi
 
-if ! command -v protoc > /dev/null 2>&1 ; then
-	echo "could not find protoc 3.5.1, is it installed + in PATH?"
-	exit 255
-fi
-
-echo "Installing Protocol Buffers plugins"
-GO111MODULE=on go mod download
-
-### INSTALL_PKGS="github.com/gogo/protobuf/protoc-gen-gogofast github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger"
-INSTALL_PKGS="github.com/gogo/protobuf/protoc-gen-gogofast"
-for pkg in ${INSTALL_PKGS}; do
-    GO111MODULE=on go install "$pkg"
-done
-
-PB_ROOT="$(GO111MODULE=on go list -f '{{.Dir}}' ./pkg/pb)"
-PB_PATH="${PB_ROOT}"
-GOGO_PROTOBUF_ROOT="$(GO111MODULE=on go list -f '{{ .Dir }}' -m github.com/gogo/protobuf)"
-GOGO_PROTOBUF_PATH="${GOGO_PROTOBUF_ROOT}:${GOGO_PROTOBUF_ROOT}/protobuf"
-GOGO_GOOGLEAPIS_ROOT="$(GO111MODULE=on go list -f '{{ .Dir }}' -m github.com/gogo/googleapis)"
-GOGO_GOOGLEAPIS_PATH="${GOGO_GOOGLEAPIS_ROOT}"
-
 DIRS="pkg/pb/synthetic_monitoring"
 
 for dir in ${DIRS}; do
 	echo "Generating Protocol Buffers code in ${dir}"
 	pushd "${dir}" > /dev/null
-		protoc --gogofast_out=plugins=grpc:. \
-			-I=. \
-			-I="${PB_PATH}" \
-			-I="${GOGO_PROTOBUF_PATH}" \
-			-I="${GOGO_GOOGLEAPIS_PATH}" \
-			./*.proto
 
-		sed -i.bak -E 's,import _ "github.com/gogo/protobuf/gogoproto",,g' -- *.pb.go
-		sed -i.bak -E 's,(import |\t)_ "google/protobuf",,g' -- *.pb.go
-		sed -i.bak -E 's,(import |\t)_ "google/api",,g' -- *.pb.go
-		sed -i.bak -E 's,golang/protobuf,gogo/protobuf,g' -- *.pb.go
-		sed -i.bak -E 's,(import |\t)protobuf "google/protobuf",protobuf "github.com/gogo/protobuf/types", g' -- *.pb.go
-		rm -f -- *.bak
-		goimports -w ./*.pb.go
+	# Check that are no breaking changes
+	buf breaking . --against ./checks.binpb
+
+	# Generate new code
+	buf generate
+	goimports -w .
+
+	# Capture the new data to validate future changes against
+	buf build --output checks.binpb
 	popd > /dev/null
 done

scripts/make/go.mk

@@ -31,7 +31,7 @@ ENUMER ?= docker run \
 		 -v '$(HOME)/.cache/go-build:/root/.cache/go-build' \
 		 --env GOFLAGS=-buildvcs=false \
 		 --workdir /mnt \
-		 '$(GO_TOOLS_IMAGE)' \
+		 '$(GBT_IMAGE)' \
 		 enumer
 endif
 endif

scripts/make/helpers.mk

@@ -119,7 +119,7 @@ else
 endif
 	$(S) false
 endif
-	$(V) ./scripts/generate-drone-yaml "$(GO_TOOLS_IMAGE)" "$(GH_REPO_NAME)" "$(ROOTDIR)/.drone.yml" "$(ROOTDIR)/scripts/configs/drone/main.jsonnet"
+	$(V) ./scripts/generate-drone-yaml "$(GBT_IMAGE)" "$(GH_REPO_NAME)" "$(ROOTDIR)/.drone.yml" "$(ROOTDIR)/scripts/configs/drone/main.jsonnet"
 
 .PHONY: dronefmt
 dronefmt: ## Format drone jsonnet files

scripts/make/linters.mk

@@ -4,15 +4,7 @@ SH_FILES ?= $(shell $(ROOTDIR)/scripts/list-sh-scripts)
 
 ifeq ($(origin GOLANGCI_LINT),undefined)
 ifneq ($(LOCAL_GOLANGCI_LINT),yes)
-GOLANGCI_LINT ?= docker run \
-		 --rm \
-		 -v '$(ROOTDIR):/mnt' \
-		 -v '$(HOME)/.cache/go-build:/root/.cache/go-build' \
-		 -v '$(HOME)/.cache/golangci-lint:/root/.cache/golangci-lint' \
-		 --env GOFLAGS=-buildvcs=false \
-		 --workdir /mnt \
-		 '$(GO_TOOLS_IMAGE)' \
-		 golangci-lint
+GOLANGCI_LINT ?= $(ROOTDIR)/scripts/docker-run env GOFLAGS=-buildvcs=false golangci-lint
 endif
 endif
 
@@ -25,12 +17,7 @@ $(GOLANGCI_LINT): scripts/go/go.mod scripts/go/go.sum
 endif
 
 ifeq ($(origin SHELLCHECK),undefined)
-SHELLCHECK ?= docker run \
-	      --rm \
-	      -v '$(ROOTDIR):/mnt' \
-	      --workdir /mnt \
-	      '$(GO_TOOLS_IMAGE)' \
-	      shellcheck
+SHELLCHECK ?= $(ROOTDIR)/scripts/docker-run shellcheck
 endif
 
 .PHONY: golangci-lint

scripts/make/testing.mk

@@ -5,21 +5,25 @@ GO_TEST_ARGS ?= $(GO_PKGS)
 TEST_OUTPUT := $(DISTDIR)/test
 
 ifeq ($(origin GOTESTSUM),undefined)
-GOTESTSUM ?= $(ROOTDIR)/scripts/go/bin/gotestsum
-LOCAL_GOTESTSUM = yes
+ifneq ($(LOCAL_GOTESTSUM),yes)
+GOTESTSUM ?= $(ROOTDIR)/scripts/docker-run gotestsum
+endif
 endif
 
 ifeq ($(LOCAL_GOTESTSUM),yes)
+GOTESTSUM ?= $(ROOTDIR)/scripts/go/bin/gotestsum
 $(GOTESTSUM): scripts/go/go.mod
 	$(S) cd scripts/go && \
 		$(GO) mod download && \
 		$(GO) build -o $(GOTESTSUM) gotest.tools/gotestsum
 endif
 
 .PHONY: test-go
-test-go: $(GOTESTSUM) ## Run Go tests.
+test-go: ## Run Go tests.
+test-go: $(if $(filter $(LOCAL_GOTESTSUM),yes),$(GOTESTSUM))
+test-go:
 	$(S) echo "test backend"
-	$(GOTESTSUM) \
+	env CGO_ENABLED=1 $(GOTESTSUM) \
 		--format standard-verbose \
 		--jsonfile $(TEST_OUTPUT).json \
 		--junitfile $(TEST_OUTPUT).xml \

scripts/make/vars.mk

@@ -23,6 +23,7 @@ endif
 # to building the tools that we need if they are missing.
 ifeq ($(USE_LOCAL_TOOLS),true)
 GOLANGCI_LINT := golangci-lint
+GOTESTSUM     := gotestsum
 SHELLCHECK    := shellcheck
 XK6           := xk6
 endif

scripts/make/xk6.mk

@@ -6,19 +6,7 @@ K6_VERSION  := v0.50.0
 LOCAL_GOPATH ?= $(shell go env GOPATH)
 
 ifeq ($(origin XK6),undefined)
-XK6 ?= docker run \
-	      --rm \
-	      -i \
-	      -u "$(shell id -u):$(shell id -g)" \
-	      -e GOOS=$(GOOS) \
-	      -e GOARCH=$(GOARCH) \
-	      -v "${HOME}/.cache:/.cache" \
-	      -v "${LOCAL_GOPATH}:/go" \
-	      -v "$(XK6_PKG_DIR):$(XK6_PKG_DIR)" \
-	      -v "$(dir $(DIST_FILENAME)):$(dir $(DIST_FILENAME))" \
-	      --workdir /mnt \
-	      '$(GO_TOOLS_IMAGE)' \
-	      xk6
+XK6 ?= $(ROOTDIR)/scripts/docker-run xk6
 endif
 
 define build_xk6_template