Go 1.16 + OTLPProto/Collector upgrades

.github/workflows/bench.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.15
+    - name: Set up Go 1.16
       uses: actions/setup-go@v1
       with:
-        go-version: 1.15
+        go-version: 1.16
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/ci.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.15
+    - name: Set up Go 1.16
       uses: actions/setup-go@v1
       with:
-        go-version: 1.15
+        go-version: 1.16
       id: go
 
     - name: Check out code into the Go module directory

Makefile

@@ -119,44 +119,61 @@ ifndef COMPONENT
 endif
 
 ### Dependencies
+DOCKER_PROTOBUF ?= otel/build-protobuf:0.2.1
+PROTOC = docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD} ${DOCKER_PROTOBUF} --proto_path=${PWD}
+PROTO_INTERMEDIATE_DIR = pkg/.patched-proto
+PROTO_INCLUDES = -I$(PROTO_INTERMEDIATE_DIR)
+PROTO_GEN = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc,paths=source_relative:$(2) $(1)
 
-# Copied from OpenTelemetry Collector Makefile
-DOCKER_PROTOBUF ?= otel/build-protobuf:0.1.0
-PROTOC := docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD}/$(PROTO_INTERMEDIATE_DIR) ${DOCKER_PROTOBUF} --proto_path=${PWD}
-PROTO_INCLUDES := -I./opentelemetry-proto/ -Ipkg/tempopb/ -I./
 
 .PHONY: gen-proto
 gen-proto:
-	git submodule init
-	git submodule update
-	rm -rf ./vendor/github.com/open-telemetry/opentelemetry-proto
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/common/v1/common.proto --gogofaster_out=plugins=grpc:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto --gogofaster_out=plugins=grpc:./vendor
-	# protoc -I opentelemetry-proto/ opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto --gogofaster_out=plugins=grpc:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto --gogofaster_out=plugins=grpc:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto --gogofaster_out=plugins=grpc:./vendor
-	# protoc -I opentelemetry-proto/ opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto --gogofaster_out=plugins=grpc:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto --gogofaster_out=plugins=grpc:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto \
-	  --grpc-gateway_out=logtostderr=true,grpc_api_configuration=opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto --gogofaster_out=plugins=grpc:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto \
-	  --grpc-gateway_out=logtostderr=true,grpc_api_configuration=opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml:./vendor
-	$(PROTOC) $(PROTO_INCLUDES) pkg/tempopb/tempo.proto --gogofaster_out=plugins=grpc:pkg/tempopb
+	@echo --
+	@echo -- Copying proto to $(PROTO_INTERMEDIATE_DIR)
+	@echo --
+
+	git submodule update --init
+	rm -rf $(PROTO_INTERMEDIATE_DIR)
+	mkdir -p $(PROTO_INTERMEDIATE_DIR)
+	cp -R opentelemetry-proto/opentelemetry/proto/* $(PROTO_INTERMEDIATE_DIR)
+
+	@echo --
+	@echo -- Editing proto
+	@echo --
+
+	@# Update package and types from opentelemetry.proto.* -> tempopb.*
+	@# giving final types like "tempopb.common.v1.InstrumentationLibrary" which
+	@# will not conflict with other usages of opentelemetry proto in downstream apps.
+	find $(PROTO_INTERMEDIATE_DIR) -name "*.proto" | xargs -L 1 sed -i $(SED_OPTS) 's+ opentelemetry.proto+ tempopb+g'
+
+	@# Update go_package
+	find $(PROTO_INTERMEDIATE_DIR) -name "*.proto" | xargs -L 1 sed -i $(SED_OPTS) 's+github.com/open-telemetry/opentelemetry-proto/gen/go+github.com/grafana/tempo/pkg/tempopb+g'
+
+	@# Update import paths
+	find $(PROTO_INTERMEDIATE_DIR) -name "*.proto" | xargs -L 1 sed -i $(SED_OPTS) 's+import "opentelemetry/proto/+import "+g'
+
+	@echo --
+	@echo -- Gen proto -- 
+	@echo --
+	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/common/v1/common.proto,./pkg/tempopb/)
+	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/resource/v1/resource.proto,./pkg/tempopb/)
+	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/trace/v1/trace.proto,./pkg/tempopb/)
+	$(call PROTO_GEN,pkg/tempopb/tempo.proto,./)
+
+	rm -rf $(PROTO_INTERMEDIATE_DIR)
 
 .PHONY: vendor-dependencies
 vendor-dependencies:
 	go mod vendor
-	go mod tidy
-	# ignore log.go b/c the proto version used by v0.6.1 doesn't actually have logs proto.
-	find . | grep 'vendor/go.opentelemetry.io.*go$\' | grep -v -e 'log.go$\' | xargs -L 1 sed -i $(SED_OPTS) 's+go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/+github.com/open-telemetry/opentelemetry-proto/gen/go/+g'
+	go mod tidy -e
 	$(MAKE) gen-proto
 
 
 .PHONE: clear-protos
 clear-protos:
 	rm -rf opentelemetry-proto
 
+
 ### Check vendored files
 .PHONY: vendor-check
 vendor-check: clear-protos vendor-dependencies

cmd/tempo-query/Dockerfile

@@ -1,4 +1,4 @@
-FROM jaegertracing/jaeger-query:1.20.0
+FROM jaegertracing/jaeger-query:1.21.0
 
 ENV SPAN_STORAGE_TYPE=grpc-plugin \
     GRPC_STORAGE_PLUGIN_BINARY=/tmp/tempo-query

cmd/tempo-query/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/grafana/tempo/cmd/tempo-query/tempo"
 	"github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
+	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
 	jaeger_config "github.com/uber/jaeger-client-go/config"
@@ -26,8 +27,6 @@ func main() {
 	flag.StringVar(&configPath, "config", "", "A path to the plugin's configuration file")
 	flag.Parse()
 
-	logger.Error(configPath)
-
 	v := viper.New()
 	v.AutomaticEnv()
 	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
@@ -51,7 +50,10 @@ func main() {
 	cfg.InitFromViper(v)
 
 	backend := tempo.New(cfg)
-	grpc.Serve(&plugin{backend: backend})
+	plugin := &plugin{backend: backend}
+	grpc.Serve(&shared.PluginServices{
+		Store: plugin,
+	})
 }
 
 type plugin struct {

cmd/tempo-query/tempo/plugin.go

@@ -7,8 +7,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/golang/protobuf/proto"
-	"github.com/grafana/tempo/pkg/tempopb"
 	"github.com/grafana/tempo/pkg/util"
 	"github.com/opentracing/opentracing-go"
 	ot_log "github.com/opentracing/opentracing-go/log"
@@ -33,7 +31,7 @@ func New(cfg *Config) *Backend {
 	}
 }
 
-func (b *Backend) GetDependencies(endTs time.Time, lookback time.Duration) ([]jaeger.DependencyLink, error) {
+func (b *Backend) GetDependencies(ctx context.Context, endTs time.Time, lookback time.Duration) ([]jaeger.DependencyLink, error) {
 	return nil, nil
 }
 func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger.Trace, error) {
@@ -76,16 +74,14 @@ func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger
 	if err != nil {
 		return nil, fmt.Errorf("error reading response from tempo: %w", err)
 	}
-	out := &tempopb.Trace{}
-	err = proto.Unmarshal(body, out)
+
+	otTrace := ot_pdata.NewTraces()
+	err = otTrace.FromOtlpProtoBytes(body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal trace proto, err: %w. Tempo response body: %s", err, string(body))
+		return nil, fmt.Errorf("Error converting tempo response to Otlp: %w", err)
 	}
 
-	span.LogFields(ot_log.String("msg", "otlp to Jaeger"))
-	otTrace := ot_pdata.TracesFromOtlp(out.Batches)
 	jaegerBatches, err := ot_jaeger.InternalTracesToJaegerProto(otTrace)
-
 	if err != nil {
 		return nil, fmt.Errorf("error translating to jaegerBatches %v: %w", hexID, err)
 	}
@@ -124,7 +120,7 @@ func (b *Backend) FindTraces(ctx context.Context, query *jaeger_spanstore.TraceQ
 func (b *Backend) FindTraceIDs(ctx context.Context, query *jaeger_spanstore.TraceQueryParameters) ([]jaeger.TraceID, error) {
 	return nil, nil
 }
-func (b *Backend) WriteSpan(span *jaeger.Span) error {
+func (b *Backend) WriteSpan(ctx context.Context, span *jaeger.Span) error {
 	return nil
 }
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/grafana/tempo
 
-go 1.15
+go 1.16
 
 require (
 	cloud.google.com/go/storage v1.12.0
@@ -16,24 +16,23 @@ require (
 	github.com/golang/protobuf v1.4.3
 	github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3
 	github.com/google/uuid v1.1.2
-	github.com/gorilla/mux v1.7.4
+	github.com/gorilla/mux v1.8.0
 	github.com/grafana/loki v1.3.0
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/hashicorp/go-hclog v0.14.0
-	github.com/jaegertracing/jaeger v1.18.2-0.20200707061226-97d2319ff2be
+	github.com/hashicorp/go-plugin v1.3.0 // indirect
+	github.com/jaegertracing/jaeger v1.21.0
 	github.com/jsternberg/zap-logfmt v1.0.0
 	github.com/klauspost/compress v1.11.7
 	github.com/minio/minio-go/v7 v7.0.5
 	github.com/olekukonko/tablewriter v0.0.2
-	github.com/open-telemetry/opentelemetry-proto v0.4.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pierrec/lz4/v4 v4.1.3
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.9.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.15.0
 	github.com/prometheus/prometheus v1.8.2-0.20210124145330-b5dfa2414b9e
-	github.com/prometheus/prometheus/discovery/config v0.0.0-00010101000000-000000000000 // indirect
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1
@@ -43,10 +42,10 @@ require (
 	github.com/willf/bitset v1.1.10 // indirect
 	github.com/willf/bloom v2.0.3+incompatible
 	go.opencensus.io v0.22.5
-	go.opentelemetry.io/collector v0.6.1
+	go.opentelemetry.io/collector v0.16.0
 	go.uber.org/atomic v1.7.0
 	go.uber.org/goleak v1.1.10
-	go.uber.org/zap v1.15.0
+	go.uber.org/zap v1.16.0
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
 	google.golang.org/api v0.36.0
 	google.golang.org/grpc v1.33.2
@@ -74,10 +73,6 @@ replace (
 	k8s.io/client-go => k8s.io/client-go v0.19.2
 )
 
-// opentelemtry collector requires older prometheus discovery config
-// copied locally because having issues referencing a subpackage any other way
-replace github.com/prometheus/prometheus/discovery/config => ./vendor-fix/github.com/prometheus/prometheus/discovery/config
-
 // Pin github.com/go-openapi versions to match Prometheus alertmanager to avoid
 // breaking changing affecting the alertmanager.
 replace (