tracing: various improvements

go/cs/beacon/BUILD.bazel

@@ -24,9 +24,11 @@ go_library(
         "//go/lib/prom:go_default_library",
         "//go/lib/serrors:go_default_library",
         "//go/lib/spath:go_default_library",
+        "//go/lib/tracing:go_default_library",
         "//go/lib/util:go_default_library",
         "//go/proto:go_default_library",
         "@com_github_opentracing_opentracing_go//:go_default_library",
+        "@com_github_opentracing_opentracing_go//ext:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@in_gopkg_yaml_v2//:go_default_library",
     ],

go/cs/beacon/metrics.go

@@ -22,13 +22,15 @@ import (
 	"time"
 
 	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/scionproto/scion/go/lib/addr"
 	"github.com/scionproto/scion/go/lib/common"
 	"github.com/scionproto/scion/go/lib/ctrl/path_mgmt"
 	"github.com/scionproto/scion/go/lib/infra/modules/db"
 	"github.com/scionproto/scion/go/lib/prom"
+	"github.com/scionproto/scion/go/lib/tracing"
 )
 
 const (
@@ -176,7 +178,12 @@ func (c *counters) Observe(ctx context.Context, op string, action func(ctx conte
 	defer span.Finish()
 	c.queriesTotal.WithLabelValues(op).Inc()
 	err := action(ctx)
-	c.resultsTotal.WithLabelValues(op, db.ErrToMetricLabel(err)).Inc()
+
+	label := db.ErrToMetricLabel(err)
+	ext.Error.Set(span, err != nil)
+	tracing.ResultLabel(span, label)
+
+	c.resultsTotal.WithLabelValues(op, label).Inc()
 }
 
 type executor struct {

go/integration/cert_req/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//go/lib/scrypto/trc:go_default_library",
         "//go/lib/snet:go_default_library",
         "//go/lib/tracing:go_default_library",
+        "@com_github_opentracing_opentracing_go//ext:go_default_library",
     ],
 )
 

go/integration/cert_req/main.go

@@ -21,6 +21,8 @@ import (
 	"net"
 	"os"
 
+	"github.com/opentracing/opentracing-go/ext"
+
 	"github.com/scionproto/scion/go/integration"
 	"github.com/scionproto/scion/go/lib/addr"
 	"github.com/scionproto/scion/go/lib/common"
@@ -112,11 +114,13 @@ func (c client) attemptRequest(n int) bool {
 	var err error
 	if chain, err = c.requestCert(ctx); err != nil {
 		log.Error("Error requesting certificate chain", "err", err)
+		ext.Error.Set(span, true)
 		return false
 	}
 	// Send TRC request
 	if err = c.requestTRC(ctx, chain); err != nil {
 		log.Error("Error requesting TRC", "err", err)
+		ext.Error.Set(span, true)
 		return false
 	}
 	return true

go/integration/end2end/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//go/lib/topology:go_default_library",
         "//go/lib/tracing:go_default_library",
         "//go/lib/util:go_default_library",
+        "@com_github_opentracing_opentracing_go//ext:go_default_library",
     ],
 )
 

go/integration/end2end/main.go

@@ -24,6 +24,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/opentracing/opentracing-go/ext"
+
 	"github.com/scionproto/scion/go/integration"
 	"github.com/scionproto/scion/go/lib/addr"
 	"github.com/scionproto/scion/go/lib/common"
@@ -184,11 +186,13 @@ func (c client) attemptRequest(n int) bool {
 	// Send ping
 	if err := c.ping(ctx, n); err != nil {
 		logger.Error("Could not send packet", "err", err)
+		ext.Error.Set(span, true)
 		return false
 	}
 	// Receive pong
 	if err := c.pong(ctx); err != nil {
 		logger.Debug("Error receiving pong", "err", err)
+		ext.Error.Set(span, true)
 		return false
 	}
 	logger.Info("Received pong")

go/lib/infra/modules/trust/trustdbmetrics/BUILD.bazel

@@ -13,7 +13,9 @@ go_library(
         "//go/lib/infra/modules/trust/internal/metrics:go_default_library",
         "//go/lib/scrypto:go_default_library",
         "//go/lib/scrypto/trc:go_default_library",
+        "//go/lib/tracing:go_default_library",
         "@com_github_opentracing_opentracing_go//:go_default_library",
+        "@com_github_opentracing_opentracing_go//ext:go_default_library",
     ],
 )
 

go/lib/infra/modules/trust/trustdbmetrics/metrics.go

@@ -22,6 +22,7 @@ import (
 	"io"
 
 	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
 
 	"github.com/scionproto/scion/go/lib/addr"
 	dblib "github.com/scionproto/scion/go/lib/infra/modules/db"
@@ -30,6 +31,7 @@ import (
 	"github.com/scionproto/scion/go/lib/infra/modules/trust/internal/metrics"
 	"github.com/scionproto/scion/go/lib/scrypto"
 	"github.com/scionproto/scion/go/lib/scrypto/trc"
+	"github.com/scionproto/scion/go/lib/tracing"
 )
 
 type observer struct {
@@ -41,10 +43,14 @@ func (o observer) Observe(ctx context.Context, op string, action func(ctx contex
 	defer span.Finish()
 	err := action(ctx)
 
+	label := errToLabel(err)
+	ext.Error.Set(span, err != nil)
+	tracing.ResultLabel(span, label)
+
 	l := metrics.QueryLabels{
 		Driver:    o.driver,
 		Operation: op,
-		Result:    errToLabel(err),
+		Result:    label,
 	}
 	metrics.DB.Queries(l).Inc()
 }
@@ -131,6 +137,9 @@ func (t *tx) Rollback() error {
 	var err error
 	t.metrics.Observe(t.ctx, metrics.RollbackTx, func(_ context.Context) error {
 		err = t.backend.Rollback()
+		if err == sql.ErrTxDone {
+			return nil
+		}
 		return err
 	})
 	return err

go/lib/pathdb/BUILD.bazel

@@ -18,7 +18,9 @@ go_library(
         "//go/lib/pathdb/query:go_default_library",
         "//go/lib/pathpol:go_default_library",
         "//go/lib/prom:go_default_library",
+        "//go/lib/tracing:go_default_library",
         "@com_github_opentracing_opentracing_go//:go_default_library",
+        "@com_github_opentracing_opentracing_go//ext:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
     ],
 )

go/lib/pathdb/metrics.go

@@ -22,13 +22,15 @@ import (
 	"time"
 
 	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/scionproto/scion/go/lib/addr"
 	"github.com/scionproto/scion/go/lib/ctrl/seg"
 	"github.com/scionproto/scion/go/lib/infra/modules/db"
 	"github.com/scionproto/scion/go/lib/pathdb/query"
 	"github.com/scionproto/scion/go/lib/prom"
+	"github.com/scionproto/scion/go/lib/tracing"
 )
 
 const (
@@ -102,7 +104,12 @@ func (c *counters) Observe(ctx context.Context, op promOp, action func(ctx conte
 	defer span.Finish()
 	c.queriesTotal.WithLabelValues(string(op)).Inc()
 	err := action(ctx)
-	c.resultsTotal.WithLabelValues(db.ErrToMetricLabel(err), string(op)).Inc()
+
+	label := db.ErrToMetricLabel(err)
+	ext.Error.Set(span, err != nil)
+	tracing.ResultLabel(span, label)
+
+	c.resultsTotal.WithLabelValues(label, string(op)).Inc()
 }
 
 var _ (PathDB) = (*metricsPathDB)(nil)
@@ -168,6 +175,9 @@ func (tx *metricsTransaction) Rollback() error {
 	var err error
 	tx.metrics.Observe(tx.ctx, promOpRollbackTx, func(_ context.Context) error {
 		err = tx.tx.Rollback()
+		if err == sql.ErrTxDone {
+			return nil
+		}
 		return err
 	})
 	return err

go/lib/periodic/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//go/lib/log:go_default_library",
         "//go/lib/periodic/internal/metrics:go_default_library",
+        "@com_github_opentracing_opentracing_go//:go_default_library",
     ],
 )
 

go/lib/periodic/periodic.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/opentracing/opentracing-go"
+
 	"github.com/scionproto/scion/go/lib/log"
 	"github.com/scionproto/scion/go/lib/periodic/internal/metrics"
 )
@@ -133,6 +135,8 @@ func (r *Runner) onTick() {
 		return
 	default:
 		ctx, cancelF := context.WithTimeout(r.ctx, r.timeout)
+		span, ctx := opentracing.StartSpanFromContext(ctx, "periodic."+r.task.Name())
+		defer span.Finish()
 		start := time.Now()
 		r.task.Run(ctx)
 		r.metric.Runtime(time.Since(start))

go/lib/svc/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//go/lib/svc/internal/ctxconn:go_default_library",
         "//go/lib/svc/internal/proto:go_default_library",
         "@com_github_opentracing_opentracing_go//:go_default_library",
+        "@com_github_opentracing_opentracing_go//ext:go_default_library",
     ],
 )
 

go/lib/svc/resolver.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
 
 	"github.com/scionproto/scion/go/lib/addr"
 	"github.com/scionproto/scion/go/lib/common"
@@ -72,6 +73,7 @@ func (r *Resolver) LookupSVC(ctx context.Context, p snet.Path, svc addr.HostSVC)
 
 	conn, port, err := r.ConnFactory.Register(ctx, r.LocalIA, u, addr.SvcNone)
 	if err != nil {
+		ext.Error.Set(span, true)
 		return nil, common.NewBasicError(errRegistration, err)
 	}
 
@@ -92,7 +94,12 @@ func (r *Resolver) LookupSVC(ctx context.Context, p snet.Path, svc addr.HostSVC)
 			Payload: common.RawBytes(r.Payload),
 		},
 	}
-	return r.getRoundTripper().RoundTrip(ctx, conn, requestPacket, p.OverlayNextHop())
+	reply, err := r.getRoundTripper().RoundTrip(ctx, conn, requestPacket, p.OverlayNextHop())
+	if err != nil {
+		ext.Error.Set(span, true)
+		return nil, err
+	}
+	return reply, nil
 }
 
 func (r *Resolver) getRoundTripper() RoundTripper {

go/lib/tracing/BUILD.bazel

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["context.go"],
+    srcs = [
+        "context.go",
+        "tag.go",
+    ],
     importpath = "github.com/scionproto/scion/go/lib/tracing",
     visibility = ["//visibility:public"],
     deps = [

go/lib/tracing/tag.go

@@ -0,0 +1,24 @@
+// Copyright 2020 Anapaya Systems
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tracing
+
+import (
+	"github.com/opentracing/opentracing-go"
+)
+
+// ResultLabel sets the operation result label on the span.
+func ResultLabel(span opentracing.Span, label string) {
+	span.SetTag("result.label", label)
+}

scion.sh

@@ -16,6 +16,7 @@ cmd_topo_clean() {
     fi
     supervisor/supervisor.sh shutdown
     stop_jaeger
+    rm -rf traces/*
     mkdir -p logs traces gen gen-cache
     find gen gen-cache -mindepth 1 -maxdepth 1 -exec rm -r {} +
 }
@@ -96,7 +97,7 @@ stop_jaeger() {
         return
     fi
     echo "Stopping jaeger..."
-    ./tools/quiet ./tools/dc jaeger down
+    ./tools/quiet ./tools/dc jaeger down -v
 }
 
 cmd_mstart() {