crash reporting: main goroutine and http handler panics (#452)

* crash reporting: track main goroutine and http handler crashes

* address comments

* add crash reporting dsn to goreleaser

* Add build args to docker image

* re-organize goreleaser ldflags

* dont collect info for now

* fix linter errors

* filter hint request

.github/workflows/release.yml

@@ -44,6 +44,9 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - run: make release/docker
+        env:
+          TELEMETRY_WRITE_KEY: ${{ secrets.TELEMETRY_WRITE_KEY }}
+          CRASH_REPORTING_DSN: ${{ secrets.CRASH_REPORTING_DSN }}
 
   release-helm:
     needs: release-docker

.goreleaser.yml

@@ -8,7 +8,7 @@ project_name: infra
 builds:
   - id: infra
     ldflags:
-      - -s -w -X github.com/infrahq/infra/internal.Version={{ .Version }} -X github.com/infrahq/infra/internal.TelemetryWriteKey={{ .Env.INFRA_TELEMETRY_WRITE_KEY }}
+      - -s -w -X github.com/infrahq/infra/internal.Version={{ .Version }}
     binary: infra
     main: ./main.go
     goos:

Dockerfile

@@ -5,10 +5,12 @@ RUN apt-get update && \
     ln -s /usr/bin/x86_64-linux-gnu-gcc /usr/bin/amd64-linux-gnu-gcc
 ARG TARGETARCH
 ARG BUILDVERSION=0.0.0-development
+ARG TELEMETRY_WRITE_KEY
+ARG CRASH_REPORTING_DSN
 WORKDIR /go/src/github.com/infrahq/infra
 COPY . .
 VOLUME ["/root/.cache/go-build", "/go/pkg/mod"]
-RUN CGO_ENABLED=1 GOOS=linux GOARCH=$TARGETARCH CC=$TARGETARCH-linux-gnu-gcc go build -ldflags '-s -w -X github.com/infrahq/infra/internal.Version='"$BUILDVERSION"' -linkmode external -w -extldflags "-static"' .
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=$TARGETARCH CC=$TARGETARCH-linux-gnu-gcc go build -ldflags '-s -X github.com/infrahq/infra/internal.Version='"$BUILDVERSION"' -X github.com/infrahq/infra/internal.TelemetryWriteKey='"$TELEMETRY_WRITE_KEY"' -X github.com/infrahq/infra/internal.CrashReportingDSN='"$CRASH_REPORTING_DSN"' -linkmode external -extldflags "-static"' .
 
 FROM alpine
 COPY --from=builder /go/src/github.com/infrahq/infra/infra /bin/infra

go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/aws/aws-sdk-go v1.41.2
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/cli/browser v1.1.0
-	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/elazarl/go-bindata-assetfs v1.0.1
+	github.com/getsentry/sentry-go v0.11.0
 	github.com/go-chi/chi v1.5.4
 	github.com/go-playground/validator/v10 v10.9.0
 	github.com/gofrs/flock v0.8.1

internal/cmd/cmd.go

@@ -257,6 +257,7 @@ func newRegistryCmd() (*cobra.Command, error) {
 	cmd.Flags().BoolVar(&options.UI, "ui", false, "enable ui")
 	cmd.Flags().StringVar(&options.UIProxy, "ui-proxy", "", "proxy ui requests to this host")
 	cmd.Flags().BoolVar(&options.EnableTelemetry, "enable-telemetry", true, "enable telemetry")
+	cmd.Flags().BoolVar(&options.EnableCrashReporting, "enable-crash-reporting", true, "enable crash reporting")
 
 	homeDir, err := os.UserHomeDir()
 	if err != nil {

internal/crash_reporting.go

@@ -0,0 +1,3 @@
+package internal
+
+var CrashReportingDSN = ""

internal/registry/registry.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/NYTimes/gziphandler"
 	assetfs "github.com/elazarl/go-bindata-assetfs"
+	"github.com/getsentry/sentry-go"
+	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/goware/urlx"
 	"github.com/infrahq/infra/internal"
 	"github.com/infrahq/infra/internal/api"
@@ -29,15 +31,16 @@ import (
 )
 
 type Options struct {
-	DBPath          string
-	TLSCache        string
-	RootAPIKey      string
-	EngineApiKey    string
-	ConfigPath      string
-	UI              bool
-	UIProxy         string
-	SyncInterval    int
-	EnableTelemetry bool
+	DBPath               string
+	TLSCache             string
+	RootAPIKey           string
+	EngineApiKey         string
+	ConfigPath           string
+	UI                   bool
+	UIProxy              string
+	SyncInterval         int
+	EnableTelemetry      bool
+	EnableCrashReporting bool
 }
 
 const (
@@ -73,11 +76,47 @@ func syncSources(db *gorm.DB, k8s *kubernetes.Kubernetes, okta Okta, logger *zap
 }
 
 func Run(options Options) error {
+	if options.EnableCrashReporting {
+		err := sentry.Init(sentry.ClientOptions{
+			Dsn:              internal.CrashReportingDSN,
+			AttachStacktrace: true,
+			Release:          internal.Version,
+			BeforeSend: func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+				event.ServerName = ""
+				event.Request = nil
+				hint.Request = nil
+				return event
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err := recover()
+			if err != nil {
+				sentry.CurrentHub().Recover(err)
+				sentry.Flush(time.Second * 5)
+			}
+		}()
+	}
+
 	db, err := NewDB(options.DBPath)
 	if err != nil {
 		return err
 	}
 
+	var settings Settings
+
+	err = db.First(&settings).Error
+	if err != nil {
+		return err
+	}
+
+	sentry.ConfigureScope(func(scope *sentry.Scope) {
+		scope.SetContext("registryId", settings.Id)
+	})
+
 	zapLogger, err := logging.Build()
 	if err != nil {
 		return err
@@ -304,9 +343,11 @@ func Run(options Options) error {
 		mux.Handle("/", h.loginRedirectMiddleware(gziphandler.GzipHandler(http.FileServer(&StaticFileSystem{base: &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, AssetInfo: AssetInfo}}))))
 	}
 
+	sentryHandler := sentryhttp.New(sentryhttp.Options{})
+
 	plaintextServer := http.Server{
 		Addr:    ":80",
-		Handler: ZapLoggerHttpMiddleware(mux),
+		Handler: ZapLoggerHttpMiddleware(sentryHandler.Handle(mux)),
 	}
 
 	go func() {
@@ -329,7 +370,7 @@ func Run(options Options) error {
 	tlsServer := &http.Server{
 		Addr:      ":443",
 		TLSConfig: tlsConfig,
-		Handler:   ZapLoggerHttpMiddleware(mux),
+		Handler:   ZapLoggerHttpMiddleware(sentryHandler.Handle(mux)),
 	}
 
 	err = tlsServer.ListenAndServeTLS("", "")