Skip to content

Commit

Permalink
chore: added stats router and grpc recovery interceptor (#80)
Browse files Browse the repository at this point in the history 
* adding git commit message truncation logic for large messages

* added recovery grpc

* added panic to test

* added panic to test

* chaining fix

* testing

* debug routes

* sringified panic

* import changes

* added panic identifier in log message

* common lib latest main
  • Loading branch information
@subhashish-devtron
subhashish-devtron authored Jan 5, 2024
1 parent 3eb0ac7 commit 872c29f
Show file tree
Hide file tree
Showing 76 changed files with 7,864 additions and 27 deletions.
18 changes: 16 additions & 2 deletions App.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"github.com/caarlos0/env"
constants "github.com/devtron-labs/common-lib/constants"
pubsub "github.com/devtron-labs/common-lib/pubsub-lib"
"github.com/devtron-labs/git-sensor/api"
"github.com/devtron-labs/git-sensor/bean"
Expand All @@ -28,14 +29,18 @@ import (
pb "github.com/devtron-labs/protos/gitSensor"
"github.com/go-pg/pg"
"github.com/gorilla/handlers"
"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery"
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/status"
"log"
"net"
"net/http"
"os"
"runtime/debug"
"time"
)

Expand Down Expand Up @@ -121,12 +126,21 @@ func (app *App) initGrpcServer(port int) error {
return err
}

grpcPanicRecoveryHandler := func(p any) (err error) {
app.Logger.Error(constants.PanicLogIdentifier, "recovered from panic", "panic", p, "stack", string(debug.Stack()))
return status.Errorf(codes.Internal, "%s", p)
}
recoveryOption := recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)
opts := []grpc.ServerOption{
grpc.KeepaliveParams(keepalive.ServerParameters{
MaxConnectionAge: 10 * time.Second,
}),
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
grpc.ChainStreamInterceptor(
grpc_prometheus.StreamServerInterceptor,
recovery.StreamServerInterceptor(recoveryOption)), // panic interceptor, should be at last
grpc.ChainUnaryInterceptor(
grpc_prometheus.UnaryServerInterceptor,
recovery.UnaryServerInterceptor(recoveryOption)), // panic interceptor, should be at last
}
// create a new gRPC grpcServer
app.grpcServer = grpc.NewServer(opts...)
Expand Down
16 changes: 11 additions & 5 deletions api/Router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package api

import (
"encoding/json"
"github.com/devtron-labs/common-lib/monitoring"
"github.com/devtron-labs/git-sensor/util"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
Expand All @@ -26,16 +27,21 @@ import (
)

type MuxRouter struct {
logger *zap.SugaredLogger
Router *mux.Router
restHandler RestHandler
logger *zap.SugaredLogger
Router *mux.Router
restHandler RestHandler
monitoringRouter *monitoring.MonitoringRouter
}

func NewMuxRouter(logger *zap.SugaredLogger, restHandler RestHandler) *MuxRouter {
return &MuxRouter{logger: logger, Router: mux.NewRouter(), restHandler: restHandler}
func NewMuxRouter(logger *zap.SugaredLogger, restHandler RestHandler, monitoringRouter *monitoring.MonitoringRouter) *MuxRouter {
return &MuxRouter{logger: logger, Router: mux.NewRouter(), restHandler: restHandler, monitoringRouter: monitoringRouter}
}

func (r MuxRouter) Init() {
pProfListenerRouter := r.Router.PathPrefix("/gitsensor/debug/pprof/").Subrouter()
statsVizRouter := r.Router.PathPrefix("/gitsensor").Subrouter()

r.monitoringRouter.InitMonitoringRouter(pProfListenerRouter, statsVizRouter)
r.Router.StrictSlash(true)
r.Router.Handle("/metrics", promhttp.Handler())
r.Router.Path("/health").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
Expand Down
7 changes: 5 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ go 1.20

require (
github.com/caarlos0/env v3.5.0+incompatible
github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5
github.com/devtron-labs/common-lib v0.0.9
github.com/devtron-labs/protos v0.0.2
github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab
github.com/go-pg/pg v6.15.1+incompatible
github.com/google/wire v0.4.0
github.com/gorilla/handlers v1.4.2
github.com/gorilla/mux v1.7.4
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/prometheus/client_golang v1.14.0
github.com/robfig/cron/v3 v3.0.0
Expand All @@ -24,12 +25,14 @@ require (
)

require (
github.com/arl/statsviz v0.6.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/gammazero/deque v0.0.0-20200124200322-7e84b94275b8 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
Expand Down
16 changes: 12 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBb
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE=
github.com/arl/statsviz v0.6.0/go.mod h1:0toboo+YGSUXDaS4g1D5TVS4dXs7S7YYT5J/qnW2h8s=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
Expand All @@ -16,8 +18,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5 h1:+Nh2SMzAdgBr1tgdKAlF5cN0CvTPUj1V/sI5aRUrZnE=
github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5/go.mod h1:pBThgympEjsza6GShqNNGCPBFXNDx0DGMc7ID/VHTAw=
github.com/devtron-labs/common-lib v0.0.9 h1:uM5NUa8BPubTO5oM5jlkOiGRpj/g4L3csSnYZ51ZEgA=
github.com/devtron-labs/common-lib v0.0.9/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU=
github.com/devtron-labs/protos v0.0.2 h1:BXrW3sXpUZKz25fbHHOAlnth1apHlALU3gfrbGd+WTY=
github.com/devtron-labs/protos v0.0.2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
Expand Down Expand Up @@ -46,8 +48,12 @@ github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg=
github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY=
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
Expand Down Expand Up @@ -104,6 +110,7 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
Expand Down Expand Up @@ -186,6 +193,7 @@ golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
2 changes: 2 additions & 0 deletions pkg/git/RepositoryManager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(repository *git.Repos
breakLoop = true
return
}

gitCommit := &GitCommit{
Author: commit.Author.String(),
Commit: commit.Hash.String(),
Expand All @@ -313,6 +314,7 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(repository *git.Repos
impl.logger.Debugw("commit dto for repo ", "repo", repository, commit)
gitCommits = append(gitCommits, gitCommit)
itrCounter = itrCounter + 1

if impl.configuration.EnableFileStats {
defer func() {
if err := recover(); err != nil {
Expand Down
11 changes: 11 additions & 0 deletions pkg/git/Watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"github.com/caarlos0/env"
"github.com/devtron-labs/common-lib/constants"
pubsub "github.com/devtron-labs/common-lib/pubsub-lib"
"github.com/devtron-labs/common-lib/pubsub-lib/model"
"github.com/devtron-labs/git-sensor/internal"
Expand All @@ -31,6 +32,7 @@ import (
"go.uber.org/zap"
"gopkg.in/src-d/go-git.v4/plumbing/object"
"regexp"
"runtime/debug"
"strings"
"time"
)
Expand Down Expand Up @@ -118,13 +120,22 @@ func (impl GitWatcherImpl) Watch() {

func (impl *GitWatcherImpl) RunOnWorker(materials []*sql.GitMaterial) {
wp := workerpool.New(impl.pollConfig.PollWorker)

handlePanic := func() {
if err := recover(); err != nil {
impl.logger.Error(constants.PanicLogIdentifier, "recovered from panic", "panic", err, "stack", string(debug.Stack()))

}
}

for _, material := range materials {
if len(material.CiPipelineMaterials) == 0 {
impl.logger.Infow("no ci pipeline, skipping", "id", material.Id, "url", material.Url)
continue
}
materialMsg := &sql.GitMaterial{Id: material.Id, Url: material.Url}
wp.Submit(func() {
defer handlePanic()
_, err := impl.pollAndUpdateGitMaterial(materialMsg)
if err != nil {
impl.logger.Errorw("error in polling git material", "material", materialMsg, "err", err)
Expand Down
1 change: 1 addition & 0 deletions vendor/github.com/arl/statsviz/.gitattributes
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions vendor/github.com/arl/statsviz/.gitignore
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

85 changes: 85 additions & 0 deletions vendor/github.com/arl/statsviz/CHANGELOG.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

66 changes: 66 additions & 0 deletions vendor/github.com/arl/statsviz/CONTRIBUTING.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 872c29f

Please sign in to comment.