feat: add mysql connection metrics (#118)

* feat: add connection metrics

* fix: comment

* fix: change metrics label

* fix: data race

* fix: data race

* fix: data race

* fix: data race

* fix: data race

* fix: add default duration

* fix: test

* fix: test flakes on slow server

* fix: test flakes

* fix: test flakes when callback twice

* fix: test flakes when callback twice

* fix: test flakes when callback twice

* fix(otgorm): metrics missing labels (#124)

* fix(otgorm): metrics missing labels

* fix: add with value, extend waiting time

* Update otgorm.go

* Update observability.go

* reame moduleIn.make to maker

* fix: func renamed

Co-authored-by: Trock <35254251+GGXXLL@users.noreply.github.com>

config/watcher/file_test.go

@@ -15,7 +15,7 @@ import (
 func TestWatch(t *testing.T) {
 	t.Run("edit", func(t *testing.T) {
 		t.Parallel()
-		ch := make(chan struct{})
+		ch := make(chan struct{}, 2)
 		f, _ := ioutil.TempFile(".", "*")
 		defer os.Remove(f.Name())
 
@@ -26,7 +26,7 @@ func TestWatch(t *testing.T) {
 		defer cancel()
 
 		go w.Watch(ctx, func() error {
-			close(ch)
+			ch <- struct{}{}
 			return nil
 		})
 		time.Sleep(time.Second)

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/go-kit/kit v0.10.0
 	github.com/go-redis/redis/v8 v8.6.0
 	github.com/gogo/protobuf v1.3.2
+	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.4.3
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -183,6 +183,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

observability/jaeger.go

@@ -13,17 +13,17 @@ type JaegerLogAdapter struct {
 	Logging log.Logger
 }
 
-// Infof implements jaeger.Logger
+// Infof implements jaeger's logger
 func (l JaegerLogAdapter) Infof(msg string, args ...interface{}) {
 	level.Info(l.Logging).Log("msg", fmt.Sprintf(msg, args...))
 }
 
-// Error implements jaeger.Logger
+// Error implements jaeger's logger
 func (l JaegerLogAdapter) Error(msg string) {
 	level.Error(l.Logging).Log("msg", msg)
 }
 
 // ProvideJaegerLogAdapter returns a valid jaeger.Logger.
 func ProvideJaegerLogAdapter(l log.Logger) jaeger.Logger {
-	return &JaegerLogAdapter{Logging: l}
+	return &JaegerLogAdapter{Logging: log.With(l, "tag", "observability")}
 }

observability/metrics.go

@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	"github.com/DoNewsCode/core/contract"
+	"github.com/DoNewsCode/core/otgorm"
 	"github.com/go-kit/kit/metrics"
 	"github.com/go-kit/kit/metrics/prometheus"
 	stdprometheus "github.com/prometheus/client_golang/prometheus"
@@ -30,3 +31,28 @@ func ProvideHistogramMetrics(appName contract.AppName, env contract.Env) metrics
 	})
 	return &his
 }
+
+// ProvideGORMMetrics returns a *otgorm.Gauges that measures the connection info in databases.
+// It is meant to be consumed by the otgorm.Providers.
+func ProvideGORMMetrics(appName contract.AppName, env contract.Env) *otgorm.Gauges {
+	return &otgorm.Gauges{
+		Idle: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "gorm_idle_connections",
+			Help:      "number of idle connections",
+		}, []string{"dbname", "driver"}),
+		Open: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "gorm_open_connections",
+			Help:      "number of open connections",
+		}, []string{"dbname", "driver"}),
+		InUse: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "gorm_in_use_connections",
+			Help:      "number of in use connections",
+		}, []string{"dbname", "driver"}),
+	}
+}

observability/observability.go

@@ -2,12 +2,7 @@ package observability
 
 import (
 	"github.com/DoNewsCode/core/config"
-	"github.com/DoNewsCode/core/contract"
 	"github.com/DoNewsCode/core/di"
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/metrics"
-	"github.com/opentracing/opentracing-go"
-	"go.uber.org/dig"
 	"gopkg.in/yaml.v3"
 )
 
@@ -24,38 +19,13 @@ Providers returns a set of providers available in package observability
 		metrics.Histogram
 */
 func Providers() di.Deps {
-	return di.Deps{provide, provideConfig}
-}
-
-// in is the injection argument of provide.
-type in struct {
-	dig.In
-
-	Logger  log.Logger
-	Conf    contract.ConfigAccessor
-	AppName contract.AppName
-	Env     contract.Env
-}
-
-// out is the result of provide
-type out struct {
-	dig.Out
-
-	Tracer opentracing.Tracer
-	Hist   metrics.Histogram
-}
-
-// provide provides the observability suite for the system. It contains a tracer and
-// a histogram to measure all incoming request.
-func provide(in in) (out, func(), error) {
-	in.Logger = log.With(in.Logger, "tag", "observability")
-	jlogger := ProvideJaegerLogAdapter(in.Logger)
-	tracer, cleanup, err := ProvideOpentracing(in.AppName, in.Env, jlogger, in.Conf)
-	hist := ProvideHistogramMetrics(in.AppName, in.Env)
-	return out{
-		Tracer: tracer,
-		Hist:   hist,
-	}, cleanup, err
+	return di.Deps{
+		ProvideJaegerLogAdapter,
+		ProvideOpentracing,
+		ProvideHistogramMetrics,
+		ProvideGORMMetrics,
+		provideConfig,
+	}
 }
 
 const sample = `
@@ -66,7 +36,7 @@ jaeger:
   reporter:
     log:
       enable: false
-    addr: ''
+    addr:
 `
 
 type configOut struct {

observability/observability_test.go

@@ -1,29 +1,64 @@
 package observability
 
 import (
-	"testing"
-
+	"github.com/DoNewsCode/core"
 	"github.com/DoNewsCode/core/config"
+	"github.com/DoNewsCode/core/otgorm"
 	"github.com/go-kit/kit/log"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/rawbytes"
 	"github.com/stretchr/testify/assert"
+	"gorm.io/gorm"
+	"testing"
 )
 
-func TestProvide(t *testing.T) {
+func TestProvideOpentracing(t *testing.T) {
 	conf, _ := config.NewConfig(config.WithProviderLayer(rawbytes.Provider([]byte(sample)), yaml.Parser()))
-	Out, cleanup, err := provide(in{
-		Conf:    conf,
-		Logger:  log.NewNopLogger(),
-		AppName: config.AppName("foo"),
-		Env:     config.EnvTesting,
-	})
+	Out, cleanup, err := ProvideOpentracing(
+		config.AppName("foo"),
+		config.EnvTesting,
+		ProvideJaegerLogAdapter(log.NewNopLogger()),
+		conf,
+	)
 	assert.NoError(t, err)
-	assert.NotNil(t, Out.Tracer)
-	assert.NotNil(t, Out.Hist)
+	assert.NotNil(t, Out)
 	cleanup()
 }
 
+func TestProvideHistogramMetrics(t *testing.T) {
+	Out := ProvideHistogramMetrics(
+		config.AppName("foo"),
+		config.EnvTesting,
+	)
+	assert.NotNil(t, Out)
+}
+
+func TestProvideGORMMetrics(t *testing.T) {
+	c := core.New()
+	c.ProvideEssentials()
+	c.Provide(Providers())
+	c.Provide(otgorm.Providers())
+	c.Invoke(func(db *gorm.DB, g *otgorm.Gauges) {
+		d, err := db.DB()
+		if err != nil {
+			t.Error(err)
+		}
+		stats := d.Stats()
+		withValues := []string{"dbname", "default", "driver", db.Name()}
+		g.Idle.
+			With(withValues...).
+			Set(float64(stats.Idle))
+
+		g.InUse.
+			With(withValues...).
+			Set(float64(stats.InUse))
+
+		g.Open.
+			With(withValues...).
+			Set(float64(stats.OpenConnections))
+	})
+}
+
 func Test_provideConfig(t *testing.T) {
 	Conf := provideConfig()
 	assert.NotEmpty(t, Conf.Config)

otgorm/dependency.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"time"
 
 	"github.com/DoNewsCode/core/config"
 	"github.com/DoNewsCode/core/contract"
@@ -25,6 +26,7 @@ the Maker, database configs and the default *gorm.DB instance.
 		log.Logger
 		GormConfigInterceptor `optional:"true"`
 		opentracing.Tracer    `optional:"true"`
+		Gauges `optional:"true"`
 	Provide:
 		Maker
 		Factory
@@ -54,7 +56,7 @@ type Maker interface {
 	Make(name string) (*gorm.DB, error)
 }
 
-// GormConfigInterceptor is a function that allows user to make last minute
+// GormConfigInterceptor is a function that allows user to Make last minute
 // change to *gorm.Config when constructing *gorm.DB.
 type GormConfigInterceptor func(name string, conf *gorm.Config)
 
@@ -86,6 +88,10 @@ type databaseConf struct {
 	} `json:"namingStrategy" yaml:"namingStrategy"`
 }
 
+type metricsConf struct {
+	Interval config.Duration `json:"interval" yaml:"interval"`
+}
+
 // provideMemoryDatabase provides a sqlite database in memory mode. This is
 // useful for testing.
 func provideMemoryDatabase() *SQLite {
@@ -111,15 +117,17 @@ type databaseIn struct {
 	Logger                log.Logger
 	GormConfigInterceptor GormConfigInterceptor `optional:"true"`
 	Tracer                opentracing.Tracer    `optional:"true"`
+	Gauges                *Gauges               `optional:"true"`
 }
 
 // databaseOut is the result of provideDatabaseFactory. *gorm.DB is not a interface
 // type. It is up to the users to define their own database repository interface.
 type databaseOut struct {
 	di.Out
 
-	Factory Factory
-	Maker   Maker
+	Factory   Factory
+	Maker     Maker
+	Collector *collector
 }
 
 // provideDialector provides a gorm.Dialector. Mean to be used as an intermediate
@@ -182,10 +190,19 @@ func provideGormDB(dialector gorm.Dialector, config *gorm.Config, tracer opentra
 // provideDatabaseFactory creates the Factory. It is a valid dependency for
 // package core.
 func provideDatabaseFactory(p databaseIn) (databaseOut, func(), error) {
+	var collector *collector
+
 	factory, cleanup := provideDBFactory(p)
+	if p.Gauges != nil {
+		var interval time.Duration
+		p.Conf.Unmarshal("gormMetrics.interval", &interval)
+		collector = newCollector(factory, p.Gauges, interval)
+	}
+
 	return databaseOut{
-		Factory: factory,
-		Maker:   factory,
+		Factory:   factory,
+		Maker:     factory,
+		Collector: collector,
 	}, cleanup, nil
 }
 
@@ -265,6 +282,9 @@ func provideConfig() configOut {
 						}{},
 					},
 				},
+				"gormMetrics": metricsConf{
+					Interval: config.Duration{Duration: 15 * time.Second},
+				},
 			},
 			Comment: "The database configuration",
 		},

otgorm/gorm_metrics.go

@@ -0,0 +1,54 @@
+//go:generate mockgen -destination=./mocks/metrics.go github.com/go-kit/kit/metrics Gauge
+
+package otgorm
+
+import (
+	"time"
+
+	"github.com/go-kit/kit/metrics"
+	"gorm.io/gorm"
+)
+
+type collector struct {
+	factory  Factory
+	gauges   *Gauges
+	interval time.Duration
+}
+
+// Gauges is a collection of metrics for database connection info.
+type Gauges struct {
+	Idle  metrics.Gauge
+	InUse metrics.Gauge
+	Open  metrics.Gauge
+}
+
+// newCollector creates a new database wrapper containing the name of the database,
+// it's driver and the (sql) database itself.
+func newCollector(factory Factory, gauges *Gauges, interval time.Duration) *collector {
+	return &collector{
+		factory:  factory,
+		gauges:   gauges,
+		interval: interval,
+	}
+}
+
+// collectConnectionStats collects database connections for Prometheus to scrape.
+func (d *collector) collectConnectionStats() {
+	for k, v := range d.factory.List() {
+		conn := v.Conn.(*gorm.DB)
+		db, _ := conn.DB()
+		stats := db.Stats()
+		withValues := []string{"dbname", k, "driver", conn.Name()}
+		d.gauges.Idle.
+			With(withValues...).
+			Set(float64(stats.Idle))
+
+		d.gauges.InUse.
+			With(withValues...).
+			Set(float64(stats.InUse))
+
+		d.gauges.Open.
+			With(withValues...).
+			Set(float64(stats.OpenConnections))
+	}
+}