From 69bce621e7ab073e92c7ec479e0708eb315d5b6a Mon Sep 17 00:00:00 2001 From: Elian Gidoni Date: Wed, 6 May 2020 08:59:36 +0200 Subject: [PATCH] Add flag for enabling metrics and use bigger duration buckets --- README.md | 4 ++++ config/config.go | 4 +++- config/config_test.go | 2 ++ main.go | 43 +++++++++++++++++++++++++++++++++---------- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2269d3a0e..9c5a7c9da 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ GLOBAL OPTIONS: --s3.region value The AWS region. Required when using s3.iam_role_endpoint. [$BAZEL_REMOTE_S3_REGION] --disable_http_ac_validation Whether to disable ActionResult validation for HTTP requests. (default: false, ie enable validation) [$BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION] --disable_grpc_ac_deps_check Whether to disable ActionResult dependency check for gRPC GetActionResult requests. (default: false, ie enable ActionCache dependency checks) [$BAZEL_REMOTE_DISABLE_GRPS_AC_DEPS_CHECK] + --enable_endpoint_metrics Whether to enable metrics for each HTTP/gRPC endpoint. (default: false, ie disable metrics) [$BAZEL_REMOTE_ENABLE_ENDPOINT_METRICS] --help, -h show help (default: false) ``` @@ -157,6 +158,9 @@ host: localhost # to by ActionResult messages are in the cache. #disable_grpc_ac_deps_check: false +# If set to true, enable metrics for each HTTP/gRPC endpoint. +#enable_endpoint_metrics: false + # At most one of the proxy backends can be selected: # #gcs_proxy: diff --git a/config/config.go b/config/config.go index 452f16eba..2ad6b3c5f 100644 --- a/config/config.go +++ b/config/config.go @@ -52,6 +52,7 @@ type Config struct { IdleTimeout time.Duration `yaml:"idle_timeout"` DisableHTTPACValidation bool `yaml:"disable_http_ac_validation"` DisableGRPCACDepsCheck bool `yaml:"disable_grpc_ac_deps_check"` + EnableEndpointMetrics bool `yaml:"enable_endpoint_metrics"` } // New returns a validated Config with the specified values, and an error @@ -60,7 +61,7 @@ func New(dir string, maxSize int, host string, port int, grpcPort int, profileHost string, profilePort int, htpasswdFile string, tlsCertFile string, tlsKeyFile string, idleTimeout time.Duration, s3 *S3CloudStorageConfig, disableHTTPACValidation bool, - disableGRPCACDepsCheck bool) (*Config, error) { + disableGRPCACDepsCheck bool, enableEndpointMetrics bool) (*Config, error) { c := Config{ Host: host, Port: port, @@ -78,6 +79,7 @@ func New(dir string, maxSize int, host string, port int, grpcPort int, IdleTimeout: idleTimeout, DisableHTTPACValidation: disableHTTPACValidation, DisableGRPCACDepsCheck: disableGRPCACDepsCheck, + EnableEndpointMetrics: enableEndpointMetrics, } err := validateConfig(&c) diff --git a/config/config_test.go b/config/config_test.go index b48f72e79..c933bc346 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -18,6 +18,7 @@ htpasswd_file: /opt/.htpasswd tls_cert_file: /opt/tls.cert tls_key_file: /opt/tls.key disable_http_ac_validation: true +enable_endpoint_metrics: true ` config, err := newFromYaml([]byte(yaml)) @@ -35,6 +36,7 @@ disable_http_ac_validation: true TLSCertFile: "/opt/tls.cert", TLSKeyFile: "/opt/tls.key", DisableHTTPACValidation: true, + EnableEndpointMetrics: true, } if !reflect.DeepEqual(config, expectedConfig) { diff --git a/main.go b/main.go index 0086c2ed2..f080a4f0a 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,9 @@ const ( // is set through linker options. var gitCommit string +// durationBuckets is the buckets used for Prometheus histograms in seconds. +var durationBuckets = []float64{.5, 1, 2.5, 5, 10, 20, 40, 80, 160, 320} + func main() { log.SetFlags(logFlags) @@ -197,6 +200,12 @@ func main() { DefaultText: "false, ie enable ActionCache dependency checks", EnvVars: []string{"BAZEL_REMOTE_DISABLE_GRPS_AC_DEPS_CHECK"}, }, + &cli.BoolFlag{ + Name: "enable_endpoint_metrics", + Usage: "Whether to enable metrics for each HTTP/gRPC endpoint.", + DefaultText: "false, ie disable metrics", + EnvVars: []string{"BAZEL_REMOTE_ENABLE_ENDPOINT_METRICS"}, + }, } app.Action = func(ctx *cli.Context) error { @@ -234,6 +243,7 @@ func main() { s3, ctx.Bool("disable_http_ac_validation"), ctx.Bool("disable_grpc_ac_deps_check"), + ctx.Bool("enable_endpoint_metrics"), ) } @@ -291,11 +301,6 @@ func main() { validateAC := !c.DisableHTTPACValidation h := server.NewHTTPCache(diskCache, accessLogger, errorLogger, validateAC, gitCommit) - metricsMdlw := httpmiddleware.New(httpmiddleware.Config{ - Recorder: httpmetrics.NewRecorder(httpmetrics.Config{}), - }) - mux.Handle("/metrics", metricsMdlw.Handler("metrics", promhttp.Handler())) - mux.Handle("/status", metricsMdlw.Handler("status", http.HandlerFunc(h.StatusPageHandler))) var htpasswdSecrets auth.SecretProvider cacheHandler := h.CacheHandler @@ -310,9 +315,21 @@ func main() { cacheHandler = wrapIdleHandler(cacheHandler, idleTimer, accessLogger, httpServer) } - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - metricsMdlw.Handler(r.Method, http.HandlerFunc(cacheHandler)).ServeHTTP(w, r) - }) + if c.EnableEndpointMetrics { + metricsMdlw := httpmiddleware.New(httpmiddleware.Config{ + Recorder: httpmetrics.NewRecorder(httpmetrics.Config{ + DurationBuckets: durationBuckets, + }), + }) + mux.Handle("/metrics", metricsMdlw.Handler("metrics", promhttp.Handler())) + mux.Handle("/status", metricsMdlw.Handler("status", http.HandlerFunc(h.StatusPageHandler))) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + metricsMdlw.Handler(r.Method, http.HandlerFunc(cacheHandler)).ServeHTTP(w, r) + }) + } else { + mux.HandleFunc("/status", h.StatusPageHandler) + mux.HandleFunc("/", cacheHandler) + } if c.GRPCPort > 0 { @@ -324,8 +341,14 @@ func main() { addr := c.Host + ":" + strconv.Itoa(c.GRPCPort) opts := []grpc.ServerOption{} - streamInterceptors := []grpc.StreamServerInterceptor{grpc_prometheus.StreamServerInterceptor} - unaryInterceptors := []grpc.UnaryServerInterceptor{grpc_prometheus.UnaryServerInterceptor} + streamInterceptors := []grpc.StreamServerInterceptor{} + unaryInterceptors := []grpc.UnaryServerInterceptor{} + + if c.EnableEndpointMetrics { + streamInterceptors = append(streamInterceptors, grpc_prometheus.StreamServerInterceptor) + unaryInterceptors = append(unaryInterceptors, grpc_prometheus.UnaryServerInterceptor) + grpc_prometheus.EnableHandlingTimeHistogram(grpc_prometheus.WithHistogramBuckets(durationBuckets)) + } if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 { creds, err2 := credentials.NewServerTLSFromFile(