Improve HTTP metrics and add ClickHouse metric (#158)

Improve the exported HTTP metrics, by using summary instead of
histograms for the request latency.

Add a new ClickHouse metric, to determine how often a field is used in
queries. With the help of this metric we can then add more fields to the
logs table to improve the query time for the fields which are used most
of the time.

CHANGELOG.md

@@ -68,6 +68,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#152](https://github.com/kobsio/kobs/pull/152): Improve performance for large dashboards and open Application page in gallery view.
 - [#155](https://github.com/kobsio/kobs/pull/155): Allow users to get all Applications from all namespaces, by allowing an empty namespace list.
 - [#157](https://github.com/kobsio/kobs/pull/157): Imporve query performance for ClickHouse plugin.
+- [#158](https://github.com/kobsio/kobs/pull/158): Improve Pormetheus HTTP metrics and create a metric to find the most used fields in queries.
 
 ## [v0.5.0](https://github.com/kobsio/kobs/releases/tag/v0.5.0) (2021-08-03)
 

pkg/api/middleware/metrics/metrics.go

@@ -2,6 +2,7 @@ package metrics
 
 import (
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/go-chi/chi/v5/middleware"
@@ -10,18 +11,18 @@ import (
 )
 
 var (
-	reqs = promauto.NewCounterVec(prometheus.CounterOpts{
+	reqMetric = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "kobs",
 		Name:      "chi_requests_total",
 		Help:      "Number of HTTP requests processed, partitioned by status code, method and path.",
-	}, []string{"code", "method", "path"})
+	}, []string{"response_code", "request_method", "request_path"})
 
-	latency = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: "kobs",
-		Name:      "chi_request_duration_milliseconds",
-		Help:      "Latency of HTTP requests processed, partitioned by status code, method and path.",
-		Buckets:   []float64{100, 500, 1000, 5000},
-	}, []string{"code", "method", "path"})
+	sumMetric = promauto.NewSummaryVec(prometheus.SummaryOpts{
+		Namespace:  "kobs",
+		Name:       "chi_request_duration_milliseconds",
+		Help:       "Latency of HTTP requests processed, partitioned by status code, method and path.",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001},
+	}, []string{"response_code", "request_method", "request_path"})
 )
 
 // Metrics is a middleware that handles the Prometheus metrics for kobs and chi.
@@ -31,8 +32,8 @@ func Metrics(next http.Handler) http.Handler {
 		wrw := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
 		next.ServeHTTP(wrw, r)
 
-		reqs.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Inc()
-		latency.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000)
+		reqMetric.WithLabelValues(strconv.Itoa(wrw.Status()), r.Method, r.URL.Path).Inc()
+		sumMetric.WithLabelValues(strconv.Itoa(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000)
 	}
 
 	return http.HandlerFunc(fn)

plugins/clickhouse/pkg/instance/logs.go

@@ -3,11 +3,20 @@ package instance
 import (
 	"fmt"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
 	defaultFields  = []string{"timestamp", "cluster", "namespace", "app", "pod_name", "container_name", "host", "log"}
 	defaultColumns = "timestamp, cluster, namespace, app, pod_name, container_name, host, fields_string.key, fields_string.value, fields_number.key, fields_number.value, log"
+
+	fieldsMetric = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "kobs",
+		Name:      "clickhouse_fields_total",
+		Help:      "Number how often a field was used in a query.",
+	}, []string{"field"})
 )
 
 // parseLogsQuery parses the given query string and return the conditions for the where statement in the sql query. We
@@ -123,6 +132,12 @@ func handleConditionParts(key, value, operator string) (string, error) {
 	key = strings.TrimSpace(key)
 	value = strings.TrimSpace(value)
 
+	// The kobs_clickhouse_fields_total metric can be used to determine how often a field is used. This information can
+	// then be used to create an additional column for this field via the following SQL commands:
+	// ALTER TABLE logs.logs ON CLUSTER '{cluster}' ADD COLUMN <FIELD> String DEFAULT fields_string.value[indexOf(fields_string.key, '<FIELD>')];
+	// ALTER TABLE logs.logs ON CLUSTER '{cluster}' ADD COLUMN <FIELD> String DEFAULT fields_number.value[indexOf(fields_number.key, '<FIELD>')];
+	fieldsMetric.WithLabelValues(key).Inc()
+
 	if contains(defaultFields, key) {
 		if operator == "=~" {
 			return fmt.Sprintf("%s ILIKE %s", key, value), nil