Improve HTTP metrics and add ClickHouse metric

CHANGELOG.md

@@ -68,6 +68,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#152](https://github.com/kobsio/kobs/pull/152): Improve performance for large dashboards and open Application page in gallery view.
 - [#155](https://github.com/kobsio/kobs/pull/155): Allow users to get all Applications from all namespaces, by allowing an empty namespace list.
 - [#157](https://github.com/kobsio/kobs/pull/157): Imporve query performance for ClickHouse plugin.
+- [#158](https://github.com/kobsio/kobs/pull/158): Improve Pormetheus HTTP metrics and create a metric to find the most used fields in queries.
 
 ## [v0.5.0](https://github.com/kobsio/kobs/releases/tag/v0.5.0) (2021-08-03)
 

pkg/api/middleware/metrics/metrics.go

@@ -2,6 +2,7 @@ package metrics
 
 import (
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/go-chi/chi/v5/middleware"
@@ -10,18 +11,18 @@ import (
 )
 
 var (
-	reqs = promauto.NewCounterVec(prometheus.CounterOpts{
+	reqMetric = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "kobs",
 		Name:      "chi_requests_total",
 		Help:      "Number of HTTP requests processed, partitioned by status code, method and path.",
-	}, []string{"code", "method", "path"})
+	}, []string{"response_code", "request_method", "request_path"})
 
-	latency = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: "kobs",
-		Name:      "chi_request_duration_milliseconds",
-		Help:      "Latency of HTTP requests processed, partitioned by status code, method and path.",
-		Buckets:   []float64{100, 500, 1000, 5000},
-	}, []string{"code", "method", "path"})
+	sumMetric = promauto.NewSummaryVec(prometheus.SummaryOpts{
+		Namespace:  "kobs",
+		Name:       "chi_request_duration_milliseconds",
+		Help:       "Latency of HTTP requests processed, partitioned by status code, method and path.",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001},
+	}, []string{"response_code", "request_method", "request_path"})
 )
 
 // Metrics is a middleware that handles the Prometheus metrics for kobs and chi.
@@ -31,8 +32,8 @@ func Metrics(next http.Handler) http.Handler {
 		wrw := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
 		next.ServeHTTP(wrw, r)
 
-		reqs.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Inc()
-		latency.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000)
+		reqMetric.WithLabelValues(strconv.Itoa(wrw.Status()), r.Method, r.URL.Path).Inc()
+		sumMetric.WithLabelValues(strconv.Itoa(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000)
 	}
 
 	return http.HandlerFunc(fn)

plugins/clickhouse/pkg/instance/logs.go

@@ -3,11 +3,20 @@ package instance
 import (
 	"fmt"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
 	defaultFields  = []string{"timestamp", "cluster", "namespace", "app", "pod_name", "container_name", "host", "log"}
 	defaultColumns = "timestamp, cluster, namespace, app, pod_name, container_name, host, fields_string.key, fields_string.value, fields_number.key, fields_number.value, log"
+
+	fieldsMetric = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "kobs",
+		Name:      "clickhouse_fields_total",
+		Help:      "Number how often a field was used in a query.",
+	}, []string{"field"})
 )
 
 // parseLogsQuery parses the given query string and return the conditions for the where statement in the sql query. We
@@ -123,6 +132,12 @@ func handleConditionParts(key, value, operator string) (string, error) {
 	key = strings.TrimSpace(key)
 	value = strings.TrimSpace(value)
 
+	// The kobs_clickhouse_fields_total metric can be used to determine how often a field is used. This information can
+	// then be used to create an additional column for this field via the following SQL commands:
+	// ALTER TABLE logs.logs ON CLUSTER '{cluster}' ADD COLUMN <FIELD> String DEFAULT fields_string.value[indexOf(fields_string.key, '<FIELD>')];
+	// ALTER TABLE logs.logs ON CLUSTER '{cluster}' ADD COLUMN <FIELD> String DEFAULT fields_number.value[indexOf(fields_number.key, '<FIELD>')];
+	fieldsMetric.WithLabelValues(key).Inc()
+
 	if contains(defaultFields, key) {
 		if operator == "=~" {
 			return fmt.Sprintf("%s ILIKE %s", key, value), nil