Cortex Service Discovery improvements & Disable SLO API

go.mod

@@ -85,7 +85,7 @@ require (
 	github.com/samber/lo v1.26.0
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/pflag v1.0.5
-	github.com/tidwall/gjson v1.14.1
+	github.com/tidwall/gjson v1.14.2
 	github.com/tidwall/sjson v1.2.4
 	github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31
 	github.com/vbauerster/mpb/v7 v7.4.2

go.sum

@@ -2715,8 +2715,8 @@ github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH
 github.com/tidwall/gjson v1.6.8/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI=
 github.com/tidwall/gjson v1.7.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
-github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=

pkg/metrics/unmarshal/unmarshal.go

@@ -3,8 +3,9 @@ package unmarshal
 import (
 	"encoding/json"
 	"fmt"
-
 	"github.com/prometheus/common/model"
+	"go.uber.org/zap"
+	"net/http"
 )
 
 // Struct for unmarshalling from github.com/prometheus/common/model
@@ -75,3 +76,56 @@ func UnmarshalPrometheusResponse(data []byte) (*queryResult, error) {
 	}
 	return &q, nil
 }
+
+//https://github.com/prometheus/prometheus/blob/main/web/api/v1/api.go
+type status string
+
+const (
+	statusSuccess status = "success"
+	statusError   status = "error"
+)
+
+// https://github.com/prometheus/prometheus/blob/main/web/api/v1/api.go
+type errorType string
+
+const (
+	errorNone        errorType = ""
+	errorTimeout     errorType = "timeout"
+	errorCanceled    errorType = "canceled"
+	errorExec        errorType = "execution"
+	errorBadData     errorType = "bad_data"
+	errorInternal    errorType = "internal"
+	errorUnavailable errorType = "unavailable"
+	errorNotFound    errorType = "not_found"
+)
+
+// Generic struct for unmarshalling prometheus http api responses
+// https://github.com/prometheus/prometheus/blob/bcd548c88b06543c8eeb19e68bef4adefb7b95fb/web/api/v1/api.go#L140
+type response struct {
+	Status    status      `json:"status"`
+	Data      interface{} `json:"data,omitempty"`
+	ErrorType errorType   `json:"errorType,omitempty"`
+	Error     string      `json:"error,omitempty"`
+	Warnings  []string    `json:"warnings,omitempty"`
+}
+
+func unmarshallPrometheusWebResponseData(data []byte) (*response, error) {
+	var r response
+	err := json.Unmarshal(data, &r)
+	if err != nil {
+		return nil, err
+	}
+	return &r, nil
+}
+
+func UnmarshallPrometheusWebResponse(resp *http.Response, lg *zap.SugaredLogger) (*response, error) {
+	var val *response
+	err := json.NewDecoder(resp.Body).Decode(&val)
+	if err != nil {
+		return nil, err
+	}
+	if val.Status != statusSuccess {
+		return nil, fmt.Errorf("well formed prometheus request failed internally with: %s", val.Error)
+	}
+	return val, nil
+}

pkg/slo/query/collectors.go

@@ -31,22 +31,22 @@ var (
 	availabilityCollector *prometheus.CounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: "http_request_duration_seconds_count",
 	},
-		[]string{"code", "job"},
+		[]string{"code"},
 	)
 	availabilityGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
 		randomStatusInt1 := rand.Intn(199)
 
 		// anything between 200-399, and yes http status codes dont'work like this
 		randomStatusCode := 200 + randomStatusInt1
 
-		availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode), MockTestServerName).Inc()
+		availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode)).Inc()
 	}
 
 	availabilityBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
 		randomStatusInt1 := rand.Intn(199)
 		// anything between 400-599, and yes http status codes dont'work like this
 		randomStatusCode := 400 + randomStatusInt1
-		availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode), MockTestServerName).Inc()
+		availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode)).Inc()
 	}
 
 	latencyCollector *prometheus.HistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{

pkg/slo/query/def.go

@@ -46,7 +46,7 @@ type templateExecutor struct {
 }
 
 // must not contain spaces
-const MockTestServerName = "MetricMockServer"
+const MockTestServerName = "MyServer"
 
 func init() {
 	// Names should be unique for each pre-configured query, as they are used as keys

pkg/test/environment.go

@@ -680,7 +680,7 @@ func (e *Environment) StartPrometheus(opniAgentPort int, override ...*overridePr
 			if resp.StatusCode == http.StatusOK {
 				break
 			}
-		}
+		} ///tmp/opni-monitoring-test-289661467/prometheus/config.yaml
 		time.Sleep(time.Second)
 	}
 	lg.With("address", fmt.Sprintf("http://localhost:%d", port)).Info("Prometheus started")
@@ -742,10 +742,11 @@ func (e *Environment) StartInstrumentationServer(ctx context.Context) (int, chan
 		panic(err)
 	}
 	mux := http.NewServeMux()
+	reg := prometheus.NewRegistry()
 
 	for queryName, queryObj := range query.AvailableQueries {
 		// register each prometheus collector
-		prometheus.MustRegister(queryObj.GetCollector())
+		reg.MustRegister(queryObj.GetCollector())
 
 		// create an endpoint simulating good events
 		mux.HandleFunc(fmt.Sprintf("/%s/%s", queryName, "good"), queryObj.GetGoodEventGenerator())
@@ -754,7 +755,10 @@ func (e *Environment) StartInstrumentationServer(ctx context.Context) (int, chan
 
 	}
 	// expose prometheus metrics
-	mux.Handle("/metrics", promhttp.Handler())
+
+	mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{
+		Registry: reg,
+	}))
 
 	autoInstrumentationServer := &http.Server{
 		Addr:           fmt.Sprintf("127.0.0.1:%d", port),

pkg/test/testdata/slo/prometheus/config.yaml

@@ -1,7 +1,7 @@
 # my global config
 global:
-  scrape_interval: 1s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 1s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
   # scrape_timeout is set to the global default (10s).
 
 # A scrape configuration containing exactly one endpoint to scrape:
@@ -21,7 +21,6 @@ scrape_configs:
   - job_name: "{{ .JobName }}"
   # metrics_path defaults to '/metrics'
   # scheme defaults to 'http'.
-
     static_configs:
       - targets:
           - "localhost:{{ .ScrapePort }}"