From 68d0ad202fbab683d0fbe1bcbefb63c86c027694 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Mon, 21 Sep 2020 18:59:00 +0300 Subject: [PATCH] querier: Add a flag to limit time range for metadata APIs (Labels and Series) (#3147) * Limit the default time window for label name and label value APIs Signed-off-by: Kemal Akkoyun * Keep the existing behaviour as default Signed-off-by: Kemal Akkoyun * Add label API endpoint tests Signed-off-by: Kemal Akkoyun * Fix changelog entry Signed-off-by: Kemal Akkoyun * Address review issues Implement time ranges for series API Signed-off-by: Kemal Akkoyun * Update changelog Signed-off-by: Kemal Akkoyun * Fix formatting Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 3 +- cmd/thanos/query.go | 6 + .../{query-frontend.go => query_frontend.go} | 3 +- docs/components/query.md | 6 + pkg/api/query/v1.go | 159 ++++--- pkg/api/query/v1_test.go | 430 +++++++++++++----- pkg/testutil/e2eutil/prometheus.go | 3 +- 7 files changed, 430 insertions(+), 180 deletions(-) rename cmd/thanos/{query-frontend.go => query_frontend.go} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cd22560bd..3a30918ff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,12 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#3133](https://github.com/thanos-io/thanos/pull/3133) Query: Allow passing a `storeMatch[]` to Labels APIs. Also time range metadata based store filtering is supported on Labels APIs. - [#3154](https://github.com/thanos-io/thanos/pull/3154) Query Frontend: Add metric `thanos_memcached_getmulti_gate_queries_max`. - [#3146](https://github.com/thanos-io/thanos/pull/3146) Sidecar: Add `thanos_sidecar_prometheus_store_received_frames` histogram metric. +- [#3147](https://github.com/thanos-io/thanos/pull/3147) Querier: Add `query.metadata.default-time-range` flag to specify the default metadata time range duration for retrieving labels through Labels and Series API when the range parameters are not specified. The zero value means range covers the time since the beginning. ### Changed - [#3136](https://github.com/thanos-io/thanos/pull/3136) Sidecar: Add metric `thanos_sidecar_reloader_config_apply_operations_total` and rename metric `thanos_sidecar_reloader_config_apply_errors_total` to `thanos_sidecar_reloader_config_apply_operations_failed_total`. -- [#3154](https://github.com/thanos-io/thanos/pull/3154) Query: Add metric `thanos_query_gate_queries_max`. Remove metric `thanos_query_concurrent_selects_gate_queries_in_flight`. +- [#3154](https://github.com/thanos-io/thanos/pull/3154) Querier: Add metric `thanos_query_gate_queries_max`. Remove metric `thanos_query_concurrent_selects_gate_queries_in_flight`. - [#3154](https://github.com/thanos-io/thanos/pull/3154) Store: Rename metric `thanos_bucket_store_queries_concurrent_max` to `thanos_bucket_store_series_gate_queries_max`. - [#3179](https://github.com/thanos-io/thanos/pull/3179) Store: context.Canceled will not increase `thanos_objstore_bucket_operation_failures_total`. diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index f3999e350e..b8c7ec3127 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" + "github.com/thanos-io/thanos/pkg/extkingpin" v1 "github.com/thanos-io/thanos/pkg/api/query" @@ -81,6 +82,8 @@ func registerQuery(app *extkingpin.App) { instantDefaultMaxSourceResolution := modelDuration(cmd.Flag("query.instant.default.max_source_resolution", "default value for max_source_resolution for instant queries. If not set, defaults to 0s only taking raw resolution into account. 1h can be a good value if you use instant queries over time ranges that incorporate times outside of your raw-retention.").Default("0s").Hidden()) + defaultMetadataTimeRange := cmd.Flag("query.metadata.default-time-range", "The default metadata time range duration for retrieving labels through Labels and Series API when the range parameters are not specified. The zero value means range covers the time since the beginning.").Default("0s").Duration() + selectorLabels := cmd.Flag("selector-label", "Query selector labels that will be exposed in info endpoint (repeated)."). PlaceHolder("=\"\"").Strings() @@ -193,6 +196,7 @@ func registerQuery(app *extkingpin.App) { *dnsSDResolver, time.Duration(*unhealthyStoreTimeout), time.Duration(*instantDefaultMaxSourceResolution), + *defaultMetadataTimeRange, *strictStores, component.Query, ) @@ -241,6 +245,7 @@ func runQuery( dnsSDResolver string, unhealthyStoreTimeout time.Duration, instantDefaultMaxSourceResolution time.Duration, + defaultMetadataTimeRange time.Duration, strictStores []string, comp component.Component, ) error { @@ -442,6 +447,7 @@ func runQuery( queryReplicaLabels, flagsMap, instantDefaultMaxSourceResolution, + defaultMetadataTimeRange, gate.New( extprom.WrapRegistererWithPrefix("thanos_query_concurrent_", reg), maxConcurrentQueries, diff --git a/cmd/thanos/query-frontend.go b/cmd/thanos/query_frontend.go similarity index 99% rename from cmd/thanos/query-frontend.go rename to cmd/thanos/query_frontend.go index fadc14053e..44d83036fa 100644 --- a/cmd/thanos/query-frontend.go +++ b/cmd/thanos/query_frontend.go @@ -17,6 +17,8 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/weaveworks/common/user" + "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/extkingpin" @@ -28,7 +30,6 @@ import ( httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/server/http/middleware" "github.com/thanos-io/thanos/pkg/tracing" - "github.com/weaveworks/common/user" ) type queryFrontendConfig struct { diff --git a/docs/components/query.md b/docs/components/query.md index 8a780fee78..5dfacab116 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -389,6 +389,12 @@ Flags: able to query without deduplication using 'dedup=false' parameter. Data includes time series, recording rules, and alerting rules. + --query.metadata.default-time-range=0s + The default metadata time range duration for + retrieving labels through Labels and Series API + when the range parameters are not specified. + The zero value means range covers the time + since the beginning. --selector-label=="" ... Query selector labels that will be exposed in info endpoint (repeated). diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index b674898ad6..47d4e8e8f5 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -70,10 +70,12 @@ type QueryAPI struct { enableAutodownsampling bool enableQueryPartialResponse bool enableRulePartialResponse bool - replicaLabels []string - storeSet *query.StoreSet + replicaLabels []string + storeSet *query.StoreSet + defaultInstantQueryMaxSourceResolution time.Duration + defaultMetadataTimeRange time.Duration } // NewQueryAPI returns an initialized QueryAPI type. @@ -89,6 +91,7 @@ func NewQueryAPI( replicaLabels []string, flagsMap map[string]string, defaultInstantQueryMaxSourceResolution time.Duration, + defaultMetadataTimeRange time.Duration, gate gate.Gate, ) *QueryAPI { return &QueryAPI{ @@ -105,6 +108,7 @@ func NewQueryAPI( replicaLabels: replicaLabels, storeSet: storeSet, defaultInstantQueryMaxSourceResolution: defaultInstantQueryMaxSourceResolution, + defaultMetadataTimeRange: defaultMetadataTimeRange, } } @@ -418,18 +422,10 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: errors.Errorf("invalid label name: %q", name)} } - start, err := parseTimeParam(r, "start", minTime) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } - end, err := parseTimeParam(r, "end", maxTime) + start, end, err := parseMetadataTimeRange(r, qapi.defaultMetadataTimeRange) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} } - if end.Before(start) { - err := errors.New("end timestamp must not be before start time") - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } enablePartialResponse, apiErr := qapi.parsePartialResponseParam(r, qapi.enableQueryPartialResponse) if apiErr != nil { @@ -462,11 +458,6 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A return vals, warnings, nil } -var ( - minTime = time.Unix(math.MinInt64/1000+62135596801, 0) - maxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999) -) - func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiError) { if err := r.ParseForm(); err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(err, "parse form")} @@ -476,18 +467,10 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: errors.New("no match[] parameter provided")} } - start, err := parseTimeParam(r, "start", minTime) + start, end, err := parseMetadataTimeRange(r, qapi.defaultMetadataTimeRange) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} } - end, err := parseTimeParam(r, "end", maxTime) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } - if end.Before(start) { - err := errors.New("end timestamp must not be before start time") - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } var matcherSets [][]*labels.Matcher for _, s := range r.Form["match[]"] { @@ -543,59 +526,11 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return metrics, set.Warnings(), nil } -func parseTimeParam(r *http.Request, paramName string, defaultValue time.Time) (time.Time, error) { - val := r.FormValue(paramName) - if val == "" { - return defaultValue, nil - } - result, err := parseTime(val) - if err != nil { - return time.Time{}, errors.Wrapf(err, "Invalid time value for '%s'", paramName) - } - return result, nil -} - -func parseTime(s string) (time.Time, error) { - if t, err := strconv.ParseFloat(s, 64); err == nil { - s, ns := math.Modf(t) - ns = math.Round(ns*1000) / 1000 - return time.Unix(int64(s), int64(ns*float64(time.Second))), nil - } - if t, err := time.Parse(time.RFC3339Nano, s); err == nil { - return t, nil - } - return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s) -} - -func parseDuration(s string) (time.Duration, error) { - if d, err := strconv.ParseFloat(s, 64); err == nil { - ts := d * float64(time.Second) - if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) { - return 0, errors.Errorf("cannot parse %q to a valid duration. It overflows int64", s) - } - return time.Duration(ts), nil - } - if d, err := model.ParseDuration(s); err == nil { - return time.Duration(d), nil - } - return 0, errors.Errorf("cannot parse %q to a valid duration", s) -} - func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.ApiError) { - ctx := r.Context() - - start, err := parseTimeParam(r, "start", minTime) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } - end, err := parseTimeParam(r, "end", maxTime) + start, end, err := parseMetadataTimeRange(r, qapi.defaultMetadataTimeRange) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} } - if end.Before(start) { - err := errors.New("end timestamp must not be before start time") - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } enablePartialResponse, apiErr := qapi.parsePartialResponseParam(r, qapi.enableQueryPartialResponse) if apiErr != nil { @@ -607,7 +542,8 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap return nil, nil, apiErr } - q, err := qapi.queryableCreate(true, nil, storeMatchers, 0, enablePartialResponse, false).Querier(ctx, timestamp.FromTime(start), timestamp.FromTime(end)) + q, err := qapi.queryableCreate(true, nil, storeMatchers, 0, enablePartialResponse, false). + Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} } @@ -659,3 +595,76 @@ func NewRulesHandler(client rules.UnaryClient, enablePartialResponse bool) func( return groups, warnings, nil } } + +var ( + infMinTime = time.Unix(math.MinInt64/1000+62135596801, 0) + infMaxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999) +) + +func parseMetadataTimeRange(r *http.Request, defaultMetadataTimeRange time.Duration) (time.Time, time.Time, error) { + // If start and end time not specified as query parameter, we get the range from the beginning of time by default. + var defaultStartTime, defaultEndTime time.Time + if defaultMetadataTimeRange == 0 { + defaultStartTime = infMinTime + defaultEndTime = infMaxTime + } else { + now := time.Now() + defaultStartTime = now.Add(-defaultMetadataTimeRange) + defaultEndTime = now + } + + start, err := parseTimeParam(r, "start", defaultStartTime) + if err != nil { + return time.Time{}, time.Time{}, &api.ApiError{Typ: api.ErrorBadData, Err: err} + } + end, err := parseTimeParam(r, "end", defaultEndTime) + if err != nil { + return time.Time{}, time.Time{}, &api.ApiError{Typ: api.ErrorBadData, Err: err} + } + if end.Before(start) { + return time.Time{}, time.Time{}, &api.ApiError{ + Typ: api.ErrorBadData, + Err: errors.New("end timestamp must not be before start time"), + } + } + + return start, end, nil +} + +func parseTimeParam(r *http.Request, paramName string, defaultValue time.Time) (time.Time, error) { + val := r.FormValue(paramName) + if val == "" { + return defaultValue, nil + } + result, err := parseTime(val) + if err != nil { + return time.Time{}, errors.Wrapf(err, "Invalid time value for '%s'", paramName) + } + return result, nil +} + +func parseTime(s string) (time.Time, error) { + if t, err := strconv.ParseFloat(s, 64); err == nil { + s, ns := math.Modf(t) + ns = math.Round(ns*1000) / 1000 + return time.Unix(int64(s), int64(ns*float64(time.Second))), nil + } + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s) +} + +func parseDuration(s string) (time.Duration, error) { + if d, err := strconv.ParseFloat(s, 64); err == nil { + ts := d * float64(time.Second) + if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) { + return 0, errors.Errorf("cannot parse %q to a valid duration. It overflows int64", s) + } + return time.Duration(ts), nil + } + if d, err := model.ParseDuration(s); err == nil { + return time.Duration(d), nil + } + return 0, errors.Errorf("cannot parse %q to a valid duration", s) +} diff --git a/pkg/api/query/v1_test.go b/pkg/api/query/v1_test.go index d3ae8dc0d7..5dba268c58 100644 --- a/pkg/api/query/v1_test.go +++ b/pkg/api/query/v1_test.go @@ -21,6 +21,8 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" + "math" "math/rand" "net/http" "net/url" @@ -37,6 +39,7 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb" baseAPI "github.com/thanos-io/thanos/pkg/api" "github.com/thanos-io/thanos/pkg/compact" @@ -55,7 +58,64 @@ func TestMain(m *testing.M) { testutil.TolerantVerifyLeakMain(m) } -func TestEndpoints(t *testing.T) { +type endpointTestCase struct { + endpoint baseAPI.ApiFunc + params map[string]string + query url.Values + method string + response interface{} + errType baseAPI.ErrorType +} + +func testEndpoint(t *testing.T, test endpointTestCase, name string) bool { + return t.Run(name, func(t *testing.T) { + // Build a context with the correct request params. + ctx := context.Background() + for p, v := range test.params { + ctx = route.WithParam(ctx, p, v) + } + + reqURL := "http://example.com" + params := test.query.Encode() + + var body io.Reader + if test.method == http.MethodPost { + body = strings.NewReader(params) + } else if test.method == "" { + test.method = "ANY" + reqURL += "?" + params + } + + req, err := http.NewRequest(test.method, reqURL, body) + if err != nil { + t.Fatal(err) + } + + if body != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + + resp, _, apiErr := test.endpoint(req.WithContext(ctx)) + if apiErr != nil { + if test.errType == baseAPI.ErrorNone { + t.Fatalf("Unexpected error: %s", apiErr) + } + if test.errType != apiErr.Typ { + t.Fatalf("Expected error of type %q but got type %q", test.errType, apiErr.Typ) + } + return + } + if test.errType != baseAPI.ErrorNone { + t.Fatalf("Expected error of type %q but got none", test.errType) + } + + if !reflect.DeepEqual(resp, test.response) { + t.Fatalf("Response does not match, expected:\n%+v\ngot:\n%+v", test.response, resp) + } + }) +} + +func TestQueryEndpoints(t *testing.T) { lbls := []labels.Labels{ { labels.Label{Name: "__name__", Value: "test_metric1"}, @@ -122,14 +182,7 @@ func TestEndpoints(t *testing.T) { start := time.Unix(0, 0) - var tests = []struct { - endpoint baseAPI.ApiFunc - params map[string]string - query url.Values - method string - response interface{} - errType baseAPI.ErrorType - }{ + var tests = []endpointTestCase{ { endpoint: api.query, query: url.Values{ @@ -513,6 +566,125 @@ func TestEndpoints(t *testing.T) { }, errType: baseAPI.ErrorBadData, }, + } + + for i, test := range tests { + if ok := testEndpoint(t, test, fmt.Sprintf("#%d %s", i, test.query.Encode())); !ok { + return + } + } +} + +func TestMetadataEndpoints(t *testing.T) { + var old = []labels.Labels{ + { + labels.Label{Name: "__name__", Value: "test_metric1"}, + labels.Label{Name: "foo", Value: "bar"}, + }, + { + labels.Label{Name: "__name__", Value: "test_metric1"}, + labels.Label{Name: "foo", Value: "boo"}, + }, + { + labels.Label{Name: "__name__", Value: "test_metric2"}, + labels.Label{Name: "foo", Value: "boo"}, + }, + } + + var recent = []labels.Labels{ + { + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "bar"}, + labels.Label{Name: "replica", Value: "a"}, + }, + { + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "boo"}, + labels.Label{Name: "replica", Value: "a"}, + }, + { + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "boo"}, + labels.Label{Name: "replica", Value: "b"}, + }, + { + labels.Label{Name: "__name__", Value: "test_metric_replica2"}, + labels.Label{Name: "foo", Value: "boo"}, + labels.Label{Name: "replica1", Value: "a"}, + }, + } + + dir, err := ioutil.TempDir("", "prometheus-test") + testutil.Ok(t, err) + + var ( + mint int64 = 0 + maxt int64 = 600_000 + ) + var metricSamples []*tsdb.MetricSample + for _, lbl := range old { + for i := int64(0); i < 10; i++ { + metricSamples = append(metricSamples, &tsdb.MetricSample{ + TimestampMs: i * 60_000, + Value: float64(i), + Labels: lbl, + }) + } + } + + _, err = tsdb.CreateBlock(metricSamples, dir, mint, maxt, nil) + testutil.Ok(t, err) + + opts := tsdb.DefaultOptions() + opts.RetentionDuration = math.MaxInt64 + db, err := tsdb.Open(dir, nil, nil, opts) + defer func() { testutil.Ok(t, db.Close()) }() + testutil.Ok(t, err) + + var ( + apiLookbackDelta = 2 * time.Hour + start = time.Now().Add(-apiLookbackDelta).Unix() * 1000 + app = db.Appender(context.Background()) + ) + for _, lbl := range recent { + for i := int64(0); i < 10; i++ { + _, err := app.Add(lbl, start+(i*60_000), float64(i)) // ms + testutil.Ok(t, err) + } + } + testutil.Ok(t, app.Commit()) + + now := time.Now() + timeout := 100 * time.Second + api := &QueryAPI{ + baseAPI: &baseAPI.BaseAPI{ + Now: func() time.Time { return now }, + }, + queryableCreate: query.NewQueryableCreator(nil, nil, store.NewTSDBStore(nil, nil, db, component.Query, nil), 2, timeout), + queryEngine: promql.NewEngine(promql.EngineOpts{ + Logger: nil, + Reg: nil, + MaxSamples: 10000, + Timeout: timeout, + }), + gate: gate.New(nil, 4), + } + apiWithLabelLookback := &QueryAPI{ + baseAPI: &baseAPI.BaseAPI{ + Now: func() time.Time { return now }, + }, + queryableCreate: query.NewQueryableCreator(nil, nil, store.NewTSDBStore(nil, nil, db, component.Query, nil), 2, timeout), + queryEngine: promql.NewEngine(promql.EngineOpts{ + Logger: nil, + Reg: nil, + MaxSamples: 10000, + Timeout: timeout, + }), + gate: gate.New(nil, 4), + defaultMetadataTimeRange: apiLookbackDelta, + } + + var tests = []endpointTestCase{ { endpoint: api.labelValues, params: map[string]string{ @@ -522,16 +694,97 @@ func TestEndpoints(t *testing.T) { "test_metric1", "test_metric2", "test_metric_replica1", + "test_metric_replica2", + }, + }, + { + endpoint: apiWithLabelLookback.labelValues, + params: map[string]string{ + "name": "__name__", + }, + response: []string{ + "test_metric_replica1", + "test_metric_replica2", }, }, { endpoint: api.labelValues, + query: url.Values{ + "start": []string{"1970-01-01T00:00:00Z"}, + "end": []string{"1970-01-01T00:09:00Z"}, + }, + params: map[string]string{ + "name": "__name__", + }, + response: []string{ + "test_metric1", + "test_metric2", + }, + }, + { + endpoint: apiWithLabelLookback.labelValues, + query: url.Values{ + "start": []string{"1970-01-01T00:00:00Z"}, + "end": []string{"1970-01-01T00:09:00Z"}, + }, + params: map[string]string{ + "name": "__name__", + }, + response: []string{ + "test_metric1", + "test_metric2", + }, + }, + { + endpoint: api.labelNames, + params: map[string]string{ + "name": "__name__", + }, + response: []string{ + "__name__", + "foo", + "replica", + "replica1", + }, + }, + { + endpoint: apiWithLabelLookback.labelNames, params: map[string]string{ "name": "foo", }, response: []string{ - "bar", - "boo", + "__name__", + "foo", + "replica", + "replica1", + }, + }, + { + endpoint: api.labelNames, + query: url.Values{ + "start": []string{"1970-01-01T00:00:00Z"}, + "end": []string{"1970-01-01T00:09:00Z"}, + }, + params: map[string]string{ + "name": "foo", + }, + response: []string{ + "__name__", + "foo", + }, + }, + { + endpoint: apiWithLabelLookback.labelNames, + query: url.Values{ + "start": []string{"1970-01-01T00:00:00Z"}, + "end": []string{"1970-01-01T00:09:00Z"}, + }, + params: map[string]string{ + "name": "foo", + }, + response: []string{ + "__name__", + "foo", }, }, // Bad name parameter. @@ -551,6 +804,24 @@ func TestEndpoints(t *testing.T) { labels.FromStrings("__name__", "test_metric2", "foo", "boo"), }, }, + { + endpoint: apiWithLabelLookback.series, + query: url.Values{ + "match[]": []string{`test_metric2`}, + }, + response: []labels.Labels{}, + }, + { + endpoint: apiWithLabelLookback.series, + query: url.Values{ + "match[]": []string{`test_metric_replica1`}, + }, + response: []labels.Labels{ + labels.FromStrings("__name__", "test_metric_replica1", "foo", "bar", "replica", "a"), + labels.FromStrings("__name__", "test_metric_replica1", "foo", "boo", "replica", "a"), + labels.FromStrings("__name__", "test_metric_replica1", "foo", "boo", "replica", "b"), + }, + }, // Series that does not exist should return an empty array. { endpoint: api.series, @@ -796,55 +1067,10 @@ func TestEndpoints(t *testing.T) { }, } - for _, test := range tests { - if ok := t.Run(test.query.Encode(), func(t *testing.T) { - // Build a context with the correct request params. - ctx := context.Background() - for p, v := range test.params { - ctx = route.WithParam(ctx, p, v) - } - - reqURL := "http://example.com" - params := test.query.Encode() - - var body io.Reader - if test.method == http.MethodPost { - body = strings.NewReader(params) - } else if test.method == "" { - test.method = "ANY" - reqURL += "?" + params - } - - req, err := http.NewRequest(test.method, reqURL, body) - if err != nil { - t.Fatal(err) - } - - if body != nil { - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - } - - resp, _, apiErr := test.endpoint(req.WithContext(ctx)) - if apiErr != nil { - if test.errType == baseAPI.ErrorNone { - t.Fatalf("Unexpected error: %s", apiErr) - } - if test.errType != apiErr.Typ { - t.Fatalf("Expected error of type %q but got type %q", test.errType, apiErr.Typ) - } - return - } - if test.errType != baseAPI.ErrorNone { - t.Fatalf("Expected error of type %q but got none", test.errType) - } - - if !reflect.DeepEqual(resp, test.response) { - t.Fatalf("Response does not match, expected:\n%+v\ngot:\n%+v", test.response, resp) - } - }); !ok { + for i, test := range tests { + if ok := testEndpoint(t, test, strings.TrimSpace(fmt.Sprintf("#%d %s", i, test.query.Encode()))); !ok { return } - } } @@ -957,40 +1183,6 @@ func TestParseDuration(t *testing.T) { } } -func BenchmarkQueryResultEncoding(b *testing.B) { - var mat promql.Matrix - for i := 0; i < 1000; i++ { - lset := labels.FromStrings( - "__name__", "my_test_metric_name", - "instance", fmt.Sprintf("abcdefghijklmnopqrstuvxyz-%d", i), - "job", "test-test", - "method", "ABCD", - "status", "199", - "namespace", "something", - "long-label", "34grnt83j0qxj309je9rgt9jf2jd-92jd-92jf9wrfjre", - ) - var points []promql.Point - for j := 0; j < b.N/1000; j++ { - points = append(points, promql.Point{ - T: int64(j * 10000), - V: rand.Float64(), - }) - } - mat = append(mat, promql.Series{ - Metric: lset, - Points: points, - }) - } - input := &queryData{ - ResultType: parser.ValueTypeMatrix, - Result: mat, - } - b.ResetTimer() - - _, err := json.Marshal(&input) - testutil.Ok(b, err) -} - func TestParseDownsamplingParamMillis(t *testing.T) { var tests = []struct { maxSourceResolutionParam string @@ -1119,16 +1311,6 @@ func TestParseStoreMatchersParam(t *testing.T) { } } -type mockedRulesClient struct { - g map[rulespb.RulesRequest_Type][]*rulespb.RuleGroup - w storage.Warnings - err error -} - -func (c mockedRulesClient) Rules(_ context.Context, req *rulespb.RulesRequest) (*rulespb.RuleGroups, storage.Warnings, error) { - return &rulespb.RuleGroups{Groups: c.g[req.Type]}, c.w, c.err -} - func TestRulesHandler(t *testing.T) { twoHAgo := time.Now().Add(-2 * time.Hour) all := []*rulespb.Rule{ @@ -1399,3 +1581,47 @@ func TestRulesHandler(t *testing.T) { }) } } + +func BenchmarkQueryResultEncoding(b *testing.B) { + var mat promql.Matrix + for i := 0; i < 1000; i++ { + lset := labels.FromStrings( + "__name__", "my_test_metric_name", + "instance", fmt.Sprintf("abcdefghijklmnopqrstuvxyz-%d", i), + "job", "test-test", + "method", "ABCD", + "status", "199", + "namespace", "something", + "long-label", "34grnt83j0qxj309je9rgt9jf2jd-92jd-92jf9wrfjre", + ) + var points []promql.Point + for j := 0; j < b.N/1000; j++ { + points = append(points, promql.Point{ + T: int64(j * 10000), + V: rand.Float64(), + }) + } + mat = append(mat, promql.Series{ + Metric: lset, + Points: points, + }) + } + input := &queryData{ + ResultType: parser.ValueTypeMatrix, + Result: mat, + } + b.ResetTimer() + + _, err := json.Marshal(&input) + testutil.Ok(b, err) +} + +type mockedRulesClient struct { + g map[rulespb.RulesRequest_Type][]*rulespb.RuleGroup + w storage.Warnings + err error +} + +func (c mockedRulesClient) Rules(_ context.Context, req *rulespb.RulesRequest) (*rulespb.RuleGroups, storage.Warnings, error) { + return &rulespb.RuleGroups{Groups: c.g[req.Type]}, c.w, c.err +} diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go index acc4482d0d..447d65425f 100644 --- a/pkg/testutil/e2eutil/prometheus.go +++ b/pkg/testutil/e2eutil/prometheus.go @@ -30,10 +30,11 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/index" + "golang.org/x/sync/errgroup" + "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" - "golang.org/x/sync/errgroup" ) const (