Support querying multiple stores by Querier (grafana#2747)

* Added support to querier to work with multiple stores.
Store queryables now have filtering function.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Querier can now use second store for querying.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Fixes for using chunks and blocks at the same time.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Added tests for new filtering queryables.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Added test for distributor querier filter.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Added test for using multiple store queryables.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Ignore linter trying to get rid of else.

This form allows q to have smaller scope,
which reduces possibility of incorrect reuse.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Allow "0" value.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Mention available formats.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Support for flagext.Time.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Extend help for querier.use-second-store-before-time.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Added CHANGELOG entry, and check to avoid using same primary and secondary engines.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Put comment why formatMatcher is used.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Mention that querying feature is experimental.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Enhance CHANGELOG.md

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Explain default value.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Added <time> placeholder, with example better examples.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Removed `buildService` function.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Mention that last two formats are specified by RFC 3339.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

* Fix docs.

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>

chunk_store.go

@@ -446,7 +446,7 @@ func (c *store) lookupChunksByMetricName(ctx context.Context, userID string, fro
 }
 
 func (c *baseStore) lookupIdsByMetricNameMatcher(ctx context.Context, from, through model.Time, userID, metricName string, matcher *labels.Matcher, filter func([]IndexQuery) []IndexQuery) ([]string, error) {
-	log, ctx := spanlogger.New(ctx, "Store.lookupIdsByMetricNameMatcher", "metricName", metricName, "matcher", matcher)
+	log, ctx := spanlogger.New(ctx, "Store.lookupIdsByMetricNameMatcher", "metricName", metricName, "matcher", formatMatcher(matcher))
 	defer log.Span.Finish()
 
 	var err error
@@ -474,11 +474,11 @@ func (c *baseStore) lookupIdsByMetricNameMatcher(ctx context.Context, from, thro
 	if err != nil {
 		return nil, err
 	}
-	level.Debug(log).Log("matcher", matcher, "queries", len(queries))
+	level.Debug(log).Log("matcher", formatMatcher(matcher), "queries", len(queries))
 
 	if filter != nil {
 		queries = filter(queries)
-		level.Debug(log).Log("matcher", matcher, "filteredQueries", len(queries))
+		level.Debug(log).Log("matcher", formatMatcher(matcher), "filteredQueries", len(queries))
 	}
 
 	entries, err := c.lookupEntriesByQueries(ctx, queries)
@@ -489,17 +489,27 @@ func (c *baseStore) lookupIdsByMetricNameMatcher(ctx context.Context, from, thro
 	} else if err != nil {
 		return nil, err
 	}
-	level.Debug(log).Log("matcher", matcher, "entries", len(entries))
+	level.Debug(log).Log("matcher", formatMatcher(matcher), "entries", len(entries))
 
 	ids, err := c.parseIndexEntries(ctx, entries, matcher)
 	if err != nil {
 		return nil, err
 	}
-	level.Debug(log).Log("matcher", matcher, "ids", len(ids))
+	level.Debug(log).Log("matcher", formatMatcher(matcher), "ids", len(ids))
 
 	return ids, nil
 }
 
+// Using this function avoids logging of nil matcher, which works, but indirectly via panic and recover.
+// That confuses attached debugger, which wants to breakpoint on each panic.
+// Using simple check is also faster.
+func formatMatcher(matcher *labels.Matcher) string {
+	if matcher == nil {
+		return "nil"
+	}
+	return matcher.String()
+}
+
 func (c *baseStore) lookupEntriesByQueries(ctx context.Context, queries []IndexQuery) ([]IndexEntry, error) {
 	log, ctx := spanlogger.New(ctx, "store.lookupEntriesByQueries")
 	defer log.Span.Finish()