Upgraded Thanos dependency

go.mod

@@ -3,17 +3,14 @@ module github.com/cortexproject/cortex
 go 1.12
 
 require (
-	cloud.google.com/go v0.44.1
+	cloud.google.com/go/bigtable v1.1.0
+	cloud.google.com/go/storage v1.3.0
 	github.com/Azure/azure-storage-blob-go v0.8.0
-	github.com/Azure/go-autorest/autorest v0.9.2 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect
-	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
-	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
 	github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5
 	github.com/NYTimes/gziphandler v1.1.1
-	github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4
-	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
-	github.com/aws/aws-sdk-go v1.25.22
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
+	github.com/armon/go-metrics v0.3.0
+	github.com/aws/aws-sdk-go v1.25.35
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
 	github.com/cenkalti/backoff v1.0.0 // indirect
@@ -27,7 +24,7 @@ require (
 	github.com/go-kit/kit v0.9.0
 	github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
 	github.com/gogo/googleapis v1.1.0 // indirect
-	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48
+	github.com/gogo/protobuf v1.3.1
 	github.com/gogo/status v1.0.3
 	github.com/golang-migrate/migrate/v4 v4.7.0
 	github.com/golang/protobuf v1.3.2
@@ -36,11 +33,11 @@ require (
 	github.com/gorilla/mux v1.7.1
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
-	github.com/hashicorp/consul/api v1.1.0
+	github.com/hashicorp/consul/api v1.3.0
 	github.com/hashicorp/go-cleanhttp v0.5.1
 	github.com/hashicorp/go-sockaddr v1.0.2
-	github.com/hashicorp/memberlist v0.1.4
-	github.com/json-iterator/go v1.1.7
+	github.com/hashicorp/memberlist v0.1.5
+	github.com/json-iterator/go v1.1.8
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect
 	github.com/lann/builder v0.0.0-20150808151131-f22ce00fd939 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
@@ -52,28 +49,28 @@ require (
 	github.com/philhofer/fwd v0.0.0-20160129035939-98c11a7a6ec8 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/alertmanager v0.19.0
-	github.com/prometheus/client_golang v1.1.0
+	github.com/prometheus/client_golang v1.2.1
 	github.com/prometheus/common v0.7.0
-	github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0
+	github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1
 	github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
 	github.com/sercand/kuberesolver v2.1.0+incompatible // indirect
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.4.0
-	github.com/thanos-io/thanos v0.8.1
+	github.com/thanos-io/thanos v0.8.1-0.20200102143048-a37ac093a67a
 	github.com/tinylib/msgp v0.0.0-20161221055906-38a6f61a768d // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
-	github.com/uber/jaeger-client-go v2.20.0+incompatible
+	github.com/uber/jaeger-client-go v2.20.1+incompatible
 	github.com/weaveworks/billing-client v0.0.0-20171006123215-be0d55e547b1
 	github.com/weaveworks/common v0.0.0-20190822150010-afb9996716e4
 	github.com/weaveworks/promrus v1.2.0 // indirect
 	go.etcd.io/bbolt v1.3.3
 	go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5
-	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
+	golang.org/x/net v0.0.0-20191112182307-2180aed22343
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
-	google.golang.org/api v0.11.0
+	google.golang.org/api v0.14.0
 	google.golang.org/grpc v1.25.1
-	gopkg.in/yaml.v2 v2.2.2
+	gopkg.in/yaml.v2 v2.2.5
 )
 
 replace github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible

pkg/cortex/modules.go

@@ -1,6 +1,7 @@
 package cortex
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -226,6 +227,8 @@ func (t *Cortex) initQuerier(cfg *Config) (err error) {
 		querier.DummyRulesRetriever{},
 		0, 0, 0, // Remote read samples and concurrency limit.
 		regexp.MustCompile(".*"),
+		func() (v1.RuntimeInfo, error) { return v1.RuntimeInfo{}, errors.New("not implemented") },
+		&v1.PrometheusVersion{},
 	)
 	promRouter := route.New().WithPrefix("/api/prom/api/v1")
 	api.Register(promRouter)

pkg/ingester/bucket.go

@@ -66,3 +66,8 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) {
 func (b *Bucket) IsObjNotFoundErr(err error) bool {
 	return b.Bucket.IsObjNotFoundErr(err)
 }
+
+// ObjectSize returns the size of the specified object.
+func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) {
+	return b.Bucket.ObjectSize(ctx, b.fullName(name))
+}

pkg/ingester/client/compat.go

@@ -194,6 +194,22 @@ func FromLabelAdaptersToLabels(ls []LabelAdapter) labels.Labels {
 	return *(*labels.Labels)(unsafe.Pointer(&ls))
 }
 
+// FromLabelAdaptersToLabelsWithCopy converts []LabelAdapter to labels.Labels.
+// Do NOT use unsafe to convert between data types because this function may
+// get in input labels whose data structure is reused.
+func FromLabelAdaptersToLabelsWithCopy(input []LabelAdapter) labels.Labels {
+	result := make(labels.Labels, len(input))
+
+	for i, l := range input {
+		result[i] = labels.Label{
+			Name:  l.Name,
+			Value: l.Value,
+		}
+	}
+
+	return result
+}
+
 // FromLabelsToLabelAdapters casts labels.Labels to []LabelAdapter.
 // It uses unsafe, but as LabelAdapter == labels.Label this should be safe.
 // This allows us to use labels.Labels directly in protos.

pkg/ingester/ingester_v2.go

@@ -13,8 +13,8 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/tsdb"
-	lbls "github.com/prometheus/prometheus/tsdb/labels"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/runutil"
@@ -119,7 +119,7 @@ func (i *Ingester) v2Push(ctx old_ctx.Context, req *client.WriteRequest) (*clien
 	app := db.Appender()
 	for _, ts := range req.Timeseries {
 		// Convert labels to the type expected by TSDB
-		lset := cortex_tsdb.FromLabelAdaptersToLabels(ts.Labels)
+		lset := client.FromLabelAdaptersToLabelsWithCopy(ts.Labels)
 
 		for _, s := range ts.Samples {
 			_, err := app.Add(lset, s.TimestampMs, s.Value)
@@ -189,12 +189,7 @@ func (i *Ingester) v2Query(ctx old_ctx.Context, req *client.QueryRequest) (*clie
 	}
 	defer q.Close()
 
-	convertedMatchers, err := cortex_tsdb.FromLegacyLabelMatchersToMatchers(matchers)
-	if err != nil {
-		return nil, err
-	}
-
-	ss, err := q.Select(convertedMatchers...)
+	ss, err := q.Select(matchers...)
 	if err != nil {
 		return nil, err
 	}
@@ -204,7 +199,7 @@ func (i *Ingester) v2Query(ctx old_ctx.Context, req *client.QueryRequest) (*clie
 		series := ss.At()
 
 		ts := client.TimeSeries{
-			Labels: cortex_tsdb.FromLabelsToLabelAdapters(series.Labels()),
+			Labels: client.FromLabelsToLabelAdapters(series.Labels()),
 		}
 
 		it := series.Iterator()
@@ -308,12 +303,7 @@ func (i *Ingester) v2MetricsForLabelMatchers(ctx old_ctx.Context, req *client.Me
 	}
 
 	for _, matchers := range matchersSet {
-		convertedMatchers, err := cortex_tsdb.FromLegacyLabelMatchersToMatchers(matchers)
-		if err != nil {
-			return nil, err
-		}
-
-		seriesSet, err := q.Select(convertedMatchers...)
+		seriesSet, err := q.Select(matchers...)
 		if err != nil {
 			return nil, err
 		}
@@ -333,7 +323,7 @@ func (i *Ingester) v2MetricsForLabelMatchers(ctx old_ctx.Context, req *client.Me
 			}
 
 			result.Metric = append(result.Metric, &client.Metric{
-				Labels: cortex_tsdb.FromLabelsToLabelAdapters(ls),
+				Labels: client.FromLabelsToLabelAdapters(ls),
 			})
 
 			added[key] = struct{}{}
@@ -398,15 +388,15 @@ func (i *Ingester) getOrCreateTSDB(userID string, force bool) (*tsdb.DB, error)
 	// Thanos shipper requires at least 1 external label to be set. For this reason,
 	// we set the tenant ID as external label and we'll filter it out when reading
 	// the series from the storage.
-	l := lbls.Labels{
+	l := labels.Labels{
 		{
 			Name:  cortex_tsdb.TenantIDExternalLabel,
 			Value: userID,
 		},
 	}
 
 	// Create a new shipper for this database
-	s := shipper.New(util.Logger, nil, udir, &Bucket{userID, i.TSDBState.bucket}, func() lbls.Labels { return l }, metadata.ReceiveSource)
+	s := shipper.New(util.Logger, nil, udir, &Bucket{userID, i.TSDBState.bucket}, func() labels.Labels { return l }, metadata.ReceiveSource)
 	i.done.Add(1)
 	go func() {
 		defer i.done.Done()

pkg/querier/block_store.go

@@ -118,7 +118,7 @@ func (u *UserStore) syncUserStores(ctx context.Context, f func(context.Context,
 
 			indexCacheSizeBytes := u.cfg.BucketStore.IndexCacheSizeBytes
 			maxItemSizeBytes := indexCacheSizeBytes / 2
-			indexCache, err := storecache.NewIndexCache(u.logger, nil, storecache.Opts{
+			indexCache, err := storecache.NewInMemoryIndexCache(u.logger, nil, storecache.Opts{
 				MaxSizeBytes:     indexCacheSizeBytes,
 				MaxItemSizeBytes: maxItemSizeBytes,
 			})

pkg/storage/tsdb/config.go

@@ -12,10 +12,15 @@ import (
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/s3"
 )
 
-// Constants for the config values
 const (
-	BackendS3  = "s3"
+	// BackendS3 is the value for the S3 storage backend
+	BackendS3 = "s3"
+
+	// BackendGCS is the value for the GCS storage backend
 	BackendGCS = "gcs"
+
+	// TenantIDExternalLabel is the external label set when shipping blocks to the storage
+	TenantIDExternalLabel = "__org_id__"
 )
 
 // Validation errors