diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 8a7daf57d7b..c5626e0e7c8 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -380,7 +380,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str } func registerBucketReplicate(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) { - cmd := root.Command("replicate", "Replicate data from one object storage to another") + cmd := root.Command("replicate", "Replicate data from one object storage to another. NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata).") httpMetricsBindAddr, _ := regHTTPFlags(cmd) toObjStoreConfig := regCommonObjStoreFlags(cmd, "-to", false, "The object storage which replicate data to.") resolution := cmd.Flag("resolution", "Only blocks with this resolution will be replicated.").Default(strconv.FormatInt(downsample.ResLevel0, 10)).Int64() diff --git a/docs/components/bucket.md b/docs/components/bucket.md index c1ca1ea9f13..819b397590a 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -323,6 +323,8 @@ Flags: `bucket replicate` is used to replicate buckets from one object storage to another. +NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata). + Example: ``` $ thanos bucket replicate --objstore.config-file="..." --objstore-to.config="..." diff --git a/go.mod b/go.mod index 0ed5f7a9ff4..beb3c75d701 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,6 @@ require ( github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.6 // indirect github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 // master ~ v2.15.2 - github.com/prometheus/tsdb v0.10.0 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/smartystreets/assertions v1.0.1 // indirect diff --git a/go.sum b/go.sum index ef05edbcd40..a4ddc0f20a9 100644 --- a/go.sum +++ b/go.sum @@ -118,7 +118,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -504,8 +503,6 @@ github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 h1:uEq+8hKI4kfycPLSKNw844YYkdMNpC2eZpov73AvlFk= github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= -github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= diff --git a/pkg/replicater/scheme.go b/pkg/replicater/scheme.go index 560d86bc085..5555b2722d5 100644 --- a/pkg/replicater/scheme.go +++ b/pkg/replicater/scheme.go @@ -51,6 +51,11 @@ func NewBlockFilter( // Filter return true if block is non-compacted and matches selector. func (bf *BlockFilter) Filter(b *metadata.Meta) bool { + if len(b.Thanos.Labels) == 0 { + level.Error(bf.logger).Log("msg", "filtering block", "reason", "labels should not be empty") + return false + } + blockLabels := labels.FromMap(b.Thanos.Labels) labelMatch := bf.labelSelector.Matches(blockLabels) @@ -174,6 +179,7 @@ func (rs *replicationScheme) execute(ctx context.Context) error { level.Debug(rs.logger).Log("msg", "scanning blocks available blocks for replication") + // TODO: Use block.MetaFetcher with filters instead. if err := rs.fromBkt.Iter(ctx, "", func(name string) error { rs.metrics.originIterations.Inc() diff --git a/pkg/replicater/scheme_test.go b/pkg/replicater/scheme_test.go index a282325eefe..2a993123f29 100644 --- a/pkg/replicater/scheme_test.go +++ b/pkg/replicater/scheme_test.go @@ -19,11 +19,11 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/tsdb/testutil" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/testutil" ) func testLogger(testName string) log.Logger {