feedback

Signed-off-by: Xiang Dai <764524258@qq.com>
thanos-io · Feb 12, 2020 · e0d13d2 · e0d13d2
1 parent 1927577
commit e0d13d2
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 8 deletions.
diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go
@@ -380,7 +380,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str
 }
 
 func registerBucketReplicate(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) {
-	cmd := root.Command("replicate", "Replicate data from one object storage to another")
+	cmd := root.Command("replicate", "Replicate data from one object storage to another. NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata).")
 	httpMetricsBindAddr, _ := regHTTPFlags(cmd)
 	toObjStoreConfig := regCommonObjStoreFlags(cmd, "-to", false, "The object storage which replicate data to.")
 	resolution := cmd.Flag("resolution", "Only blocks with this resolution will be replicated.").Default(strconv.FormatInt(downsample.ResLevel0, 10)).Int64()

diff --git a/docs/components/bucket.md b/docs/components/bucket.md
@@ -75,7 +75,8 @@ Subcommands:
     Web interface for remote storage bucket
 
   bucket replicate [<flags>]
-    Replicate data from one object storage to another
+    Replicate data from one object storage to another. NOTE: Currently it works
+    only with Thanos blocks (meta.json has to have Thanos metadata).
 
 
 ```
@@ -323,6 +324,8 @@ Flags:
 
 `bucket replicate` is used to replicate buckets from one object storage to another.
 
+NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata).
+
 Example:
 ```
 $ thanos bucket replicate --objstore.config-file="..." --objstore-to.config="..."
@@ -332,7 +335,8 @@ $ thanos bucket replicate --objstore.config-file="..." --objstore-to.config="...
 ```txt
 usage: thanos bucket replicate [<flags>]
 
-Replicate data from one object storage to another
+Replicate data from one object storage to another. NOTE: Currently it works only
+with Thanos blocks (meta.json has to have Thanos metadata).
 
 Flags:
   -h, --help                     Show context-sensitive help (also try

diff --git a/go.mod b/go.mod
@@ -73,7 +73,6 @@ require (
 	github.com/prometheus/common v0.7.0
 	github.com/prometheus/procfs v0.0.6 // indirect
 	github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 // master ~ v2.15.2
-	github.com/prometheus/tsdb v0.10.0
 	github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/smartystreets/assertions v1.0.1 // indirect

diff --git a/go.sum b/go.sum
@@ -118,7 +118,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -504,8 +503,6 @@ github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
 github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 h1:uEq+8hKI4kfycPLSKNw844YYkdMNpC2eZpov73AvlFk=
 github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI=
-github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic=
-github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

diff --git a/pkg/replicater/scheme.go b/pkg/replicater/scheme.go
@@ -51,6 +51,11 @@ func NewBlockFilter(
 
 // Filter return true if block is non-compacted and matches selector.
 func (bf *BlockFilter) Filter(b *metadata.Meta) bool {
+	if len(b.Thanos.Labels) == 0 {
+		level.Error(bf.logger).Log("msg", "filtering block", "reason", "labels should not be empty")
+		return false
+	}
+
 	blockLabels := labels.FromMap(b.Thanos.Labels)
 
 	labelMatch := bf.labelSelector.Matches(blockLabels)
@@ -174,6 +179,7 @@ func (rs *replicationScheme) execute(ctx context.Context) error {
 
 	level.Debug(rs.logger).Log("msg", "scanning blocks available blocks for replication")
 
+	// TODO: Use block.MetaFetcher with filters instead.
 	if err := rs.fromBkt.Iter(ctx, "", func(name string) error {
 		rs.metrics.originIterations.Inc()
 

diff --git a/pkg/replicater/scheme_test.go b/pkg/replicater/scheme_test.go
@@ -19,11 +19,11 @@ import (
 	"github.com/oklog/ulid"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/tsdb"
-	"github.com/prometheus/tsdb/testutil"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/compact"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
+	"github.com/thanos-io/thanos/pkg/testutil"
 )
 
 func testLogger(testName string) log.Logger {