Store: Make initial sync more robust

Added re-try mechanism for store inital sync, where if the initial sync fails, it tries to do the initial sync again every 5 seconds for 15 seconds duration (total 3 re-tries for initial sync of store).

Signed-off-by: Kartik-Garg <kartik.garg@infracloud.io>

cmd/thanos/store.go

@@ -49,10 +49,10 @@ import (
 	"github.com/thanos-io/thanos/pkg/ui"
 )
 
-// const (
-// 	timeoutDuration  = 15
-// 	intervalDuration = 5
-// )
+const (
+	timeoutDuration  = 15
+	intervalDuration = 5
+)
 
 type storeConfig struct {
 	indexCacheConfigs           extflag.PathOrContent
@@ -388,11 +388,11 @@ func runStore(
 			begin := time.Now()
 
 			//This will stop retrying after 15 seconds.
-			initialSyncCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			initialSyncCtx, cancel := context.WithTimeout(context.Background(), timeoutDuration*time.Second)
 			defer cancel()
 
 			//If error occurs, it will re-try after every 5 seconds, but only for 15 seconds, (so total re-try is three).
-			err := runutil.Retry(5*time.Second, initialSyncCtx.Done(), func() error {
+			err := runutil.Retry(intervalDuration*time.Second, initialSyncCtx.Done(), func() error {
 				return bs.InitialSync(ctx)
 			})