Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(restore): Do not retry restore proposal #8058

Merged
merged 2 commits into from
Sep 30, 2021
Merged

fix(restore): Do not retry restore proposal #8058

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4d9d9cf
Do not retry restore proposal
ahsanbarkati Sep 29, 2021
8659ff1
Minor change
ahsanbarkati Sep 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 1 addition & 37 deletions worker/online_restore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"path/filepath"
"strings"
"sync"
"time"

"github.com/golang/glog"
"github.com/minio/minio-go/v6/pkg/credentials"
Expand Down Expand Up @@ -177,7 +176,7 @@ func ProcessRestoreRequest(ctx context.Context, req *pb.RestoreRequest, wg *sync
reqCopy.GroupId = gid
wg.Add(1)
go func() {
errCh <- tryRestoreProposal(ctx, reqCopy)
errCh <- proposeRestoreOrSend(ctx, reqCopy)
}()
}

Expand Down Expand Up @@ -209,41 +208,6 @@ func proposeRestoreOrSend(ctx context.Context, req *pb.RestoreRequest) error {
return err
}

func retriableRestoreError(err error) bool {
switch {
case err == conn.ErrNoConnection:
// Try to recover from temporary connection issues.
return true
case strings.Contains(err.Error(), "Raft isn't initialized yet"):
// Try to recover if raft has not been initialized.
return true
case strings.Contains(err.Error(), errRestoreProposal):
// Do not try to recover from other errors when sending the proposal.
return false
default:
// Try to recover from other errors (e.g wrong group, waiting for timestamp, etc).
return true
}
}

func tryRestoreProposal(ctx context.Context, req *pb.RestoreRequest) error {
var err error
for i := 0; i < 10; i++ {
err = proposeRestoreOrSend(ctx, req)
if err == nil {
glog.Info("proposeRestoreOrSend done.")
return nil
}
glog.Errorf("Got error while proposeRestoreOrSend: %v, will retry...\n", err)
if retriableRestoreError(err) {
time.Sleep(time.Second)
continue
}
return err
}
return err
}

// Restore implements the Worker interface.
func (w *grpcWorker) Restore(ctx context.Context, req *pb.RestoreRequest) (*pb.Status, error) {
var emptyRes pb.Status
Expand Down