From 39b90a6f9b3e7678319b44dcb68a8c185ff73d47 Mon Sep 17 00:00:00 2001 From: fanmin shi Date: Mon, 5 Feb 2018 15:27:06 -0800 Subject: [PATCH] *: support backup timeout in backup operator --- pkg/controller/backup-operator/abs_backup.go | 9 +++++++-- pkg/controller/backup-operator/s3_backup.go | 9 +++++++-- pkg/controller/backup-operator/sync.go | 12 ++++++++++-- pkg/util/constants/constants.go | 7 ++++--- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/controller/backup-operator/abs_backup.go b/pkg/controller/backup-operator/abs_backup.go index 1d339e20b..bff8df2b6 100644 --- a/pkg/controller/backup-operator/abs_backup.go +++ b/pkg/controller/backup-operator/abs_backup.go @@ -15,8 +15,10 @@ package controller import ( + "context" "crypto/tls" "fmt" + "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -27,7 +29,7 @@ import ( ) // handleABS saves etcd cluster's backup to specificed ABS path. -func handleABS(kubecli kubernetes.Interface, s *api.ABSBackupSource, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { +func handleABS(kubecli kubernetes.Interface, s *api.ABSBackupSource, backupTimeout time.Duration, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { cli, err := absfactory.NewClientFromSecret(kubecli, namespace, s.ABSSecret) if err != nil { return nil, err @@ -39,7 +41,10 @@ func handleABS(kubecli kubernetes.Interface, s *api.ABSBackupSource, endpoints [ } bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewABSWriter(cli.ABS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(s.Path) + + ctx, cancel := context.WithTimeout(context.Background(), backupTimeout) + defer cancel() + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } diff --git a/pkg/controller/backup-operator/s3_backup.go b/pkg/controller/backup-operator/s3_backup.go index 48bfd66a1..088cb3429 100644 --- a/pkg/controller/backup-operator/s3_backup.go +++ b/pkg/controller/backup-operator/s3_backup.go @@ -15,8 +15,10 @@ package controller import ( + "context" "crypto/tls" "fmt" + "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -28,7 +30,7 @@ import ( // TODO: replace this with generic backend interface for other options (PV, Azure) // handleS3 saves etcd cluster's backup to specificed S3 path. -func handleS3(kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { +func handleS3(kubecli kubernetes.Interface, s *api.S3BackupSource, backupTimeout time.Duration, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { cli, err := s3factory.NewClientFromSecret(kubecli, namespace, s.Endpoint, s.AWSSecret) if err != nil { return nil, err @@ -41,7 +43,10 @@ func handleS3(kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []s } bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewS3Writer(cli.S3), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(s.Path) + + ctx, cancel := context.WithTimeout(context.Background(), backupTimeout) + defer cancel() + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index 45d652d19..1b7010fc0 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -16,8 +16,10 @@ package controller import ( "errors" + "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" + "github.com/coreos/etcd-operator/pkg/util/constants" "github.com/sirupsen/logrus" ) @@ -119,15 +121,21 @@ func (b *Backup) handleBackup(spec *api.BackupSpec) (*api.BackupStatus, error) { return nil, err } + // When BackupPolicy.Timeout <= 0, use default DefaultBackupTimeout. + backupTimeout := time.Duration(constants.DefaultBackupTimeout) + if spec.BackupPolicy != nil && spec.BackupPolicy.Timeout > 0 { + backupTimeout = time.Duration(spec.BackupPolicy.Timeout) * time.Second + } + switch spec.StorageType { case api.BackupStorageTypeS3: - bs, err := handleS3(b.kubecli, spec.S3, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) + bs, err := handleS3(b.kubecli, spec.S3, backupTimeout, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeABS: - bs, err := handleABS(b.kubecli, spec.ABS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) + bs, err := handleABS(b.kubecli, spec.ABS, backupTimeout, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) if err != nil { return nil, err } diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index c42f74c28..661875f8a 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -17,9 +17,10 @@ package constants import "time" const ( - DefaultDialTimeout = 5 * time.Second - DefaultRequestTimeout = 5 * time.Second - DefaultSnapshotTimeout = 1 * time.Minute + DefaultDialTimeout = 5 * time.Second + DefaultRequestTimeout = 5 * time.Second + // DefaultBackupTimeout is the maximal allowed time of retriving and saving etcd backup. + DefaultBackupTimeout = 1 * time.Minute DefaultSnapshotInterval = 1800 * time.Second DefaultBackupPodHTTPPort = 19999