Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v1.5] add timeout to etcd member add #3536

Merged
merged 1 commit into from
Jul 2, 2024
Merged

[v1.5] add timeout to etcd member add #3536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion services/etcd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/rancher/rke/util"
"github.com/sirupsen/logrus"
etcdclientv2 "go.etcd.io/etcd/client/v2"
etcdclientv3 "go.etcd.io/etcd/client/v3"
"golang.org/x/sync/errgroup"
)

Expand All @@ -29,6 +30,7 @@ const (
EtcdDataDir = "/var/lib/rancher/etcd/"
EtcdInitWaitTime = 10
EtcdSnapshotWaitTime = 5
EtcdRequestWaitTimeout = 10
EtcdPermFixContainerName = "etcd-fix-perm"
)

Expand Down Expand Up @@ -169,6 +171,14 @@ func RemoveEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, force bool) e
return nil
}

func memberAdd(ctx context.Context, etcdClient *etcdclientv3.Client, peerURL string) error {
ctxTimeout, cancel := context.WithTimeout(ctx, EtcdRequestWaitTimeout*time.Second)
_, err := etcdClient.MemberAdd(ctxTimeout, []string{peerURL})
cancel()
_ = etcdClient.Close()
return err
}

func AddEtcdMember(ctx context.Context, toAddEtcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory,
k8sVersion string, cert, key []byte) error {
log.Infof(ctx, "[add/%s] Adding member [etcd-%s] to etcd cluster", ETCDRole, toAddEtcdHost.HostnameOverride)
Expand All @@ -184,7 +194,7 @@ func AddEtcdMember(ctx context.Context, toAddEtcdHost *hosts.Host, etcdHosts []*
logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err)
continue
}
if _, err := etcdClient.MemberAdd(ctx, []string{peerURL}); err != nil {
if err := memberAdd(ctx, etcdClient, peerURL); err != nil {
logrus.Debugf("Failed to Add etcd member [%s] from host: %v", host.Address, err)
continue
}
Expand Down Expand Up @@ -240,6 +250,7 @@ func RemoveEtcdMember(ctx context.Context, toDeleteEtcdHost *hosts.Host, etcdHos
logrus.Debugf("Failed to list etcd members from host [%s]: %v", host.Address, err)
continue
}
_ = etcdClient.Close()
for _, member := range members.Members {
if member.Name == fmt.Sprintf("etcd-%s", toDeleteEtcdHost.HostnameOverride) {
mIDv3 = member.ID
Expand Down Expand Up @@ -347,6 +358,7 @@ func IsEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.
continue
}
members, err := etcdClient.MemberList(ctx)
_ = etcdClient.Close()
if err != nil {
listErr = errors.Wrapf(err, "Failed to create etcd client for host [%s]", host.Address)
logrus.Debugf("Failed to list etcd cluster members [%s]: %v", etcdHost.Address, err)
Expand Down
1 change: 1 addition & 0 deletions services/etcd_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func getEtcdClientV3(ctx context.Context, etcdHost *hosts.Host, localConnDialerF
Endpoints: []string{"https://" + etcdHost.InternalAddress + ":2379"},
TLS: tlsConfig,
DialOptions: []grpc.DialOption{grpc.WithContextDialer(wrapper(dialer))},
DialTimeout: 5 * time.Second,
}

return etcdclientv3.New(cfg)
Expand Down