From 579a791349429bc38ec1a7df36f194176b830917 Mon Sep 17 00:00:00 2001 From: Zach <44234405+zjhuber@users.noreply.github.com> Date: Mon, 25 Oct 2021 07:22:22 -0700 Subject: [PATCH] feat(dot/telemetry): Added connection retry (#1904) --- dot/telemetry/telemetry.go | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/dot/telemetry/telemetry.go b/dot/telemetry/telemetry.go index 2b9fd4081f..af08860190 100644 --- a/dot/telemetry/telemetry.go +++ b/dot/telemetry/telemetry.go @@ -43,6 +43,8 @@ type Handler struct { connections []*telemetryConnection log log.Logger sendMessageTimeout time.Duration + maxRetries int + retryDelay time.Duration } // Instance interface that telemetry handler instance needs to implement @@ -61,7 +63,11 @@ var ( initilised sync.Once ) -const defaultMessageTimeout = time.Second +const ( + defaultMessageTimeout = time.Second + defaultMaxRetries = 5 + defaultRetryDelay = time.Second * 15 +) // GetInstance singleton pattern to for accessing TelemetryHandler func GetInstance() Instance { @@ -72,6 +78,8 @@ func GetInstance() Instance { msg: make(chan Message, 256), log: log.New("pkg", "telemetry"), sendMessageTimeout: defaultMessageTimeout, + maxRetries: defaultMaxRetries, + retryDelay: defaultRetryDelay, } go handlerInstance.startListening() }) @@ -94,17 +102,19 @@ func (h *Handler) Initialise(e bool) { // AddConnections adds the given telemetry endpoint as listeners that will receive telemetry data func (h *Handler) AddConnections(conns []*genesis.TelemetryEndpoint) { for _, v := range conns { - c, _, err := websocket.DefaultDialer.Dial(v.Endpoint, nil) - if err != nil { - // TODO: try reconnecting if there is an error connecting (#1862) - h.log.Debug("issue adding telemetry connection", "error", err) - continue - } - tConn := &telemetryConnection{ - wsconn: c, - verbosity: v.Verbosity, + for connAttempts := 0; connAttempts < h.maxRetries; connAttempts++ { + c, _, err := websocket.DefaultDialer.Dial(v.Endpoint, nil) + if err != nil { + h.log.Debug("issue adding telemetry connection", "error", err) + time.Sleep(h.retryDelay) + continue + } + h.connections = append(h.connections, &telemetryConnection{ + wsconn: c, + verbosity: v.Verbosity, + }) + break } - h.connections = append(h.connections, tConn) } }