[FAB-4743] Harden backoff policy code in deliverClient

The delivery client has a backoff policy that specifies
an amount of time to sleep and whether to continue to retry
 based on an attempt number.

Currently the code that calculates the amount of time to sleep
is prone to overflow to a negative number.

However, the fact that the code only retries a limited number
of times (not indefinitely) saves us from the overflow.

I would like to make the code that calculates the time duration
to sleep to never overflow in order for if someone in the future
to change the attempt threshold and forget about the overflow problem
lurking around the corner, we won't have a problem.

Change-Id: Ie0d365e879eb71141c278231819e52e341b73fe5
Signed-off-by: Yacov Manevich <yacovm@il.ibm.com>

core/deliverservice/deliveryclient.go

@@ -31,6 +31,7 @@ func init() {
 var (
 	reConnectTotalTimeThreshold = time.Second * 60 * 5
 	connTimeout                 = time.Second * 3
+	reConnectBackoffThreshold   = float64(time.Hour)
 )
 
 // DeliverService used to communicate with orderers to obtain
@@ -181,7 +182,9 @@ func (d *deliverServiceImpl) newClient(chainID string, ledgerInfoProvider blocks
 		if elapsedTime.Nanoseconds() > reConnectTotalTimeThreshold.Nanoseconds() {
 			return 0, false
 		}
-		return time.Duration(math.Pow(2, float64(attemptNum))) * time.Millisecond * 500, true
+		sleepIncrement := float64(time.Millisecond * 500)
+		attempt := float64(attemptNum)
+		return time.Duration(math.Min(math.Pow(2, attempt)*sleepIncrement, reConnectBackoffThreshold)), true
 	}
 	connProd := comm.NewConnectionProducer(d.conf.ConnFactory(chainID), d.conf.Endpoints)
 	bClient := NewBroadcastClient(connProd, d.conf.ABCFactory, broadcastSetup, backoffPolicy)

core/deliverservice/deliveryclient_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package deliverclient
 
 import (
+	"errors"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -406,6 +407,20 @@ func TestDeliverServiceBadConfig(t *testing.T) {
 	assert.Nil(t, service)
 }
 
+func TestRetryPolicyOverflow(t *testing.T) {
+	connFactory := func(channelID string) func(endpoint string) (*grpc.ClientConn, error) {
+		return func(_ string) (*grpc.ClientConn, error) {
+			return nil, errors.New("")
+		}
+	}
+	client := (&deliverServiceImpl{conf: &Config{ConnFactory: connFactory}}).newClient("TEST", &mocks.MockLedgerInfo{Height: uint64(100)})
+	assert.NotNil(t, client.shouldRetry)
+	for i := 0; i < 100; i++ {
+		retryTime, _ := client.shouldRetry(i, time.Second)
+		assert.True(t, retryTime <= time.Hour && retryTime > 0)
+	}
+}
+
 func assertBlockDissemination(expectedSeq uint64, ch chan uint64, t *testing.T) {
 	select {
 	case seq := <-ch: