fix(test): retry MockBroker Listen for EADDRINUSE (#2721)

Some of the FV tests involve shutting down a MockBroker and then
re-creating a new one on the same ip:port. Sometimes this fails because
the port shows as still being in-use. Add some rudimentary retry
attempts for this case to try and close the FV flakiness in that area.

Signed-off-by: Dominic Evans <dominic.evans@uk.ibm.com>

mockbroker.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"strconv"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
@@ -410,10 +411,29 @@ func NewMockBroker(t TestReporter, brokerID int32) *MockBroker {
 // NewMockBrokerAddr behaves like newMockBroker but listens on the address you give
 // it rather than just some ephemeral port.
 func NewMockBrokerAddr(t TestReporter, brokerID int32, addr string) *MockBroker {
-	listener, err := net.Listen("tcp", addr)
+	var (
+		listener net.Listener
+		err      error
+	)
+
+	// retry up to 20 times if address already in use (e.g., if replacing broker which hasn't cleanly shutdown)
+	for i := 0; i < 20; i++ {
+		listener, err = net.Listen("tcp", addr)
+		if err != nil {
+			if errors.Is(err, syscall.EADDRINUSE) {
+				Logger.Printf("*** mockbroker/%d waiting for %s (address already in use)", brokerID, addr)
+				time.Sleep(time.Millisecond * 100)
+				continue
+			}
+			t.Fatal(err)
+		}
+		break
+	}
+
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	return NewMockBrokerListener(t, brokerID, listener)
 }