diff --git a/pkg/kopia/repository/client.go b/pkg/kopia/repository/client.go
index 1e14f1514a..7ed33a8290 100644
--- a/pkg/kopia/repository/client.go
+++ b/pkg/kopia/repository/client.go
@@ -33,9 +33,20 @@ import (
 
 const (
 	defaultConnectMaxListCacheDuration time.Duration = time.Second * 600
-	kopiaGetRepoParametersError                      = "unable to get repository parameters"
+	connectionRefusedError                           = "connection refused"
+
+	// maxConnectRetries with value 100 results in ~23 total minutes of
+	// retries with the apiConnectBackoff settings defined below.
+	maxConnectionRetries = 100
 )
 
+var apiConnectBackoff = backoff.Backoff{
+	Factor: 2,
+	Jitter: false,
+	Min:    100 * time.Millisecond,
+	Max:    15 * time.Second,
+}
+
 // ConnectToAPIServer connects to the Kopia API server running at the given address
 func ConnectToAPIServer(
 	ctx context.Context,
@@ -73,19 +84,11 @@ func ConnectToAPIServer(
 		},
 	}
 
-	err = poll.WaitWithBackoff(ctx, backoff.Backoff{
-		Factor: 2,
-		Jitter: false,
-		Min:    100 * time.Millisecond,
-		Max:    15 * time.Second,
-	}, func(c context.Context) (bool, error) {
+	err = poll.WaitWithBackoffWithRetries(ctx, apiConnectBackoff, maxConnectionRetries, isErrConnectionRefused, func(c context.Context) (bool, error) {
 		// TODO(@pavan): Modify this to use custom config file path, if required
 		err := repo.ConnectAPIServer(ctx, kopia.DefaultClientConfigFilePath, serverInfo, userPassphrase, opts)
-		switch {
-		case isGetRepoParametersError(err):
+		if err != nil {
 			log.Debug().WithError(err).Print("Connecting to the Kopia API Server")
-			return false, nil
-		case err != nil:
 			return false, err
 		}
 		return true, nil
@@ -126,6 +129,6 @@ func repositoryConfigFileName(configFile string) string {
 	return filepath.Join(os.Getenv("HOME"), ".config", "kopia", "repository.config")
 }
 
-func isGetRepoParametersError(err error) bool {
-	return err != nil && strings.Contains(err.Error(), kopiaGetRepoParametersError)
+func isErrConnectionRefused(err error) bool {
+	return err != nil && strings.Contains(err.Error(), connectionRefusedError)
 }