Retry failed registry HTTP requests

ivanilves · Nov 8, 2017 · 1c1172b · 1c1172b
1 parent d2e171b
commit 1c1172b
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 19 deletions.
diff --git a/main.go b/main.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/jessevdk/go-flags"
 
@@ -19,17 +20,19 @@ import (
 
 // Options represents configuration options we extract from passed command line arguments
 type Options struct {
-	DockerJSON         string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"`
-	Pull               bool   `short:"p" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"`
-	Push               bool   `short:"P" long:"push" description:"Push Docker images matched by filter to some registry (See 'push-registry')" env:"PUSH"`
-	PushRegistry       string `short:"r" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"`
-	PushPrefix         string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
-	PushUpdate         bool   `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"`
-	ConcurrentRequests int    `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
-	InsecureRegistryEx string `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"`
-	TraceRequests      bool   `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
-	DoNotFail          bool   `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`
-	Version            bool   `short:"V" long:"version" description:"Show version and exit"`
+	DockerJSON         string        `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"`
+	Pull               bool          `short:"p" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"`
+	Push               bool          `short:"P" long:"push" description:"Push Docker images matched by filter to some registry (See 'push-registry')" env:"PUSH"`
+	PushRegistry       string        `short:"r" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"`
+	PushPrefix         string        `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
+	PushUpdate         bool          `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"`
+	ConcurrentRequests int           `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
+	RetryRequests      int           `short:"y" long:"retry-requests" default:"2" description:"Number of retries for failed registry HTTP requests" env:"RETRY_REQUESTS"`
+	RetryDelay         time.Duration `short:"D" long:"retry-delay" default:"30s" description:"Delay between retries of failed registry requests" env:"RETRY_DELAY"`
+	InsecureRegistryEx string        `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"`
+	TraceRequests      bool          `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
+	DoNotFail          bool          `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`
+	Version            bool          `short:"V" long:"version" description:"Show version and exit"`
 	Positional         struct {
 		Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"`
 	} `positional-args:"yes" required:"yes"`
@@ -74,6 +77,10 @@ func parseFlags() (*Options, error) {
 
 	remote.ConcurrentRequests = o.ConcurrentRequests
 
+	remote.RetryRequests = o.RetryRequests
+
+	remote.RetryDelay = o.RetryDelay
+
 	if o.InsecureRegistryEx != "" {
 		docker.InsecureRegistryEx = o.InsecureRegistryEx
 	}

diff --git a/tag/remote/remote.go b/tag/remote/remote.go
@@ -23,13 +23,15 @@ import (
 // ConcurrentRequests defines maximum number of concurrent requests we could maintain against the registry
 var ConcurrentRequests = 32
 
+// RetryRequests is a number of retries we do in case of request failure
+var RetryRequests = 0
+
+// RetryDelay is a delay between retries of failed requests to the registry
+var RetryDelay = 5 * time.Second
+
 // TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
 var TraceRequests = false
 
-func getAuthorizationType(authorization string) string {
-	return strings.Split(authorization, " ")[0]
-}
-
 func getRequestID() string {
 	data := make([]byte, 10)
 
@@ -73,7 +75,7 @@ func httpRequest(url, authorization, mode string) (*http.Response, error) {
 		return nil, err
 	}
 	if resp.StatusCode != 200 {
-		return nil, errors.New("Bad response status: " + resp.Status + " >> " + url)
+		return resp, errors.New("Bad response status: " + resp.Status + " >> " + url)
 	}
 
 	if TraceRequests {
@@ -91,6 +93,43 @@ func httpRequest(url, authorization, mode string) (*http.Response, error) {
 	return resp, nil
 }
 
+func httpRetriableRequest(url, authorization, mode string) (*http.Response, error) {
+	tries := 1
+
+	if RetryRequests > 0 {
+		tries = tries + RetryRequests
+	}
+
+	var resp *http.Response
+	var err error
+
+	for try := 1; try <= tries; try++ {
+		resp, err := httpRequest(url, authorization, mode)
+
+		if err == nil {
+			return resp, nil
+		}
+
+		if resp.StatusCode >= 400 && resp.StatusCode < 500 {
+			return nil, err
+		}
+
+		if try < tries {
+			fmt.Printf(
+				"Will retry '%s' [%s] in a %v\n=> Error: %s\n",
+				url,
+				mode,
+				RetryDelay,
+				err.Error(),
+			)
+
+			time.Sleep(RetryDelay)
+		}
+	}
+
+	return resp, err
+}
+
 type tagNameInfo struct {
 	TagNames []string `json:"tags"`
 }
@@ -109,7 +148,7 @@ func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
 func fetchTagNames(registry, repoPath, authorization string) ([]string, error) {
 	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/tags/list"
 
-	resp, err := httpRequest(url, authorization, "v2")
+	resp, err := httpRetriableRequest(url, authorization, "v2")
 	if err != nil {
 		return nil, err
 	}
@@ -144,7 +183,7 @@ func extractMetadataFromHistory(s string) (imageMetadata, error) {
 }
 
 func fetchMetadata(url, authorization string) (imageMetadata, error) {
-	resp, err := httpRequest(url, authorization, "v1")
+	resp, err := httpRetriableRequest(url, authorization, "v1")
 	if err != nil {
 		return imageMetadata{}, nil
 	}
@@ -171,7 +210,7 @@ func fetchMetadata(url, authorization string) (imageMetadata, error) {
 }
 
 func fetchDigest(url, authorization string) (string, error) {
-	resp, err := httpRequest(url, authorization, "v2")
+	resp, err := httpRetriableRequest(url, authorization, "v2")
 	if err != nil {
 		return "", err
 	}