Add configurable wait time between each HTTP request

cmd/gitjacker/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -19,11 +20,13 @@ import (
 
 var outputDir string
 var verbose bool
+var wait time.Duration
 
 func main() {
 
 	rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", verbose, "Enable verbose logging")
 	rootCmd.Flags().StringVarP(&outputDir, "output-dir", "o", outputDir, "Directory to output retrieved git repository - defaults to a temporary directory")
+	rootCmd.Flags().DurationVarP(&wait, "wait", "w", wait, "Wait this long between HTTP requests (to avoid throttling)")
 
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
@@ -89,7 +92,7 @@ Output Dir: %s
 			_ = tml.Printf("\n<yellow>Gitjacking in progress...")
 		}
 
-		summary, err := gitjacker.New(u, outputDir).Run()
+		summary, err := gitjacker.New(u, outputDir, wait).Run()
 		if err != nil {
 			if !verbose {
 				fmt.Printf("\x1b[2K\r")

go.mod

@@ -9,5 +9,6 @@ require (
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/cobra v1.0.0
+	golang.org/x/time v0.3.0 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
 )

go.sum

@@ -162,6 +162,8 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

internal/pkg/gitjacker/retriever.go

@@ -1,6 +1,7 @@
 package gitjacker
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -14,6 +15,7 @@ import (
     "crypto/tls"
 
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 )
 
 var paths = []string{
@@ -37,6 +39,7 @@ type retriever struct {
 	baseURL    *url.URL
 	outputDir  string
 	http       *http.Client
+	limiter    *rate.Limiter
 	downloaded map[string]bool
 	summary    Summary
 }
@@ -88,7 +91,7 @@ type Branch struct {
 	Remote string
 }
 
-func New(target *url.URL, outputDir string) *retriever {
+func New(target *url.URL, outputDir string, wait time.Duration) *retriever {
 
 	relative, _ := url.Parse(".git/")
 	target = target.ResolveReference(relative)
@@ -103,6 +106,7 @@ func New(target *url.URL, outputDir string) *retriever {
 			Timeout: time.Second * 10,
             Transport: customTransport,
 		},
+		limiter:    rate.NewLimiter(rate.Every(wait), 1),
 		downloaded: make(map[string]bool),
 		summary: Summary{
 			OutputDirectory: outputDir,
@@ -172,6 +176,11 @@ func (r *retriever) downloadFile(path string) error {
 		return err
 	}
 
+	err = r.limiter.Wait(context.Background())
+	if err != nil {
+		return err
+	}
+
 	absolute := r.baseURL.ResolveReference(relative)
 	resp, err := r.http.Get(absolute.String())
 	if err != nil {