reusable reader & bug fixes

client_test.go

@@ -46,6 +46,28 @@ func TestRequest(t *testing.T) {
 	}
 }
 
+// TestRequestBody reads request body multiple times
+// using httputil.DumpRequestOut
+func TestRequestBody(t *testing.T) {
+	body := bytes.NewReader([]byte("yo"))
+	req, err := NewRequest("GET", "https://projectdiscovery.io", body)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	for i := 0; i < 10; i++ {
+		bin, err := httputil.DumpRequestOut(req.Request, true)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if bytes.Equal([]byte("yo"), bin) {
+			t.Errorf("expected %v but got %v", "yo", string(bin))
+		}
+	}
+
+}
+
 // TestFromRequest cloning from an existing request
 func TestFromRequest(t *testing.T) {
 	// Works with no request body
@@ -108,9 +130,7 @@ func (c *custReader) Read(p []byte) (n int, err error) {
 func TestClient_Do(t *testing.T) {
 	testBytes := []byte("hello")
 	// Native func
-	testClientSuccess_Do(t, ReaderFunc(func() (io.Reader, error) {
-		return bytes.NewReader(testBytes), nil
-	}))
+	testClientSuccess_Do(t, testBytes)
 	// Native func, different Go type
 	testClientSuccess_Do(t, func() (io.Reader, error) {
 		return bytes.NewReader(testBytes), nil
@@ -156,7 +176,7 @@ func testClientSuccess_Do(t *testing.T, body interface{}) {
 
 		dumpBytes, err := httputil.DumpRequestOut(req, false)
 		if err != nil {
-			t.Fatal("Dumping requests failed")
+			t.Fatalf("Dumping requests failed %v", err)
 		}
 
 		dumpString := string(dumpBytes)
@@ -220,6 +240,45 @@ func TestClientRetry_Do(t *testing.T) {
 		// if at the end we get a failure then it's unexpected behavior
 		t.Fatalf("err: %v", err)
 	}
+
+	// Validate Metrics
+	if req.Metrics.Retries != expectedRetries {
+		t.Fatalf("err: retries do not match expected %v but got %v", expectedRetries, req.Metrics.Retries)
+	}
+}
+
+// TestClientRetryWithBody_Do does same as TestClientRetry_Do but with request body and 5 retries
+func TestClientRetryWithBody_Do(t *testing.T) {
+	// start buggyhttp
+	buggyhttp.Listen(8080)
+	defer buggyhttp.Stop()
+
+	expectedRetries := 5
+	// Create a generic request towards /successAfter passing the number of times before the same request is successful
+	req, err := NewRequest("GET", fmt.Sprintf("http://127.0.0.1:8080/successAfter?successAfter=%d", expectedRetries), "request with body")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	var options Options
+	options.RetryWaitMin = 10 * time.Millisecond
+	options.RetryWaitMax = 50 * time.Millisecond
+	options.RetryMax = 6
+
+	// Create the client. Use short retry windows.
+	client := NewClient(options)
+
+	// In this point the retry strategy should kick in until a response is succesful
+	_, err = client.Do(req)
+	if err != nil {
+		// if at the end we get a failure then it's unexpected behavior
+		t.Fatalf("err: %v", err)
+	}
+
+	// Validate Metrics
+	if req.Metrics.Retries != expectedRetries {
+		t.Fatalf("err: retries do not match expected %v but got %v", expectedRetries, req.Metrics.Retries)
+	}
 }
 
 // TestClientEmptyResponse_Do tests a generic endpoint that simulates the server hanging connection immediately (http connection closed by peer)

do.go

@@ -36,19 +36,8 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
 	}
 
 	for i := 0; ; i++ {
-		// Always rewind the request body when non-nil.
-		if req.body != nil {
-			body, err := req.body()
-			if err != nil {
-				c.closeIdleConnections()
-				return resp, err
-			}
-			if c, ok := body.(io.ReadCloser); ok {
-				req.Body = c
-			} else {
-				req.Body = io.NopCloser(body)
-			}
-		}
+		// request body can be read multiple times
+		// hence no need to rewind it
 
 		if c.RequestLogHook != nil {
 			c.RequestLogHook(req.Request, i)
@@ -114,9 +103,11 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
 
 		// Exit if the main context or the request context is done
 		// Otherwise, wait for the duration and try again.
+		// use label to explicitly specify what to break
+	selectstatement:
 		select {
 		case <-mainCtx.Done():
-			break
+			break selectstatement
 		case <-req.Context().Done():
 			c.closeIdleConnections()
 			return nil, req.Context().Err()

go.mod

@@ -4,18 +4,8 @@ go 1.18
 
 require (
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809
-	github.com/projectdiscovery/utils v0.0.4-0.20221130140026-1b86b537bf1c
+	github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3
 	golang.org/x/net v0.1.0
 )
 
-require (
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.21 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
-	golang.org/x/text v0.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
+require golang.org/x/text v0.4.0 // indirect

go.sum

@@ -1,35 +1,12 @@
 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
-github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
-github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
-github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
-github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
-github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/projectdiscovery/utils v0.0.4-0.20221130140026-1b86b537bf1c h1:5Kb6Tnq9YUr8uHhnuIUIXYwc0/w61abZUnSCTJxYr0w=
-github.com/projectdiscovery/utils v0.0.4-0.20221130140026-1b86b537bf1c/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3 h1:sOvfN3xHLiBMb6GJ3yDxBmPnN0dh3xllaQXQYo7CFUo=
+github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=