fix double url encoding in path

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809
-	github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d
+	github.com/projectdiscovery/utils v0.0.7
 	golang.org/x/net v0.5.0
 )
 

go.sum

@@ -8,14 +8,10 @@ github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl
 github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
 github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3 h1:sOvfN3xHLiBMb6GJ3yDxBmPnN0dh3xllaQXQYo7CFUo=
-github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
-github.com/projectdiscovery/utils v0.0.4-0.20230117121210-1eaffe0d0834 h1:ehoX21rVDm+i7/o8OpTTtDdbesHshF0AD13gbc21wBA=
-github.com/projectdiscovery/utils v0.0.4-0.20230117121210-1eaffe0d0834/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
-github.com/projectdiscovery/utils v0.0.4-0.20230117132455-e51a5b2e562c h1:+iHkNvGP/1Cbq6lo8htaQZd3fWch8E9OKD0xTUwS+Zo=
-github.com/projectdiscovery/utils v0.0.4-0.20230117132455-e51a5b2e562c/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
-github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d h1:iB/n2/NL4oh1IaEcqX6pBxj0WHfYN7finzNOKVNVISM=
-github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
+github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDkst9DA2M=
+github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
+github.com/projectdiscovery/utils v0.0.8-0.20230206142604-469c07cf5050 h1:PwtYD40LMJag5jpB3F2bi1y4tLAMUPIeuWO37txfbOI=
+github.com/projectdiscovery/utils v0.0.8-0.20230206142604-469c07cf5050/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=

request.go

@@ -107,6 +107,18 @@ func (r *Request) Update() {
 	updateScheme(r.URL.URL)
 }
 
+// Prepares request (applies hot patch if any. Ex: Path Unescape to prevent double url encoding)
+// calling multiple times may have unexpected results unlike Update() method
+func (r *Request) Prepare() {
+	// hot patch to avoid url path encoding issues
+	// by default we decode encoded/escaped path and internally http.Request encodes them again
+	// this avoid double url encoding (or reencoding or path)
+	if rawPath, err := url.QueryUnescape(r.URL.Path); err == nil {
+		r.URL.Path = rawPath
+	}
+	r.Update()
+}
+
 // SetURL updates request url (i.e http.Request.URL) with given url
 func (r *Request) SetURL(u *urlutil.URL) {
 	r.URL = u

request_test.go

@@ -1,7 +1,10 @@
 package retryablehttp_test
 
 import (
+	"bufio"
+	"bytes"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -35,3 +38,51 @@ func TestRequestUrls(t *testing.T) {
 		}
 	}
 }
+
+func TestEncodedPaths(t *testing.T) {
+
+	// test this on all valid crlf payloads
+	payloads := []string{"%00", "%0a", "%0a%20", "%0d", "%0d%09", "%0d%0a", "%0d%0a%09", "%0d%0a%20", "%0d%20", "%20", "%20%0a", "%20%0d", "%20%0d%0a", "%23%0a", "%23%0a%20", "%23%0d", "%23%0d%0a", "%23%0a", "%25%30", "%25%30%61", "%2e%2e%2f%0d%0a", "%2f%2e%2e%0d%0a", "%2f..%0d%0a", "%3f", "%3f%0a", "%3f%0d", "%3f%0d%0a", "%e5%98%8a%e5%98%8d", "%e5%98%8a%e5%98%8d%0a", "%e5%98%8a%e5%98%8d%0d", "%e5%98%8a%e5%98%8d%0d%0a", "%e5%98%8a%e5%98%8d%e5%98%8a%e5%98%8d"}
+
+	// create url using below data and payload
+	suffix := "/path?param=true"
+
+	for _, v := range payloads {
+		exURL := "https://scanme.sh/" + v + suffix
+		req, err := retryablehttp.NewRequest("GET", exURL, nil)
+		if err != nil {
+			t.Fatalf("got %v with payload %v", err.Error(), v)
+		}
+
+		req.Prepare()
+		bin, err := req.Dump()
+		if err != nil {
+			t.Errorf("failed to dump request body for payload %v got %v", v, err)
+		}
+
+		relPath := getPathFromRaw(bin)
+		payload := strings.TrimSuffix(relPath, suffix)
+		payload = strings.TrimPrefix(payload, "/")
+
+		if v != payload {
+			t.Errorf("something went wrong expected `%v` in outgoing request but got-----\n%v\n------", v, string(bin))
+		}
+	}
+}
+
+func getPathFromRaw(bin []byte) (relpath string) {
+	buff := bufio.NewReader(bytes.NewReader(bin))
+readline:
+	line, err := buff.ReadString('\n')
+	if err != nil {
+		return
+	}
+	if strings.Contains(line, "HTTP/1.1") {
+		parts := strings.Split(line, " ")
+		if len(parts) == 3 {
+			relpath = parts[1]
+			return
+		}
+	}
+	goto readline
+}