aws/signer/v4: Add support for URL.RawPath to signer

Adds support for the URL.RawPath added in Go1.5. This allows you to
hint to the signer and Go HTTP client what the escaped form of the
request's URI path will be. This is needed when using the AWS v4 Signer
outside of the context of the SDK on http.Requests you manage.

Also adds documentation to the signer that pre-escaping of the URI path
is needed, and suggestions how how to do this.

aws/signer/v4 TestStandaloneSign test function is an example
pre-escaping the URI path before generating a signature.

aws/signer/v4/functional_1_4_test.go

@@ -0,0 +1,44 @@
+// +build !go1.5
+
+package v4_test
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/awstesting/unit"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStandaloneSign(t *testing.T) {
+	creds := unit.Session.Config.Credentials
+	signer := v4.NewSigner(creds, func(s *v4.Signer) {
+		s.Debug = aws.LogDebugWithSigning
+		s.Logger = aws.NewDefaultLogger()
+	})
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", fmt.Sprintf("https://%s", host), nil)
+		assert.NoError(t, err)
+
+		req.URL.Path = c.OrigURI
+		req.URL.RawQuery = c.OrigQuery
+		req.URL.Opaque = fmt.Sprintf("//%s%s", host, c.EscapedURI)
+		opaqueURI := req.URL.Opaque
+
+		_, err = signer.Sign(req, nil, c.Service, c.Region, time.Unix(0, 0))
+		assert.NoError(t, err)
+
+		actual := req.Header.Get("Authorization")
+		assert.Equal(t, c.ExpSig, actual)
+		assert.Equal(t, c.OrigURI, req.URL.Path)
+		assert.Equal(t, opaqueURI, req.URL.Opaque)
+	}
+}

aws/signer/v4/functional_1_5_test.go

@@ -0,0 +1,46 @@
+// +build go1.5
+
+package v4_test
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/awstesting/unit"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStandaloneSign(t *testing.T) {
+	creds := unit.Session.Config.Credentials
+	signer := v4.NewSigner(creds, func(s *v4.Signer) {
+		s.Debug = aws.LogDebugWithSigning
+		s.Logger = aws.NewDefaultLogger()
+	})
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		assert.NoError(t, err)
+
+		req.URL.Path = c.OrigURI
+		req.URL.RawQuery = c.OrigQuery
+
+		// Set the Raw, pre-escaped path of the URL so the signer will known
+		// what will be sent to the service when building the canonical string.
+		req.URL.RawPath = req.URL.EscapedPath()
+
+		_, err = signer.Sign(req, nil, c.Service, c.Region, time.Unix(0, 0))
+		assert.NoError(t, err)
+
+		actual := req.Header.Get("Authorization")
+		assert.Equal(t, c.ExpSig, actual)
+		assert.Equal(t, c.OrigURI, req.URL.Path)
+		assert.Equal(t, c.EscapedURI, req.URL.RawPath)
+	}
+}

aws/signer/v4/functional_test.go

@@ -1,17 +1,35 @@
 package v4_test
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/awstesting/unit"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/stretchr/testify/assert"
 )
 
+var standaloneSignCases = []struct {
+	OrigURI                    string
+	OrigQuery                  string
+	Region, Service, SubDomain string
+	ExpSig                     string
+	EscapedURI                 string
+}{
+	{
+		OrigURI:   `/logs-*/_search`,
+		OrigQuery: `pretty=true`,
+		Region:    "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
+		EscapedURI: `/logs-%2A/_search`,
+		ExpSig:     `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
+	},
+}
+
 func TestPresignHandler(t *testing.T) {
 	svc := s3.New(unit.Session)
 	req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
@@ -75,3 +93,30 @@ func TestPresignRequest(t *testing.T) {
 
 	assert.NotContains(t, urlstr, "+") // + encoded as %20
 }
+
+func TestStandaloneSign_CustomURIEscape(t *testing.T) {
+	creds := unit.Session.Config.Credentials
+	signer := v4.NewSigner(creds, func(s *v4.Signer) {
+		s.URIPathEscapeFn = func(in string) string {
+			return v4.DefaultURIPathEscape(v4.DefaultURIPathEscape(in))
+		}
+	})
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		assert.NoError(t, err)
+
+		req.URL.Path = c.OrigURI
+		req.URL.RawQuery = c.OrigQuery
+
+		_, err = signer.Sign(req, nil, c.Service, c.Region, time.Unix(0, 0))
+		assert.NoError(t, err)
+
+		actual := req.Header.Get("Authorization")
+		assert.Equal(t, c.ExpSig, actual)
+		assert.Equal(t, c.OrigURI, req.URL.Path)
+	}
+}

aws/signer/v4/uri_path.go

@@ -0,0 +1,28 @@
+// +build go1.5
+
+package v4
+
+import (
+	"net/url"
+	"strings"
+)
+
+func getURIPath(u *url.URL) string {
+	var uri string
+
+	if len(u.Opaque) > 0 {
+		uri = "/" + strings.Join(strings.Split(u.Opaque, "/")[3:], "/")
+	} else if len(u.RawPath) > 0 {
+		// RawPath was added in Go 1.5. It like Opaque, but applies to just
+		// the URI path not the full URL.
+		uri = u.RawPath
+	} else {
+		uri = u.Path
+	}
+
+	if len(uri) == 0 {
+		uri = "/"
+	}
+
+	return uri
+}

aws/signer/v4/uri_path_1_4.go

@@ -0,0 +1,24 @@
+// +build !go1.5
+
+package v4
+
+import (
+	"net/url"
+	"strings"
+)
+
+func getURIPath(u *url.URL) string {
+	var uri string
+
+	if len(u.Opaque) > 0 {
+		uri = "/" + strings.Join(strings.Split(u.Opaque, "/")[3:], "/")
+	} else {
+		uri = u.Path
+	}
+
+	if len(uri) == 0 {
+		uri = "/"
+	}
+
+	return uri
+}

aws/signer/v4/v4.go

@@ -2,6 +2,46 @@
 //
 // Provides request signing for request that need to be signed with
 // AWS V4 Signatures.
+//
+// Standalone Signer
+//
+// Using the signer by it self, outside of the SDK may need additional logic
+// to correctly escape the request's URI path for the signature. All APIs in
+// the SDK will escape their requests prior to being given signed. This process
+// ensures that the signature is valid, and what the receiving service expects.
+//
+// AWS v4 signature validation requires that the canonical string's URI path
+// element must be the URI escaped form of the HTTP request's path.
+// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+//
+// This means that if the HTTP client would perform escaping automatically under
+// the hood prior to sending the request the signature's canonical string must
+// be the escaped form of the HTTP request's path as it is sent across the wire.
+//
+// The Go HTTP client will perform escaping automatically on the request. Some
+// of these escaping may cause signature validation errors because the HTTP
+// request differs from the URI path or query that the signature was generated.
+// https://golang.org/pkg/net/url/#URL.EscapedPath
+//
+// Because of this, it is recommended that when using the signer outside of the
+// SDK that explicitly escaping the request prior to being signed is preferable,
+// and will help prevent signature validation errors.
+//
+// When using Go +1.5 you can use the URL.RawPath field to set the escaped path
+// to. The easiest way to get the escaped path is to call URL.EscapedPath().
+// https://golang.org/pkg/net/url/#URL.EscapedPath
+//
+// Test `TestStandaloneSign` provides a complete example of using the signer
+// outside of the SDK and pre-escaping the URI path.
+//
+// Using Elastic Search for example, this shows how the Go HTTP client will
+// automatically escape the path, and why additional logic is needed with using
+// the signer. The request's path is `/log-*/_search`. The Go HTTP client will
+// automatically escape the path to `/log-%2A/_search` when sent. This means
+// the signer must generate the canonical string as `/log-%252A/_search` in order
+// for the signature to be valid. Which is a double escaping of the original
+// request's path. If the request's path is not pre-escaped prior to signing,
+// the generated signature will be invalid.
 package v4
 
 import (
@@ -120,12 +160,43 @@ type Signer struct {
 	// request's query string.
 	DisableHeaderHoisting bool
 
+	// The func to use for escaping URI paths. Default for AWS signed request
+	// is to escape all but [a-zA-Z0-9-_.~]. The escaping of '+' is not a part
+	// of this func and is done for all requests. If not set, DefaultURIPathEscape
+	// will be used.
+	//
+	// Generally the escaping function does not need to be set. The only time
+	// this may be useful to set is if custom escaping logic is needed for the
+	// request's URI path escaping.
+	//
+	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+	URIPathEscapeFn URIPathEscapeFunc
+
 	// currentTimeFn returns the time value which represents the current time.
 	// This value should only be used for testing. If it is nil the default
 	// time.Now will be used.
 	currentTimeFn func() time.Time
 }
 
+// URIPathEscapeFunc is the type for strategies that can be used by the signer
+// for escaping URI path. This is not needed when signing requests generated with
+// the SDK's API operations.
+type URIPathEscapeFunc func(u string) string
+
+// DefaultURIPathEscape is the default escape URI path func the AWS signer
+// will use. Performs escaping based on the AWS Signature V4 rules.
+//
+// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+func DefaultURIPathEscape(u string) string {
+	return rest.EscapePath(u, false)
+}
+
+// noURIPathEscape performs no escaping. Used for services like S3 that do not
+// need double escaping for the signature validation to function properly.
+func noURIPathEscape(u string) string {
+	return u
+}
+
 // NewSigner returns a Signer pointer configured with the credentials and optional
 // option values provided. If not options are provided the Signer will use its
 // default configuration.
@@ -151,6 +222,8 @@ type signingCtx struct {
 	ExpireTime       time.Duration
 	SignedHeaderVals http.Header
 
+	URIPathEscapeFn URIPathEscapeFunc
+
 	credValues         credentials.Value
 	isPresign          bool
 	formattedTime      string
@@ -236,14 +309,15 @@ func (v4 Signer) signWithBody(r *http.Request, body io.ReadSeeker, service, regi
 	}
 
 	ctx := &signingCtx{
-		Request:     r,
-		Body:        body,
-		Query:       r.URL.Query(),
-		Time:        signTime,
-		ExpireTime:  exp,
-		isPresign:   exp != 0,
-		ServiceName: service,
-		Region:      region,
+		Request:         r,
+		Body:            body,
+		Query:           r.URL.Query(),
+		Time:            signTime,
+		ExpireTime:      exp,
+		isPresign:       exp != 0,
+		ServiceName:     service,
+		Region:          region,
+		URIPathEscapeFn: v4.URIPathEscapeFn,
 	}
 
 	if ctx.isRequestSigned() {
@@ -257,6 +331,10 @@ func (v4 Signer) signWithBody(r *http.Request, body io.ReadSeeker, service, regi
 		return http.Header{}, err
 	}
 
+	if ctx.URIPathEscapeFn == nil {
+		ctx.URIPathEscapeFn = DefaultURIPathEscape
+	}
+
 	ctx.assignAmzQueryValues()
 	ctx.build(v4.DisableHeaderHoisting)
 
@@ -354,6 +432,10 @@ func signSDKRequestWithCurrTime(req *request.Request, curTimeFn func() time.Time
 		v4.Logger = req.Config.Logger
 		v4.DisableHeaderHoisting = req.NotHoist
 		v4.currentTimeFn = curTimeFn
+		if name == "s3" {
+			// S3 service should not have any escapting applied
+			v4.URIPathEscapeFn = noURIPathEscape
+		}
 	})
 
 	signingTime := req.Time
@@ -510,19 +592,10 @@ func (ctx *signingCtx) buildCanonicalHeaders(r rule, header http.Header) {
 
 func (ctx *signingCtx) buildCanonicalString() {
 	ctx.Request.URL.RawQuery = strings.Replace(ctx.Query.Encode(), "+", "%20", -1)
-	uri := ctx.Request.URL.Opaque
-	if uri != "" {
-		uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/")
-	} else {
-		uri = ctx.Request.URL.Path
-	}
-	if uri == "" {
-		uri = "/"
-	}
 
-	if ctx.ServiceName != "s3" {
-		uri = rest.EscapePath(uri, false)
-	}
+	uri := getURIPath(ctx.Request.URL)
+
+	uri = ctx.URIPathEscapeFn(uri)
 
 	ctx.canonicalString = strings.Join([]string{
 		ctx.Request.Method,

aws/signer/v4/v4_test.go

@@ -2,7 +2,6 @@ package v4
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -217,7 +216,6 @@ func TestIgnorePreResignRequestWithValidCreds(t *testing.T) {
 	SignSDKRequest(r)
 	sig := r.HTTPRequest.URL.Query().Get("X-Amz-Signature")
 
-	fmt.Println(sig)
 	signSDKRequestWithCurrTime(r, func() time.Time {
 		// Simulate one second has passed so that signature's date changes
 		// when it is resigned.