Skip to content

Commit

Permalink
fix(storage): respect STORAGE_EMULATOR_HOST in signedURL (#5673)
Browse files Browse the repository at this point in the history 
Co-authored-by: Christian Bruckmayer <christian.bruckmayer@shopify.com>
  • Loading branch information
@BrennaEpp @ChrisBr
BrennaEpp and ChrisBr committed Mar 11, 2022
1 parent c2df3c1 commit 1c249ae
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 5 deletions.
5 changes: 1 addition & 4 deletions storage/grpc_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package storage
import (
"context"
"os"
"strings"

gapic "cloud.google.com/go/storage/internal/apiv2"
"google.golang.org/api/option"
Expand Down Expand Up @@ -56,9 +55,7 @@ func defaultGRPCOptions() []option.ClientOption {
if host := os.Getenv("STORAGE_EMULATOR_HOST_GRPC"); host != "" {
// Strip the scheme from the emulator host. WithEndpoint does not take a
// scheme for gRPC.
if strings.Contains(host, "://") {
host = strings.SplitN(host, "://", 2)[1]
}
host = stripScheme(host)

defaults = append(defaults,
option.WithEndpoint(host),
Expand Down
18 changes: 17 additions & 1 deletion storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,10 +276,18 @@ type bucketBoundHostname struct {
}

func (s pathStyle) host(bucket string) string {
if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" {
return stripScheme(host)
}

return "storage.googleapis.com"
}

func (s virtualHostedStyle) host(bucket string) string {
if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" {
return bucket + "." + stripScheme(host)
}

return bucket + ".storage.googleapis.com"
}

Expand Down Expand Up @@ -327,6 +335,14 @@ func BucketBoundHostname(hostname string) URLStyle {
return bucketBoundHostname{hostname: hostname}
}

// Strips the scheme from a host if it contains it
func stripScheme(host string) string {
if strings.Contains(host, "://") {
host = strings.SplitN(host, "://", 2)[1]
}
return host
}

// SignedURLOptions allows you to restrict the access to the signed URL.
type SignedURLOptions struct {
// GoogleAccessID represents the authorizer of the signed URL generation.
Expand Down Expand Up @@ -823,7 +839,7 @@ func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) {
}
encoded := base64.StdEncoding.EncodeToString(b)
u.Scheme = "https"
u.Host = "storage.googleapis.com"
u.Host = PathStyle().host(bucket)
q := u.Query()
q.Set("GoogleAccessId", opts.GoogleAccessID)
q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix()))
Expand Down
156 changes: 156 additions & 0 deletions storage/storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,162 @@ func TestSignedURLV4(t *testing.T) {
}
}

// TestSignedURL_EmulatorHost tests that SignedURl respects the host set in
// STORAGE_EMULATOR_HOST
func TestSignedURL_EmulatorHost(t *testing.T) {
expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00")
bucketName := "bucket-name"
objectName := "obj-name"

emulatorHost := os.Getenv("STORAGE_EMULATOR_HOST")
defer os.Setenv("STORAGE_EMULATOR_HOST", emulatorHost)

tests := []struct {
desc string
emulatorHost string
now time.Time
opts *SignedURLOptions
// Note for future implementors: X-Goog-Signature generated by having
// the client run through its algorithm with pre-defined input and copy
// pasting the output. These tests are not great for testing whether
// the right signature is calculated - instead we rely on the backend
// and integration tests for that.
want string
}{
{
desc: "SignURLV4 creates link to resources in emulator",
emulatorHost: "localhost:9000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Insecure: true,
},
want: "http://localhost:9000/" + bucketName + "/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "using SigningSchemeV2",
emulatorHost: "localhost:9000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV2,
},
want: "https://localhost:9000/" + bucketName + "/" + objectName +
"?Expires=1033570800" +
"&GoogleAccessId=xxx%40clientid" +
"&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D",
},
{
desc: "using VirtualHostedStyle",
emulatorHost: "localhost:8000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Style: VirtualHostedStyle(),
Insecure: true,
},
want: "http://" + bucketName + ".localhost:8000/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=35e0b9d33901a2518956821175f88c2c4eb3f4461b725af74b37c36d23f8bbe927558ac57b0be40d345f20bca55ba0652d38b7a620f8da68d4f733706ad104da468c3a039459acf35f3022e388760cd49893c998c33fe3ccc8c022d7034ab98bdbdcac4b680bb24ae5ed586a42ee9495a873ffc484e297853a8a3892d0d6385c980cb7e3c5c8bdd4939b4c17105f10fe8b5b9744017bf59431ff176c1550ae1c64ddd6628096eb6895c97c5da4d850aca72c14b7f5018c15b34d4b00ec63ff2ccb688ddbef2d32648e247ffd0137498080f320f293eb811a94fb526227324bbbd01335446388797803e67d802f97b52565deba3d2387ecabf4f3094662236017" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "using BucketBoundHostname",
emulatorHost: "localhost:8000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Style: BucketBoundHostname("myhost"),
},
want: "https://" + "myhost/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=15fe19f6c61bcbdbd6473c32f2bec29caa8a5fa3b2ce32cfb5329a71edaa0d4e5ffe6469f32ed4c23ca2fbed3882fdf1ed107c6a98c2c4995dda6036c64bae51e6cb542c353618f483832aa1f3ef85342ddadd69c13ad4c69fd3f573ea5cf325a58056e3d5a37005217662af63b49fef8688de3c5c7a2f7b43651a030edd0813eb7f7713989a4c29a8add65133ce652895fea9de7dbc6248ee11b4d7c6c1e152df87700100e896e544ba8eeea96584078f56e699665140b750e90550b9b79633f4e7c8409efa807be5670d6e987eeee04a4180be9b9e30bb8557597beaf390a3805cc602c87a3e34800f8bc01449c3dd10ac2f2263e55e55b91e445052548d5e" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "emulator host specifies scheme",
emulatorHost: "https://localhost:6000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Insecure: true,
},
want: "http://localhost:6000/" + bucketName + "/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "emulator host specifies scheme using SigningSchemeV2",
emulatorHost: "https://localhost:8000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV2,
},
want: "https://localhost:8000/" + bucketName + "/" + objectName +
"?Expires=1033570800" +
"&GoogleAccessId=xxx%40clientid" +
"&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D",
},
}
oldUTCNow := utcNow
defer func() {
utcNow = oldUTCNow
}()

for _, test := range tests {
t.Run(test.desc, func(s *testing.T) {
utcNow = func() time.Time {
return test.now
}

os.Setenv("STORAGE_EMULATOR_HOST", test.emulatorHost)

got, err := SignedURL(bucketName, objectName, test.opts)
if err != nil {
s.Fatal(err)
}
if got != test.want {
s.Fatalf("\n\tgot:\t%v\n\twant:\t%v", got, test.want)
}
})
}
}

func TestSignedURL_MissingOptions(t *testing.T) {
now, _ := time.Parse(time.RFC3339, "2002-10-01T00:00:00-05:00")
expires, _ := time.Parse(time.RFC3339, "2002-10-15T00:00:00-05:00")
Expand Down

0 comments on commit 1c249ae

Please sign in to comment.