Skip to content

Commit

Permalink
progress on first-done backup url logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@danenania
danenania committed Sep 12, 2018
1 parent 418668c commit 7f32392
Show file tree
Hide file tree
Showing 2 changed files with 105 additions and 43 deletions.
116 changes: 86 additions & 30 deletions fetch/fetch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ import (
"time"

"github.com/certifi/gocertifi"
"github.com/davecgh/go-spew/spew"
"github.com/envkey/envkey-fetch/cache"
"github.com/envkey/envkey-fetch/parser"
"github.com/envkey/envkey-fetch/version"
"github.com/envkey/myhttp"
multierror "github.com/hashicorp/go-multierror"
)

type FetchOptions struct {
Expand All @@ -31,7 +33,8 @@ type FetchOptions struct {
}

var DefaultHost = "env.envkey.com"
var BackupDefaultHost = "s3-eu-west-1.amazonaws.com/envkey-backup/envs"
var BackupHost = "s3-eu-west-1.amazonaws.com/envkey-backup/envs"
var BackupHostRestricted = "me66hg5t17.execute-api.eu-west-1.amazonaws.com/default/envBackup"
var ApiVersion = 1
var HttpGetter = myhttp.New(time.Second * 6)

Expand Down Expand Up @@ -83,7 +86,40 @@ func Fetch(envkey string, options FetchOptions) (string, error) {
return res, nil
}

func UrlWithLoggingParams(baseUrl string, options FetchOptions) string {
clientName := options.ClientName
if clientName == "" {
clientName = "envkey-fetch"
}

clientVersion := options.ClientVersion
if clientVersion == "" {
clientVersion = version.Version
}

var querySep string
if strings.Contains(baseUrl, "?") {
querySep = "&"
} else {
querySep = "?"
}

fmtStr := "%s%sclientName=%s&clientVersion=%s&clientOs=%s&clientArch=%s"
return fmt.Sprintf(
fmtStr,
baseUrl,
querySep,
url.QueryEscape(clientName),
url.QueryEscape(clientVersion),
url.QueryEscape(runtime.GOOS),
url.QueryEscape(runtime.GOARCH),
)
}

func httpGet(url string) (*http.Response, error) {
fmt.Println("httpGet:")
fmt.Println(url)

r, err := HttpGetter.Get(url)

// if error caused by missing root certificates, pull in gocertifi certs (which come from Mozilla) and try again with those
Expand Down Expand Up @@ -158,33 +194,53 @@ func getBaseUrl(envkeyHost string, envkeyParam string) string {

func getJsonUrl(envkeyHost string, envkeyParam string, options FetchOptions) string {
baseUrl := getBaseUrl(envkeyHost, envkeyParam)
return UrlWithLoggingParams(baseUrl, options)
}

clientName := options.ClientName
if clientName == "" {
clientName = "envkey-fetch"
func getBackupUrls(envkeyParam string) []string {
protocol := "https://"
apiVersion := strconv.Itoa(ApiVersion)
return []string{
strings.Join([]string{protocol + BackupHost, "v" + apiVersion, envkeyParam}, "/"),
fmt.Sprintf("%s?v=%s&id=%s", protocol+BackupHostRestricted, apiVersion, envkeyParam),
}
}

clientVersion := options.ClientVersion
if clientVersion == "" {
clientVersion = version.Version
func fetchBackup(envkeyParam string, options FetchOptions) (*http.Response, error) {
backupUrls := getBackupUrls(envkeyParam)

if options.VerboseOutput {
fmt.Fprintf(os.Stderr, "Attempting to load encrypted config from backup urls: %s\n", backupUrls)
}

fmtStr := "%s?clientName=%s&clientVersion=%s&clientOs=%s&clientArch=%s"
return fmt.Sprintf(
fmtStr,
baseUrl,
url.QueryEscape(clientName),
url.QueryEscape(clientVersion),
url.QueryEscape(runtime.GOOS),
url.QueryEscape(runtime.GOARCH),
)
}
success, failed := make(chan *http.Response), make(chan error)

func getBackupUrl(envkeyParam string) string {
host := BackupDefaultHost
protocol := "https://"
apiVersion := "v" + strconv.Itoa(ApiVersion)
return strings.Join([]string{protocol + host, apiVersion, envkeyParam}, "/")
for _, backupUrl := range backupUrls {
go func(backupUrl string) {
urlWithParams := UrlWithLoggingParams(backupUrl, options)
r, err := httpGet(urlWithParams)
logRequestIfVerbose(urlWithParams, options, err, r)
if err == nil {
success <- r
} else {
failed <- err
}
}(backupUrl)
}

var err error

for {
r, any := <-success

if any {
return r, nil
} else {
err = multierror.Append(err, <-failed)
}
}

return nil, err
}

func getJson(envkeyHost string, envkeyParam string, options FetchOptions, response *parser.EnvServiceResponse, fetchCache *cache.Cache) error {
Expand All @@ -203,28 +259,28 @@ func getJson(envkeyHost string, envkeyParam string, options FetchOptions, respon
fmt.Fprintf(os.Stderr, "Attempting to load encrypted config from default url: %s\n", url)
}

// If http request failed and we're using the default host, now try backup host
// If http request failed and we're using the default host, now try backup hosts
if fetchErr != nil || r.StatusCode >= 500 {
logRequestIfVerbose(url, options, fetchErr, r)

if envkeyHost == "" || envkeyHost == DefaultHost {
backupUrl := getBackupUrl(envkeyParam)
r, backupFetchErr = fetchBackup(envkeyParam, options)

if options.VerboseOutput {
fmt.Fprintf(os.Stderr, "Attempting to load encrypted config from backup url: %s\n", backupUrl)
}

r, backupFetchErr = httpGet(backupUrl)
if r != nil {
defer r.Body.Close()
}

logRequestIfVerbose(backupUrl, options, backupFetchErr, r)
fmt.Println("backupFetchErr")
spew.Dump(backupFetchErr)
fmt.Println("r.StatusCode")
spew.Dump(r.StatusCode)

}
}

if backupFetchErr == nil && (r != nil && r.StatusCode == 200) {
body, err = ioutil.ReadAll(r.Body)

if err != nil {
if options.VerboseOutput {
fmt.Fprintln(os.Stderr, "Error reading response body:")
Expand Down
32 changes: 19 additions & 13 deletions fetch/fetch_test/fetch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package fetch_test

import (
"fmt"
"net/url"
"runtime"
"strconv"
"strings"
"testing"
Expand Down Expand Up @@ -85,22 +83,19 @@ func TestFetch(t *testing.T) {
var envkeyParam = strings.Split(test.envkey, "-")[0]

baseUrl := (test.protocol + "://" + test.host + "/v" + strconv.Itoa(fetch.ApiVersion) + "/" + envkeyParam)
fmtStr := "%s?clientName=envkey-fetch&clientVersion=%s&clientOs=%s&clientArch=%s"
url := fmt.Sprintf(
fmtStr,
baseUrl,
url.QueryEscape(version.Version),
url.QueryEscape(runtime.GOOS),
url.QueryEscape(runtime.GOARCH),
)
opts := fetch.FetchOptions{true, "", "envkey-fetch", version.Version, false, 2.0}
url := fetch.UrlWithLoggingParams(baseUrl, opts)

fmt.Println("TestFetch url:")
fmt.Println(url)

httpmock.RegisterResponder(
"GET",
url,
httpmock.NewStringResponder(test.responseStatus, test.response),
)

res, err := fetch.Fetch(test.envkey, fetch.FetchOptions{true, "", "envkey-fetch", version.Version, false, 2.0})
res, err := fetch.Fetch(test.envkey, opts)

if test.expectErr {
assert.NotNil(err)
Expand Down Expand Up @@ -167,15 +162,26 @@ func TestBackup(t *testing.T) {

// Test with backup
fetch.DefaultHost = "localhost:61034"
url := ("https://" + fetch.BackupDefaultHost + "/v" + strconv.Itoa(fetch.ApiVersion) + "/validkey")
opts := fetch.FetchOptions{false, "", "envkey-fetch", version.Version, false, 2.0}
url := fetch.UrlWithLoggingParams("https://"+fetch.BackupHost+"/v"+strconv.Itoa(fetch.ApiVersion)+"/validkey", opts)
restrictedUrl := fetch.UrlWithLoggingParams(fmt.Sprintf("%s?v=%s&id=%s", ("https://"+fetch.BackupHostRestricted), strconv.Itoa(fetch.ApiVersion), "validkey"), opts)

fmt.Println(url)
fmt.Println(restrictedUrl)

httpmock.RegisterResponder(
"GET",
url,
httpmock.NewStringResponder(200, responseSimple),
)

res, err := fetch.Fetch(validEnvkeySimple, fetch.FetchOptions{false, "", "envkey-fetch", version.Version, false, 2.0})
httpmock.RegisterResponder(
"GET",
restrictedUrl,
httpmock.NewStringResponder(200, responseSimple),
)

res, err := fetch.Fetch(validEnvkeySimple, opts)

assert.Nil(err)
assert.Equal(validResult, res, "Backup")
Expand Down

0 comments on commit 7f32392

Please sign in to comment.