diff --git a/CHANGELOG.md b/CHANGELOG.md index 493a03aa370e..5941ddd960c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ IMPROVEMENTS: * driver/docker: Upgrade pause container and detect architecture [[GH-8957](https://github.com/hashicorp/nomad/pull/8957)] * driver/docker: Support pinning tasks to specific CPUs with `cpuset_cpus` option. [[GH-8291](https://github.com/hashicorp/nomad/pull/8291)] * jobspec: Lowered minimum CPU allowed from 10 to 1. [[GH-8996](https://github.com/hashicorp/nomad/issues/8996)] + * jobspec: Added support for `headers` option in `artifact` stanza [[GH-9306](https://github.com/hashicorp/nomad/issues/9306)] __BACKWARDS INCOMPATIBILITIES:__ * core: null characters are prohibited in region, datacenter, job name/ID, task group name, and task name [[GH-9020](https://github.com/hashicorp/nomad/issues/9020)] diff --git a/api/tasks.go b/api/tasks.go index 2406a9bc3921..4f1c34448770 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -726,6 +726,7 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) { type TaskArtifact struct { GetterSource *string `mapstructure:"source" hcl:"source,optional"` GetterOptions map[string]string `mapstructure:"options" hcl:"options,block"` + GetterHeaders map[string]string `mapstructure:"headers" hcl:"headers,block"` GetterMode *string `mapstructure:"mode" hcl:"mode,optional"` RelativeDest *string `mapstructure:"destination" hcl:"destination,optional"` } @@ -738,6 +739,12 @@ func (a *TaskArtifact) Canonicalize() { // Shouldn't be possible, but we don't want to panic a.GetterSource = stringToPtr("") } + if len(a.GetterOptions) == 0 { + a.GetterOptions = nil + } + if len(a.GetterHeaders) == 0 { + a.GetterHeaders = nil + } if a.RelativeDest == nil { switch *a.GetterMode { case "file": diff --git a/api/tasks_test.go b/api/tasks_test.go index 9e15bd82d68e..844972fe5088 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -356,16 +356,16 @@ func TestTask_AddAffinity(t *testing.T) { func TestTask_Artifact(t *testing.T) { t.Parallel() a := TaskArtifact{ - GetterSource: stringToPtr("http://localhost/foo.txt"), - GetterMode: stringToPtr("file"), + GetterSource: stringToPtr("http://localhost/foo.txt"), + GetterMode: stringToPtr("file"), + GetterHeaders: make(map[string]string), + GetterOptions: make(map[string]string), } a.Canonicalize() - if *a.GetterMode != "file" { - t.Errorf("expected file but found %q", *a.GetterMode) - } - if filepath.ToSlash(*a.RelativeDest) != "local/foo.txt" { - t.Errorf("expected local/foo.txt but found %q", *a.RelativeDest) - } + require.Equal(t, "file", *a.GetterMode) + require.Equal(t, "local/foo.txt", filepath.ToSlash(*a.RelativeDest)) + require.Nil(t, a.GetterOptions) + require.Nil(t, a.GetterHeaders) } func TestTask_VolumeMount(t *testing.T) { diff --git a/client/allocrunner/taskrunner/getter/getter.go b/client/allocrunner/taskrunner/getter/getter.go index a1abb456a9cf..3a8ce82b86be 100644 --- a/client/allocrunner/taskrunner/getter/getter.go +++ b/client/allocrunner/taskrunner/getter/getter.go @@ -3,6 +3,7 @@ package getter import ( "errors" "fmt" + "net/http" "net/url" "path/filepath" "strings" @@ -34,28 +35,53 @@ type EnvReplacer interface { ReplaceEnv(string) string } -// getClient returns a client that is suitable for Nomad downloading artifacts. -func getClient(src string, mode gg.ClientMode, dst string) *gg.Client { - lock.Lock() - defer lock.Unlock() - - // Return the pre-initialized client - if getters == nil { - getters = make(map[string]gg.Getter, len(supported)) - for _, getter := range supported { - if impl, ok := gg.Getters[getter]; ok { - getters[getter] = impl +func makeGetters(headers http.Header) map[string]gg.Getter { + getters := make(map[string]gg.Getter, len(supported)) + for _, getter := range supported { + switch { + case getter == "http" && len(headers) > 0: + fallthrough + case getter == "https" && len(headers) > 0: + getters[getter] = &gg.HttpGetter{ + Netrc: true, + Header: headers, + } + default: + if defaultGetter, ok := gg.Getters[getter]; ok { + getters[getter] = defaultGetter } } } + return getters +} - return &gg.Client{ - Src: src, - Dst: dst, - Mode: mode, - Getters: getters, - Umask: 060000000, +// getClient returns a client that is suitable for Nomad downloading artifacts. +func getClient(src string, headers http.Header, mode gg.ClientMode, dst string) *gg.Client { + client := &gg.Client{ + Src: src, + Dst: dst, + Mode: mode, + Umask: 060000000, + } + + switch len(headers) { + case 0: + // When no headers are present use the memoized getters, creating them + // on demand if they do not exist yet. + lock.Lock() + if getters == nil { + getters = makeGetters(nil) + } + lock.Unlock() + client.Getters = getters + default: + // When there are headers present, we must create fresh gg.HttpGetter + // objects, because that is where gg stores the headers to use in its + // artifact HTTP GET requests. + client.Getters = makeGetters(headers) } + + return client } // getGetterUrl returns the go-getter URL to download the artifact. @@ -83,17 +109,29 @@ func getGetterUrl(taskEnv EnvReplacer, artifact *structs.TaskArtifact) (string, u.RawQuery = q.Encode() // Add the prefix back - url := u.String() + ggURL := u.String() if gitSSH { - url = fmt.Sprintf("%s%s", gitSSHPrefix, url) + ggURL = fmt.Sprintf("%s%s", gitSSHPrefix, ggURL) + } + + return ggURL, nil +} + +func getHeaders(env EnvReplacer, m map[string]string) http.Header { + if len(m) == 0 { + return nil } - return url, nil + headers := make(http.Header, len(m)) + for k, v := range m { + headers.Set(k, env.ReplaceEnv(v)) + } + return headers } // GetArtifact downloads an artifact into the specified task directory. func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir string) error { - url, err := getGetterUrl(taskEnv, artifact) + ggURL, err := getGetterUrl(taskEnv, artifact) if err != nil { return newGetError(artifact.GetterSource, err, false) } @@ -118,8 +156,9 @@ func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir st mode = gg.ClientModeDir } - if err := getClient(url, mode, dest).Get(); err != nil { - return newGetError(url, err, true) + headers := getHeaders(taskEnv, artifact.GetterHeaders) + if err := getClient(ggURL, headers, mode, dest).Get(); err != nil { + return newGetError(ggURL, err, true) } return nil diff --git a/client/allocrunner/taskrunner/getter/getter_test.go b/client/allocrunner/taskrunner/getter/getter_test.go index cfac7a0106e0..3931bc52c66d 100644 --- a/client/allocrunner/taskrunner/getter/getter_test.go +++ b/client/allocrunner/taskrunner/getter/getter_test.go @@ -2,7 +2,9 @@ package getter import ( "fmt" + "io" "io/ioutil" + "mime" "net/http" "net/http/httptest" "os" @@ -18,14 +20,92 @@ import ( "github.com/stretchr/testify/require" ) -// fakeReplacer is a noop version of taskenv.TaskEnv.ReplaceEnv -type fakeReplacer struct{} +// noopReplacer is a noop version of taskenv.TaskEnv.ReplaceEnv. +type noopReplacer struct{} -func (fakeReplacer) ReplaceEnv(s string) string { +func (noopReplacer) ReplaceEnv(s string) string { return s } -var taskEnv = fakeReplacer{} +var noopTaskEnv = noopReplacer{} + +// upperReplacer is a version of taskenv.TaskEnv.ReplaceEnv that upper-cases +// the given input. +type upperReplacer struct{} + +func (upperReplacer) ReplaceEnv(s string) string { + return strings.ToUpper(s) +} + +func removeAllT(t *testing.T, path string) { + require.NoError(t, os.RemoveAll(path)) +} + +func TestGetArtifact_getHeaders(t *testing.T) { + t.Run("nil", func(t *testing.T) { + require.Nil(t, getHeaders(noopTaskEnv, nil)) + }) + + t.Run("empty", func(t *testing.T) { + require.Nil(t, getHeaders(noopTaskEnv, make(map[string]string))) + }) + + t.Run("set", func(t *testing.T) { + upperTaskEnv := new(upperReplacer) + expected := make(http.Header) + expected.Set("foo", "BAR") + result := getHeaders(upperTaskEnv, map[string]string{ + "foo": "bar", + }) + require.Equal(t, expected, result) + }) +} + +func TestGetArtifact_Headers(t *testing.T) { + file := "output.txt" + + // Create the test server with a handler that will validate headers are set. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Validate the expected value for our header. + value := r.Header.Get("X-Some-Value") + require.Equal(t, "FOOBAR", value) + + // Write the value to the file that is our artifact, for fun. + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(file))) + w.WriteHeader(http.StatusOK) + _, err := io.Copy(w, strings.NewReader(value)) + require.NoError(t, err) + })) + defer ts.Close() + + // Create a temp directory to download into. + taskDir, err := ioutil.TempDir("", "nomad-test") + require.NoError(t, err) + defer removeAllT(t, taskDir) + + // Create the artifact. + artifact := &structs.TaskArtifact{ + GetterSource: fmt.Sprintf("%s/%s", ts.URL, file), + GetterHeaders: map[string]string{ + "X-Some-Value": "foobar", + }, + RelativeDest: file, + GetterMode: "file", + } + + // Download the artifact. + taskEnv := new(upperReplacer) + err = GetArtifact(taskEnv, artifact, taskDir) + require.NoError(t, err) + + // Verify artifact exists. + b, err := ioutil.ReadFile(filepath.Join(taskDir, file)) + require.NoError(t, err) + + // Verify we wrote the interpolated header value into the file that is our + // artifact. + require.Equal(t, "FOOBAR", string(b)) +} func TestGetArtifact_FileAndChecksum(t *testing.T) { // Create the test server hosting the file to download @@ -37,7 +117,7 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) { if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(taskDir) + defer removeAllT(t, taskDir) // Create the artifact file := "test.sh" @@ -49,7 +129,7 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) { } // Download the artifact - if err := GetArtifact(taskEnv, artifact, taskDir); err != nil { + if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -69,7 +149,7 @@ func TestGetArtifact_File_RelativeDest(t *testing.T) { if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(taskDir) + defer removeAllT(t, taskDir) // Create the artifact file := "test.sh" @@ -83,7 +163,7 @@ func TestGetArtifact_File_RelativeDest(t *testing.T) { } // Download the artifact - if err := GetArtifact(taskEnv, artifact, taskDir); err != nil { + if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -103,7 +183,7 @@ func TestGetArtifact_File_EscapeDest(t *testing.T) { if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(taskDir) + defer removeAllT(t, taskDir) // Create the artifact file := "test.sh" @@ -117,7 +197,7 @@ func TestGetArtifact_File_EscapeDest(t *testing.T) { } // attempt to download the artifact - err = GetArtifact(taskEnv, artifact, taskDir) + err = GetArtifact(noopTaskEnv, artifact, taskDir) if err == nil || !strings.Contains(err.Error(), "escapes") { t.Fatalf("expected GetArtifact to disallow sandbox escape: %v", err) } @@ -155,7 +235,7 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) { if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(taskDir) + defer removeAllT(t, taskDir) // Create the artifact with an incorrect checksum file := "test.sh" @@ -167,7 +247,7 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) { } // Download the artifact and expect an error - if err := GetArtifact(taskEnv, artifact, taskDir); err == nil { + if err := GetArtifact(noopTaskEnv, artifact, taskDir); err == nil { t.Fatalf("GetArtifact should have failed") } } @@ -216,7 +296,7 @@ func TestGetArtifact_Archive(t *testing.T) { if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(taskDir) + defer removeAllT(t, taskDir) create := map[string]string{ "exist/my.config": "to be replaced", @@ -232,7 +312,7 @@ func TestGetArtifact_Archive(t *testing.T) { }, } - if err := GetArtifact(taskEnv, artifact, taskDir); err != nil { + if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -255,7 +335,7 @@ func TestGetArtifact_Setuid(t *testing.T) { // files that exist in the artifact to ensure they are overridden taskDir, err := ioutil.TempDir("", "nomad-test") require.NoError(t, err) - defer os.RemoveAll(taskDir) + defer removeAllT(t, taskDir) file := "setuid.tgz" artifact := &structs.TaskArtifact{ @@ -265,7 +345,7 @@ func TestGetArtifact_Setuid(t *testing.T) { }, } - require.NoError(t, GetArtifact(taskEnv, artifact, taskDir)) + require.NoError(t, GetArtifact(noopTaskEnv, artifact, taskDir)) var expected map[string]int @@ -390,7 +470,7 @@ func TestGetGetterUrl_Queries(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - act, err := getGetterUrl(taskEnv, c.artifact) + act, err := getGetterUrl(noopTaskEnv, c.artifact) if err != nil { t.Fatalf("want %q; got err %v", c.output, err) } else if act != c.output { diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 35b1df6f83d9..75cff06b6f93 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1103,7 +1103,8 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, for k, ta := range apiTask.Artifacts { structsTask.Artifacts[k] = &structs.TaskArtifact{ GetterSource: *ta.GetterSource, - GetterOptions: ta.GetterOptions, + GetterOptions: helper.CopyMapStringString(ta.GetterOptions), + GetterHeaders: helper.CopyMapStringString(ta.GetterHeaders), GetterMode: *ta.GetterMode, RelativeDest: *ta.RelativeDest, } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 990c71434930..8c6f92dd4a4a 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2628,12 +2628,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, Artifacts: []*api.TaskArtifact{ { - GetterSource: helper.StringToPtr("source"), - GetterOptions: map[string]string{ - "a": "b", - }, - GetterMode: helper.StringToPtr("dir"), - RelativeDest: helper.StringToPtr("dest"), + GetterSource: helper.StringToPtr("source"), + GetterOptions: map[string]string{"a": "b"}, + GetterHeaders: map[string]string{"User-Agent": "nomad"}, + GetterMode: helper.StringToPtr("dir"), + RelativeDest: helper.StringToPtr("dest"), }, }, DispatchPayload: &api.DispatchPayloadConfig{ @@ -2752,12 +2751,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, Artifacts: []*structs.TaskArtifact{ { - GetterSource: "source", - GetterOptions: map[string]string{ - "a": "b", - }, - GetterMode: "dir", - RelativeDest: "dest", + GetterSource: "source", + GetterOptions: map[string]string{"a": "b"}, + GetterHeaders: map[string]string{"User-Agent": "nomad"}, + GetterMode: "dir", + RelativeDest: "dest", }, }, DispatchPayload: &structs.DispatchPayloadConfig{ diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 4a7ad6592e86..e89537693e94 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -4056,6 +4056,9 @@ func TestTaskDiff(t *testing.T) { GetterOptions: map[string]string{ "bar": "baz", }, + GetterHeaders: map[string]string{ + "User": "user1", + }, GetterMode: "dir", RelativeDest: "bar", }, @@ -4075,6 +4078,10 @@ func TestTaskDiff(t *testing.T) { GetterOptions: map[string]string{ "bam": "baz", }, + GetterHeaders: map[string]string{ + "User": "user2", + "User-Agent": "nomad", + }, GetterMode: "file", RelativeDest: "bam", }, @@ -4087,6 +4094,18 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeAdded, Name: "Artifact", Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "GetterHeaders[User-Agent]", + Old: "", + New: "nomad", + }, + { + Type: DiffTypeAdded, + Name: "GetterHeaders[User]", + Old: "", + New: "user2", + }, { Type: DiffTypeAdded, Name: "GetterMode", @@ -4117,6 +4136,12 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeDeleted, Name: "Artifact", Fields: []*FieldDiff{ + { + Type: DiffTypeDeleted, + Name: "GetterHeaders[User]", + Old: "user1", + New: "", + }, { Type: DiffTypeDeleted, Name: "GetterMode", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index e67cdd2d87a5..1d6b2f480bde 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -12,6 +12,7 @@ import ( "encoding/hex" "errors" "fmt" + "hash" "hash/crc32" "math" "net" @@ -7866,6 +7867,10 @@ type TaskArtifact struct { // go-getter. GetterOptions map[string]string + // GetterHeaders are headers to use when downloading the artifact using + // go-getter. + GetterHeaders map[string]string + // GetterMode is the go-getter.ClientMode for fetching resources. // Defaults to "any" but can be set to "file" or "dir". GetterMode string @@ -7879,40 +7884,48 @@ func (ta *TaskArtifact) Copy() *TaskArtifact { if ta == nil { return nil } - nta := new(TaskArtifact) - *nta = *ta - nta.GetterOptions = helper.CopyMapStringString(ta.GetterOptions) - return nta + return &TaskArtifact{ + GetterSource: ta.GetterSource, + GetterOptions: helper.CopyMapStringString(ta.GetterOptions), + GetterHeaders: helper.CopyMapStringString(ta.GetterHeaders), + GetterMode: ta.GetterMode, + RelativeDest: ta.RelativeDest, + } } func (ta *TaskArtifact) GoString() string { return fmt.Sprintf("%+v", ta) } +// hashStringMap appends a deterministic hash of m onto h. +func hashStringMap(h hash.Hash, m map[string]string) { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + h.Write([]byte(k)) + h.Write([]byte(m[k])) + } +} + // Hash creates a unique identifier for a TaskArtifact as the same GetterSource // may be specified multiple times with different destinations. func (ta *TaskArtifact) Hash() string { - hash, err := blake2b.New256(nil) + h, err := blake2b.New256(nil) if err != nil { panic(err) } - hash.Write([]byte(ta.GetterSource)) + h.Write([]byte(ta.GetterSource)) - // Must iterate over keys in a consistent order - keys := make([]string, 0, len(ta.GetterOptions)) - for k := range ta.GetterOptions { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - hash.Write([]byte(k)) - hash.Write([]byte(ta.GetterOptions[k])) - } + hashStringMap(h, ta.GetterOptions) + hashStringMap(h, ta.GetterHeaders) - hash.Write([]byte(ta.GetterMode)) - hash.Write([]byte(ta.RelativeDest)) - return base64.RawStdEncoding.EncodeToString(hash.Sum(nil)) + h.Write([]byte(ta.GetterMode)) + h.Write([]byte(ta.RelativeDest)) + return base64.RawStdEncoding.EncodeToString(h.Sum(nil)) } // PathEscapesAllocDir returns if the given path escapes the allocation diff --git a/vendor/github.com/hashicorp/nomad/api/tasks.go b/vendor/github.com/hashicorp/nomad/api/tasks.go index 2406a9bc3921..4f1c34448770 100644 --- a/vendor/github.com/hashicorp/nomad/api/tasks.go +++ b/vendor/github.com/hashicorp/nomad/api/tasks.go @@ -726,6 +726,7 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) { type TaskArtifact struct { GetterSource *string `mapstructure:"source" hcl:"source,optional"` GetterOptions map[string]string `mapstructure:"options" hcl:"options,block"` + GetterHeaders map[string]string `mapstructure:"headers" hcl:"headers,block"` GetterMode *string `mapstructure:"mode" hcl:"mode,optional"` RelativeDest *string `mapstructure:"destination" hcl:"destination,optional"` } @@ -738,6 +739,12 @@ func (a *TaskArtifact) Canonicalize() { // Shouldn't be possible, but we don't want to panic a.GetterSource = stringToPtr("") } + if len(a.GetterOptions) == 0 { + a.GetterOptions = nil + } + if len(a.GetterHeaders) == 0 { + a.GetterHeaders = nil + } if a.RelativeDest == nil { switch *a.GetterMode { case "file": diff --git a/website/pages/docs/job-specification/artifact.mdx b/website/pages/docs/job-specification/artifact.mdx index 6516e7ad6b30..2efa9d5550c5 100644 --- a/website/pages/docs/job-specification/artifact.mdx +++ b/website/pages/docs/job-specification/artifact.mdx @@ -54,7 +54,11 @@ automatically unarchived before the starting the task. - `options` `(map: nil)` - Specifies configuration parameters to fetch the artifact. The key-value pairs map directly to parameters appended to the supplied `source` URL. Please see the [`go-getter` - documentation][go-getter] for a complete list of options and examples + documentation][go-getter] for a complete list of options and examples. + +- `headers` `(map: nil)` - Specifies HTTP headers to set when + fetching the artifact using `http` or `https` protocol. Please see the + [`go-getter` headers documentation][go-getter-headers] for more information. - `source` `(string: )` - Specifies the URL of the artifact to download. See [`go-getter`][go-getter] for details. @@ -76,6 +80,20 @@ artifact { } ``` +To set HTTP headers in the request for the source the optional `headers` field +can be configured. + +```hcl +artifact { + source = "https://example.com/file.txt" + + headers { + User-Agent = "nomad-[${NOMAD_JOB_ID}]-[${NOMAD_GROUP_NAME}]-[${NOMAD_TASK_NAME}]" + X-Nomad-Alloc = "${NOMAD_ALLOC_ID}" + } +} +``` + ### Download using git This example downloads the artifact from the provided GitHub URL and places it at @@ -186,6 +204,7 @@ artifact { ``` [go-getter]: https://github.com/hashicorp/go-getter 'HashiCorp go-getter Library' +[go-getter-headers]: https://github.com/hashicorp/go-getter#headers 'HashiCorp go-getter Headers' [minio]: https://www.minio.io/ [s3-bucket-addr]: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro 'Amazon S3 Bucket Addressing' [s3-region-endpoints]: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region 'Amazon S3 Region Endpoints'