From 450e347708766afbfdfa624c2c2bec8100b2aa0a Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Wed, 5 Jul 2017 20:44:49 -0700
Subject: [PATCH 1/6] Add support for go-getter modes

Fixes #2678
---
 api/tasks.go                                  | 23 ++++++++++++++++++-
 api/tasks_test.go                             | 14 +++++++++++
 client/getter/getter.go                       | 16 ++++++++++---
 command/agent/job_endpoint.go                 |  1 +
 jobspec/parse.go                              |  1 +
 jobspec/parse_test.go                         |  1 +
 jobspec/test-fixtures/basic.hcl               |  1 +
 nomad/structs/diff_test.go                    | 14 +++++++++++
 nomad/structs/structs.go                      | 19 +++++++++++++++
 .../hashicorp/go-getter/appveyor.yml          |  2 +-
 10 files changed, 87 insertions(+), 5 deletions(-)

diff --git a/api/tasks.go b/api/tasks.go
index aac90ee40709..bedbaa14972a 100644
--- a/api/tasks.go
+++ b/api/tasks.go
@@ -2,6 +2,9 @@ package api
 
 import (
 	"fmt"
+	"path"
+
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -323,12 +326,30 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) {
 type TaskArtifact struct {
 	GetterSource  *string           `mapstructure:"source"`
 	GetterOptions map[string]string `mapstructure:"options"`
+	GetterMode    *string           `mapstructure:"mode"`
 	RelativeDest  *string           `mapstructure:"destination"`
 }
 
 func (a *TaskArtifact) Canonicalize() {
+	if a.GetterMode == nil {
+		a.GetterMode = helper.StringToPtr("any")
+	}
+	if a.GetterSource == nil {
+		// Shouldn't be possible, but we don't want to panic
+		a.GetterSource = helper.StringToPtr("")
+	}
 	if a.RelativeDest == nil {
-		a.RelativeDest = helper.StringToPtr("local/")
+		switch *a.GetterMode {
+		case "file":
+			// File mode should default to local/filename
+			dest := *a.GetterSource
+			dest = path.Base(dest)
+			dest = filepath.Join("local", dest)
+			a.RelativeDest = &dest
+		default:
+			// Default to a directory
+			a.RelativeDest = helper.StringToPtr("local/")
+		}
 	}
 }
 
diff --git a/api/tasks_test.go b/api/tasks_test.go
index 7756bfe528bc..334a1cdf3889 100644
--- a/api/tasks_test.go
+++ b/api/tasks_test.go
@@ -219,3 +219,17 @@ func TestTask_Constrain(t *testing.T) {
 		t.Fatalf("expect: %#v, got: %#v", expect, task.Constraints)
 	}
 }
+
+func TestTask_Artifact(t *testing.T) {
+	a := TaskArtifact{
+		GetterSource: helper.StringToPtr("http://localhost/foo.txt"),
+		GetterMode:   helper.StringToPtr("file"),
+	}
+	a.Canonicalize()
+	if *a.GetterMode != "file" {
+		t.Errorf("expected file but found %q", *a.GetterMode)
+	}
+	if *a.RelativeDest != "local/foo.txt" {
+		t.Errorf("expected local/foo.txt but found %q", *a.RelativeDest)
+	}
+}
diff --git a/client/getter/getter.go b/client/getter/getter.go
index c55dafd2776b..eeb9bb62c363 100644
--- a/client/getter/getter.go
+++ b/client/getter/getter.go
@@ -33,7 +33,7 @@ type EnvReplacer interface {
 }
 
 // getClient returns a client that is suitable for Nomad downloading artifacts.
-func getClient(src, dst string) *gg.Client {
+func getClient(src string, mode gg.ClientMode, dst string) *gg.Client {
 	lock.Lock()
 	defer lock.Unlock()
 
@@ -50,7 +50,7 @@ func getClient(src, dst string) *gg.Client {
 	return &gg.Client{
 		Src:     src,
 		Dst:     dst,
-		Mode:    gg.ClientModeAny,
+		Mode:    mode,
 		Getters: getters,
 	}
 }
@@ -97,7 +97,17 @@ func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir st
 
 	// Download the artifact
 	dest := filepath.Join(taskDir, artifact.RelativeDest)
-	if err := getClient(url, dest).Get(); err != nil {
+
+	// Convert from string getter mode to go-getter const
+	mode := gg.ClientModeAny
+	switch artifact.GetterMode {
+	case structs.GetterModeFile:
+		mode = gg.ClientModeFile
+	case structs.GetterModeDir:
+		mode = gg.ClientModeDir
+	}
+
+	if err := getClient(url, mode, dest).Get(); err != nil {
 		return newGetError(url, err, true)
 	}
 
diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go
index 7c975d617bd8..b5bed9142373 100644
--- a/command/agent/job_endpoint.go
+++ b/command/agent/job_endpoint.go
@@ -654,6 +654,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) {
 			structsTask.Artifacts[k] = &structs.TaskArtifact{
 				GetterSource:  *ta.GetterSource,
 				GetterOptions: ta.GetterOptions,
+				GetterMode:    *ta.GetterMode,
 				RelativeDest:  *ta.RelativeDest,
 			}
 		}
diff --git a/jobspec/parse.go b/jobspec/parse.go
index 02f1dcbf1f9f..34b6f32f4e97 100644
--- a/jobspec/parse.go
+++ b/jobspec/parse.go
@@ -788,6 +788,7 @@ func parseArtifacts(result *[]*api.TaskArtifact, list *ast.ObjectList) error {
 		valid := []string{
 			"source",
 			"options",
+			"mode",
 			"destination",
 		}
 		if err := checkHCLKeys(o.Val, valid); err != nil {
diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go
index 9fc57ba92c09..012108be5217 100644
--- a/jobspec/parse_test.go
+++ b/jobspec/parse_test.go
@@ -166,6 +166,7 @@ func TestParse(t *testing.T) {
 										GetterOptions: map[string]string{
 											"checksum": "md5:ff1cc0d3432dad54d607c1505fb7245c",
 										},
+										GetterMode: helper.StringToPtr("file"),
 									},
 								},
 								Vault: &api.Vault{
diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl
index f7a38ee84f06..cfcfc5a1d814 100644
--- a/jobspec/test-fixtures/basic.hcl
+++ b/jobspec/test-fixtures/basic.hcl
@@ -140,6 +140,7 @@ job "binstore-storagelocker" {
       artifact {
         source = "http://bar.com/artifact"
         destination = "test/foo/"
+        mode = "file"
 
         options {
           checksum = "md5:ff1cc0d3432dad54d607c1505fb7245c"
diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go
index 4ba261de3de0..7e90dd9b00df 100644
--- a/nomad/structs/diff_test.go
+++ b/nomad/structs/diff_test.go
@@ -2469,6 +2469,7 @@ func TestTaskDiff(t *testing.T) {
 						GetterOptions: map[string]string{
 							"bar": "baz",
 						},
+						GetterMode:   "dir",
 						RelativeDest: "bar",
 					},
 				},
@@ -2487,6 +2488,7 @@ func TestTaskDiff(t *testing.T) {
 						GetterOptions: map[string]string{
 							"bam": "baz",
 						},
+						GetterMode:   "file",
 						RelativeDest: "bam",
 					},
 				},
@@ -2498,6 +2500,12 @@ func TestTaskDiff(t *testing.T) {
 						Type: DiffTypeAdded,
 						Name: "Artifact",
 						Fields: []*FieldDiff{
+							{
+								Type: DiffTypeAdded,
+								Name: "GetterMode",
+								Old:  "",
+								New:  "file",
+							},
 							{
 								Type: DiffTypeAdded,
 								Name: "GetterOptions[bam]",
@@ -2522,6 +2530,12 @@ func TestTaskDiff(t *testing.T) {
 						Type: DiffTypeDeleted,
 						Name: "Artifact",
 						Fields: []*FieldDiff{
+							{
+								Type: DiffTypeDeleted,
+								Name: "GetterMode",
+								Old:  "dir",
+								New:  "",
+							},
 							{
 								Type: DiffTypeDeleted,
 								Name: "GetterOptions[bar]",
diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go
index daa461fb4bee..13a2c5015d97 100644
--- a/nomad/structs/structs.go
+++ b/nomad/structs/structs.go
@@ -78,6 +78,10 @@ const (
 	ProtocolVersion = "protocol"
 	APIMajorVersion = "api.major"
 	APIMinorVersion = "api.minor"
+
+	GetterModeAny  = "any"
+	GetterModeFile = "file"
+	GetterModeDir  = "dir"
 )
 
 // RPCInfo is used to describe common information about query
@@ -3405,6 +3409,10 @@ type TaskArtifact struct {
 	// go-getter.
 	GetterOptions map[string]string
 
+	// GetterMode is the go-getter.ClientMode for fetching resources.
+	// Defaults to "any" but can be set to "file" or "dir".
+	GetterMode string
+
 	// RelativeDest is the download destination given relative to the task's
 	// directory.
 	RelativeDest string
@@ -3453,6 +3461,17 @@ func (ta *TaskArtifact) Validate() error {
 		mErr.Errors = append(mErr.Errors, fmt.Errorf("source must be specified"))
 	}
 
+	switch ta.GetterMode {
+	case "":
+		// Default to any
+		ta.GetterMode = GetterModeAny
+	case GetterModeAny, GetterModeFile, GetterModeDir:
+		// Ok
+	default:
+		mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid artifact mode %q; must be one of: %s, %s, %s",
+			ta.GetterMode, GetterModeAny, GetterModeFile, GetterModeDir))
+	}
+
 	escaped, err := PathEscapesAllocDir("task", ta.RelativeDest)
 	if err != nil {
 		mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid destination path: %v", err))
diff --git a/vendor/github.com/hashicorp/go-getter/appveyor.yml b/vendor/github.com/hashicorp/go-getter/appveyor.yml
index 159dad4dc25a..ec48d45ec3dd 100644
--- a/vendor/github.com/hashicorp/go-getter/appveyor.yml
+++ b/vendor/github.com/hashicorp/go-getter/appveyor.yml
@@ -1,5 +1,5 @@
 version: "build-{branch}-{build}"
-image: Visual Studio 2015
+image: Visual Studio 2017
 clone_folder: c:\gopath\github.com\hashicorp\go-getter
 environment:
   GOPATH: c:\gopath

From 56d41f11d9c5988700525eccebd20a04a37f2eb1 Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Wed, 5 Jul 2017 20:45:54 -0700
Subject: [PATCH 2/6] Update go-getter for minio support

---
 .../github.com/hashicorp/go-getter/README.md  | 19 ++++-
 .../hashicorp/go-getter/decompress_testing.go |  9 ++-
 .../github.com/hashicorp/go-getter/get_s3.go  | 75 +++++++++++++------
 vendor/vendor.json                            | 10 +--
 4 files changed, 77 insertions(+), 36 deletions(-)

diff --git a/vendor/github.com/hashicorp/go-getter/README.md b/vendor/github.com/hashicorp/go-getter/README.md
index 4a0b6a625d15..78239da54c9b 100644
--- a/vendor/github.com/hashicorp/go-getter/README.md
+++ b/vendor/github.com/hashicorp/go-getter/README.md
@@ -222,13 +222,17 @@ None
 
 ### HTTP (`http`)
 
-None
+#### Basic Authentication
+
+To use HTTP basic authentication with go-getter, simply prepend `username:password@` to the
+hostname in the URL such as `https://Aladdin:OpenSesame@www.example.com/index.html`. All special
+characters, including the username and password, must be URL encoded.
 
 ### S3 (`s3`)
 
 S3 takes various access configurations in the URL. Note that it will also
-read these from standard AWS environment variables if they're set. If
-the query parameters are present, these take priority.
+read these from standard AWS environment variables if they're set. S3 compliant servers like Minio
+are also supported. If the query parameters are present, these take priority.
 
   * `aws_access_key_id` - AWS access key.
   * `aws_access_key_secret` - AWS access key secret.
@@ -240,6 +244,14 @@ If you use go-getter and want to use an EC2 IAM Instance Profile to avoid
 using credentials, then just omit these and the profile, if available will
 be used automatically.
 
+### Using S3 with Minio
+ If you use go-gitter for Minio support, you must consider the following:
+
+  * `aws_access_key_id` (required) - Minio access key.
+  * `aws_access_key_secret` (required) - Minio access key secret.
+  * `region` (optional - defaults to us-east-1) - Region identifier to use.
+  * `version` (optional - fefaults to Minio default) - Configuration file format.
+
 #### S3 Bucket Examples
 
 S3 has several addressing schemes used to reference your bucket. These are
@@ -250,4 +262,5 @@ Some examples for these addressing schemes:
 - s3::https://s3-eu-west-1.amazonaws.com/bucket/foo
 - bucket.s3.amazonaws.com/foo
 - bucket.s3-eu-west-1.amazonaws.com/foo/bar
+- "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=KEYID&aws_access_key_secret=SECRETKEY&region=us-east-2"
 
diff --git a/vendor/github.com/hashicorp/go-getter/decompress_testing.go b/vendor/github.com/hashicorp/go-getter/decompress_testing.go
index a2390e5d0773..82b8ab4f6e8c 100644
--- a/vendor/github.com/hashicorp/go-getter/decompress_testing.go
+++ b/vendor/github.com/hashicorp/go-getter/decompress_testing.go
@@ -11,7 +11,8 @@ import (
 	"runtime"
 	"sort"
 	"strings"
-	"testing"
+
+	"github.com/mitchellh/go-testing-interface"
 )
 
 // TestDecompressCase is a single test case for testing decompressors
@@ -24,7 +25,7 @@ type TestDecompressCase struct {
 }
 
 // TestDecompressor is a helper function for testing generic decompressors.
-func TestDecompressor(t *testing.T, d Decompressor, cases []TestDecompressCase) {
+func TestDecompressor(t testing.T, d Decompressor, cases []TestDecompressCase) {
 	for _, tc := range cases {
 		t.Logf("Testing: %s", tc.Input)
 
@@ -87,7 +88,7 @@ func TestDecompressor(t *testing.T, d Decompressor, cases []TestDecompressCase)
 	}
 }
 
-func testListDir(t *testing.T, path string) []string {
+func testListDir(t testing.T, path string) []string {
 	var result []string
 	err := filepath.Walk(path, func(sub string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -116,7 +117,7 @@ func testListDir(t *testing.T, path string) []string {
 	return result
 }
 
-func testMD5(t *testing.T, path string) string {
+func testMD5(t testing.T, path string) string {
 	f, err := os.Open(path)
 	if err != nil {
 		t.Fatalf("err: %s", err)
diff --git a/vendor/github.com/hashicorp/go-getter/get_s3.go b/vendor/github.com/hashicorp/go-getter/get_s3.go
index d3bffeb17375..ebb3217417d2 100644
--- a/vendor/github.com/hashicorp/go-getter/get_s3.go
+++ b/vendor/github.com/hashicorp/go-getter/get_s3.go
@@ -28,7 +28,7 @@ func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
 	}
 
 	// Create client config
-	config := g.getAWSConfig(region, creds)
+	config := g.getAWSConfig(region, u, creds)
 	sess := session.New(config)
 	client := s3.New(sess)
 
@@ -84,7 +84,7 @@ func (g *S3Getter) Get(dst string, u *url.URL) error {
 		return err
 	}
 
-	config := g.getAWSConfig(region, creds)
+	config := g.getAWSConfig(region, u, creds)
 	sess := session.New(config)
 	client := s3.New(sess)
 
@@ -139,7 +139,7 @@ func (g *S3Getter) GetFile(dst string, u *url.URL) error {
 		return err
 	}
 
-	config := g.getAWSConfig(region, creds)
+	config := g.getAWSConfig(region, u, creds)
 	sess := session.New(config)
 	client := s3.New(sess)
 	return g.getObject(client, dst, bucket, path, version)
@@ -174,7 +174,7 @@ func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) er
 	return err
 }
 
-func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) *aws.Config {
+func (g *S3Getter) getAWSConfig(region string, url *url.URL, creds *credentials.Credentials) *aws.Config {
 	conf := &aws.Config{}
 	if creds == nil {
 		// Grab the metadata URL
@@ -195,6 +195,14 @@ func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) *
 			})
 	}
 
+	if creds != nil {
+		conf.Endpoint = &url.Host
+		conf.S3ForcePathStyle = aws.Bool(true)
+		if url.Scheme == "http" {
+			conf.DisableSSL = aws.Bool(true)
+		}
+	}
+
 	conf.Credentials = creds
 	if region != "" {
 		conf.Region = aws.String(region)
@@ -204,29 +212,48 @@ func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) *
 }
 
 func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.Credentials, err error) {
-	// Expected host style: s3.amazonaws.com. They always have 3 parts,
-	// although the first may differ if we're accessing a specific region.
-	hostParts := strings.Split(u.Host, ".")
-	if len(hostParts) != 3 {
-		err = fmt.Errorf("URL is not a valid S3 URL")
-		return
-	}
+	// This just check whether we are dealing with S3 or
+	// any other S3 compliant service. S3 has a predictable
+	// url as others do not
+	if strings.Contains(u.Host, "amazonaws.com") {
+		// Expected host style: s3.amazonaws.com. They always have 3 parts,
+		// although the first may differ if we're accessing a specific region.
+		hostParts := strings.Split(u.Host, ".")
+		if len(hostParts) != 3 {
+			err = fmt.Errorf("URL is not a valid S3 URL")
+			return
+		}
 
-	// Parse the region out of the first part of the host
-	region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3")
-	if region == "" {
-		region = "us-east-1"
-	}
+		// Parse the region out of the first part of the host
+		region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3")
+		if region == "" {
+			region = "us-east-1"
+		}
 
-	pathParts := strings.SplitN(u.Path, "/", 3)
-	if len(pathParts) != 3 {
-		err = fmt.Errorf("URL is not a valid S3 URL")
-		return
-	}
+		pathParts := strings.SplitN(u.Path, "/", 3)
+		if len(pathParts) != 3 {
+			err = fmt.Errorf("URL is not a valid S3 URL")
+			return
+		}
+
+		bucket = pathParts[1]
+		path = pathParts[2]
+		version = u.Query().Get("version")
 
-	bucket = pathParts[1]
-	path = pathParts[2]
-	version = u.Query().Get("version")
+	} else {
+		pathParts := strings.SplitN(u.Path, "/", 3)
+		if len(pathParts) != 3 {
+			err = fmt.Errorf("URL is not a valid S3 complaint URL")
+			return
+		}
+		bucket = pathParts[1]
+		path = pathParts[2]
+		version = u.Query().Get("version")
+		region = u.Query().Get("region")
+		if region == "" {
+			region = "us-east-1"
+		}
+	}
 
 	_, hasAwsId := u.Query()["aws_access_key_id"]
 	_, hasAwsSecret := u.Query()["aws_access_key_secret"]
diff --git a/vendor/vendor.json b/vendor/vendor.json
index b8da1cd876d7..27d1276f3881 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -689,16 +689,16 @@
 			"revisionTime": "2017-06-02T22:43:19Z"
 		},
 		{
-			"checksumSHA1": "hHoNT6CfqcDF1+yXRKnz/duuwIk=",
+			"checksumSHA1": "Sozy3aNAPBleUfiECj0jnaYjw2k=",
 			"path": "github.com/hashicorp/go-getter",
-			"revision": "e48f67b534e614bf7fbd978fd0020f61a17b7527",
-			"revisionTime": "2017-04-05T22:15:29Z"
+			"revision": "2814e6fb2ca5b3bd950c97eff22553ecb3c7f77b",
+			"revisionTime": "2017-07-06T02:51:20Z"
 		},
 		{
 			"checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=",
 			"path": "github.com/hashicorp/go-getter/helper/url",
-			"revision": "e48f67b534e614bf7fbd978fd0020f61a17b7527",
-			"revisionTime": "2017-04-05T22:15:29Z"
+			"revision": "2814e6fb2ca5b3bd950c97eff22553ecb3c7f77b",
+			"revisionTime": "2017-07-06T02:51:20Z"
 		},
 		{
 			"checksumSHA1": "zvmksNyW6g+Fd/bywd4vcn8rp+M=",

From b212ddac937dffe0abdb076906153647198f4fcd Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Wed, 5 Jul 2017 20:57:11 -0700
Subject: [PATCH 3/6] Update docs and readme

---
 CHANGELOG.md                                  |  2 ++
 .../docs/job-specification/artifact.html.md   | 20 +++++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eca054be043d..1ace5e6d12a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,6 +44,8 @@ BUG FIXES:
    allocation destroy fixing a corner case in which an allocation may be blocked
    till destroy [GH-2563]
  * client/artifact: Honor netrc [GH-2524]
+ * client/artifact: Allow specifying a go-gettter mode [GH-2781]
+ * client/artifact: Support non-Amazon S3-compatible sources [GH-2781]
  * client/artifact: Handle tars where file in directory is listed before
    directory [GH-2524]
  * client/config: Use `cpu_total_compute` whenever it is set [GH-2745]
diff --git a/website/source/docs/job-specification/artifact.html.md b/website/source/docs/job-specification/artifact.html.md
index 6ecbfb3861bb..cc083e31a11b 100644
--- a/website/source/docs/job-specification/artifact.html.md
+++ b/website/source/docs/job-specification/artifact.html.md
@@ -46,10 +46,15 @@ unarchived before the starting the task.
 
 ## `artifact` Parameters
 
-- `destination` `(string: "local/$1")` - Specifies the directory path to download the
-  artifact, relative to the root of the task's directory. If omitted, the
-  default value is to place the binary in `local/`. The destination is treated
-  as a directory and source files will be downloaded into that directory path.
+- `destination` `(string: "local/")` - Specifies the directory path to download
+  the artifact, relative to the root of the task's directory. If omitted, the
+  default value is to place the artifact in `local/`. The destination is treated
+  as a directory unless `mode` is set to `file`. Source files will be downloaded
+  into that directory path.
+
+- `mode` `(string: "any")` - One of `any`, `file`, or `dir`. If set to `file`
+  the `destination` must be a file, not a directory. By default the
+  `destination` will be `local/<filename>`.
 
 - `options` `(map<string|string>: nil)` - Specifies configuration parameters to
   fetch the artifact. The key-value pairs map directly to parameters appended to
@@ -150,11 +155,13 @@ artifact {
 }
 ```
 
-### Download from an S3 Bucket
+### Download from an S3-compatible Bucket
 
 These examples download artifacts from Amazon S3. There are several different
 types of [S3 bucket addressing][s3-bucket-addr] and [S3 region-specific
-endpoints][s3-region-endpoints].
+endpoints][s3-region-endpoints]. As of Nomad 0.6 non-Amazon S3-compatible
+endpoints like [Minio] are supported, but you must explicitly set the "s3::"
+prefix.
 
 This example uses path-based notation on a publicly-accessible bucket:
 
@@ -194,5 +201,6 @@ artifact {
 ```
 
 [go-getter]: https://github.com/hashicorp/go-getter "HashiCorp go-getter Library"
+[Minio]: https://www.minio.io/
 [s3-bucket-addr]: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro "Amazon S3 Bucket Addressing"
 [s3-region-endpoints]: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region "Amazon S3 Region Endpoints"

From 8d2bea4b454b2faa0be0ec016452f2eff5d63a9a Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Thu, 6 Jul 2017 09:53:56 -0700
Subject: [PATCH 4/6] New go-getter test dep

---
 .../hashicorp/go-sockaddr/GNUmakefile         | 65 +++++++++++++++++
 .../mitchellh/go-testing-interface/LICENSE    | 21 ++++++
 .../mitchellh/go-testing-interface/README.md  | 52 ++++++++++++++
 .../mitchellh/go-testing-interface/testing.go | 70 +++++++++++++++++++
 vendor/vendor.json                            |  6 ++
 5 files changed, 214 insertions(+)
 create mode 100644 vendor/github.com/hashicorp/go-sockaddr/GNUmakefile
 create mode 100644 vendor/github.com/mitchellh/go-testing-interface/LICENSE
 create mode 100644 vendor/github.com/mitchellh/go-testing-interface/README.md
 create mode 100644 vendor/github.com/mitchellh/go-testing-interface/testing.go

diff --git a/vendor/github.com/hashicorp/go-sockaddr/GNUmakefile b/vendor/github.com/hashicorp/go-sockaddr/GNUmakefile
new file mode 100644
index 000000000000..f3dfd24cfd36
--- /dev/null
+++ b/vendor/github.com/hashicorp/go-sockaddr/GNUmakefile
@@ -0,0 +1,65 @@
+TOOLS= golang.org/x/tools/cover
+GOCOVER_TMPFILE?=	$(GOCOVER_FILE).tmp
+GOCOVER_FILE?=	.cover.out
+GOCOVERHTML?=	coverage.html
+FIND=`/usr/bin/which 2> /dev/null gfind find | /usr/bin/grep -v ^no | /usr/bin/head -n 1`
+XARGS=`/usr/bin/which 2> /dev/null gxargs xargs | /usr/bin/grep -v ^no | /usr/bin/head -n 1`
+
+test:: $(GOCOVER_FILE)
+	@$(MAKE) -C cmd/sockaddr test
+
+cover:: coverage_report
+
+$(GOCOVER_FILE)::
+	@${FIND} . -type d ! -path '*cmd*' ! -path '*.git*' -print0 | ${XARGS} -0 -I % sh -ec "cd % && rm -f $(GOCOVER_TMPFILE) && go test -coverprofile=$(GOCOVER_TMPFILE)"
+
+	@echo 'mode: set' > $(GOCOVER_FILE)
+	@${FIND} . -type f ! -path '*cmd*' ! -path '*.git*' -name "$(GOCOVER_TMPFILE)" -print0 | ${XARGS} -0 -n1 cat $(GOCOVER_TMPFILE) | grep -v '^mode: ' >> ${PWD}/$(GOCOVER_FILE)
+
+$(GOCOVERHTML): $(GOCOVER_FILE)
+	go tool cover -html=$(GOCOVER_FILE) -o $(GOCOVERHTML)
+
+coverage_report:: $(GOCOVER_FILE)
+	go tool cover -html=$(GOCOVER_FILE)
+
+audit_tools::
+	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
+	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
+	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
+	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
+	@go get -u github.com/gordonklaus/ineffassign && echo "Installed ineffassign:"
+
+audit::
+	deadcode
+	go tool vet -all *.go
+	go tool vet -shadow=true *.go
+	golint *.go
+	ineffassign .
+	gocyclo -over 65 *.go
+	misspell *.go
+
+clean::
+	rm -f $(GOCOVER_FILE) $(GOCOVERHTML)
+
+dev::
+	@go build
+	@$(MAKE) -B -C cmd/sockaddr sockaddr
+
+install::
+	@go install
+	@$(MAKE) -C cmd/sockaddr install
+
+doc::
+	@echo Visit: http://127.0.0.1:6161/pkg/github.com/hashicorp/go-sockaddr/
+	godoc -http=:6161 -goroot $GOROOT
+
+world::
+	@set -e; \
+	for os in solaris darwin freebsd linux windows; do \
+		for arch in amd64; do \
+			printf "Building on %s-%s\n" "$${os}" "$${arch}" ; \
+			env GOOS="$${os}" GOARCH="$${arch}" go build -o /dev/null; \
+		done; \
+	done
+
+	$(MAKE) -C cmd/sockaddr world
diff --git a/vendor/github.com/mitchellh/go-testing-interface/LICENSE b/vendor/github.com/mitchellh/go-testing-interface/LICENSE
new file mode 100644
index 000000000000..a3866a291fd1
--- /dev/null
+++ b/vendor/github.com/mitchellh/go-testing-interface/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Mitchell Hashimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/vendor/github.com/mitchellh/go-testing-interface/README.md b/vendor/github.com/mitchellh/go-testing-interface/README.md
new file mode 100644
index 000000000000..26781bbae887
--- /dev/null
+++ b/vendor/github.com/mitchellh/go-testing-interface/README.md
@@ -0,0 +1,52 @@
+# go-testing-interface
+
+go-testing-interface is a Go library that exports an interface that
+`*testing.T` implements as well as a runtime version you can use in its
+place.
+
+The purpose of this library is so that you can export test helpers as a
+public API without depending on the "testing" package, since you can't
+create a `*testing.T` struct manually. This lets you, for example, use the
+public testing APIs to generate mock data at runtime, rather than just at
+test time.
+
+## Usage & Example
+
+For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/go-testing-interface).
+
+Given a test helper written using `go-testing-interface` like this:
+
+    import "github.com/mitchellh/go-testing-interface"
+
+    func TestHelper(t testing.T) {
+        t.Fatal("I failed")
+    }
+
+You can call the test helper in a real test easily:
+
+    import "testing"
+
+    func TestThing(t *testing.T) {
+        TestHelper(t)
+    }
+
+You can also call the test helper at runtime if needed:
+
+    import "github.com/mitchellh/go-testing-interface"
+
+    func main() {
+        TestHelper(&testing.RuntimeT{})
+    }
+
+## Why?!
+
+**Why would I call a test helper that takes a *testing.T at runtime?**
+
+You probably shouldn't. The only use case I've seen (and I've had) for this
+is to implement a "dev mode" for a service where the test helpers are used
+to populate mock data, create a mock DB, perhaps run service dependencies
+in-memory, etc.
+
+Outside of a "dev mode", I've never seen a use case for this and I think
+there shouldn't be one since the point of the `testing.T` interface is that
+you can fail immediately.
diff --git a/vendor/github.com/mitchellh/go-testing-interface/testing.go b/vendor/github.com/mitchellh/go-testing-interface/testing.go
new file mode 100644
index 000000000000..af6426a4dfd6
--- /dev/null
+++ b/vendor/github.com/mitchellh/go-testing-interface/testing.go
@@ -0,0 +1,70 @@
+package testing
+
+import (
+	"fmt"
+	"log"
+)
+
+// T is the interface that mimics the standard library *testing.T.
+//
+// In unit tests you can just pass a *testing.T struct. At runtime, outside
+// of tests, you can pass in a RuntimeT struct from this package.
+type T interface {
+	Error(args ...interface{})
+	Errorf(format string, args ...interface{})
+	Fatal(args ...interface{})
+	Fatalf(format string, args ...interface{})
+	Fail()
+	FailNow()
+	Failed() bool
+	Log(args ...interface{})
+	Logf(format string, args ...interface{})
+}
+
+// RuntimeT implements T and can be instantiated and run at runtime to
+// mimic *testing.T behavior. Unlike *testing.T, this will simply panic
+// for calls to Fatal. For calls to Error, you'll have to check the errors
+// list to determine whether to exit yourself.
+type RuntimeT struct {
+	failed bool
+}
+
+func (t *RuntimeT) Error(args ...interface{}) {
+	log.Println(fmt.Sprintln(args...))
+	t.Fail()
+}
+
+func (t *RuntimeT) Errorf(format string, args ...interface{}) {
+	log.Println(fmt.Sprintf(format, args...))
+	t.Fail()
+}
+
+func (t *RuntimeT) Fatal(args ...interface{}) {
+	log.Println(fmt.Sprintln(args...))
+	t.FailNow()
+}
+
+func (t *RuntimeT) Fatalf(format string, args ...interface{}) {
+	log.Println(fmt.Sprintf(format, args...))
+	t.FailNow()
+}
+
+func (t *RuntimeT) Fail() {
+	t.failed = true
+}
+
+func (t *RuntimeT) FailNow() {
+	panic("testing.T failed, see logs for output (if any)")
+}
+
+func (t *RuntimeT) Failed() bool {
+	return t.failed
+}
+
+func (t *RuntimeT) Log(args ...interface{}) {
+	log.Println(fmt.Sprintln(args...))
+}
+
+func (t *RuntimeT) Logf(format string, args ...interface{}) {
+	log.Println(fmt.Sprintf(format, args...))
+}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 27d1276f3881..974466edf35a 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -981,6 +981,12 @@
 			"revision": "4fdf99ab29366514c69ccccddab5dc58b8d84062",
 			"revisionTime": "2017-03-09T13:30:38Z"
 		},
+		{
+			"checksumSHA1": "j83WZHiKiPF86Hj5QMQdWboizls=",
+			"path": "github.com/mitchellh/go-testing-interface",
+			"revision": "477c2d05a845d8b55912a5a7993b9b24abcc5ef8",
+			"revisionTime": "2017-04-30T14:07:22Z"
+		},
 		{
 			"path": "github.com/mitchellh/hashstructure",
 			"revision": "1ef5c71b025aef149d12346356ac5973992860bc"

From 1078df4fdeddf47a0ecab4ae770b62fcadc26327 Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Thu, 6 Jul 2017 10:10:06 -0700
Subject: [PATCH 5/6] Improvements, not bug fixes! Also spelttting

---
 CHANGELOG.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1ace5e6d12a6..db48fcbe09be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,8 @@ IMPROVEMENTS:
    [GH-2610]
  * client: Fingerprint all routable addresses on an interface including IPv6
    addresses [GH-2536]
+ * client/artifact: Allow specifying a go-getter mode [GH-2781]
+ * client/artifact: Support non-Amazon S3-compatible sources [GH-2781]
  * client/template: Support reading env vars from templates [GH-2654]
  * config: Support Unix socket addresses for Consul [GH-2622]
  * discovery: Advertise driver-specified IP address and port [GH-2709]
@@ -44,8 +46,6 @@ BUG FIXES:
    allocation destroy fixing a corner case in which an allocation may be blocked
    till destroy [GH-2563]
  * client/artifact: Honor netrc [GH-2524]
- * client/artifact: Allow specifying a go-gettter mode [GH-2781]
- * client/artifact: Support non-Amazon S3-compatible sources [GH-2781]
  * client/artifact: Handle tars where file in directory is listed before
    directory [GH-2524]
  * client/config: Use `cpu_total_compute` whenever it is set [GH-2745]

From 9d2682e1bb420ecb55e41075abe3fad39abef828 Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Thu, 6 Jul 2017 10:17:19 -0700
Subject: [PATCH 6/6] Fix api endpoint test

---
 command/agent/job_endpoint_test.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go
index 75b8103d79a3..3d189773a9ce 100644
--- a/command/agent/job_endpoint_test.go
+++ b/command/agent/job_endpoint_test.go
@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -12,6 +13,7 @@ import (
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/mock"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/kr/pretty"
 )
 
 func TestHTTP_JobsList(t *testing.T) {
@@ -993,6 +995,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 								GetterOptions: map[string]string{
 									"a": "b",
 								},
+								GetterMode:   helper.StringToPtr("dir"),
 								RelativeDest: helper.StringToPtr("dest"),
 							},
 						},
@@ -1178,6 +1181,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 								GetterOptions: map[string]string{
 									"a": "b",
 								},
+								GetterMode:   "dir",
 								RelativeDest: "dest",
 							},
 						},
@@ -1213,7 +1217,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 
 	structsJob := ApiJobToStructJob(apiJob)
 
-	if !reflect.DeepEqual(expected, structsJob) {
-		t.Fatalf("bad %#v", structsJob)
+	if diff := pretty.Diff(expected, structsJob); len(diff) > 0 {
+		t.Fatalf("bad:\n%s", strings.Join(diff, "\n"))
 	}
 }