diff --git a/client/getter/getter.go b/client/getter/getter.go index c534964a8f1f..48ca4d0b4b6d 100644 --- a/client/getter/getter.go +++ b/client/getter/getter.go @@ -18,7 +18,7 @@ var ( lock sync.Mutex // supported is the set of download schemes supported by Nomad - supported = []string{"http", "https", "s3"} + supported = []string{"http", "https", "s3", "hg", "git"} ) // getClient returns a client that is suitable for Nomad downloading artifacts. diff --git a/client/task_runner.go b/client/task_runner.go index a18769503162..6329cb65396c 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -793,6 +793,7 @@ func (r *TaskRunner) prestart(resultCh chan bool) { for _, artifact := range r.task.Artifacts { if err := getter.GetArtifact(r.getTaskEnv(), artifact, r.taskDir.Dir); err != nil { wrapped := fmt.Errorf("failed to download artifact %q: %v", artifact.GetterSource, err) + r.logger.Printf("[DEBUG] client: %v", wrapped) r.setState(structs.TaskStatePending, structs.NewTaskEvent(structs.TaskArtifactDownloadFailed).SetDownloadError(wrapped)) r.restartTracker.SetStartError(structs.NewRecoverableError(wrapped, true)) diff --git a/vendor/github.com/bgentry/go-netrc/LICENSE b/vendor/github.com/bgentry/go-netrc/LICENSE new file mode 100644 index 000000000000..aade9a58b13e --- /dev/null +++ b/vendor/github.com/bgentry/go-netrc/LICENSE @@ -0,0 +1,20 @@ +Original version Copyright © 2010 Fazlul Shahriar . Newer +portions Copyright © 2014 Blake Gentry . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/bgentry/go-netrc/netrc/netrc.go b/vendor/github.com/bgentry/go-netrc/netrc/netrc.go new file mode 100644 index 000000000000..ea49987c081d --- /dev/null +++ b/vendor/github.com/bgentry/go-netrc/netrc/netrc.go @@ -0,0 +1,510 @@ +package netrc + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +type tkType int + +const ( + tkMachine tkType = iota + tkDefault + tkLogin + tkPassword + tkAccount + tkMacdef + tkComment + tkWhitespace +) + +var keywords = map[string]tkType{ + "machine": tkMachine, + "default": tkDefault, + "login": tkLogin, + "password": tkPassword, + "account": tkAccount, + "macdef": tkMacdef, + "#": tkComment, +} + +type Netrc struct { + tokens []*token + machines []*Machine + macros Macros + updateLock sync.Mutex +} + +// FindMachine returns the Machine in n named by name. If a machine named by +// name exists, it is returned. If no Machine with name name is found and there +// is a ``default'' machine, the ``default'' machine is returned. Otherwise, nil +// is returned. +func (n *Netrc) FindMachine(name string) (m *Machine) { + // TODO(bgentry): not safe for concurrency + var def *Machine + for _, m = range n.machines { + if m.Name == name { + return m + } + if m.IsDefault() { + def = m + } + } + if def == nil { + return nil + } + return def +} + +// MarshalText implements the encoding.TextMarshaler interface to encode a +// Netrc into text format. +func (n *Netrc) MarshalText() (text []byte, err error) { + // TODO(bgentry): not safe for concurrency + for i := range n.tokens { + switch n.tokens[i].kind { + case tkComment, tkDefault, tkWhitespace: // always append these types + text = append(text, n.tokens[i].rawkind...) + default: + if n.tokens[i].value != "" { // skip empty-value tokens + text = append(text, n.tokens[i].rawkind...) + } + } + if n.tokens[i].kind == tkMacdef { + text = append(text, ' ') + text = append(text, n.tokens[i].macroName...) + } + text = append(text, n.tokens[i].rawvalue...) + } + return +} + +func (n *Netrc) NewMachine(name, login, password, account string) *Machine { + n.updateLock.Lock() + defer n.updateLock.Unlock() + + prefix := "\n" + if len(n.tokens) == 0 { + prefix = "" + } + m := &Machine{ + Name: name, + Login: login, + Password: password, + Account: account, + + nametoken: &token{ + kind: tkMachine, + rawkind: []byte(prefix + "machine"), + value: name, + rawvalue: []byte(" " + name), + }, + logintoken: &token{ + kind: tkLogin, + rawkind: []byte("\n\tlogin"), + value: login, + rawvalue: []byte(" " + login), + }, + passtoken: &token{ + kind: tkPassword, + rawkind: []byte("\n\tpassword"), + value: password, + rawvalue: []byte(" " + password), + }, + accounttoken: &token{ + kind: tkAccount, + rawkind: []byte("\n\taccount"), + value: account, + rawvalue: []byte(" " + account), + }, + } + n.insertMachineTokensBeforeDefault(m) + for i := range n.machines { + if n.machines[i].IsDefault() { + n.machines = append(append(n.machines[:i], m), n.machines[i:]...) + return m + } + } + n.machines = append(n.machines, m) + return m +} + +func (n *Netrc) insertMachineTokensBeforeDefault(m *Machine) { + newtokens := []*token{m.nametoken} + if m.logintoken.value != "" { + newtokens = append(newtokens, m.logintoken) + } + if m.passtoken.value != "" { + newtokens = append(newtokens, m.passtoken) + } + if m.accounttoken.value != "" { + newtokens = append(newtokens, m.accounttoken) + } + for i := range n.tokens { + if n.tokens[i].kind == tkDefault { + // found the default, now insert tokens before it + n.tokens = append(n.tokens[:i], append(newtokens, n.tokens[i:]...)...) + return + } + } + // didn't find a default, just add the newtokens to the end + n.tokens = append(n.tokens, newtokens...) + return +} + +func (n *Netrc) RemoveMachine(name string) { + n.updateLock.Lock() + defer n.updateLock.Unlock() + + for i := range n.machines { + if n.machines[i] != nil && n.machines[i].Name == name { + m := n.machines[i] + for _, t := range []*token{ + m.nametoken, m.logintoken, m.passtoken, m.accounttoken, + } { + n.removeToken(t) + } + n.machines = append(n.machines[:i], n.machines[i+1:]...) + return + } + } +} + +func (n *Netrc) removeToken(t *token) { + if t != nil { + for i := range n.tokens { + if n.tokens[i] == t { + n.tokens = append(n.tokens[:i], n.tokens[i+1:]...) + return + } + } + } +} + +// Machine contains information about a remote machine. +type Machine struct { + Name string + Login string + Password string + Account string + + nametoken *token + logintoken *token + passtoken *token + accounttoken *token +} + +// IsDefault returns true if the machine is a "default" token, denoted by an +// empty name. +func (m *Machine) IsDefault() bool { + return m.Name == "" +} + +// UpdatePassword sets the password for the Machine m. +func (m *Machine) UpdatePassword(newpass string) { + m.Password = newpass + updateTokenValue(m.passtoken, newpass) +} + +// UpdateLogin sets the login for the Machine m. +func (m *Machine) UpdateLogin(newlogin string) { + m.Login = newlogin + updateTokenValue(m.logintoken, newlogin) +} + +// UpdateAccount sets the login for the Machine m. +func (m *Machine) UpdateAccount(newaccount string) { + m.Account = newaccount + updateTokenValue(m.accounttoken, newaccount) +} + +func updateTokenValue(t *token, value string) { + oldvalue := t.value + t.value = value + newraw := make([]byte, len(t.rawvalue)) + copy(newraw, t.rawvalue) + t.rawvalue = append( + bytes.TrimSuffix(newraw, []byte(oldvalue)), + []byte(value)..., + ) +} + +// Macros contains all the macro definitions in a netrc file. +type Macros map[string]string + +type token struct { + kind tkType + macroName string + value string + rawkind []byte + rawvalue []byte +} + +// Error represents a netrc file parse error. +type Error struct { + LineNum int // Line number + Msg string // Error message +} + +// Error returns a string representation of error e. +func (e *Error) Error() string { + return fmt.Sprintf("line %d: %s", e.LineNum, e.Msg) +} + +func (e *Error) BadDefaultOrder() bool { + return e.Msg == errBadDefaultOrder +} + +const errBadDefaultOrder = "default token must appear after all machine tokens" + +// scanLinesKeepPrefix is a split function for a Scanner that returns each line +// of text. The returned token may include newlines if they are before the +// first non-space character. The returned line may be empty. The end-of-line +// marker is one optional carriage return followed by one mandatory newline. In +// regular expression notation, it is `\r?\n`. The last non-empty line of +// input will be returned even if it has no newline. +func scanLinesKeepPrefix(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + // Skip leading spaces. + start := 0 + for width := 0; start < len(data); start += width { + var r rune + r, width = utf8.DecodeRune(data[start:]) + if !unicode.IsSpace(r) { + break + } + } + if i := bytes.IndexByte(data[start:], '\n'); i >= 0 { + // We have a full newline-terminated line. + return start + i, data[0 : start+i], nil + } + // If we're at EOF, we have a final, non-terminated line. Return it. + if atEOF { + return len(data), data, nil + } + // Request more data. + return 0, nil, nil +} + +// scanWordsKeepPrefix is a split function for a Scanner that returns each +// space-separated word of text, with prefixing spaces included. It will never +// return an empty string. The definition of space is set by unicode.IsSpace. +// +// Adapted from bufio.ScanWords(). +func scanTokensKeepPrefix(data []byte, atEOF bool) (advance int, token []byte, err error) { + // Skip leading spaces. + start := 0 + for width := 0; start < len(data); start += width { + var r rune + r, width = utf8.DecodeRune(data[start:]) + if !unicode.IsSpace(r) { + break + } + } + if atEOF && len(data) == 0 || start == len(data) { + return len(data), data, nil + } + if len(data) > start && data[start] == '#' { + return scanLinesKeepPrefix(data, atEOF) + } + // Scan until space, marking end of word. + for width, i := 0, start; i < len(data); i += width { + var r rune + r, width = utf8.DecodeRune(data[i:]) + if unicode.IsSpace(r) { + return i, data[:i], nil + } + } + // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. + if atEOF && len(data) > start { + return len(data), data, nil + } + // Request more data. + return 0, nil, nil +} + +func newToken(rawb []byte) (*token, error) { + _, tkind, err := bufio.ScanWords(rawb, true) + if err != nil { + return nil, err + } + var ok bool + t := token{rawkind: rawb} + t.kind, ok = keywords[string(tkind)] + if !ok { + trimmed := strings.TrimSpace(string(tkind)) + if trimmed == "" { + t.kind = tkWhitespace // whitespace-only, should happen only at EOF + return &t, nil + } + if strings.HasPrefix(trimmed, "#") { + t.kind = tkComment // this is a comment + return &t, nil + } + return &t, fmt.Errorf("keyword expected; got " + string(tkind)) + } + return &t, nil +} + +func scanValue(scanner *bufio.Scanner, pos int) ([]byte, string, int, error) { + if scanner.Scan() { + raw := scanner.Bytes() + pos += bytes.Count(raw, []byte{'\n'}) + return raw, strings.TrimSpace(string(raw)), pos, nil + } + if err := scanner.Err(); err != nil { + return nil, "", pos, &Error{pos, err.Error()} + } + return nil, "", pos, nil +} + +func parse(r io.Reader, pos int) (*Netrc, error) { + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + nrc := Netrc{machines: make([]*Machine, 0, 20), macros: make(Macros, 10)} + + defaultSeen := false + var currentMacro *token + var m *Machine + var t *token + scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner.Split(scanTokensKeepPrefix) + + for scanner.Scan() { + rawb := scanner.Bytes() + if len(rawb) == 0 { + break + } + pos += bytes.Count(rawb, []byte{'\n'}) + t, err = newToken(rawb) + if err != nil { + if currentMacro == nil { + return nil, &Error{pos, err.Error()} + } + currentMacro.rawvalue = append(currentMacro.rawvalue, rawb...) + continue + } + + if currentMacro != nil && bytes.Contains(rawb, []byte{'\n', '\n'}) { + // if macro rawvalue + rawb would contain \n\n, then macro def is over + currentMacro.value = strings.TrimLeft(string(currentMacro.rawvalue), "\r\n") + nrc.macros[currentMacro.macroName] = currentMacro.value + currentMacro = nil + } + + switch t.kind { + case tkMacdef: + if _, t.macroName, pos, err = scanValue(scanner, pos); err != nil { + return nil, &Error{pos, err.Error()} + } + currentMacro = t + case tkDefault: + if defaultSeen { + return nil, &Error{pos, "multiple default token"} + } + if m != nil { + nrc.machines, m = append(nrc.machines, m), nil + } + m = new(Machine) + m.Name = "" + defaultSeen = true + case tkMachine: + if defaultSeen { + return nil, &Error{pos, errBadDefaultOrder} + } + if m != nil { + nrc.machines, m = append(nrc.machines, m), nil + } + m = new(Machine) + if t.rawvalue, m.Name, pos, err = scanValue(scanner, pos); err != nil { + return nil, &Error{pos, err.Error()} + } + t.value = m.Name + m.nametoken = t + case tkLogin: + if m == nil || m.Login != "" { + return nil, &Error{pos, "unexpected token login "} + } + if t.rawvalue, m.Login, pos, err = scanValue(scanner, pos); err != nil { + return nil, &Error{pos, err.Error()} + } + t.value = m.Login + m.logintoken = t + case tkPassword: + if m == nil || m.Password != "" { + return nil, &Error{pos, "unexpected token password"} + } + if t.rawvalue, m.Password, pos, err = scanValue(scanner, pos); err != nil { + return nil, &Error{pos, err.Error()} + } + t.value = m.Password + m.passtoken = t + case tkAccount: + if m == nil || m.Account != "" { + return nil, &Error{pos, "unexpected token account"} + } + if t.rawvalue, m.Account, pos, err = scanValue(scanner, pos); err != nil { + return nil, &Error{pos, err.Error()} + } + t.value = m.Account + m.accounttoken = t + } + + nrc.tokens = append(nrc.tokens, t) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + if m != nil { + nrc.machines, m = append(nrc.machines, m), nil + } + return &nrc, nil +} + +// ParseFile opens the file at filename and then passes its io.Reader to +// Parse(). +func ParseFile(filename string) (*Netrc, error) { + fd, err := os.Open(filename) + if err != nil { + return nil, err + } + defer fd.Close() + return Parse(fd) +} + +// Parse parses from the the Reader r as a netrc file and returns the set of +// machine information and macros defined in it. The ``default'' machine, +// which is intended to be used when no machine name matches, is identified +// by an empty machine name. There can be only one ``default'' machine. +// +// If there is a parsing error, an Error is returned. +func Parse(r io.Reader) (*Netrc, error) { + return parse(r, 1) +} + +// FindMachine parses the netrc file identified by filename and returns the +// Machine named by name. If a problem occurs parsing the file at filename, an +// error is returned. If a machine named by name exists, it is returned. If no +// Machine with name name is found and there is a ``default'' machine, the +// ``default'' machine is returned. Otherwise, nil is returned. +func FindMachine(filename, name string) (m *Machine, err error) { + n, err := ParseFile(filename) + if err != nil { + return nil, err + } + return n.FindMachine(name), nil +} diff --git a/vendor/github.com/hashicorp/go-getter/.travis.yml b/vendor/github.com/hashicorp/go-getter/.travis.yml deleted file mode 100644 index 59bf168556a0..000000000000 --- a/vendor/github.com/hashicorp/go-getter/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -sudo: false - -language: go - -go: - - 1.5 - -branches: - only: - - master diff --git a/vendor/github.com/hashicorp/go-getter/README.md b/vendor/github.com/hashicorp/go-getter/README.md index 1e37892bb74d..4a0b6a625d15 100644 --- a/vendor/github.com/hashicorp/go-getter/README.md +++ b/vendor/github.com/hashicorp/go-getter/README.md @@ -210,6 +210,12 @@ None a commit SHA, a branch name, etc. If it is a named ref such as a branch name, go-getter will update it to the latest on each get. + * `sshkey` - An SSH private key to use during clones. The provided key must + be a base64-encoded string. For example, to generate a suitable `sshkey` + from a private key file on disk, you would run `base64 -w0 `. + + **Note**: Git 2.3+ is required to use this feature. + ### Mercurial (`hg`) * `rev` - The Mercurial revision to checkout. @@ -227,3 +233,21 @@ the query parameters are present, these take priority. * `aws_access_key_id` - AWS access key. * `aws_access_key_secret` - AWS access key secret. * `aws_access_token` - AWS access token if this is being used. + +#### Using IAM Instance Profiles with S3 + +If you use go-getter and want to use an EC2 IAM Instance Profile to avoid +using credentials, then just omit these and the profile, if available will +be used automatically. + +#### S3 Bucket Examples + +S3 has several addressing schemes used to reference your bucket. These are +listed here: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro + +Some examples for these addressing schemes: +- s3::https://s3.amazonaws.com/bucket/foo +- s3::https://s3-eu-west-1.amazonaws.com/bucket/foo +- bucket.s3.amazonaws.com/foo +- bucket.s3-eu-west-1.amazonaws.com/foo/bar + diff --git a/vendor/github.com/hashicorp/go-getter/client.go b/vendor/github.com/hashicorp/go-getter/client.go index 5a039dbacb67..876812a0a268 100644 --- a/vendor/github.com/hashicorp/go-getter/client.go +++ b/vendor/github.com/hashicorp/go-getter/client.go @@ -153,7 +153,7 @@ func (c *Client) Get() error { // We don't appear to... but is it part of the filename? matchingLen := 0 for k, _ := range decompressors { - if strings.HasSuffix(u.Path, k) && len(k) > matchingLen { + if strings.HasSuffix(u.Path, "."+k) && len(k) > matchingLen { archiveV = k matchingLen = len(k) } @@ -222,13 +222,18 @@ func (c *Client) Get() error { checksumValue = b } - // For now, any means file. In the future, we'll ask the getter - // what it thinks it is. if mode == ClientModeAny { - mode = ClientModeFile + // Ask the getter which client mode to use + mode, err = g.ClientMode(u) + if err != nil { + return err + } - // Destination is the base name of the URL path - dst = filepath.Join(dst, filepath.Base(u.Path)) + // Destination is the base name of the URL path in "any" mode when + // a file source is detected. + if mode == ClientModeFile { + dst = filepath.Join(dst, filepath.Base(u.Path)) + } } // If we're not downloading a directory, then just download the file diff --git a/vendor/github.com/hashicorp/go-getter/detect_file.go b/vendor/github.com/hashicorp/go-getter/detect_file.go index ddeedc1dd08d..756ea43f83aa 100644 --- a/vendor/github.com/hashicorp/go-getter/detect_file.go +++ b/vendor/github.com/hashicorp/go-getter/detect_file.go @@ -36,6 +36,13 @@ func (d *FileDetector) Detect(src, pwd string) (string, bool, error) { if err != nil { return "", true, err } + + // The symlink itself might be a relative path, so we have to + // resolve this to have a correctly rooted URL. + pwd, err = filepath.Abs(pwd) + if err != nil { + return "", true, err + } } } diff --git a/vendor/github.com/hashicorp/go-getter/get.go b/vendor/github.com/hashicorp/go-getter/get.go index 75d813cdb218..c3236f553e43 100644 --- a/vendor/github.com/hashicorp/go-getter/get.go +++ b/vendor/github.com/hashicorp/go-getter/get.go @@ -35,6 +35,10 @@ type Getter interface { // reference a single file. If possible, the Getter should check if // the remote end contains the same file and no-op this operation. GetFile(string, *url.URL) error + + // ClientMode returns the mode based on the given URL. This is used to + // allow clients to let the getters decide which mode to use. + ClientMode(*url.URL) (ClientMode, error) } // Getters is the mapping of scheme to the Getter implementation that will @@ -46,7 +50,7 @@ var Getters map[string]Getter var forcedRegexp = regexp.MustCompile(`^([A-Za-z0-9]+)::(.+)$`) func init() { - httpGetter := new(HttpGetter) + httpGetter := &HttpGetter{Netrc: true} Getters = map[string]Getter{ "file": new(FileGetter), diff --git a/vendor/github.com/hashicorp/go-getter/get_file.go b/vendor/github.com/hashicorp/go-getter/get_file.go index 341cd0ed8f26..e5d2d61d7d87 100644 --- a/vendor/github.com/hashicorp/go-getter/get_file.go +++ b/vendor/github.com/hashicorp/go-getter/get_file.go @@ -1,8 +1,32 @@ package getter +import ( + "net/url" + "os" +) + // FileGetter is a Getter implementation that will download a module from // a file scheme. type FileGetter struct { // Copy, if set to true, will copy data instead of using a symlink Copy bool } + +func (g *FileGetter) ClientMode(u *url.URL) (ClientMode, error) { + path := u.Path + if u.RawPath != "" { + path = u.RawPath + } + + fi, err := os.Stat(path) + if err != nil { + return 0, err + } + + // Check if the source is a directory. + if fi.IsDir() { + return ClientModeDir, nil + } + + return ClientModeFile, nil +} diff --git a/vendor/github.com/hashicorp/go-getter/get_file_unix.go b/vendor/github.com/hashicorp/go-getter/get_file_unix.go index c69d34ad3b79..c89a2d5a4385 100644 --- a/vendor/github.com/hashicorp/go-getter/get_file_unix.go +++ b/vendor/github.com/hashicorp/go-getter/get_file_unix.go @@ -11,8 +11,13 @@ import ( ) func (g *FileGetter) Get(dst string, u *url.URL) error { + path := u.Path + if u.RawPath != "" { + path = u.RawPath + } + // The source path must exist and be a directory to be usable. - if fi, err := os.Stat(u.Path); err != nil { + if fi, err := os.Stat(path); err != nil { return fmt.Errorf("source path error: %s", err) } else if !fi.IsDir() { return fmt.Errorf("source path must be a directory") @@ -41,12 +46,17 @@ func (g *FileGetter) Get(dst string, u *url.URL) error { return err } - return os.Symlink(u.Path, dst) + return os.Symlink(path, dst) } func (g *FileGetter) GetFile(dst string, u *url.URL) error { - // The source path must exist and be a directory to be usable. - if fi, err := os.Stat(u.Path); err != nil { + path := u.Path + if u.RawPath != "" { + path = u.RawPath + } + + // The source path must exist and be a file to be usable. + if fi, err := os.Stat(path); err != nil { return fmt.Errorf("source path error: %s", err) } else if fi.IsDir() { return fmt.Errorf("source path must be a file") @@ -72,11 +82,11 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error { // If we're not copying, just symlink and we're done if !g.Copy { - return os.Symlink(u.Path, dst) + return os.Symlink(path, dst) } // Copy - srcF, err := os.Open(u.Path) + srcF, err := os.Open(path) if err != nil { return err } diff --git a/vendor/github.com/hashicorp/go-getter/get_file_windows.go b/vendor/github.com/hashicorp/go-getter/get_file_windows.go index cc50ae7e1fe2..f87ed0a0becb 100644 --- a/vendor/github.com/hashicorp/go-getter/get_file_windows.go +++ b/vendor/github.com/hashicorp/go-getter/get_file_windows.go @@ -13,8 +13,13 @@ import ( ) func (g *FileGetter) Get(dst string, u *url.URL) error { + path := u.Path + if u.RawPath != "" { + path = u.RawPath + } + // The source path must exist and be a directory to be usable. - if fi, err := os.Stat(u.Path); err != nil { + if fi, err := os.Stat(path); err != nil { return fmt.Errorf("source path error: %s", err) } else if !fi.IsDir() { return fmt.Errorf("source path must be a directory") @@ -43,7 +48,7 @@ func (g *FileGetter) Get(dst string, u *url.URL) error { return err } - sourcePath := toBackslash(u.Path) + sourcePath := toBackslash(path) // Use mklink to create a junction point output, err := exec.Command("cmd", "/c", "mklink", "/J", dst, sourcePath).CombinedOutput() @@ -55,8 +60,13 @@ func (g *FileGetter) Get(dst string, u *url.URL) error { } func (g *FileGetter) GetFile(dst string, u *url.URL) error { + path := u.Path + if u.RawPath != "" { + path = u.RawPath + } + // The source path must exist and be a directory to be usable. - if fi, err := os.Stat(u.Path); err != nil { + if fi, err := os.Stat(path); err != nil { return fmt.Errorf("source path error: %s", err) } else if fi.IsDir() { return fmt.Errorf("source path must be a file") @@ -82,11 +92,11 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error { // If we're not copying, just symlink and we're done if !g.Copy { - return os.Symlink(u.Path, dst) + return os.Symlink(path, dst) } // Copy - srcF, err := os.Open(u.Path) + srcF, err := os.Open(path) if err != nil { return err } diff --git a/vendor/github.com/hashicorp/go-getter/get_git.go b/vendor/github.com/hashicorp/go-getter/get_git.go index 1bf0dc7e6492..07281398322b 100644 --- a/vendor/github.com/hashicorp/go-getter/get_git.go +++ b/vendor/github.com/hashicorp/go-getter/get_git.go @@ -1,58 +1,105 @@ package getter import ( + "encoding/base64" "fmt" "io/ioutil" "net/url" "os" "os/exec" "path/filepath" + "strings" urlhelper "github.com/hashicorp/go-getter/helper/url" + "github.com/hashicorp/go-version" ) // GitGetter is a Getter implementation that will download a module from // a git repository. type GitGetter struct{} +func (g *GitGetter) ClientMode(_ *url.URL) (ClientMode, error) { + return ClientModeDir, nil +} + func (g *GitGetter) Get(dst string, u *url.URL) error { if _, err := exec.LookPath("git"); err != nil { return fmt.Errorf("git must be available and on the PATH") } // Extract some query parameters we use - var ref string + var ref, sshKey string q := u.Query() if len(q) > 0 { ref = q.Get("ref") q.Del("ref") + sshKey = q.Get("sshkey") + q.Del("sshkey") + // Copy the URL var newU url.URL = *u u = &newU u.RawQuery = q.Encode() } - // First: clone or update the repository + var sshKeyFile string + if sshKey != "" { + // Check that the git version is sufficiently new. + if err := checkGitVersion("2.3"); err != nil { + return fmt.Errorf("Error using ssh key: %v", err) + } + + // We have an SSH key - decode it. + raw, err := base64.StdEncoding.DecodeString(sshKey) + if err != nil { + return err + } + + // Create a temp file for the key and ensure it is removed. + fh, err := ioutil.TempFile("", "go-getter") + if err != nil { + return err + } + sshKeyFile = fh.Name() + defer os.Remove(sshKeyFile) + + // Set the permissions prior to writing the key material. + if err := os.Chmod(sshKeyFile, 0600); err != nil { + return err + } + + // Write the raw key into the temp file. + _, err = fh.Write(raw) + fh.Close() + if err != nil { + return err + } + } + + // Clone or update the repository _, err := os.Stat(dst) if err != nil && !os.IsNotExist(err) { return err } if err == nil { - err = g.update(dst, ref) + err = g.update(dst, sshKeyFile, ref) } else { - err = g.clone(dst, u) + err = g.clone(dst, sshKeyFile, u) } if err != nil { return err } // Next: check out the proper tag/branch if it is specified, and checkout - if ref == "" { - return nil + if ref != "" { + if err := g.checkout(dst, ref); err != nil { + return err + } } - return g.checkout(dst, ref) + // Lastly, download any/all submodules. + return g.fetchSubmodules(dst, sshKeyFile) } // GetFile for Git doesn't support updating at this time. It will download @@ -92,16 +139,18 @@ func (g *GitGetter) checkout(dst string, ref string) error { return getRunCommand(cmd) } -func (g *GitGetter) clone(dst string, u *url.URL) error { +func (g *GitGetter) clone(dst, sshKeyFile string, u *url.URL) error { cmd := exec.Command("git", "clone", u.String(), dst) + setupGitEnv(cmd, sshKeyFile) return getRunCommand(cmd) } -func (g *GitGetter) update(dst string, ref string) error { +func (g *GitGetter) update(dst, sshKeyFile, ref string) error { // Determine if we're a branch. If we're NOT a branch, then we just // switch to master prior to checking out cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref) cmd.Dir = dst + if getRunCommand(cmd) != nil { // Not a branch, switch to master. This will also catch non-existent // branches, in which case we want to switch to master and then @@ -116,5 +165,61 @@ func (g *GitGetter) update(dst string, ref string) error { cmd = exec.Command("git", "pull", "--ff-only") cmd.Dir = dst + setupGitEnv(cmd, sshKeyFile) return getRunCommand(cmd) } + +// fetchSubmodules downloads any configured submodules recursively. +func (g *GitGetter) fetchSubmodules(dst, sshKeyFile string) error { + cmd := exec.Command("git", "submodule", "update", "--init", "--recursive") + cmd.Dir = dst + setupGitEnv(cmd, sshKeyFile) + return getRunCommand(cmd) +} + +// setupGitEnv sets up the environment for the given command. This is used to +// pass configuration data to git and ssh and enables advanced cloning methods. +func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) { + var sshOpts []string + + if sshKeyFile != "" { + // We have an SSH key temp file configured, tell ssh about this. + sshOpts = append(sshOpts, "-i", sshKeyFile) + } + + cmd.Env = append(os.Environ(), + // Set the ssh command to use for clones. + "GIT_SSH_COMMAND=ssh "+strings.Join(sshOpts, " "), + ) +} + +// checkGitVersion is used to check the version of git installed on the system +// against a known minimum version. Returns an error if the installed version +// is older than the given minimum. +func checkGitVersion(min string) error { + want, err := version.NewVersion(min) + if err != nil { + return err + } + + out, err := exec.Command("git", "version").Output() + if err != nil { + return err + } + + fields := strings.Fields(string(out)) + if len(fields) != 3 { + return fmt.Errorf("Unexpected 'git version' output: %q", string(out)) + } + + have, err := version.NewVersion(fields[2]) + if err != nil { + return err + } + + if have.LessThan(want) { + return fmt.Errorf("Required git version = %s, have %s", want, have) + } + + return nil +} diff --git a/vendor/github.com/hashicorp/go-getter/get_hg.go b/vendor/github.com/hashicorp/go-getter/get_hg.go index 542bef1f9c82..820bdd488e1c 100644 --- a/vendor/github.com/hashicorp/go-getter/get_hg.go +++ b/vendor/github.com/hashicorp/go-getter/get_hg.go @@ -16,6 +16,10 @@ import ( // a Mercurial repository. type HgGetter struct{} +func (g *HgGetter) ClientMode(_ *url.URL) (ClientMode, error) { + return ClientModeDir, nil +} + func (g *HgGetter) Get(dst string, u *url.URL) error { if _, err := exec.LookPath("hg"); err != nil { return fmt.Errorf("hg must be available and on the PATH") diff --git a/vendor/github.com/hashicorp/go-getter/get_http.go b/vendor/github.com/hashicorp/go-getter/get_http.go index 47b3e7e42133..3c020343eead 100644 --- a/vendor/github.com/hashicorp/go-getter/get_http.go +++ b/vendor/github.com/hashicorp/go-getter/get_http.go @@ -32,13 +32,31 @@ import ( // The source URL, whether from the header or meta tag, must be a fully // formed URL. The shorthand syntax of "github.com/foo/bar" or relative // paths are not allowed. -type HttpGetter struct{} +type HttpGetter struct { + // Netrc, if true, will lookup and use auth information found + // in the user's netrc file if available. + Netrc bool +} + +func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) { + if strings.HasSuffix(u.Path, "/") { + return ClientModeDir, nil + } + return ClientModeFile, nil +} func (g *HttpGetter) Get(dst string, u *url.URL) error { // Copy the URL so we can modify it var newU url.URL = *u u = &newU + if g.Netrc { + // Add auth from netrc if we can + if err := addAuthFromNetrc(u); err != nil { + return err + } + } + // Add terraform-get to the parameter. q := u.Query() q.Add("terraform-get", "1") diff --git a/vendor/github.com/hashicorp/go-getter/get_mock.go b/vendor/github.com/hashicorp/go-getter/get_mock.go index a7d3d3052013..882e694dce3b 100644 --- a/vendor/github.com/hashicorp/go-getter/get_mock.go +++ b/vendor/github.com/hashicorp/go-getter/get_mock.go @@ -43,3 +43,10 @@ func (g *MockGetter) GetFile(dst string, u *url.URL) error { } return g.GetFileErr } + +func (g *MockGetter) ClientMode(u *url.URL) (ClientMode, error) { + if l := len(u.Path); l > 0 && u.Path[l-1:] == "/" { + return ClientModeDir, nil + } + return ClientModeFile, nil +} diff --git a/vendor/github.com/hashicorp/go-getter/get_s3.go b/vendor/github.com/hashicorp/go-getter/get_s3.go index bcfcbfc90323..d3bffeb17375 100644 --- a/vendor/github.com/hashicorp/go-getter/get_s3.go +++ b/vendor/github.com/hashicorp/go-getter/get_s3.go @@ -20,6 +20,45 @@ import ( // a S3 bucket. type S3Getter struct{} +func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) { + // Parse URL + region, bucket, path, _, creds, err := g.parseUrl(u) + if err != nil { + return 0, err + } + + // Create client config + config := g.getAWSConfig(region, creds) + sess := session.New(config) + client := s3.New(sess) + + // List the object(s) at the given prefix + req := &s3.ListObjectsInput{ + Bucket: aws.String(bucket), + Prefix: aws.String(path), + } + resp, err := client.ListObjects(req) + if err != nil { + return 0, err + } + + for _, o := range resp.Contents { + // Use file mode on exact match. + if *o.Key == path { + return ClientModeFile, nil + } + + // Use dir mode if child keys are found. + if strings.HasPrefix(*o.Key, path+"/") { + return ClientModeDir, nil + } + } + + // There was no match, so just return file mode. The download is going + // to fail but we will let S3 return the proper error later. + return ClientModeFile, nil +} + func (g *S3Getter) Get(dst string, u *url.URL) error { // Parse URL region, bucket, path, _, creds, err := g.parseUrl(u) diff --git a/vendor/github.com/hashicorp/go-getter/netrc.go b/vendor/github.com/hashicorp/go-getter/netrc.go new file mode 100644 index 000000000000..c7f6a3fb3fb0 --- /dev/null +++ b/vendor/github.com/hashicorp/go-getter/netrc.go @@ -0,0 +1,67 @@ +package getter + +import ( + "fmt" + "net/url" + "os" + "runtime" + + "github.com/bgentry/go-netrc/netrc" + "github.com/mitchellh/go-homedir" +) + +// addAuthFromNetrc adds auth information to the URL from the user's +// netrc file if it can be found. This will only add the auth info +// if the URL doesn't already have auth info specified and the +// the username is blank. +func addAuthFromNetrc(u *url.URL) error { + // If the URL already has auth information, do nothing + if u.User != nil && u.User.Username() != "" { + return nil + } + + // Get the netrc file path + path := os.Getenv("NETRC") + if path == "" { + filename := ".netrc" + if runtime.GOOS == "windows" { + filename = "_netrc" + } + + var err error + path, err = homedir.Expand("~/" + filename) + if err != nil { + return err + } + } + + // If the file is not a file, then do nothing + if fi, err := os.Stat(path); err != nil { + // File doesn't exist, do nothing + if os.IsNotExist(err) { + return nil + } + + // Some other error! + return err + } else if fi.IsDir() { + // File is directory, ignore + return nil + } + + // Load up the netrc file + net, err := netrc.ParseFile(path) + if err != nil { + return fmt.Errorf("Error parsing netrc file at %q: %s", path, err) + } + + machine := net.FindMachine(u.Host) + if machine == nil { + // Machine not found, no problem + return nil + } + + // Set the user info + u.User = url.UserPassword(machine.Login, machine.Password) + return nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 1f13bf03b6c0..7636e0f397ab 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -200,6 +200,12 @@ "path": "github.com/aws/aws-sdk-go/service/s3", "revision": "80dd4951fdb3f711d31843b8d87871130ef2df67" }, + { + "checksumSHA1": "nqw2Qn5xUklssHTubS5HDvEL9L4=", + "path": "github.com/bgentry/go-netrc/netrc", + "revision": "9fd32a8b3d3d3f9d43c341bfe098430e07609480", + "revisionTime": "2014-04-22T17:41:19Z" + }, { "path": "github.com/bgentry/speakeasy", "revision": "36e9cfdd690967f4f690c6edcc9ffacd006014a0" @@ -354,14 +360,14 @@ { "checksumSHA1": "iP5slJJPRZUm0rfdII8OiATAACA=", "path": "github.com/docker/docker/pkg/idtools", - "revision": "52debcd58ac91bf68503ce60561536911b74ff05", - "revisionTime": "2016-05-20T15:17:10Z" + "revision": "02caa73df411debed164f520a6a1304778f8b88c", + "revisionTime": "2016-05-28T10:48:36Z" }, { "checksumSHA1": "iP5slJJPRZUm0rfdII8OiATAACA=", "path": "github.com/docker/docker/pkg/idtools", - "revision": "02caa73df411debed164f520a6a1304778f8b88c", - "revisionTime": "2016-05-28T10:48:36Z" + "revision": "52debcd58ac91bf68503ce60561536911b74ff05", + "revisionTime": "2016-05-20T15:17:10Z" }, { "checksumSHA1": "tdhmIGUaoOMEDymMC23qTS7bt0g=", @@ -393,12 +399,6 @@ "revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5", "revisionTime": "2016-05-28T08:11:04Z" }, - { - "checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=", - "path": "github.com/docker/docker/pkg/pools", - "revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5", - "revisionTime": "2016-05-28T08:11:04Z" - }, { "checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=", "path": "github.com/docker/docker/pkg/pools", @@ -406,8 +406,8 @@ "revisionTime": "2016-05-20T15:17:10Z" }, { - "checksumSHA1": "txf3EORYff4hO6PEvwBm2lyh1MU=", - "path": "github.com/docker/docker/pkg/promise", + "checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=", + "path": "github.com/docker/docker/pkg/pools", "revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5", "revisionTime": "2016-05-28T08:11:04Z" }, @@ -417,6 +417,12 @@ "revision": "52debcd58ac91bf68503ce60561536911b74ff05", "revisionTime": "2016-05-20T15:17:10Z" }, + { + "checksumSHA1": "txf3EORYff4hO6PEvwBm2lyh1MU=", + "path": "github.com/docker/docker/pkg/promise", + "revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5", + "revisionTime": "2016-05-28T08:11:04Z" + }, { "checksumSHA1": "lThih54jzz9A4zHKEFb9SIV3Ed0=", "path": "github.com/docker/docker/pkg/random", @@ -678,8 +684,10 @@ "revisionTime": "2017-02-11T00:33:01Z" }, { + "checksumSHA1": "nsL2kI426RMuq1jw15e7igFqdIY=", "path": "github.com/hashicorp/go-getter", - "revision": "3142ddc1d627a166970ddd301bc09cb510c74edc" + "revision": "c3d66e76678dce180a7b452653472f949aedfbcd", + "revisionTime": "2017-02-07T21:55:32Z" }, { "path": "github.com/hashicorp/go-getter/helper/url", diff --git a/website/source/docs/job-specification/artifact.html.md b/website/source/docs/job-specification/artifact.html.md index bcd4ef394782..3bc170477696 100644 --- a/website/source/docs/job-specification/artifact.html.md +++ b/website/source/docs/job-specification/artifact.html.md @@ -40,9 +40,9 @@ job "docs" { } ``` -Nomad supports downloading `http`, `https`, and `S3` artifacts. If these -artifacts are archived (`zip`, `tgz`, `bz2`), they are automatically unarchived -before the starting the task. +Nomad supports downloading `http`, `https`, `git`, `hg` and `S3` artifacts. If +these artifacts are archived (`zip`, `tgz`, `bz2`), they are automatically +unarchived before the starting the task. ## `artifact` Parameters