Skip to content

Commit

Permalink
cmd/go: add preliminary support for vendor directories
Browse files Browse the repository at this point in the history
When GO15VENDOREXPERIMENT=1 is in the environment,
this CL changes the resolution of import paths according to
the Go 1.5 vendor proposal:

	If there is a source directory d/vendor, then,
	when compiling a source file within the subtree rooted at d,
	import "p" is interpreted as import "d/vendor/p" if that exists.

	When there are multiple possible resolutions,
	the most specific (longest) path wins.

	The short form must always be used: no import path can
	contain “/vendor/” explicitly.

	Import comments are ignored in vendored packages.

The goal of these changes is to allow authors to vendor (copy) external
packages into their source trees without any modifications to the code.
This functionality has been achieved in tools like godep, nut, and gb by
requiring GOPATH manipulation. This alternate directory-based approach
eliminates the need for GOPATH manipulation and in keeping with the
go command's use of directory layout-based configuration.

The flag allows experimentation with these vendoring semantics once
Go 1.5 is released, without forcing them on by default. If the experiment
is deemed a success, the flag will default to true in Go 1.6 and then be
removed in Go 1.7.

For more details, see the original proposal by Keith Rarick at
https://groups.google.com/d/msg/golang-dev/74zjMON9glU/dGhnoi2IMzsJ.

Change-Id: I2c6527e777d14ac6dc43c53e4b3ff24f3279216e
Reviewed-on: https://go-review.googlesource.com/10923
Reviewed-by: Andrew Gerrand <adg@golang.org>
  • Loading branch information
rsc committed Jun 19, 2015
1 parent 7bcc6a1 commit 183cc0c
Show file tree
Hide file tree
Showing 17 changed files with 403 additions and 11 deletions.
8 changes: 8 additions & 0 deletions src/cmd/go/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -2135,6 +2135,14 @@ func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool,
gcargs = append(gcargs, "-buildid", p.buildID)
}

for _, path := range p.Imports {
if i := strings.LastIndex(path, "/vendor/"); i >= 0 {
gcargs = append(gcargs, "-importmap", path[i+len("/vendor/"):]+"="+path)
} else if strings.HasPrefix(path, "vendor/") {
gcargs = append(gcargs, "-importmap", path[len("vendor/"):]+"="+path)
}
}

args := []interface{}{buildToolExec, tool("compile"), "-o", ofile, "-trimpath", b.work, buildGcflags, gcargs, "-D", p.localPrefix, importArgs}
if ofile == archive {
args = append(args, "-pack")
Expand Down
219 changes: 213 additions & 6 deletions src/cmd/go/pkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,12 @@ func reloadPackage(arg string, stk *importStack) *Package {
return loadPackage(arg, stk)
}

// The Go 1.5 vendoring experiment is enabled by setting GO15VENDOREXPERIMENT=1.
// The variable is obnoxiously long so that years from now when people find it in
// their profiles and wonder what it does, there is some chance that a web search
// might answer the question.
var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1"

// dirToImportPath returns the pseudo-import path we use for a package
// outside the Go path. It begins with _/ and then contains the full path
// to the directory. If the package lives in c:\home\gopher\my\pkg then
Expand All @@ -239,23 +245,33 @@ func makeImportValid(r rune) rune {
// but possibly a local import path (an absolute file system path or one beginning
// with ./ or ../). A local relative path is interpreted relative to srcDir.
// It returns a *Package describing the package found in that directory.
func loadImport(path string, srcDir string, stk *importStack, importPos []token.Position) *Package {
func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position) *Package {
stk.push(path)
defer stk.pop()

// Determine canonical identifier for this package.
// For a local import the identifier is the pseudo-import path
// we create from the full directory to the package.
// Otherwise it is the usual import path.
// For vendored imports, it is the expanded form.
importPath := path
origPath := path
isLocal := build.IsLocalImport(path)
var vendorSearch []string
if isLocal {
importPath = dirToImportPath(filepath.Join(srcDir, path))
} else {
path, vendorSearch = vendoredImportPath(parent, path)
importPath = path
}

if p := packageCache[importPath]; p != nil {
if perr := disallowInternal(srcDir, p, stk); perr != p {
return perr
}
if perr := disallowVendor(srcDir, origPath, p, stk); perr != p {
return perr
}
return reusePackage(p, stk)
}

Expand All @@ -271,11 +287,33 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token.
// TODO: After Go 1, decide when to pass build.AllowBinary here.
// See issue 3268 for mistakes to avoid.
bp, err := buildContext.Import(path, srcDir, build.ImportComment)

// If we got an error from go/build about package not found,
// it contains the directories from $GOROOT and $GOPATH that
// were searched. Add to that message the vendor directories
// that were searched.
if err != nil && len(vendorSearch) > 0 {
// NOTE(rsc): The direct text manipulation here is fairly awful,
// but it avoids defining new go/build API (an exported error type)
// late in the Go 1.5 release cycle. If this turns out to be a more general
// problem we could define a real error type when the decision can be
// considered more carefully.
text := err.Error()
if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") {
old := strings.SplitAfter(text, "\n")
lines := []string{old[0]}
for _, dir := range vendorSearch {
lines = append(lines, "\t"+dir+" (vendor tree)\n")
}
lines = append(lines, old[1:]...)
err = errors.New(strings.Join(lines, ""))
}
}
bp.ImportPath = importPath
if gobin != "" {
bp.BinDir = gobin
}
if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path {
if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && (!go15VendorExperiment || !strings.Contains(path, "/vendor/")) {
err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment)
}
p.load(stk, bp, err)
Expand All @@ -288,10 +326,81 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token.
if perr := disallowInternal(srcDir, p, stk); perr != p {
return perr
}
if perr := disallowVendor(srcDir, origPath, p, stk); perr != p {
return perr
}

return p
}

var isDirCache = map[string]bool{}

func isDir(path string) bool {
result, ok := isDirCache[path]
if ok {
return result
}

fi, err := os.Stat(path)
result = err == nil && fi.IsDir()
isDirCache[path] = result
return result
}

// vendoredImportPath returns the expansion of path when it appears in parent.
// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path,
// x/vendor/path, vendor/path, or else stay x/y/z if none of those exist.
// vendoredImportPath returns the expanded path or, if no expansion is found, the original.
// If no epxansion is found, vendoredImportPath also returns a list of vendor directories
// it searched along the way, to help prepare a useful error message should path turn
// out not to exist.
func vendoredImportPath(parent *Package, path string) (found string, searched []string) {
if parent == nil || !go15VendorExperiment {
return path, nil
}
dir := filepath.Clean(parent.Dir)
root := filepath.Clean(parent.Root)
if !strings.HasPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator {
fatalf("invalid vendoredImportPath: dir=%q root=%q separator=%q", dir, root, string(filepath.Separator))
}
vpath := "vendor/" + path
for i := len(dir); i >= len(root); i-- {
if i < len(dir) && dir[i] != filepath.Separator {
continue
}
// Note: checking for the vendor directory before checking
// for the vendor/path directory helps us hit the
// isDir cache more often. It also helps us prepare a more useful
// list of places we looked, to report when an import is not found.
if !isDir(filepath.Join(dir[:i], "vendor")) {
continue
}
targ := filepath.Join(dir[:i], vpath)
if isDir(targ) {
// We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy.
// We know the import path for parent's dir.
// We chopped off some number of path elements and
// added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path.
// Now we want to know the import path for that directory.
// Construct it by chopping the same number of path elements
// (actually the same number of bytes) from parent's import path
// and then append /vendor/path.
chopped := len(dir) - i
if chopped == len(parent.ImportPath)+1 {
// We walked up from c:\gopath\src\foo\bar
// and found c:\gopath\src\vendor\path.
// We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7).
// Use "vendor/path" without any prefix.
return vpath, nil
}
return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil
}
// Note the existence of a vendor directory in case path is not found anywhere.
searched = append(searched, targ)
}
return path, searched
}

// reusePackage reuses package p to satisfy the import at the top
// of the import stack stk. If this use causes an import loop,
// reusePackage updates p's error information to record the loop.
Expand Down Expand Up @@ -384,6 +493,101 @@ func findInternal(path string) (index int, ok bool) {
return 0, false
}

// disallowVendor checks that srcDir is allowed to import p as path.
// If the import is allowed, disallowVendor returns the original package p.
// If not, it returns a new package containing just an appropriate error.
func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package {
if !go15VendorExperiment {
return p
}

// The stack includes p.ImportPath.
// If that's the only thing on the stack, we started
// with a name given on the command line, not an
// import. Anything listed on the command line is fine.
if len(*stk) == 1 {
return p
}

if perr := disallowVendorVisibility(srcDir, p, stk); perr != p {
return perr
}

// Paths like x/vendor/y must be imported as y, never as x/vendor/y.
if i, ok := findVendor(path); ok {
perr := *p
perr.Error = &PackageError{
ImportStack: stk.copy(),
Err: "must be imported as " + path[i+len("vendor/"):],
}
perr.Incomplete = true
return &perr
}

return p
}

// disallowVendorVisibility checks that srcDir is allowed to import p.
// The rules are the same as for /internal/ except that a path ending in /vendor
// is not subject to the rules, only subdirectories of vendor.
// This allows people to have packages and commands named vendor,
// for maximal compatibility with existing source trees.
func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package {
// The stack includes p.ImportPath.
// If that's the only thing on the stack, we started
// with a name given on the command line, not an
// import. Anything listed on the command line is fine.
if len(*stk) == 1 {
return p
}

// Check for "vendor" element.
i, ok := findVendor(p.ImportPath)
if !ok {
return p
}

// Vendor is present.
// Map import path back to directory corresponding to parent of vendor.
if i > 0 {
i-- // rewind over slash in ".../vendor"
}
parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)]
if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) {
return p
}

// Vendor is present, and srcDir is outside parent's tree. Not allowed.
perr := *p
perr.Error = &PackageError{
ImportStack: stk.copy(),
Err: "use of vendored package not allowed",
}
perr.Incomplete = true
return &perr
}

// findVendor looks for the last non-terminating "vendor" path element in the given import path.
// If there isn't one, findVendor returns ok=false.
// Otherwise, findInternal returns ok=true and the index of the "vendor".
//
// Note that terminating "vendor" elements don't count: "x/vendor" is its own package,
// not the vendored copy of an import "" (the empty import path).
// This will allow people to have packages or commands named vendor.
// This may help reduce breakage, or it may just be confusing. We'll see.
func findVendor(path string) (index int, ok bool) {
// Two cases, depending on internal at start of string or not.
// The order matters: we must return the index of the final element,
// because the final one is where the effective import path starts.
switch {
case strings.Contains(path, "/vendor/"):
return strings.LastIndex(path, "/vendor/") + 1, true
case strings.HasPrefix(path, "vendor/"):
return 0, true
}
return 0, false
}

type targetDir int

const (
Expand Down Expand Up @@ -630,7 +834,7 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
if path == "C" {
continue
}
p1 := loadImport(path, p.Dir, stk, p.build.ImportPos[path])
p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path])
if p1.Name == "main" {
p.Error = &PackageError{
ImportStack: stk.copy(),
Expand All @@ -652,8 +856,11 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
p.Error.Pos = pos[0].String()
}
}
path = p1.ImportPath
importPaths[i] = path
}
path = p1.ImportPath
importPaths[i] = path
if i < len(p.Imports) {
p.Imports[i] = path
}
deps[path] = p1
imports = append(imports, p1)
Expand Down Expand Up @@ -1294,7 +1501,7 @@ func loadPackage(arg string, stk *importStack) *Package {
}
}

return loadImport(arg, cwd, stk, nil)
return loadImport(arg, cwd, nil, stk, nil)
}

// packages returns the packages named by the
Expand Down
12 changes: 7 additions & 5 deletions src/cmd/go/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -573,8 +573,8 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
var imports, ximports []*Package
var stk importStack
stk.push(p.ImportPath + " (test)")
for _, path := range p.TestImports {
p1 := loadImport(path, p.Dir, &stk, p.build.TestImportPos[path])
for i, path := range p.TestImports {
p1 := loadImport(path, p.Dir, p, &stk, p.build.TestImportPos[path])
if p1.Error != nil {
return nil, nil, nil, p1.Error
}
Expand All @@ -589,21 +589,23 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
}
return nil, nil, nil, err
}
p.TestImports[i] = p1.ImportPath
imports = append(imports, p1)
}
stk.pop()
stk.push(p.ImportPath + "_test")
pxtestNeedsPtest := false
for _, path := range p.XTestImports {
for i, path := range p.XTestImports {
if path == p.ImportPath {
pxtestNeedsPtest = true
continue
}
p1 := loadImport(path, p.Dir, &stk, p.build.XTestImportPos[path])
p1 := loadImport(path, p.Dir, p, &stk, p.build.XTestImportPos[path])
if p1.Error != nil {
return nil, nil, nil, p1.Error
}
ximports = append(ximports, p1)
p.XTestImports[i] = p1.ImportPath
}
stk.pop()

Expand Down Expand Up @@ -728,7 +730,7 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
if dep == ptest.ImportPath {
pmain.imports = append(pmain.imports, ptest)
} else {
p1 := loadImport(dep, "", &stk, nil)
p1 := loadImport(dep, "", nil, &stk, nil)
if p1.Error != nil {
return nil, nil, nil, p1.Error
}
Expand Down
3 changes: 3 additions & 0 deletions src/cmd/go/testdata/src/vend/bad.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package vend

import _ "r"
3 changes: 3 additions & 0 deletions src/cmd/go/testdata/src/vend/good.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package vend

import _ "p"
10 changes: 10 additions & 0 deletions src/cmd/go/testdata/src/vend/hello/hello.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package main

import (
"fmt"
"strings" // really ../vendor/strings
)

func main() {
fmt.Printf("%s\n", strings.Msg)
}
12 changes: 12 additions & 0 deletions src/cmd/go/testdata/src/vend/hello/hello_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package main

import (
"strings" // really ../vendor/strings
"testing"
)

func TestMsgInternal(t *testing.T) {
if strings.Msg != "hello, world" {
t.Fatal("unexpected msg: %v", strings.Msg)
}
}
Loading

0 comments on commit 183cc0c

Please sign in to comment.