diff --git a/builders/bower.go b/builders/bower.go index 4c6176ffb6..bd0c9a3001 100644 --- a/builders/bower.go +++ b/builders/bower.go @@ -80,8 +80,8 @@ type bowerJSONManifest struct { Version string } -func normalizeBowerComponents(parent module.ImportPath, c bowerListManifest) []module.Dependency { - var deps []module.Dependency +func normalizeBowerComponents(parent module.ImportPath, c bowerListManifest) []Imported { + var deps []Imported for _, dep := range c.Dependencies { deps = append( deps, @@ -95,13 +95,13 @@ func normalizeBowerComponents(parent module.ImportPath, c bowerListManifest) []m )...) } - return append(deps, module.Dependency{ + return append(deps, Imported{ Locator: module.Locator{ Fetcher: "bower", Project: c.PkgMeta.Name, Revision: c.PkgMeta.Version, }, - Via: []module.ImportPath{parent}, + From: parent, }) } @@ -121,7 +121,7 @@ func (builder *BowerBuilder) Analyze(m module.Module, allowUnresolved bool) ([]m return nil, errors.Wrap(err, "could not parse `bower ls --json` output") } - var depList []module.Dependency + var depList []Imported for _, dep := range output.Dependencies { depList = append( depList, diff --git a/builders/common.go b/builders/common.go index b66ea2a4b6..d7849c7389 100644 --- a/builders/common.go +++ b/builders/common.go @@ -170,15 +170,20 @@ func which(versionFlags string, cmds ...string) (string, string, error) { }) } +type Imported struct { + module.Locator + From module.ImportPath +} + // Utilities for computing import paths -func computeImportPaths(deps []module.Dependency) []module.Dependency { +func computeImportPaths(deps []Imported) []module.Dependency { pathsSet := make(map[module.Locator]map[module.ImportPathString]bool) for _, dep := range deps { _, ok := pathsSet[dep.Locator] if !ok { pathsSet[dep.Locator] = make(map[module.ImportPathString]bool) } - pathsSet[dep.Locator][dep.Via[0].String()] = true + pathsSet[dep.Locator][dep.From.String()] = true } var out []module.Dependency diff --git a/builders/composer.go b/builders/composer.go index 8d64b371bd..6eb3d77ef3 100644 --- a/builders/composer.go +++ b/builders/composer.go @@ -98,12 +98,13 @@ func (builder *ComposerBuilder) Analyze(m module.Module, allowUnresolved bool) ( if err != nil { return nil, fmt.Errorf("could not get dependency list from Composer: %s", err.Error()) } - var depList []module.Dependency - depContext := module.ImportPath{module.Locator{ + var depList []Imported + root := module.Locator{ Fetcher: "root", Project: "root", Revision: "root", - }} + } + depContext := module.ImportPath{root} lastDepth := 0 lines := strings.Split(treeOutput, "\n") for _, line := range lines { @@ -120,7 +121,7 @@ func (builder *ComposerBuilder) Analyze(m module.Module, allowUnresolved bool) ( Project: parts[0], Revision: "revision-placeholder", } - depContext = []module.Locator{locator} + depContext = module.ImportPath{root, locator} depth = 0 } else { // We're somewhere in the tree. @@ -145,9 +146,9 @@ func (builder *ComposerBuilder) Analyze(m module.Module, allowUnresolved bool) ( } else { depContext = depContext[:depth/3+1] } - depList = append(depList, module.Dependency{ + depList = append(depList, Imported{ Locator: locator, - Via: []module.ImportPath{depContext}, + From: depContext, }) lastDepth = depth } diff --git a/builders/golang.go b/builders/golang.go index b7573668fb..c67deff015 100644 --- a/builders/golang.go +++ b/builders/golang.go @@ -257,6 +257,7 @@ func (builder *GoBuilder) Build(m module.Module, force bool) error { } var goInternalPackages = map[string]bool{ + "C": true, "archive": true, "archive/tar": true, "archive/zip": true, @@ -423,30 +424,54 @@ func goImportIsInternal(pkg string) bool { if pkg == "." { return false } - if goInternalPackages[pkg] || strings.Index(pkg, "internal") != -1 { + if goInternalPackages[pkg] { return true } + // TEST: This is for packages like `crypto/internal/cipherhw` return goImportIsInternal(path.Dir(pkg)) } -// Build a dependency list given an entry point. -func getGoImports(builder *GoBuilder, m module.Module) ([]goPkg, error) { - stdout, _, err := runLogged(goLogger, m.Dir, builder.GoCmd, "list", "-f", "{{ join .Deps \"\\n\" }}") +// NOTE: we don't really need the module.Module argument, that's just a hack so I can use runLogged easily. +func getGoImportsRecurse(builder *GoBuilder, m module.Module, memo map[module.Locator][]Imported, context module.ImportPath, pkg string) ([]Imported, error) { + if pkg == "" || goImportIsInternal(pkg) { + return []Imported{}, nil + } + + stdout, _, err := runLogged(goLogger, m.Dir, builder.GoCmd, "list", "-f", "{{ join .Imports \"\\n\" }}", pkg) if err != nil { return nil, fmt.Errorf("could not trace imports: %s", err.Error()) } - var deps []goPkg - for _, line := range strings.Split(stdout, "\n") { - if line != "" { - deps = append(deps, goPkg{ - ImportPath: line, - isInternal: goImportIsInternal(line), - }) + locator := module.Locator{ + Fetcher: "go", + Project: pkg, + Revision: "", + } + var imports []Imported + for _, dep := range strings.Split(stdout, "\n") { + transitive, err := getGoImportsRecurse(builder, m, memo, append(context, locator), dep) + if err != nil { + return nil, err } + imports = append(imports, transitive...) } + imports = append(imports, Imported{ + Locator: locator, + From: context, + }) - return deps, nil + return imports, nil +} + +// Build a dependency list given an entry point. +func getGoImports(builder *GoBuilder, m module.Module) ([]Imported, error) { + return getGoImportsRecurse( + builder, + m, + make(map[module.Locator][]Imported), + module.ImportPath{}, + m.Target, + ) } // Lockfile structs for JSON unmarshalling @@ -521,51 +546,21 @@ func findRevisionRecurse(projects map[string]string, importPath string) (string, return findRevisionRecurse(projects, path.Dir(importPath)) } -// Analyze traces imports and then looks up revisions in lockfiles -func (builder *GoBuilder) Analyze(m module.Module, allowUnresolved bool) ([]module.Dependency, error) { - goLogger.Debugf("Running Go analysis: %#v %#v", m, allowUnresolved) - - // Trace imports - traced, err := getGoImports(builder, m) - if err != nil { - return nil, fmt.Errorf("could not trace go imports: %#v", err.Error()) - } - goLogger.Debugf("Traced imports: %#v", traced) - - // Find project folder (this is an ancestor of the module folder) - projectFolder, ok, err := findGoProjectFolder(m.Dir) - if err != nil { - return nil, err - } - if !ok { - goLogger.Debugf("Could not find project folder") - - if allowUnresolved { - var deps []module.Dependency - for _, dep := range traced { - deps = append(deps, module.Dependency{ - Locator: module.Locator{ - Fetcher: "go", - Project: dep.ImportPath, - Revision: dep.Revision, - }, - Via: nil, - }) - } - return deps, err - } - return nil, errors.New("could not find project folder") - } - goLogger.Debugf("Found project folder: %#v", projectFolder) +var errNoLockfile = errors.New("could not find lockfile") +// TODO: there might actually be a more sane way of doing this: search upwards +// in the import path for every /vendor/ folder, which should accompany every +// package manifest (unless, of course, you're using legacy Godeps or another +// import path rewriting tool...) +func readLockfile(dir string) (map[string]string, error) { // If possible, read lockfiles for versions lockfileVersions := make(map[string]string) - if ok, err := hasDepManifest(projectFolder); err == nil && ok { + if ok, err := hasDepManifest(dir); err == nil && ok { goLogger.Debugf("Found Dep manifest") var lockfile depLockfile - parseLoggedWithUnmarshaller(goLogger, filepath.Join(projectFolder, "Gopkg.lock"), &lockfile, func(data []byte, v interface{}) error { + parseLoggedWithUnmarshaller(goLogger, filepath.Join(dir, "Gopkg.lock"), &lockfile, func(data []byte, v interface{}) error { _, err := toml.Decode(string(data), v) return err }) @@ -576,11 +571,11 @@ func (builder *GoBuilder) Analyze(m module.Module, allowUnresolved bool) ([]modu goLogger.Debugf("Parsed Dep manifest: %#v", lockfile.Projects) } - if ok, err := hasGlideManifest(projectFolder); err == nil && ok { + if ok, err := hasGlideManifest(dir); err == nil && ok { goLogger.Debugf("Found Glide manifest") var lockfile glideLockfile - parseLoggedWithUnmarshaller(goLogger, filepath.Join(projectFolder, "glide.lock"), &lockfile, yaml.Unmarshal) + parseLoggedWithUnmarshaller(goLogger, filepath.Join(dir, "glide.lock"), &lockfile, yaml.Unmarshal) for _, dependency := range lockfile.Imports { lockfileVersions[dependency.Name] = dependency.Version } @@ -588,11 +583,11 @@ func (builder *GoBuilder) Analyze(m module.Module, allowUnresolved bool) ([]modu goLogger.Debugf("Parsed Glide manifest: %#v", lockfile.Imports) } - if ok, err := hasGodepManifest(projectFolder); err == nil && ok { + if ok, err := hasGodepManifest(dir); err == nil && ok { goLogger.Debugf("Found Godeps manifest") var lockfile godepLockfile - parseLogged(goLogger, filepath.Join(projectFolder, "Godeps", "Godeps.json"), &lockfile) + parseLogged(goLogger, filepath.Join(dir, "Godeps", "Godeps.json"), &lockfile) for _, dependency := range lockfile.Deps { lockfileVersions[dependency.ImportPath] = dependency.Rev } @@ -600,11 +595,11 @@ func (builder *GoBuilder) Analyze(m module.Module, allowUnresolved bool) ([]modu goLogger.Debugf("Parsed Godeps manifest: %#v", lockfile.Deps) } - if ok, err := hasGovendorManifest(projectFolder); err == nil && ok { + if ok, err := hasGovendorManifest(dir); err == nil && ok { goLogger.Debugf("Found Govendor manifest") var lockfile govendorLockfile - parseLogged(goLogger, filepath.Join(projectFolder, "vendor", "vendor.json"), &lockfile) + parseLogged(goLogger, filepath.Join(dir, "vendor", "vendor.json"), &lockfile) for _, dependency := range lockfile.Package { lockfileVersions[dependency.Path] = dependency.Revision } @@ -612,78 +607,103 @@ func (builder *GoBuilder) Analyze(m module.Module, allowUnresolved bool) ([]modu goLogger.Debugf("Parsed Godeps manifest: %#v", lockfile.Package) } - if ok, err := hasVndrManifest(projectFolder); err == nil && ok { + if ok, err := hasVndrManifest(dir); err == nil && ok { goLogger.Debugf("Found Vndr manifest") - parseGPMLockfile(&lockfileVersions, projectFolder, "vendor.conf") + parseGPMLockfile(&lockfileVersions, dir, "vendor.conf") goLogger.Debugf("Parsed Vndr manifest: %#v", lockfileVersions) } // gdm rolls its own format as well - if ok, err := hasGdmManifest(projectFolder); err == nil && ok { + if ok, err := hasGdmManifest(dir); err == nil && ok { goLogger.Debugf("Found Gdm manifest") - parseGPMLockfile(&lockfileVersions, projectFolder, "Godeps") + parseGPMLockfile(&lockfileVersions, dir, "Godeps") goLogger.Debugf("Parsed Gndr manifest: %#v", lockfileVersions) } - goLogger.Debugf("Parsed lockfiles: %#v", lockfileVersions) - depSet := make(map[goPkg]bool) - projectImports := strings.TrimPrefix(projectFolder, filepath.Join(os.Getenv("GOPATH"), "src")+string(filepath.Separator)) - for _, dep := range traced { - goLogger.Debugf("Resolving raw import: %s", dep.ImportPath) + return lockfileVersions, nil +} - // Strip out `/vendor/` weirdness in import paths. - const vendorPrefix = "/vendor/" - vendoredPathSections := strings.Split(dep.ImportPath, vendorPrefix) - importPath := vendoredPathSections[len(vendoredPathSections)-1] +func goImportToDir(pkg string) string { + return filepath.Join(os.Getenv("GOPATH"), "src", pkg) +} + +// Analyze traces imports and then looks up revisions in lockfiles +func (builder *GoBuilder) Analyze(m module.Module, allowUnresolved bool) ([]module.Dependency, error) { + goLogger.Debugf("Running Go analysis: %#v %#v", m, allowUnresolved) + + // Trace imports + traced, err := getGoImports(builder, m) + if err != nil { + return nil, fmt.Errorf("could not trace go imports: %#v", err.Error()) + } + goLogger.Debugf("Traced imports: %#v", traced) + + // Resolve the version of each import by finding its appropriate lockfile and reading it. + for i, pkg := range traced { + // Get the project folder + packageDir := goImportToDir(pkg.Project) + projectFolder, ok, err := findGoProjectFolder(packageDir) + if err != nil { + return nil, err + } + if !ok { + if allowUnresolved { + goLogger.Warningf("Could not find lockfile for package %s", pkg.Project) + continue + } else { + return nil, err + } + } + goLogger.Debugf("Found project folder: %#v", projectFolder) + + // Get the lockfile + lockfile, err := readLockfile(projectFolder) + if err != nil { + if allowUnresolved { + goLogger.Warningf("Could not find lockfile for package %s", pkg.Project) + continue + } else { + return nil, err + } + } + // Process the import path + projectGopath := strings.TrimPrefix(projectFolder, filepath.Join(os.Getenv("GOPATH"), "src")+string(filepath.Separator)) // Work around awful Go compiler hack: see https://github.com/golang/go/issues/16333 - if strings.HasPrefix(dep.ImportPath, "vendor/golang_org") { + if strings.HasPrefix(pkg.Project, "vendor/golang_org") { continue } - + // Strip `/vendor/` folders. + const vendorPrefix = "/vendor/" + vendoredPathSections := strings.Split(pkg.Project, vendorPrefix) + importPath := vendoredPathSections[len(vendoredPathSections)-1] goLogger.Debugf("Resolving import: %s", importPath) - // Get revisions (often these are scoped to repository, not package) - project, err := findRevision(lockfileVersions, importPath) + importedProject, err := findRevision(lockfile, importPath) if err != nil { - if dep.isInternal || - strings.Index(importPath, projectImports) == 0 || - dep.ImportPath == "C" { - goLogger.Debugf("Did not resolve import: %#v", dep) - depSet[goPkg{ImportPath: importPath, Revision: "", isInternal: dep.isInternal}] = true + if strings.Index(importPath, projectGopath) == 0 { + goLogger.Debugf("Did not resolve import: %#v", pkg.Project) + traced[i].Revision = lockfile[importedProject] } else if allowUnresolved { - goLogger.Warningf("Could not resolve import: %#v", dep) - depSet[goPkg{ImportPath: importPath, Revision: "", isInternal: dep.isInternal}] = true + goLogger.Warningf("Could not resolve import: %#v", pkg.Project) + traced[i].Revision = lockfile[importedProject] } else { - goLogger.Errorf("Could not resolve import: %#v", dep) + goLogger.Errorf("Could not resolve import: %#v", pkg.Project) goLogger.Debugf("Project folder: %#v", projectFolder) goLogger.Debugf("$GOPATH: %#v", os.Getenv("GOPATH")) - goLogger.Debugf("Project folder relative to $GOPATH: %#v", projectImports) - goLogger.Debugf("Lockfile versions: %#v", lockfileVersions) + goLogger.Debugf("Project folder relative to $GOPATH: %#v", projectGopath) + goLogger.Debugf("Lockfile versions: %#v", lockfile) return nil, fmt.Errorf("could not resolve import: %#v", importPath) } } else { - depSet[goPkg{ImportPath: project, Revision: lockfileVersions[project], isInternal: dep.isInternal}] = true - } - } - - var deps []module.Dependency - for dep, ok := range depSet { - if ok { - deps = append(deps, module.Dependency{ - Locator: module.Locator{ - Fetcher: "go", - Project: dep.ImportPath, - Revision: dep.Revision, - }, - Via: nil, - }) + traced[i].Revision = lockfile[importedProject] } } + deps := computeImportPaths(traced) goLogger.Debugf("Done running Go analysis: %#v", deps) return deps, nil