From 443f6f046998f789ed5d5ee7fc52b2525d8bf81b Mon Sep 17 00:00:00 2001 From: mstrong Date: Sun, 1 Oct 2017 13:18:08 -0500 Subject: [PATCH] Prefetch and log output during InitializeRootManifestAndLock --- cmd/dep/root_analyzer.go | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/cmd/dep/root_analyzer.go b/cmd/dep/root_analyzer.go index c300eadad8..055164a88b 100644 --- a/cmd/dep/root_analyzer.go +++ b/cmd/dep/root_analyzer.go @@ -5,9 +5,12 @@ package main import ( + "context" "io/ioutil" "log" + "golang.org/x/sync/errgroup" + "github.com/golang/dep" fb "github.com/golang/dep/internal/feedback" "github.com/golang/dep/internal/gps" @@ -45,6 +48,13 @@ func (a *rootAnalyzer) InitializeRootManifestAndLock(dir string, pr gps.ProjectR if rootM == nil { rootM = dep.NewManifest() + + // Since we didn't find anything to import, dep's cache is empty. + // We are prefetching dependencies and logging so that the subsequent solve step + // doesn't spend a long time retrieving dependencies without feedback for the user. + if err := a.cacheDeps(pr); err != nil { + return nil, nil, err + } } if rootL == nil { rootL = &dep.Lock{} @@ -53,6 +63,58 @@ func (a *rootAnalyzer) InitializeRootManifestAndLock(dir string, pr gps.ProjectR return } +func (a *rootAnalyzer) cacheDeps(pr gps.ProjectRoot) error { + logger := a.ctx.Err + g, _ := errgroup.WithContext(context.TODO()) + concurrency := 4 + + syncDep := func(pr gps.ProjectRoot, sm gps.SourceManager) error { + if err := sm.SyncSourceFor(gps.ProjectIdentifier{ProjectRoot: pr}); err != nil { + logger.Printf("Unable to cache %s - %s", pr, err) + return err + } + logger.Printf("Cached %s", pr) + return nil + } + + deps := make(chan gps.ProjectRoot) + + for i := 0; i < concurrency; i++ { + g.Go(func() error { + for d := range deps { + err := syncDep(d, a.sm) + if err != nil { + return err + } + } + return nil + }) + } + + g.Go(func() error { + defer close(deps) + for ip := range a.directDeps { + logger.Printf("Package %q, analyzing...", ip) + pr, err := a.sm.DeduceProjectRoot(ip) + if err != nil { + return err + } + deps <- pr + } + return nil + }) + + go func() { + g.Wait() + }() + + if err := g.Wait(); err != nil { + return err + } + logger.Printf("Successfully cached all deps.") + return nil +} + func (a *rootAnalyzer) importManifestAndLock(dir string, pr gps.ProjectRoot, suppressLogs bool) (*dep.Manifest, *dep.Lock, error) { logger := a.ctx.Err if suppressLogs {