diff --git a/internal/gps/result.go b/internal/gps/result.go index eb9d6ad0e1..7e3dc348b3 100644 --- a/internal/gps/result.go +++ b/internal/gps/result.go @@ -10,6 +10,10 @@ import ( "path/filepath" ) +// NumDepTreeWorkers determines the number of +// WriteDepTree worker goroutines +var NumDepTreeWorkers = 2 + // A Solution is returned by a solver run. It is mostly just a Lock, with some // additional methods that report information about the solve run. type Solution interface { @@ -42,6 +46,11 @@ type solution struct { solv Solver } +type done struct { + root ProjectRoot + err error +} + // WriteDepTree takes a basedir and a Lock, and exports all the projects // listed in the lock to the appropriate target location within the basedir. // @@ -61,24 +70,72 @@ func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool) error { return err } - // TODO(sdboyer) parallelize - for _, p := range l.Projects() { - to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot))) + job := depTreeJob{sm, l.Projects(), basedir, sv, NumDepTreeWorkers} - err = sm.ExportProject(p.Ident(), p.Version(), to) - if err != nil { - removeAll(basedir) - return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err) - } - if sv { - filepath.Walk(to, stripVendor) + return job.exec() +} + +type depTreeJob struct { + sm SourceManager + projects []LockedProject + basedir string + stripVendor bool + workerCount int +} + +func (dj depTreeJob) exec() error { + dc := dj.spawnDepTreeWorkers() + + // TODO(tonto) refactor to correct behaviour + for i := 0; i < len(dj.projects); i++ { + d := <-dc + if d.err != nil { + removeAll(dj.basedir) + return d.err } - // TODO(sdboyer) dump version metadata file } return nil } +func (dj depTreeJob) spawnDepTreeWorkers() chan done { + dc := make(chan done) + wc := make(chan LockedProject) + + // TODO(tonto) brake down further if go ahead received + for i := 0; i < dj.workerCount; i++ { + go func() { + for p := range wc { + root := p.Ident().ProjectRoot + to := filepath.FromSlash(filepath.Join(dj.basedir, string(p.Ident().ProjectRoot))) + + err := dj.sm.ExportProject(p.Ident(), p.Version(), to) + if err != nil { + // doneChan <- fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err) + dc <- done{root, err} + continue + } + + if dj.stripVendor { + filepath.Walk(to, stripVendor) + } + + dc <- done{root, nil} + } + // TODO(sdboyer) dump version metadata file + }() + } + + go func() { + for _, p := range dj.projects { + wc <- p + } + close(wc) + }() + + return dc +} + func (r solution) Projects() []LockedProject { return r.p } diff --git a/internal/gps/result_test.go b/internal/gps/result_test.go index cc96a8e2cb..1bb59db0d8 100644 --- a/internal/gps/result_test.go +++ b/internal/gps/result_test.go @@ -114,8 +114,40 @@ func testWriteDepTree(t *testing.T) { func BenchmarkCreateVendorTree(b *testing.B) { // We're fs-bound here, so restrict to single parallelism b.SetParallelism(1) + NumDepTreeWorkers = 1 r := basicResult + benchmarkCreateVendorTree(b, r) +} + +func BenchmarkCreateVendorTreeParallel(b *testing.B) { + r := solution{ + att: 1, + p: []LockedProject{ + pa2lp(atom{ + id: pi("github.com/sdboyer/testrepo"), + v: NewBranch("master").Pair(Revision("4d59fb584b15a94d7401e356d2875c472d76ef45")), + }, nil), + pa2lp(atom{ + id: pi("github.com/Masterminds/VCSTestRepo"), + v: NewVersion("1.0.0").Pair(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")), + }, nil), + pa2lp(atom{ + id: pi("launchpad.net/govcstestbzrrepo"), + v: NewVersion("1.0.0").Pair(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68")), + }, nil), + pa2lp(atom{ + id: pi("bitbucket.org/sdboyer/withbm"), + v: NewVersion("v1.0.0").Pair(Revision("aa110802a0c64195d0a6c375c9f66668827c90b4")), + }, nil), + }, + } + r.analyzerInfo = (naiveAnalyzer{}).Info() + + benchmarkCreateVendorTree(b, r) +} + +func benchmarkCreateVendorTree(b *testing.B, r solution) { tmp := path.Join(os.TempDir(), "vsolvtest") clean := true