Paralellized gps.WriteDepTree

gps.WriteDepTree now writes vendor/ tree in parallel with
configurable number of worker routines

Fixed accidental err race condition

Broken down WriteDepTree to child functions in order
to receive feedback if on the right track

internal/gps/result.go

@@ -10,6 +10,10 @@ import (
 	"path/filepath"
 )
 
+// NumDepTreeWorkers determines the number of
+// WriteDepTree worker goroutines
+var NumDepTreeWorkers = 2
+
 // A Solution is returned by a solver run. It is mostly just a Lock, with some
 // additional methods that report information about the solve run.
 type Solution interface {
@@ -42,6 +46,11 @@ type solution struct {
 	solv Solver
 }
 
+type done struct {
+	root ProjectRoot
+	err  error
+}
+
 // WriteDepTree takes a basedir and a Lock, and exports all the projects
 // listed in the lock to the appropriate target location within the basedir.
 //
@@ -61,24 +70,72 @@ func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool) error {
 		return err
 	}
 
-	// TODO(sdboyer) parallelize
-	for _, p := range l.Projects() {
-		to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
+	job := depTreeJob{sm, l.Projects(), basedir, sv, NumDepTreeWorkers}
 
-		err = sm.ExportProject(p.Ident(), p.Version(), to)
-		if err != nil {
-			removeAll(basedir)
-			return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
-		}
-		if sv {
-			filepath.Walk(to, stripVendor)
+	return job.exec()
+}
+
+type depTreeJob struct {
+	sm          SourceManager
+	projects    []LockedProject
+	basedir     string
+	stripVendor bool
+	workerCount int
+}
+
+func (dj depTreeJob) exec() error {
+	dc := dj.spawnDepTreeWorkers()
+
+	// TODO(tonto) refactor to correct behaviour
+	for i := 0; i < len(dj.projects); i++ {
+		d := <-dc
+		if d.err != nil {
+			removeAll(dj.basedir)
+			return d.err
 		}
-		// TODO(sdboyer) dump version metadata file
 	}
 
 	return nil
 }
 
+func (dj depTreeJob) spawnDepTreeWorkers() chan done {
+	dc := make(chan done)
+	wc := make(chan LockedProject)
+
+	// TODO(tonto) brake down further if go ahead received
+	for i := 0; i < dj.workerCount; i++ {
+		go func() {
+			for p := range wc {
+				root := p.Ident().ProjectRoot
+				to := filepath.FromSlash(filepath.Join(dj.basedir, string(p.Ident().ProjectRoot)))
+
+				err := dj.sm.ExportProject(p.Ident(), p.Version(), to)
+				if err != nil {
+					// doneChan <- fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
+					dc <- done{root, err}
+					continue
+				}
+
+				if dj.stripVendor {
+					filepath.Walk(to, stripVendor)
+				}
+
+				dc <- done{root, nil}
+			}
+			// TODO(sdboyer) dump version metadata file
+		}()
+	}
+
+	go func() {
+		for _, p := range dj.projects {
+			wc <- p
+		}
+		close(wc)
+	}()
+
+	return dc
+}
+
 func (r solution) Projects() []LockedProject {
 	return r.p
 }

internal/gps/result_test.go

@@ -114,8 +114,40 @@ func testWriteDepTree(t *testing.T) {
 func BenchmarkCreateVendorTree(b *testing.B) {
 	// We're fs-bound here, so restrict to single parallelism
 	b.SetParallelism(1)
+	NumDepTreeWorkers = 1
 
 	r := basicResult
+	benchmarkCreateVendorTree(b, r)
+}
+
+func BenchmarkCreateVendorTreeParallel(b *testing.B) {
+	r := solution{
+		att: 1,
+		p: []LockedProject{
+			pa2lp(atom{
+				id: pi("github.com/sdboyer/testrepo"),
+				v:  NewBranch("master").Pair(Revision("4d59fb584b15a94d7401e356d2875c472d76ef45")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("github.com/Masterminds/VCSTestRepo"),
+				v:  NewVersion("1.0.0").Pair(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("launchpad.net/govcstestbzrrepo"),
+				v:  NewVersion("1.0.0").Pair(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("bitbucket.org/sdboyer/withbm"),
+				v:  NewVersion("v1.0.0").Pair(Revision("aa110802a0c64195d0a6c375c9f66668827c90b4")),
+			}, nil),
+		},
+	}
+	r.analyzerInfo = (naiveAnalyzer{}).Info()
+
+	benchmarkCreateVendorTree(b, r)
+}
+
+func benchmarkCreateVendorTree(b *testing.B, r solution) {
 	tmp := path.Join(os.TempDir(), "vsolvtest")
 
 	clean := true