From bc9e9d362f09dcfe185a403ccc46f5e0d1d799f8 Mon Sep 17 00:00:00 2001
From: Anes Hasicic <anes.hasicic@gmail.com>
Date: Wed, 26 Jul 2017 17:58:51 +0200
Subject: [PATCH] Paralellized gps.WriteDepTree

gps.WriteDepTree now writes vendor/ tree in parallel with
configurable number of worker routines

Fixed accidental err race condition
---
 internal/gps/result.go      | 48 ++++++++++++++++++++++++++++++-------
 internal/gps/result_test.go | 32 +++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 9 deletions(-)

diff --git a/internal/gps/result.go b/internal/gps/result.go
index eb9d6ad0e1..54b351b158 100644
--- a/internal/gps/result.go
+++ b/internal/gps/result.go
@@ -10,6 +10,10 @@ import (
 	"path/filepath"
 )
 
+// NumDepTreeWorkers determines the number of
+// WriteDepTree worker goroutines
+var NumDepTreeWorkers = 2
+
 // A Solution is returned by a solver run. It is mostly just a Lock, with some
 // additional methods that report information about the solve run.
 type Solution interface {
@@ -61,19 +65,45 @@ func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool) error {
 		return err
 	}
 
-	// TODO(sdboyer) parallelize
-	for _, p := range l.Projects() {
-		to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
+	projects := l.Projects()
+	worksz := len(projects)
+
+	workChan := make(chan LockedProject, worksz)
+	doneChan := make(chan error, worksz)
+
+	for i := 0; i < NumDepTreeWorkers; i++ {
+		go func() {
+			for p := range workChan {
+				to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
 
-		err = sm.ExportProject(p.Ident(), p.Version(), to)
+				err := sm.ExportProject(p.Ident(), p.Version(), to)
+				if err != nil {
+					doneChan <- fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
+					continue
+				}
+
+				if sv {
+					filepath.Walk(to, stripVendor)
+				}
+
+				doneChan <- nil
+			}
+			// TODO(sdboyer) dump version metadata file
+		}()
+	}
+
+	for _, p := range projects {
+		workChan <- p
+	}
+
+	close(workChan)
+
+	for i := 0; i < worksz; i++ {
+		err := <-doneChan
 		if err != nil {
 			removeAll(basedir)
-			return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
-		}
-		if sv {
-			filepath.Walk(to, stripVendor)
+			return err
 		}
-		// TODO(sdboyer) dump version metadata file
 	}
 
 	return nil
diff --git a/internal/gps/result_test.go b/internal/gps/result_test.go
index cc96a8e2cb..1bb59db0d8 100644
--- a/internal/gps/result_test.go
+++ b/internal/gps/result_test.go
@@ -114,8 +114,40 @@ func testWriteDepTree(t *testing.T) {
 func BenchmarkCreateVendorTree(b *testing.B) {
 	// We're fs-bound here, so restrict to single parallelism
 	b.SetParallelism(1)
+	NumDepTreeWorkers = 1
 
 	r := basicResult
+	benchmarkCreateVendorTree(b, r)
+}
+
+func BenchmarkCreateVendorTreeParallel(b *testing.B) {
+	r := solution{
+		att: 1,
+		p: []LockedProject{
+			pa2lp(atom{
+				id: pi("github.com/sdboyer/testrepo"),
+				v:  NewBranch("master").Pair(Revision("4d59fb584b15a94d7401e356d2875c472d76ef45")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("github.com/Masterminds/VCSTestRepo"),
+				v:  NewVersion("1.0.0").Pair(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("launchpad.net/govcstestbzrrepo"),
+				v:  NewVersion("1.0.0").Pair(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("bitbucket.org/sdboyer/withbm"),
+				v:  NewVersion("v1.0.0").Pair(Revision("aa110802a0c64195d0a6c375c9f66668827c90b4")),
+			}, nil),
+		},
+	}
+	r.analyzerInfo = (naiveAnalyzer{}).Info()
+
+	benchmarkCreateVendorTree(b, r)
+}
+
+func benchmarkCreateVendorTree(b *testing.B, r solution) {
 	tmp := path.Join(os.TempDir(), "vsolvtest")
 
 	clean := true