From b0bedbc1705076bbdc79dbc81d4242e28aa9a172 Mon Sep 17 00:00:00 2001 From: Ibrahim AshShohail Date: Tue, 8 Aug 2017 21:42:08 +0300 Subject: [PATCH] dep, internal/gps: update SafeWriter.Write and gps.WriteDepTree Signed-off-by: Ibrahim AshShohail --- cmd/dep/ensure.go | 37 +++++++++++++++++--- cmd/dep/init.go | 9 ++++- internal/gps/result.go | 67 ++++++++++++++++++++++++++----------- internal/gps/result_test.go | 14 ++++---- txn_writer.go | 9 +++-- txn_writer_test.go | 28 ++++++++-------- 6 files changed, 115 insertions(+), 49 deletions(-) diff --git a/cmd/dep/ensure.go b/cmd/dep/ensure.go index 49e5a3f555..d9b18df149 100644 --- a/cmd/dep/ensure.go +++ b/cmd/dep/ensure.go @@ -8,6 +8,8 @@ import ( "bytes" "flag" "go/build" + "io/ioutil" + "log" "os" "path/filepath" "sort" @@ -252,7 +254,13 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project return nil } - return errors.WithMessage(sw.Write(p.AbsRoot, sm, true), "grouped write of manifest, lock and vendor") + logger := ctx.Err + if !ctx.Verbose { + logger = log.New(ioutil.Discard, "", 0) + } + + err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, logger) + return errors.WithMessage(err, "grouped write of manifest, lock and vendor") } solution, err := solver.Solve() @@ -269,7 +277,13 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project return sw.PrintPreparedActions(ctx.Out) } - return errors.Wrap(sw.Write(p.AbsRoot, sm, false), "grouped write of manifest, lock and vendor") + logger := ctx.Err + if !ctx.Verbose { + logger = log.New(ioutil.Discard, "", 0) + } + + err = sw.Write(p.AbsRoot, sm, false, p.Manifest.PruneOptions, logger) + return errors.Wrap(err, "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { @@ -292,7 +306,13 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj return nil } - return errors.WithMessage(sw.Write(p.AbsRoot, sm, true), "grouped write of manifest, lock and vendor") + logger := ctx.Err + if !ctx.Verbose { + logger = log.New(ioutil.Discard, "", 0) + } + + err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, logger) + return errors.WithMessage(err, "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { @@ -379,7 +399,13 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, return sw.PrintPreparedActions(ctx.Out) } - return errors.Wrap(sw.Write(p.AbsRoot, sm, false), "grouped write of manifest, lock and vendor") + logger := ctx.Err + if !ctx.Verbose { + logger = log.New(ioutil.Discard, "", 0) + } + + err = sw.Write(p.AbsRoot, sm, false, p.Manifest.PruneOptions, logger) + return errors.Wrap(err, "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { @@ -609,7 +635,8 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm return sw.PrintPreparedActions(ctx.Out) } - if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true), "grouped write of manifest, lock and vendor"); err != nil { + err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, nil) + if err := errors.Wrap(err, "grouped write of manifest, lock and vendor"); err != nil { return err } diff --git a/cmd/dep/init.go b/cmd/dep/init.go index d49ef58b98..b686099753 100644 --- a/cmd/dep/init.go +++ b/cmd/dep/init.go @@ -6,6 +6,8 @@ package main import ( "flag" + "io/ioutil" + "log" "os" "path/filepath" "time" @@ -203,7 +205,12 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { return err } - if err := sw.Write(root, sm, !cmd.noExamples); err != nil { + logger := ctx.Err + if !ctx.Verbose { + logger = log.New(ioutil.Discard, "", 0) + } + + if err := sw.Write(root, sm, !cmd.noExamples, p.Manifest.PruneOptions, logger); err != nil { return errors.Wrap(err, "safe write of manifest and lock") } diff --git a/internal/gps/result.go b/internal/gps/result.go index 504b02368b..0a512505ca 100644 --- a/internal/gps/result.go +++ b/internal/gps/result.go @@ -5,9 +5,12 @@ package gps import ( - "fmt" + "log" "os" "path/filepath" + "sync" + + "github.com/pkg/errors" ) // A Solution is returned by a solver run. It is mostly just a Lock, with some @@ -42,37 +45,61 @@ type solution struct { solv Solver } -// WriteDepTree takes a basedir and a Lock, and exports all the projects -// listed in the lock to the appropriate target location within the basedir. +// WriteDepTree takes a baseDir and a Lock, and exports all the projects +// listed in the lock to the appropriate target location within baseDir. // -// If the goal is to populate a vendor directory, basedir should be the absolute +// If the goal is to populate a vendor directory, baseDir should be the absolute // path to that vendor directory, not its parent (a project root, typically). // -// It requires a SourceManager to do the work, and takes a flag indicating -// whether or not to strip vendor directories contained in the exported -// dependencies. -func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool) error { +// It requires a SourceManager to do the work, and takes a PruneOptions +// indicating the pruning options required for the exported dependencies. +func WriteDepTree(baseDir string, l Lock, sm SourceManager, prune PruneOptions, logger *log.Logger) error { + if baseDir == "" { + return errors.New("must provide a non-empty baseDir") + } if l == nil { - return fmt.Errorf("must provide non-nil Lock to WriteDepTree") + return errors.New("must provide a non-nil Lock to WriteDepTree") + } + if sm == nil { + return errors.New("must provide a non-nil SourceManager to WriteDepTree") } - err := os.MkdirAll(basedir, 0777) - if err != nil { + if err := os.MkdirAll(baseDir, 0777); err != nil { return err } - // TODO(sdboyer) parallelize + var wg sync.WaitGroup + errCh := make(chan error, len(l.Projects())) + for _, p := range l.Projects() { - to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot))) + wg.Add(1) + go func(p LockedProject) { + to := filepath.FromSlash(filepath.Join(baseDir, string(p.Ident().ProjectRoot))) - err = sm.ExportProject(p.Ident(), p.Version(), to) - if err != nil { - removeAll(basedir) - return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err) - } - if sv { - filepath.Walk(to, stripVendor) + if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil { + removeAll(to) + errCh <- errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot) + } + + wg.Done() + }(p) + } + + wg.Wait() + + if len(errCh) > 0 { + logger.Println("Failed to write dep tree. The following errors occurred:") + for err := range errCh { + logger.Println(" * ", err) } + removeAll(baseDir) + return <-errCh + } + + if err := Prune(baseDir, prune, l, logger); err != nil { + logger.Println("Failed to prune dep tree.") + removeAll(baseDir) + return err } return nil diff --git a/internal/gps/result_test.go b/internal/gps/result_test.go index cc96a8e2cb..2aaecc8771 100644 --- a/internal/gps/result_test.go +++ b/internal/gps/result_test.go @@ -6,13 +6,17 @@ package gps import ( "io/ioutil" + "log" "os" "path" "path/filepath" "testing" ) -var basicResult solution +var ( + basicResult solution + discardLogger = log.New(ioutil.Discard, "", 0) +) func pi(n string) ProjectIdentifier { return ProjectIdentifier{ @@ -90,13 +94,11 @@ func testWriteDepTree(t *testing.T) { } // nil lock/result should err immediately - err = WriteDepTree(tmp, nil, sm, true) - if err == nil { + if err := WriteDepTree(tmp, nil, sm, PruneNestedVendorDirs, discardLogger); err == nil { t.Errorf("Should error if nil lock is passed to WriteDepTree") } - err = WriteDepTree(tmp, r, sm, true) - if err != nil { + if err := WriteDepTree(tmp, r, sm, PruneNestedVendorDirs, discardLogger); err != nil { t.Errorf("Unexpected error while creating vendor tree: %s", err) } @@ -143,7 +145,7 @@ func BenchmarkCreateVendorTree(b *testing.B) { // ease manual inspection os.RemoveAll(exp) b.StartTimer() - err = WriteDepTree(exp, r, sm, true) + err = WriteDepTree(exp, r, sm, PruneNestedVendorDirs, discardLogger) b.StopTimer() if err != nil { b.Errorf("unexpected error after %v iterations: %s", i, err) diff --git a/txn_writer.go b/txn_writer.go index aeb8134168..cb8c1d2bed 100644 --- a/txn_writer.go +++ b/txn_writer.go @@ -253,7 +253,7 @@ func (sw SafeWriter) validate(root string, sm gps.SourceManager) error { return nil } -// Write saves some combination of config yaml, lock, and a vendor tree. +// Write saves some combination of a manifest, a lock, and a vendor tree. // root is the absolute path of root dir in which to write. // sm is only required if vendor is being written. // @@ -261,7 +261,7 @@ func (sw SafeWriter) validate(root string, sm gps.SourceManager) error { // operations succeeded. It also does its best to roll back if any moves fail. // This mostly guarantees that dep cannot exit with a partial write that would // leave an undefined state on disk. -func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool) error { +func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, prune gps.PruneOptions, logger *log.Logger) error { err := sw.validate(root, sm) if err != nil { return err @@ -313,7 +313,10 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool) er } if sw.writeVendor { - err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, true) + // Ensure that gps.PruneNestedVendorDirs is toggled on. + prune |= gps.PruneNestedVendorDirs + + err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, prune, logger) if err != nil { return errors.Wrap(err, "error while writing out vendor tree") } diff --git a/txn_writer_test.go b/txn_writer_test.go index ad550cd92d..ffc90dc6a3 100644 --- a/txn_writer_test.go +++ b/txn_writer_test.go @@ -26,7 +26,7 @@ func TestSafeWriter_BadInput_MissingRoot(t *testing.T) { defer pc.Release() sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged) - err := sw.Write("", pc.SourceManager, true) + err := sw.Write("", pc.SourceManager, true, 0, discardLogger) if err == nil { t.Fatal("should have errored without a root path, but did not") @@ -44,7 +44,7 @@ func TestSafeWriter_BadInput_MissingSourceManager(t *testing.T) { pc.Load() sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways) - err := sw.Write(pc.Project.AbsRoot, nil, true) + err := sw.Write(pc.Project.AbsRoot, nil, true, 0, discardLogger) if err == nil { t.Fatal("should have errored without a source manager when forceVendor is true, but did not") @@ -92,7 +92,7 @@ func TestSafeWriter_BadInput_NonexistentRoot(t *testing.T) { sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged) missingroot := filepath.Join(pc.Project.AbsRoot, "nonexistent") - err := sw.Write(missingroot, pc.SourceManager, true) + err := sw.Write(missingroot, pc.SourceManager, true, 0, discardLogger) if err == nil { t.Fatal("should have errored with nonexistent dir for root path, but did not") @@ -110,7 +110,7 @@ func TestSafeWriter_BadInput_RootIsFile(t *testing.T) { sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged) fileroot := pc.CopyFile("fileroot", "txn_writer/badinput_fileroot") - err := sw.Write(fileroot, pc.SourceManager, true) + err := sw.Write(fileroot, pc.SourceManager, true, 0, discardLogger) if err == nil { t.Fatal("should have errored when root path is a file, but did not") @@ -145,7 +145,7 @@ func TestSafeWriter_Manifest(t *testing.T) { } // Write changes - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -190,7 +190,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLock(t *testing.T) { } // Write changes - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -235,7 +235,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLockWithForceVendor(t *testing.T) { } // Write changes - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -285,7 +285,7 @@ func TestSafeWriter_ModifiedLock(t *testing.T) { } // Write changes - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -335,7 +335,7 @@ func TestSafeWriter_ModifiedLockSkipVendor(t *testing.T) { } // Write changes - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -363,7 +363,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) { pc.Load() sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways) - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify prepared actions @@ -381,7 +381,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) { t.Fatal("Expected the payload to contain the vendor directory ") } - err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -431,7 +431,7 @@ func TestSafeWriter_NewLock(t *testing.T) { } // Write changes - err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -478,7 +478,7 @@ func TestSafeWriter_NewLockSkipVendor(t *testing.T) { } // Write changes - err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes @@ -571,7 +571,7 @@ func TestSafeWriter_VendorDotGitPreservedWithForceVendor(t *testing.T) { t.Fatal("Expected the payload to contain the vendor directory") } - err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true) + err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, discardLogger) h.Must(errors.Wrap(err, "SafeWriter.Write failed")) // Verify file system changes