Skip to content
This repository has been archived by the owner on Sep 9, 2020. It is now read-only.

Commit

Permalink
dep, internal/gps: update SafeWriter.Write and gps.WriteDepTree
Browse files Browse the repository at this point in the history 
Signed-off-by: Ibrahim AshShohail <ibra.sho@gmail.com>
  • Loading branch information
@ibrasho
ibrasho committed Aug 8, 2017
1 parent 8344181 commit 7db1353
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 48 deletions.
15 changes: 10 additions & 5 deletions cmd/dep/ensure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,8 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
return nil
}

return errors.WithMessage(sw.Write(p.AbsRoot, sm, true), "grouped write of manifest, lock and vendor")
err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, nil)
return errors.WithMessage(err, "grouped write of manifest, lock and vendor")
}

solution, err := solver.Solve()
Expand All @@ -269,7 +270,8 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
return sw.PrintPreparedActions(ctx.Out)
}

return errors.Wrap(sw.Write(p.AbsRoot, sm, false), "grouped write of manifest, lock and vendor")
err = sw.Write(p.AbsRoot, sm, false, p.Manifest.PruneOptions, nil)
return errors.Wrap(err, "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand All @@ -292,7 +294,8 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
return nil
}

return errors.WithMessage(sw.Write(p.AbsRoot, sm, true), "grouped write of manifest, lock and vendor")
err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, nil)
return errors.WithMessage(err, "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand Down Expand Up @@ -379,7 +382,8 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
return sw.PrintPreparedActions(ctx.Out)
}

return errors.Wrap(sw.Write(p.AbsRoot, sm, false), "grouped write of manifest, lock and vendor")
err = sw.Write(p.AbsRoot, sm, false, p.Manifest.PruneOptions, nil)
return errors.Wrap(err, "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand Down Expand Up @@ -609,7 +613,8 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
return sw.PrintPreparedActions(ctx.Out)
}

if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true), "grouped write of manifest, lock and vendor"); err != nil {
err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, nil)
if err := errors.Wrap(err, "grouped write of manifest, lock and vendor"); err != nil {
return err
}

Expand Down
2 changes: 1 addition & 1 deletion cmd/dep/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
return err
}

if err := sw.Write(root, sm, !cmd.noExamples); err != nil {
if err := sw.Write(root, sm, !cmd.noExamples, p.Manifest.PruneOptions, nil); err != nil {
return errors.Wrap(err, "safe write of manifest and lock")
}

Expand Down
68 changes: 48 additions & 20 deletions internal/gps/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
package gps

import (
"fmt"
"log"
"os"
"path/filepath"
"sync"

"github.com/pkg/errors"
)

// A Solution is returned by a solver run. It is mostly just a Lock, with some
Expand Down Expand Up @@ -42,37 +45,62 @@ type solution struct {
solv Solver
}

// WriteDepTree takes a basedir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within the basedir.
// WriteDepTree takes a baseDir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within baseDir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// If the goal is to populate a vendor directory, baseDir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
//
// It requires a SourceManager to do the work, and takes a flag indicating
// whether or not to strip vendor directories contained in the exported
// dependencies.
func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool) error {
// It requires a SourceManager to do the work, and takes a PruneOptions
// indicating the pruning options required for the exported dependencies.
func WriteDepTree(baseDir string, l Lock, sm SourceManager, prune PruneOptions, logger *log.Logger) error {
if baseDir == "" {
return errors.New("must provide a non-empty baseDir")
}
if l == nil {
return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
return errors.New("must provide a non-nil Lock to WriteDepTree")
}
if sm == nil {
return errors.New("must provide a non-nil SourceManager to WriteDepTree")
}

err := os.MkdirAll(basedir, 0777)
if err != nil {
if err := os.MkdirAll(baseDir, 0777); err != nil {
return err
}

// TODO(sdboyer) parallelize
var wg sync.WaitGroup
errCh := make(chan error, len(l.Projects()))

for _, p := range l.Projects() {
to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
wg.Add(1)
go func(p LockedProject) {
projectPath := filepath.Join(baseDir, string(p.Ident().ProjectRoot))
to := filepath.FromSlash(projectPath)

if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
removeAll(projectPath)
errCh <- errors.Wrapf(err, "failed to export %s: %s", p.Ident().ProjectRoot)
}

wg.Done()
}(p)
}

err = sm.ExportProject(p.Ident(), p.Version(), to)
if err != nil {
removeAll(basedir)
return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
}
if sv {
filepath.Walk(to, stripVendor)
wg.Wait()

if len(errCh) > 0 {
logger.Println("Failed to write dep tree. The following errors occurred:")
for err := range errCh {
logger.Println(" * ", err)
}
removeAll(baseDir)
return <-errCh
}

if err := Prune(baseDir, prune, l, logger); err != nil {
logger.Println("Failed to prune dep tree.")
removeAll(baseDir)
return err
}

return nil
Expand Down
8 changes: 3 additions & 5 deletions internal/gps/result_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,11 @@ func testWriteDepTree(t *testing.T) {
}

// nil lock/result should err immediately
err = WriteDepTree(tmp, nil, sm, true)
if err == nil {
if err := WriteDepTree(tmp, nil, sm, PruneNestedVendorDirs, nil); err == nil {
t.Errorf("Should error if nil lock is passed to WriteDepTree")
}

err = WriteDepTree(tmp, r, sm, true)
if err != nil {
if err := WriteDepTree(tmp, r, sm, PruneNestedVendorDirs, nil); err != nil {
t.Errorf("Unexpected error while creating vendor tree: %s", err)
}

Expand Down Expand Up @@ -143,7 +141,7 @@ func BenchmarkCreateVendorTree(b *testing.B) {
// ease manual inspection
os.RemoveAll(exp)
b.StartTimer()
err = WriteDepTree(exp, r, sm, true)
err = WriteDepTree(exp, r, sm, PruneNestedVendorDirs, nil)
b.StopTimer()
if err != nil {
b.Errorf("unexpected error after %v iterations: %s", i, err)
Expand Down
9 changes: 6 additions & 3 deletions txn_writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,15 +253,15 @@ func (sw SafeWriter) validate(root string, sm gps.SourceManager) error {
return nil
}

// Write saves some combination of config yaml, lock, and a vendor tree.
// Write saves some combination of a manifest, a lock, and a vendor tree.
// root is the absolute path of root dir in which to write.
// sm is only required if vendor is being written.
//
// It first writes to a temp dir, then moves them in place if and only if all the write
// operations succeeded. It also does its best to roll back if any moves fail.
// This mostly guarantees that dep cannot exit with a partial write that would
// leave an undefined state on disk.
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool) error {
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, prune gps.PruneOptions, logger *log.Logger) error {
err := sw.validate(root, sm)
if err != nil {
return err
Expand Down Expand Up @@ -313,7 +313,10 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool) er
}

if sw.writeVendor {
err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, true)
// Ensure that gps.PruneNestedVendorDirs is toggled on.
prune |= gps.PruneNestedVendorDirs

err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, prune, logger)
if err != nil {
return errors.Wrap(err, "error while writing out vendor tree")
}
Expand Down
28 changes: 14 additions & 14 deletions txn_writer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestSafeWriter_BadInput_MissingRoot(t *testing.T) {
defer pc.Release()

sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
err := sw.Write("", pc.SourceManager, true)
err := sw.Write("", pc.SourceManager, true, 0, nil)

if err == nil {
t.Fatal("should have errored without a root path, but did not")
Expand All @@ -44,7 +44,7 @@ func TestSafeWriter_BadInput_MissingSourceManager(t *testing.T) {
pc.Load()

sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways)
err := sw.Write(pc.Project.AbsRoot, nil, true)
err := sw.Write(pc.Project.AbsRoot, nil, true, 0, nil)

if err == nil {
t.Fatal("should have errored without a source manager when forceVendor is true, but did not")
Expand Down Expand Up @@ -92,7 +92,7 @@ func TestSafeWriter_BadInput_NonexistentRoot(t *testing.T) {
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)

missingroot := filepath.Join(pc.Project.AbsRoot, "nonexistent")
err := sw.Write(missingroot, pc.SourceManager, true)
err := sw.Write(missingroot, pc.SourceManager, true, 0, nil)

if err == nil {
t.Fatal("should have errored with nonexistent dir for root path, but did not")
Expand All @@ -110,7 +110,7 @@ func TestSafeWriter_BadInput_RootIsFile(t *testing.T) {
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)

fileroot := pc.CopyFile("fileroot", "txn_writer/badinput_fileroot")
err := sw.Write(fileroot, pc.SourceManager, true)
err := sw.Write(fileroot, pc.SourceManager, true, 0, nil)

if err == nil {
t.Fatal("should have errored when root path is a file, but did not")
Expand Down Expand Up @@ -145,7 +145,7 @@ func TestSafeWriter_Manifest(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -190,7 +190,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLock(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -235,7 +235,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLockWithForceVendor(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -285,7 +285,7 @@ func TestSafeWriter_ModifiedLock(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -335,7 +335,7 @@ func TestSafeWriter_ModifiedLockSkipVendor(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -363,7 +363,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
pc.Load()

sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify prepared actions
Expand All @@ -381,7 +381,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
t.Fatal("Expected the payload to contain the vendor directory ")
}

err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -431,7 +431,7 @@ func TestSafeWriter_NewLock(t *testing.T) {
}

// Write changes
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -478,7 +478,7 @@ func TestSafeWriter_NewLockSkipVendor(t *testing.T) {
}

// Write changes
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -571,7 +571,7 @@ func TestSafeWriter_VendorDotGitPreservedWithForceVendor(t *testing.T) {
t.Fatal("Expected the payload to contain the vendor directory")
}

err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, 0, nil)
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down

0 comments on commit 7db1353

Please sign in to comment.