Skip to content

Commit

Permalink
write partial manifest upon init failure
Browse files Browse the repository at this point in the history 
The proposed changes write a partial manifest and lock
in the case of a failed solve. The manifest contains a
note describing that the manifest is incomplete.
Solves golang#909

Signed-off-by: Koby Picker <jkp46@case.edu>
  • Loading branch information
@JKobyP
JKobyP committed Oct 17, 2017
1 parent 33bda2f commit 38ab399
Show file tree
Hide file tree
Showing 5 changed files with 84 additions and 63 deletions.
10 changes: 5 additions & 5 deletions cmd/dep/ensure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, false, logger), "grouped write of manifest, lock and vendor")
}

if cmd.noVendor && cmd.dryRun {
Expand Down Expand Up @@ -299,7 +299,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, false, logger), "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand All @@ -325,7 +325,7 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, false, logger), "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand Down Expand Up @@ -390,7 +390,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, false, logger), "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand Down Expand Up @@ -691,7 +691,7 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil {
if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, false, logger), "grouped write of manifest, lock and vendor"); err != nil {
return err
}

Expand Down
27 changes: 17 additions & 10 deletions cmd/dep/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (

const initShortHelp = `Initialize a new project with manifest and lock files`
const initLongHelp = `
Wizard
Initialize the project at filepath root by parsing its dependencies, writing
manifest and lock files, and vendoring the dependencies. If root isn't
specified, use the current directory.
Expand Down Expand Up @@ -183,10 +185,13 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
return errors.Wrap(err, "prepare solver")
}

partial := true
vendorBehavior := dep.VendorAlways
soln, err := s.Solve()
if err != nil {
handleAllTheFailuresOfTheWorld(err)
return err
partial = true
vendorBehavior = dep.VendorNever
}
p.Lock = dep.LockFromSolution(soln)

Expand All @@ -201,16 +206,18 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {

p.Lock.SolveMeta.InputsDigest = s.HashInputs()

// Pass timestamp (yyyyMMddHHmmss format) as suffix to backup name.
vendorbak, err := dep.BackupVendor(vpath, time.Now().Format("20060102150405"))
if err != nil {
return err
}
if vendorbak != "" {
ctx.Err.Printf("Old vendor backed up to %v", vendorbak)
if vendorBehavior == dep.VendorAlways {
// Pass timestamp (yyyyMMddHHmmss format) as suffix to backup name.
vendorbak, err := dep.BackupVendor(vpath, time.Now().Format("20060102150405"))
if err != nil {
return err
}
if vendorbak != "" {
ctx.Err.Printf("Old vendor backed up to %v", vendorbak)
}
}

sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, dep.VendorAlways)
sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, vendorBehavior)
if err != nil {
return err
}
Expand All @@ -219,7 +226,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
if err := sw.Write(root, sm, !cmd.noExamples, logger); err != nil {
if err := sw.Write(root, sm, !cmd.noExamples, partial, logger); err != nil {
return errors.Wrap(err, "safe write of manifest and lock")
}

Expand Down
70 changes: 37 additions & 33 deletions internal/gps/solver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,22 +440,20 @@ func (s *solver) Solve() (Solution, error) {
all, err := s.solve()

s.mtr.pop()
var soln solution
if err == nil {
soln = solution{
att: s.attempts,
solv: s,
}
soln.analyzerInfo = s.rd.an.Info()
soln.hd = s.HashInputs()

// Convert ProjectAtoms into LockedProjects
soln.p = make([]LockedProject, len(all))
k := 0
for pa, pl := range all {
soln.p[k] = pa2lp(pa, pl)
k++
}

soln := solution{
att: s.attempts,
solv: s,
}
soln.analyzerInfo = s.rd.an.Info()
soln.hd = s.HashInputs()

// Convert ProjectAtoms into LockedProjects
soln.p = make([]LockedProject, len(all))
k := 0
for pa, pl := range all {
soln.p[k] = pa2lp(pa, pl)
k++
}

s.traceFinish(soln, err)
Expand All @@ -465,6 +463,26 @@ func (s *solver) Solve() (Solution, error) {
return soln, err
}

func (s *solver) combine() map[atom]map[string]struct{} {
projs := make(map[atom]map[string]struct{})

// Skip the first project. It's always the root, and that shouldn't be
// included in results.
for _, sel := range s.sel.projects[1:] {
pm, exists := projs[sel.a.a]
if !exists {
pm = make(map[string]struct{})
projs[sel.a.a] = pm
}

for _, path := range sel.a.pl {
pm[path] = struct{}{}
}
}

return projs
}

// solve is the top-level loop for the solving process.
func (s *solver) solve() (map[atom]map[string]struct{}, error) {
// Main solving loop
Expand Down Expand Up @@ -496,7 +514,7 @@ func (s *solver) solve() (map[atom]map[string]struct{}, error) {
// backtracking succeeded, move to the next unselected id
continue
}
return nil, err
return s.combine(), err
}

if queue.current() == nil {
Expand Down Expand Up @@ -544,7 +562,7 @@ func (s *solver) solve() (map[atom]map[string]struct{}, error) {
continue
}
s.mtr.pop()
return nil, err
return s.combine(), err
}
s.selectAtom(nawp, true)
// We don't add anything to the stack of version queues because the
Expand All @@ -556,22 +574,8 @@ func (s *solver) solve() (map[atom]map[string]struct{}, error) {

// Getting this far means we successfully found a solution. Combine the
// selected projects and packages.
projs := make(map[atom]map[string]struct{})

// Skip the first project. It's always the root, and that shouldn't be
// included in results.
for _, sel := range s.sel.projects[1:] {
pm, exists := projs[sel.a.a]
if !exists {
pm = make(map[string]struct{})
projs[sel.a.a] = pm
}

for _, path := range sel.a.pl {
pm[path] = struct{}{}
}
}
return projs, nil
return s.combine(), nil
}

// selectRoot is a specialized selectAtom, used solely to initially
Expand Down
12 changes: 11 additions & 1 deletion txn_writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ var exampleTOML = []byte(`
`)

var partialTOML = []byte(`
# The solver has failed (probably due to conflicting constraints).
# The output below is a partial TOML file which represents the closest
# solution that could be found.
`)

// String added on top of lock file
var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
Expand Down Expand Up @@ -261,7 +268,7 @@ func (sw SafeWriter) validate(root string, sm gps.SourceManager) error {
// operations succeeded. It also does its best to roll back if any moves fail.
// This mostly guarantees that dep cannot exit with a partial write that would
// leave an undefined state on disk.
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, logger *log.Logger) error {
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, partial bool, logger *log.Logger) error {
err := sw.validate(root, sm)
if err != nil {
return err
Expand Down Expand Up @@ -295,6 +302,9 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo
if examples {
initOutput = exampleTOML
}
if partial {
initOutput = append(partialTOML, initOutput...)
}

if err = ioutil.WriteFile(filepath.Join(td, ManifestName), append(initOutput, tb...), 0666); err != nil {
return errors.Wrap(err, "failed to write manifest file to temp dir")
Expand Down
28 changes: 14 additions & 14 deletions txn_writer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestSafeWriter_BadInput_MissingRoot(t *testing.T) {
defer pc.Release()

sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
err := sw.Write("", pc.SourceManager, true, discardLogger())
err := sw.Write("", pc.SourceManager, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored without a root path, but did not")
Expand All @@ -44,7 +44,7 @@ func TestSafeWriter_BadInput_MissingSourceManager(t *testing.T) {
pc.Load()

sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways)
err := sw.Write(pc.Project.AbsRoot, nil, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, nil, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored without a source manager when forceVendor is true, but did not")
Expand Down Expand Up @@ -92,7 +92,7 @@ func TestSafeWriter_BadInput_NonexistentRoot(t *testing.T) {
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)

missingroot := filepath.Join(pc.Project.AbsRoot, "nonexistent")
err := sw.Write(missingroot, pc.SourceManager, true, discardLogger())
err := sw.Write(missingroot, pc.SourceManager, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored with nonexistent dir for root path, but did not")
Expand All @@ -110,7 +110,7 @@ func TestSafeWriter_BadInput_RootIsFile(t *testing.T) {
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)

fileroot := pc.CopyFile("fileroot", "txn_writer/badinput_fileroot")
err := sw.Write(fileroot, pc.SourceManager, true, discardLogger())
err := sw.Write(fileroot, pc.SourceManager, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored when root path is a file, but did not")
Expand Down Expand Up @@ -145,7 +145,7 @@ func TestSafeWriter_Manifest(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -190,7 +190,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLock(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -235,7 +235,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLockWithForceVendor(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -285,7 +285,7 @@ func TestSafeWriter_ModifiedLock(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -335,7 +335,7 @@ func TestSafeWriter_ModifiedLockSkipVendor(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -363,7 +363,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
pc.Load()

sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify prepared actions
Expand All @@ -381,7 +381,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
t.Fatal("Expected the payload to contain the vendor directory ")
}

err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -431,7 +431,7 @@ func TestSafeWriter_NewLock(t *testing.T) {
}

// Write changes
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -478,7 +478,7 @@ func TestSafeWriter_NewLockSkipVendor(t *testing.T) {
}

// Write changes
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -571,7 +571,7 @@ func TestSafeWriter_VendorDotGitPreservedWithForceVendor(t *testing.T) {
t.Fatal("Expected the payload to contain the vendor directory")
}

err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down

0 comments on commit 38ab399

Please sign in to comment.