Skip to content
This repository has been archived by the owner on Sep 9, 2020. It is now read-only.

vendor directory writes: add counts to verbose logging, limits writers, abort on error #1043

Merged
merged 3 commits into from
Aug 30, 2017
Merged

vendor directory writes: add counts to verbose logging, limits writers, abort on error #1043

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
76e1854
verbose logging: add counts to vendor directory lists
jmank88 Aug 21, 2017
ef47c48
limit # routines writing dep tree; abort on error
jmank88 Aug 27, 2017
1b50999
gps: change WriteDepTree concurrent writers from GOMAXPROCS to 16
jmank88 Aug 29, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions internal/gps/identifier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ func (i ProjectIdentifier) errString() string {
return fmt.Sprintf("%s (from %s)", i.ProjectRoot, i.Source)
}

func (i ProjectIdentifier) String() string {
return i.errString()
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any objections to this errString format becoming the canonical exported String method format? fmt.Sprintf("%s (from %s)", i.ProjectRoot, i.Source)

errString() has 36 usages, all of which are passed to formatting functions with %s, so I'd like to do a follow up PR to just absorb that method into this one and let them all implicitly call this.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mmm yes, i like this. the individual properties are accessible already - callers can easily construct their own output if they so choose. 👍

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(separate PR, though)

}

func (i ProjectIdentifier) normalize() ProjectIdentifier {
if i.Source == "" {
i.Source = string(i.ProjectRoot)
Expand Down
57 changes: 39 additions & 18 deletions internal/gps/result.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"log"
"os"
"path/filepath"
"sync"

"github.com/pkg/errors"
)
Expand Down Expand Up @@ -65,37 +64,59 @@ func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log
return err
}

var wg sync.WaitGroup
errCh := make(chan error, len(l.Projects()))
lps := l.Projects()

type resp struct {
i int
err error
}
respCh := make(chan resp, len(lps))

for i := range lps {
go func(i int) {
p := lps[i]

var err error
defer func() {
if r := recover(); r != nil {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we get panics here?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't seen one, but I wanted to be careful not to return a false positive if one occurs.

err = errors.Errorf("recovered from panic exporting %s: %s", p.Ident().ProjectRoot, r)
}
respCh <- resp{i, err}
}()

for _, p := range l.Projects() {
wg.Add(1)
go func(p LockedProject) {
defer wg.Done()
to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
logger.Printf("Writing out %s@%s", p.Ident().errString(), p.Version())

if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
errCh <- errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)
if err = sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
err = errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)
return
}

if sv {
err := filepath.Walk(to, stripVendor)
err = filepath.Walk(to, stripVendor)
if err != nil {
errCh <- errors.Wrapf(err, "failed to strip vendor from %s", p.Ident().ProjectRoot)
err = errors.Wrapf(err, "failed to strip vendor from %s", p.Ident().ProjectRoot)
}
}
}(p)
}(i)
}

wg.Wait()
close(errCh)
var errs []error
for i := 0; i < len(lps); i++ {
resp := <-respCh
msg := "Wrote"
if resp.err != nil {
errs = append(errs, resp.err)
msg = "Failed to write"
}
p := lps[resp.i]
logger.Printf("(%d/%d) %s %s@%s\n", i+1, len(lps), msg, p.Ident(), p.Version())
}
close(respCh)

if len(errCh) > 0 {
if len(errs) > 0 {
logger.Println("Failed to write dep tree. The following errors occurred:")
for err := range errCh {
logger.Println(" * ", err)
for i, err := range errs {
logger.Printf("(%d/%d) %s\n", i+1, len(errs), err)
}

removeAll(basedir)
Expand Down
5 changes: 3 additions & 2 deletions txn_writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,8 +455,9 @@ func (sw *SafeWriter) PrintPreparedActions(output *log.Logger, verbose bool) err
if sw.writeVendor {
if verbose {
output.Printf("Would have written the following %d projects to the vendor directory:\n", len(sw.lock.Projects()))
for _, project := range sw.lock.Projects() {
output.Println(project)
lps := sw.lock.Projects()
for i, p := range lps {
output.Printf("(%d/%d) %s@%s\n", i+1, len(lps), p.Ident(), p.Version())
}
} else {
output.Printf("Would have written %d projects to the vendor directory.\n", len(sw.lock.Projects()))
Expand Down