Skip to content
This repository has been archived by the owner on Sep 9, 2020. It is now read-only.

fix(status): concurrent BasicStatus creation #1135

Merged
merged 8 commits into from
Sep 21, 2017
Merged

fix(status): concurrent BasicStatus creation #1135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6cec5a8
fix(status): concurrent BasicStatus creation
darkowlzz Sep 7, 2017
9de9063
status: log errors and failure feedback
darkowlzz Sep 9, 2017
152a82a
status: add failure count for update failures
darkowlzz Sep 10, 2017
cf9a971
fix(status): add shortRev and longRev for latest
darkowlzz Sep 10, 2017
d1f2cdc
status: add test for getConsolidatedLatest()
darkowlzz Sep 10, 2017
bce09f4
status: document runStatusAll() result
darkowlzz Sep 11, 2017
88230aa
status: refactor return statements
darkowlzz Sep 11, 2017
17bb4d6
status: cleanup
darkowlzz Sep 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
264 changes: 188 additions & 76 deletions cmd/dep/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"io/ioutil"
"log"
"sort"
"sync"
"text/tabwriter"

"github.com/golang/dep"
Expand Down Expand Up @@ -42,6 +43,17 @@ print an extended status output for each dependency of the project.
Status returns exit code zero if all dependencies are in a "good state".
`

const (
shortRev uint8 = iota
longRev
)

var (
errFailedUpdate = errors.New("failed to fetch updates")
errFailedListPkg = errors.New("failed to list packages")
errMultipleFailures = errors.New("multiple sources of failure")
)

func (cmd *statusCommand) Name() string { return "status" }
func (cmd *statusCommand) Args() string { return "[package...]" }
func (cmd *statusCommand) ShortHelp() string { return statusShortHelp }
Expand Down Expand Up @@ -97,7 +109,7 @@ func (out *tableOutput) BasicLine(bs *BasicStatus) {
bs.getConsolidatedConstraint(),
formatVersion(bs.Version),
formatVersion(bs.Revision),
formatVersion(bs.Latest),
bs.getConsolidatedLatest(shortRev),
bs.PackageCount,
)
}
Expand Down Expand Up @@ -223,8 +235,21 @@ func (cmd *statusCommand) Run(ctx *dep.Ctx, args []string) error {
}
}

digestMismatch, hasMissingPkgs, err := runStatusAll(ctx, out, p, sm)
digestMismatch, hasMissingPkgs, errCount, err := runStatusAll(ctx, out, p, sm)
if err != nil {
// If it's only update errors
if err == errFailedUpdate {
// Print the results with unknown data
ctx.Out.Println(buf.String())

// Print the help when in non-verbose mode
if !ctx.Verbose {
ctx.Out.Printf("The status of %d projects are unknown due to errors. Rerun with `-v` flag to see details.\n", errCount)
}
} else {
// List package failure or multiple failures
ctx.Out.Println("Failed to get status. Rerun with `-v` flag to see details.")
}
return err
}

Expand All @@ -248,8 +273,8 @@ type rawStatus struct {
ProjectRoot string
Constraint string
Version string
Revision gps.Revision
Latest gps.Version
Revision string
Latest string
PackageCount int
}

Expand All @@ -264,6 +289,7 @@ type BasicStatus struct {
Latest gps.Version
PackageCount int
hasOverride bool
hasError bool
}

func (bs *BasicStatus) getConsolidatedConstraint() string {
Expand Down Expand Up @@ -291,13 +317,31 @@ func (bs *BasicStatus) getConsolidatedVersion() string {
return version
}

func (bs *BasicStatus) getConsolidatedLatest(revSize uint8) string {
latest := ""
if bs.Latest != nil {
switch revSize {
case shortRev:
latest = formatVersion(bs.Latest)
case longRev:
latest = bs.Latest.String()
}
}

if bs.hasError {
latest += "unknown"
}

return latest
}

func (bs *BasicStatus) marshalJSON() *rawStatus {
return &rawStatus{
ProjectRoot: bs.ProjectRoot,
Constraint: bs.getConsolidatedConstraint(),
Version: formatVersion(bs.Version),
Revision: bs.Revision,
Latest: bs.Latest,
Revision: string(bs.Revision),
Latest: bs.getConsolidatedLatest(longRev),
PackageCount: bs.PackageCount,
}
}
Expand All @@ -308,18 +352,16 @@ type MissingStatus struct {
MissingPackages []string
}

func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (bool, bool, error) {
var digestMismatch, hasMissingPkgs bool

func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (digestMismatch bool, hasMissingPkgs bool, errCount int, err error) {
if p.Lock == nil {
return digestMismatch, hasMissingPkgs, errors.Errorf("no Gopkg.lock found. Run `dep ensure` to generate lock file")
return false, false, 0, errors.Errorf("no Gopkg.lock found. Run `dep ensure` to generate lock file")
}

// While the network churns on ListVersions() requests, statically analyze
// code from the current project.
ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
if err != nil {
return digestMismatch, hasMissingPkgs, errors.Wrapf(err, "analysis of local packages failed")
return false, false, 0, errors.Wrapf(err, "analysis of local packages failed")
}

// Set up a solver in order to check the InputHash.
Expand All @@ -339,12 +381,12 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
}

if err := ctx.ValidateParams(sm, params); err != nil {
return digestMismatch, hasMissingPkgs, err
return false, false, 0, err
}

s, err := gps.Prepare(params, sm)
if err != nil {
return digestMismatch, hasMissingPkgs, errors.Wrapf(err, "could not set up solver for input hashing")
return false, false, 0, errors.Wrapf(err, "could not set up solver for input hashing")
}

cm := collectConstraints(ptree, p, sm)
Expand All @@ -366,85 +408,156 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana

logger.Println("Checking upstream projects:")

// BasicStatus channel to collect all the BasicStatus.
bsCh := make(chan *BasicStatus, len(slp))

// Error channels to collect different errors.
errListPkgCh := make(chan error, len(slp))
errListVerCh := make(chan error, len(slp))

var wg sync.WaitGroup

for i, proj := range slp {
wg.Add(1)
logger.Printf("(%d/%d) %s\n", i+1, len(slp), proj.Ident().ProjectRoot)

bs := BasicStatus{
ProjectRoot: string(proj.Ident().ProjectRoot),
PackageCount: len(proj.Packages()),
}
go func(proj gps.LockedProject) {
bs := BasicStatus{
ProjectRoot: string(proj.Ident().ProjectRoot),
PackageCount: len(proj.Packages()),
}

// Get children only for specific outputers
// in order to avoid slower status process.
switch out.(type) {
case *dotOutput:
ptr, err := sm.ListPackages(proj.Ident(), proj.Version())

// Get children only for specific outputers
// in order to avoid slower status process
switch out.(type) {
case *dotOutput:
ptr, err := sm.ListPackages(proj.Ident(), proj.Version())
if err != nil {
bs.hasError = true
errListPkgCh <- err
}

if err != nil {
return digestMismatch, hasMissingPkgs, errors.Wrapf(err, "analysis of %s package failed", proj.Ident().ProjectRoot)
prm, _ := ptr.ToReachMap(true, false, false, nil)
bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
}

prm, _ := ptr.ToReachMap(true, false, false, nil)
bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
}
// Split apart the version from the lock into its constituent parts.
switch tv := proj.Version().(type) {
case gps.UnpairedVersion:
bs.Version = tv
case gps.Revision:
bs.Revision = tv
case gps.PairedVersion:
bs.Version = tv.Unpair()
bs.Revision = tv.Revision()
}

// Split apart the version from the lock into its constituent parts
switch tv := proj.Version().(type) {
case gps.UnpairedVersion:
bs.Version = tv
case gps.Revision:
bs.Revision = tv
case gps.PairedVersion:
bs.Version = tv.Unpair()
bs.Revision = tv.Revision()
// Check if the manifest has an override for this project. If so,
// set that as the constraint.
if pp, has := p.Manifest.Ovr[proj.Ident().ProjectRoot]; has && pp.Constraint != nil {
bs.hasOverride = true
bs.Constraint = pp.Constraint
} else {
bs.Constraint = gps.Any()
for _, c := range cm[bs.ProjectRoot] {
bs.Constraint = c.Intersect(bs.Constraint)
}
}

// Only if we have a non-rev and non-plain version do/can we display
// anything wrt the version's updateability.
if bs.Version != nil && bs.Version.Type() != gps.IsVersion {
c, has := p.Manifest.Constraints[proj.Ident().ProjectRoot]
if !has {
c.Constraint = gps.Any()
}
// TODO: This constraint is only the constraint imposed by the
// current project, not by any transitive deps. As a result,
// transitive project deps will always show "any" here.
bs.Constraint = c.Constraint

vl, err := sm.ListVersions(proj.Ident())
if err == nil {
gps.SortPairedForUpgrade(vl)

for _, v := range vl {
// Because we've sorted the version list for
// upgrade, the first version we encounter that
// matches our constraint will be what we want.
if c.Constraint.Matches(v) {
bs.Latest = v.Revision()
break
}
}
} else {
// Failed to fetch version list (could happen due to
// network issue).
bs.hasError = true
errListVerCh <- err
}
}

bsCh <- &bs

wg.Done()
}(proj)
}

wg.Wait()
close(bsCh)
close(errListPkgCh)
close(errListVerCh)

// Newline after printing the status progress output.
logger.Println()

// List Packages errors. This would happen only for dot output.
if len(errListPkgCh) > 0 {
err = errFailedListPkg
if ctx.Verbose {
for err := range errListPkgCh {
ctx.Err.Println(err.Error())
}
ctx.Err.Println()
}
}

// Check if the manifest has an override for this project. If so,
// set that as the constraint.
if pp, has := p.Manifest.Ovr[proj.Ident().ProjectRoot]; has && pp.Constraint != nil {
bs.hasOverride = true
bs.Constraint = pp.Constraint
// List Version errors.
if len(errListVerCh) > 0 {
if err == nil {
err = errFailedUpdate
} else {
bs.Constraint = gps.Any()
for _, c := range cm[bs.ProjectRoot] {
bs.Constraint = c.Intersect(bs.Constraint)
}
err = errMultipleFailures
}

// Only if we have a non-rev and non-plain version do/can we display
// anything wrt the version's updateability.
if bs.Version != nil && bs.Version.Type() != gps.IsVersion {
c, has := p.Manifest.Constraints[proj.Ident().ProjectRoot]
if !has {
c.Constraint = gps.Any()
}
// TODO: This constraint is only the constraint imposed by the
// current project, not by any transitive deps. As a result,
// transitive project deps will always show "any" here.
bs.Constraint = c.Constraint

vl, err := sm.ListVersions(proj.Ident())
if err == nil {
gps.SortPairedForUpgrade(vl)

for _, v := range vl {
// Because we've sorted the version list for
// upgrade, the first version we encounter that
// matches our constraint will be what we want.
if c.Constraint.Matches(v) {
bs.Latest = v.Revision()
break
}
}
// Count ListVersions error because we get partial results when
// this happens.
errCount = len(errListVerCh)
if ctx.Verbose {
for err := range errListVerCh {
ctx.Err.Println(err.Error())
}
ctx.Err.Println()
}
}

out.BasicLine(&bs)
// A map of ProjectRoot and *BasicStatus. This is used in maintain the
// order of BasicStatus in output by collecting all the BasicStatus and
// then using them in order.
bsMap := make(map[string]*BasicStatus)
for bs := range bsCh {
bsMap[bs.ProjectRoot] = bs
}
logger.Println()

// Use the collected BasicStatus in outputter.
for _, proj := range slp {
out.BasicLine(bsMap[string(proj.Ident().ProjectRoot)])
}

out.BasicFooter()

return digestMismatch, hasMissingPkgs, nil
return false, false, errCount, err
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this be false, false, 0, nil?

Everything else looks good!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

err... no, I don't think so. That's the only place where we are counting and returning error, for partial results.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whoops thought I was looking at the final return.

}

// Hash digest mismatch may indicate that some deps are no longer
Expand All @@ -453,7 +566,6 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
//
// It's possible for digests to not match, but still have a correct
// lock.
digestMismatch = true
rm, _ := ptree.ToReachMap(true, true, false, nil)

external := rm.FlattenFn(paths.IsStandardImportPath)
Expand Down Expand Up @@ -486,7 +598,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
ctx.Err.Printf("\t%s: %s\n", fail.ex, fail.err.Error())
}

return digestMismatch, hasMissingPkgs, errors.New("address issues with undeducible import paths to get more status information")
return true, false, 0, errors.New("address issues with undeducible import paths to get more status information")
}

out.MissingHeader()
Expand All @@ -506,7 +618,7 @@ outer:
}
out.MissingFooter()

return digestMismatch, hasMissingPkgs, nil
return true, hasMissingPkgs, 0, nil
}

func formatVersion(v gps.Version) string {
Expand Down