Skip to content
This repository has been archived by the owner on Sep 9, 2020. It is now read-only.

Commit

Permalink
Merge pull request #1135 from darkowlzz/concurrent-status
Browse files Browse the repository at this point in the history 
fix(status): concurrent BasicStatus creation
  • Loading branch information
@darkowlzz
darkowlzz committed Sep 21, 2017
2 parents 5e004f4 + 17bb4d6 commit 88b7ad9
Show file tree
Hide file tree
Showing 2 changed files with 255 additions and 76 deletions.
264 changes: 188 additions & 76 deletions cmd/dep/status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"io/ioutil"
"log"
"sort"
"sync"
"text/tabwriter"

"github.com/golang/dep"
Expand Down Expand Up @@ -42,6 +43,17 @@ print an extended status output for each dependency of the project.
Status returns exit code zero if all dependencies are in a "good state".
`

const (
shortRev uint8 = iota
longRev
)

var (
errFailedUpdate = errors.New("failed to fetch updates")
errFailedListPkg = errors.New("failed to list packages")
errMultipleFailures = errors.New("multiple sources of failure")
)

func (cmd *statusCommand) Name() string { return "status" }
func (cmd *statusCommand) Args() string { return "[package...]" }
func (cmd *statusCommand) ShortHelp() string { return statusShortHelp }
Expand Down Expand Up @@ -97,7 +109,7 @@ func (out *tableOutput) BasicLine(bs *BasicStatus) {
bs.getConsolidatedConstraint(),
formatVersion(bs.Version),
formatVersion(bs.Revision),
formatVersion(bs.Latest),
bs.getConsolidatedLatest(shortRev),
bs.PackageCount,
)
}
Expand Down Expand Up @@ -223,8 +235,21 @@ func (cmd *statusCommand) Run(ctx *dep.Ctx, args []string) error {
}
}

digestMismatch, hasMissingPkgs, err := runStatusAll(ctx, out, p, sm)
digestMismatch, hasMissingPkgs, errCount, err := runStatusAll(ctx, out, p, sm)
if err != nil {
// If it's only update errors
if err == errFailedUpdate {
// Print the results with unknown data
ctx.Out.Println(buf.String())

// Print the help when in non-verbose mode
if !ctx.Verbose {
ctx.Out.Printf("The status of %d projects are unknown due to errors. Rerun with `-v` flag to see details.\n", errCount)
}
} else {
// List package failure or multiple failures
ctx.Out.Println("Failed to get status. Rerun with `-v` flag to see details.")
}
return err
}

Expand All @@ -248,8 +273,8 @@ type rawStatus struct {
ProjectRoot string
Constraint string
Version string
Revision gps.Revision
Latest gps.Version
Revision string
Latest string
PackageCount int
}

Expand All @@ -264,6 +289,7 @@ type BasicStatus struct {
Latest gps.Version
PackageCount int
hasOverride bool
hasError bool
}

func (bs *BasicStatus) getConsolidatedConstraint() string {
Expand Down Expand Up @@ -291,13 +317,31 @@ func (bs *BasicStatus) getConsolidatedVersion() string {
return version
}

func (bs *BasicStatus) getConsolidatedLatest(revSize uint8) string {
latest := ""
if bs.Latest != nil {
switch revSize {
case shortRev:
latest = formatVersion(bs.Latest)
case longRev:
latest = bs.Latest.String()
}
}

if bs.hasError {
latest += "unknown"
}

return latest
}

func (bs *BasicStatus) marshalJSON() *rawStatus {
return &rawStatus{
ProjectRoot: bs.ProjectRoot,
Constraint: bs.getConsolidatedConstraint(),
Version: formatVersion(bs.Version),
Revision: bs.Revision,
Latest: bs.Latest,
Revision: string(bs.Revision),
Latest: bs.getConsolidatedLatest(longRev),
PackageCount: bs.PackageCount,
}
}
Expand All @@ -308,18 +352,16 @@ type MissingStatus struct {
MissingPackages []string
}

func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (bool, bool, error) {
var digestMismatch, hasMissingPkgs bool

func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (digestMismatch bool, hasMissingPkgs bool, errCount int, err error) {
if p.Lock == nil {
return digestMismatch, hasMissingPkgs, errors.Errorf("no Gopkg.lock found. Run `dep ensure` to generate lock file")
return false, false, 0, errors.Errorf("no Gopkg.lock found. Run `dep ensure` to generate lock file")
}

// While the network churns on ListVersions() requests, statically analyze
// code from the current project.
ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
if err != nil {
return digestMismatch, hasMissingPkgs, errors.Wrapf(err, "analysis of local packages failed")
return false, false, 0, errors.Wrapf(err, "analysis of local packages failed")
}

// Set up a solver in order to check the InputHash.
Expand All @@ -339,12 +381,12 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
}

if err := ctx.ValidateParams(sm, params); err != nil {
return digestMismatch, hasMissingPkgs, err
return false, false, 0, err
}

s, err := gps.Prepare(params, sm)
if err != nil {
return digestMismatch, hasMissingPkgs, errors.Wrapf(err, "could not set up solver for input hashing")
return false, false, 0, errors.Wrapf(err, "could not set up solver for input hashing")
}

cm := collectConstraints(ptree, p, sm)
Expand All @@ -366,85 +408,156 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana

logger.Println("Checking upstream projects:")

// BasicStatus channel to collect all the BasicStatus.
bsCh := make(chan *BasicStatus, len(slp))

// Error channels to collect different errors.
errListPkgCh := make(chan error, len(slp))
errListVerCh := make(chan error, len(slp))

var wg sync.WaitGroup

for i, proj := range slp {
wg.Add(1)
logger.Printf("(%d/%d) %s\n", i+1, len(slp), proj.Ident().ProjectRoot)

bs := BasicStatus{
ProjectRoot: string(proj.Ident().ProjectRoot),
PackageCount: len(proj.Packages()),
}
go func(proj gps.LockedProject) {
bs := BasicStatus{
ProjectRoot: string(proj.Ident().ProjectRoot),
PackageCount: len(proj.Packages()),
}

// Get children only for specific outputers
// in order to avoid slower status process.
switch out.(type) {
case *dotOutput:
ptr, err := sm.ListPackages(proj.Ident(), proj.Version())

// Get children only for specific outputers
// in order to avoid slower status process
switch out.(type) {
case *dotOutput:
ptr, err := sm.ListPackages(proj.Ident(), proj.Version())
if err != nil {
bs.hasError = true
errListPkgCh <- err
}

if err != nil {
return digestMismatch, hasMissingPkgs, errors.Wrapf(err, "analysis of %s package failed", proj.Ident().ProjectRoot)
prm, _ := ptr.ToReachMap(true, false, false, nil)
bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
}

prm, _ := ptr.ToReachMap(true, false, false, nil)
bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
}
// Split apart the version from the lock into its constituent parts.
switch tv := proj.Version().(type) {
case gps.UnpairedVersion:
bs.Version = tv
case gps.Revision:
bs.Revision = tv
case gps.PairedVersion:
bs.Version = tv.Unpair()
bs.Revision = tv.Revision()
}

// Split apart the version from the lock into its constituent parts
switch tv := proj.Version().(type) {
case gps.UnpairedVersion:
bs.Version = tv
case gps.Revision:
bs.Revision = tv
case gps.PairedVersion:
bs.Version = tv.Unpair()
bs.Revision = tv.Revision()
// Check if the manifest has an override for this project. If so,
// set that as the constraint.
if pp, has := p.Manifest.Ovr[proj.Ident().ProjectRoot]; has && pp.Constraint != nil {
bs.hasOverride = true
bs.Constraint = pp.Constraint
} else {
bs.Constraint = gps.Any()
for _, c := range cm[bs.ProjectRoot] {
bs.Constraint = c.Intersect(bs.Constraint)
}
}

// Only if we have a non-rev and non-plain version do/can we display
// anything wrt the version's updateability.
if bs.Version != nil && bs.Version.Type() != gps.IsVersion {
c, has := p.Manifest.Constraints[proj.Ident().ProjectRoot]
if !has {
c.Constraint = gps.Any()
}
// TODO: This constraint is only the constraint imposed by the
// current project, not by any transitive deps. As a result,
// transitive project deps will always show "any" here.
bs.Constraint = c.Constraint

vl, err := sm.ListVersions(proj.Ident())
if err == nil {
gps.SortPairedForUpgrade(vl)

for _, v := range vl {
// Because we've sorted the version list for
// upgrade, the first version we encounter that
// matches our constraint will be what we want.
if c.Constraint.Matches(v) {
bs.Latest = v.Revision()
break
}
}
} else {
// Failed to fetch version list (could happen due to
// network issue).
bs.hasError = true
errListVerCh <- err
}
}

bsCh <- &bs

wg.Done()
}(proj)
}

wg.Wait()
close(bsCh)
close(errListPkgCh)
close(errListVerCh)

// Newline after printing the status progress output.
logger.Println()

// List Packages errors. This would happen only for dot output.
if len(errListPkgCh) > 0 {
err = errFailedListPkg
if ctx.Verbose {
for err := range errListPkgCh {
ctx.Err.Println(err.Error())
}
ctx.Err.Println()
}
}

// Check if the manifest has an override for this project. If so,
// set that as the constraint.
if pp, has := p.Manifest.Ovr[proj.Ident().ProjectRoot]; has && pp.Constraint != nil {
bs.hasOverride = true
bs.Constraint = pp.Constraint
// List Version errors.
if len(errListVerCh) > 0 {
if err == nil {
err = errFailedUpdate
} else {
bs.Constraint = gps.Any()
for _, c := range cm[bs.ProjectRoot] {
bs.Constraint = c.Intersect(bs.Constraint)
}
err = errMultipleFailures
}

// Only if we have a non-rev and non-plain version do/can we display
// anything wrt the version's updateability.
if bs.Version != nil && bs.Version.Type() != gps.IsVersion {
c, has := p.Manifest.Constraints[proj.Ident().ProjectRoot]
if !has {
c.Constraint = gps.Any()
}
// TODO: This constraint is only the constraint imposed by the
// current project, not by any transitive deps. As a result,
// transitive project deps will always show "any" here.
bs.Constraint = c.Constraint

vl, err := sm.ListVersions(proj.Ident())
if err == nil {
gps.SortPairedForUpgrade(vl)

for _, v := range vl {
// Because we've sorted the version list for
// upgrade, the first version we encounter that
// matches our constraint will be what we want.
if c.Constraint.Matches(v) {
bs.Latest = v.Revision()
break
}
}
// Count ListVersions error because we get partial results when
// this happens.
errCount = len(errListVerCh)
if ctx.Verbose {
for err := range errListVerCh {
ctx.Err.Println(err.Error())
}
ctx.Err.Println()
}
}

out.BasicLine(&bs)
// A map of ProjectRoot and *BasicStatus. This is used in maintain the
// order of BasicStatus in output by collecting all the BasicStatus and
// then using them in order.
bsMap := make(map[string]*BasicStatus)
for bs := range bsCh {
bsMap[bs.ProjectRoot] = bs
}
logger.Println()

// Use the collected BasicStatus in outputter.
for _, proj := range slp {
out.BasicLine(bsMap[string(proj.Ident().ProjectRoot)])
}

out.BasicFooter()

return digestMismatch, hasMissingPkgs, nil
return false, false, errCount, err
}

// Hash digest mismatch may indicate that some deps are no longer
Expand All @@ -453,7 +566,6 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
//
// It's possible for digests to not match, but still have a correct
// lock.
digestMismatch = true
rm, _ := ptree.ToReachMap(true, true, false, nil)

external := rm.FlattenFn(paths.IsStandardImportPath)
Expand Down Expand Up @@ -486,7 +598,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
ctx.Err.Printf("\t%s: %s\n", fail.ex, fail.err.Error())
}

return digestMismatch, hasMissingPkgs, errors.New("address issues with undeducible import paths to get more status information")
return true, false, 0, errors.New("address issues with undeducible import paths to get more status information")
}

out.MissingHeader()
Expand All @@ -506,7 +618,7 @@ outer:
}
out.MissingFooter()

return digestMismatch, hasMissingPkgs, nil
return true, hasMissingPkgs, 0, nil
}

func formatVersion(v gps.Version) string {
Expand Down

0 comments on commit 88b7ad9

Please sign in to comment.