From 505c236d895ec60ae546e4264068ba6fac5f1d5c Mon Sep 17 00:00:00 2001 From: Ibrahim AshShohail Date: Fri, 29 Sep 2017 11:49:51 +0300 Subject: [PATCH] dep: add prune options to manifests Signed-off-by: Ibrahim AshShohail --- Gopkg.toml | 5 + analyzer_test.go | 2 +- internal/gps/prune.go | 5 +- manifest.go | 209 +++++++++++++++++++++++++++++++++++++----- manifest_test.go | 39 +++++++- 5 files changed, 231 insertions(+), 29 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 054a3cf59a..2e181e69ab 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -30,3 +30,8 @@ [[constraint]] name = "github.com/golang/protobuf" branch = "master" + +[prune] + non-go = true + go-tests = true + unused-packages = true diff --git a/analyzer_test.go b/analyzer_test.go index 4e453620a4..40b7307422 100644 --- a/analyzer_test.go +++ b/analyzer_test.go @@ -38,7 +38,7 @@ func TestAnalyzerDeriveManifestAndLock(t *testing.T) { t.Fatal(err) } } else { - t.Fatalf("expected %s\n got %s", want, string(got)) + t.Fatalf("(WNT):\n%s\n(GOT):\n%s", want, string(got)) } } diff --git a/internal/gps/prune.go b/internal/gps/prune.go index b67e01b1ff..7fdff4f053 100644 --- a/internal/gps/prune.go +++ b/internal/gps/prune.go @@ -16,9 +16,12 @@ import ( // PruneOptions represents the pruning options used to write the dependecy tree. type PruneOptions uint8 +// PruneProjectOptions is map of prune options per project name. +type PruneProjectOptions map[ProjectRoot]PruneOptions + const ( // PruneNestedVendorDirs indicates if nested vendor directories should be pruned. - PruneNestedVendorDirs = 1 << iota + PruneNestedVendorDirs PruneOptions = 1 << iota // PruneUnusedPackages indicates if unused Go packages should be pruned. PruneUnusedPackages // PruneNonGoFiles indicates if non-Go files should be pruned. diff --git a/manifest.go b/manifest.go index 3ab4439b62..0d49c6be66 100644 --- a/manifest.go +++ b/manifest.go @@ -24,26 +24,39 @@ const ManifestName = "Gopkg.toml" // Errors var ( - errInvalidConstraint = errors.New("\"constraint\" must be a TOML array of tables") - errInvalidOverride = errors.New("\"override\" must be a TOML array of tables") - errInvalidRequired = errors.New("\"required\" must be a TOML list of strings") - errInvalidIgnored = errors.New("\"ignored\" must be a TOML list of strings") - errInvalidProjectRoot = errors.New("ProjectRoot name validation failed") + errInvalidConstraint = errors.Errorf("%q must be a TOML array of tables", "constraint") + errInvalidOverride = errors.Errorf("%q must be a TOML array of tables", "override") + errInvalidRequired = errors.Errorf("%q must be a TOML list of strings", "required") + errInvalidIgnored = errors.Errorf("%q must be a TOML list of strings", "ignored") + errInvalidPrune = errors.Errorf("%q must be a TOML table of booleans", "prune") + + errInvalidProjectRoot = errors.New("ProjectRoot name validation failed") + errInvalidPruneValue = errors.New("prune options values must be booleans") + errInvalidPruneProject = errors.Errorf("%q must be a TOML array of tables", "prune.project") + errPruneSubProject = errors.New("prune projects should not contain sub projects") + + errInvalidRootPruneValue = errors.New("root prune options must be omitted instead of being set to false") + errInvalidPruneProjectName = errors.Errorf("%q in %q must be a string", "name", "prune.project") ) // Manifest holds manifest file data and implements gps.RootManifest. type Manifest struct { Constraints gps.ProjectConstraints Ovr gps.ProjectConstraints - Ignored []string - Required []string + + Ignored []string + Required []string + + PruneOptions gps.PruneOptions + PruneProjectOptions gps.PruneProjectOptions } type rawManifest struct { - Constraints []rawProject `toml:"constraint,omitempty"` - Overrides []rawProject `toml:"override,omitempty"` - Ignored []string `toml:"ignored,omitempty"` - Required []string `toml:"required,omitempty"` + Constraints []rawProject `toml:"constraint,omitempty"` + Overrides []rawProject `toml:"override,omitempty"` + Ignored []string `toml:"ignored,omitempty"` + Required []string `toml:"required,omitempty"` + PruneOptions rawPruneOptions `toml:"prune,omitempty"` } type rawProject struct { @@ -54,11 +67,33 @@ type rawProject struct { Source string `toml:"source,omitempty"` } -// NewManifest instantiates a new manifest. +type rawPruneOptions struct { + UnusedPackages bool `toml:"unused-packages,omitempty"` + NonGoFiles bool `toml:"non-go,omitempty"` + GoTests bool `toml:"go-tests,omitempty"` + + Projects []rawPruneProjectOptions `toml:"project,omitempty"` +} + +type rawPruneProjectOptions struct { + Name string `toml:"name"` + UnusedPackages bool `toml:"unused-packages,omitempty"` + NonGoFiles bool `toml:"non-go,omitempty"` + GoTests bool `toml:"go-tests,omitempty"` +} + +const ( + pruneOptionUnusedPackages = "unused-packages" + pruneOptionGoTests = "go-tests" + pruneOptionNonGo = "non-go" +) + +// NewManifest instantites a new manifest. func NewManifest() *Manifest { return &Manifest{ - Constraints: make(gps.ProjectConstraints), - Ovr: make(gps.ProjectConstraints), + Constraints: make(gps.ProjectConstraints), + Ovr: make(gps.ProjectConstraints), + PruneOptions: gps.PruneNestedVendorDirs, } } @@ -151,6 +186,12 @@ func validateManifest(s string) ([]error, error) { return warns, errInvalidRequired } } + case "prune": + pruneWarns, err := validatePruneOptions(val, true) + warns = append(warns, pruneWarns...) + if err != nil { + return warns, err + } default: warns = append(warns, fmt.Errorf("unknown field in manifest: %v", prop)) } @@ -159,6 +200,70 @@ func validateManifest(s string) ([]error, error) { return warns, nil } +func validatePruneOptions(val interface{}, root bool) (warns []error, err error) { + if reflect.TypeOf(val).Kind() != reflect.Map { + return warns, errInvalidPrune + } + + for key, value := range val.(map[string]interface{}) { + switch key { + case pruneOptionNonGo, pruneOptionGoTests, pruneOptionUnusedPackages: + if option, ok := value.(bool); !ok { + return warns, errInvalidPruneValue + } else if root && !option { + return warns, errInvalidRootPruneValue + } + case "name": + if root { + warns = append(warns, errors.Errorf("%q should not include a name", "prune")) + } else if _, ok := value.(string); !ok { + return warns, errInvalidPruneProjectName + } + case "project": + if !root { + return warns, errPruneSubProject + } + if reflect.TypeOf(value).Kind() != reflect.Slice { + return warns, errInvalidPruneProject + } + for _, project := range value.([]interface{}) { + projectWarns, err := validatePruneOptions(project, false) + warns = append(warns, projectWarns...) + if err != nil { + return nil, err + } + } + + default: + if root { + warns = append(warns, errors.Errorf("unknown field %q in %q", key, "prune")) + } else { + warns = append(warns, errors.Errorf("unknown field %q in %q", key, "prune.project")) + } + } + } + + return warns, err +} + +func checkRedundantPruneOptions(raw rawManifest) (warns []error) { + rootOptions := raw.PruneOptions + + for _, project := range raw.PruneOptions.Projects { + if rootOptions.GoTests && project.GoTests { + warns = append(warns, errors.Errorf("redundant prune option %q set for %q", pruneOptionGoTests, project.Name)) + } + if rootOptions.NonGoFiles && project.NonGoFiles { + warns = append(warns, errors.Errorf("redundant prune option %q set for %q", pruneOptionNonGo, project.Name)) + } + if rootOptions.UnusedPackages && project.UnusedPackages { + warns = append(warns, errors.Errorf("redundant prune option %q set for %q", pruneOptionUnusedPackages, project.Name)) + } + } + + return warns +} + // ValidateProjectRoots validates the project roots present in manifest. func ValidateProjectRoots(c *Ctx, m *Manifest, sm gps.SourceManager) error { // Channel to receive all the errors @@ -184,6 +289,10 @@ func ValidateProjectRoots(c *Ctx, m *Manifest, sm gps.SourceManager) error { wg.Add(1) go validate(pr) } + for pr := range m.PruneProjectOptions { + wg.Add(1) + go validate(pr) + } wg.Wait() close(errorCh) @@ -220,6 +329,8 @@ func readManifest(r io.Reader) (*Manifest, []error, error) { return nil, warns, errors.Wrap(err, "unable to parse the manifest as TOML") } + warns = append(warns, checkRedundantPruneOptions(raw)...) + m, err := fromRawManifest(raw) return m, warns, err } @@ -254,9 +365,43 @@ func fromRawManifest(raw rawManifest) (*Manifest, error) { m.Ovr[name] = prj } + m.PruneOptions, m.PruneProjectOptions = fromRawPruneOptions(raw.PruneOptions) + return m, nil } +func fromRawPruneOptions(raw rawPruneOptions) (gps.PruneOptions, gps.PruneProjectOptions) { + rootOptions := gps.PruneNestedVendorDirs + pruneProjects := make(gps.PruneProjectOptions) + + if raw.UnusedPackages { + rootOptions |= gps.PruneUnusedPackages + } + if raw.GoTests { + rootOptions |= gps.PruneGoTestFiles + } + if raw.NonGoFiles { + rootOptions |= gps.PruneNonGoFiles + } + + for _, p := range raw.Projects { + pr := gps.ProjectRoot(p.Name) + pruneProjects[pr] = gps.PruneNestedVendorDirs + + if raw.UnusedPackages { + pruneProjects[pr] |= gps.PruneUnusedPackages + } + if raw.GoTests { + pruneProjects[pr] |= gps.PruneGoTestFiles + } + if raw.NonGoFiles { + pruneProjects[pr] |= gps.PruneNonGoFiles + } + } + + return rootOptions, pruneProjects +} + // toProject interprets the string representations of project information held in // a rawProject, converting them into a proper gps.ProjectProperties. An // error is returned if the rawProject contains some invalid combination - @@ -288,17 +433,27 @@ func toProject(raw rawProject) (n gps.ProjectRoot, pp gps.ProjectProperties, err } pp.Source = raw.Source + return n, pp, nil } +// MarshalTOML serializes this manifest into TOML via an intermediate raw form. +func (m *Manifest) MarshalTOML() ([]byte, error) { + raw := m.toRaw() + result, err := toml.Marshal(raw) + return result, errors.Wrap(err, "unable to marshal the lock to a TOML string") +} + // toRaw converts the manifest into a representation suitable to write to the manifest file func (m *Manifest) toRaw() rawManifest { raw := rawManifest{ - Constraints: make([]rawProject, 0, len(m.Constraints)), - Overrides: make([]rawProject, 0, len(m.Ovr)), - Ignored: m.Ignored, - Required: m.Required, + Constraints: make([]rawProject, 0, len(m.Constraints)), + Overrides: make([]rawProject, 0, len(m.Ovr)), + Ignored: m.Ignored, + Required: m.Required, + PruneOptions: rawPruneOptions{}, } + for n, prj := range m.Constraints { raw.Constraints = append(raw.Constraints, toRawProject(n, prj)) } @@ -309,6 +464,8 @@ func (m *Manifest) toRaw() rawManifest { } sort.Sort(sortedRawProjects(raw.Overrides)) + // TODO(ibrasho): write out prune options. + return raw } @@ -329,13 +486,6 @@ func (s sortedRawProjects) Less(i, j int) bool { return l.Source < r.Source } -// MarshalTOML serializes this manifest into TOML via an intermediate raw form. -func (m *Manifest) MarshalTOML() ([]byte, error) { - raw := m.toRaw() - result, err := toml.Marshal(raw) - return result, errors.Wrap(err, "Unable to marshal the lock to a TOML string") -} - func toRawProject(name gps.ProjectRoot, project gps.ProjectProperties) rawProject { raw := rawProject{ Name: string(name), @@ -363,6 +513,7 @@ func toRawProject(name gps.ProjectRoot, project gps.ProjectProperties) rawProjec // Has to be a semver range. raw.Version = project.Constraint.ImpliedCaretString() } + return raw } @@ -407,3 +558,11 @@ func (m *Manifest) RequiredPackages() map[string]bool { return mp } + +func (m *Manifest) PruneOptionsFor(pr gps.ProjectRoot) gps.PruneOptions { + if po, ok := m.PruneProjectOptions[pr]; ok { + return po + } + + return m.PruneOptions +} diff --git a/manifest_test.go b/manifest_test.go index 617edd22b3..3fb7d070d3 100644 --- a/manifest_test.go +++ b/manifest_test.go @@ -44,7 +44,11 @@ func TestReadManifest(t *testing.T) { Constraint: gps.NewBranch("master"), }, }, - Ignored: []string{"github.com/foo/bar"}, + Ignored: []string{"github.com/foo/bar"}, + PruneOptions: gps.PruneNestedVendorDirs | gps.PruneNonGoFiles, + PruneProjectOptions: gps.PruneProjectOptions{ + gps.ProjectRoot("github.com/golang/dep"): gps.PruneNestedVendorDirs, + }, } if !reflect.DeepEqual(got.Constraints, want.Constraints) { @@ -78,6 +82,11 @@ func TestWriteManifest(t *testing.T) { } m.Ignored = []string{"github.com/foo/bar"} + m.PruneOptions = gps.PruneNestedVendorDirs | gps.PruneNonGoFiles + m.PruneProjectOptions = gps.PruneProjectOptions{ + gps.ProjectRoot("github.com/golang/dep"): gps.PruneNestedVendorDirs, + } + got, err := m.MarshalTOML() if err != nil { t.Fatalf("error while marshaling valid manifest to TOML: %q", err) @@ -89,7 +98,7 @@ func TestWriteManifest(t *testing.T) { t.Fatal(err) } } else { - t.Errorf("valid manifest did not marshal to TOML as expected:\n\t(GOT): %s\n\t(WNT): %s", string(got), want) + t.Errorf("valid manifest did not marshal to TOML as expected:\n(GOT):\n%s\n(WNT):\n%s", string(got), want) } } } @@ -368,6 +377,32 @@ func TestValidateManifest(t *testing.T) { wantWarn: []error{errors.New("revision \"8d43f8c0b836\" should not be in abbreviated form")}, wantError: nil, }, + { + name: "valid prune options", + tomlString: ` + [[constraint]] + name = "github.com/foo/bar" + version = "1.0.0" + + [prune] + non-go = true + `, + wantWarn: []error{}, + wantError: nil, + }, + { + name: "invalid root prune options", + tomlString: ` + [[constraint]] + name = "github.com/foo/bar" + version = "1.0.0" + + [prune] + non-go = false + `, + wantWarn: []error{}, + wantError: errInvalidRootPruneValue, + }, } // contains for error