From 78a47e610d125b044f8a82f880741bf5ed54da13 Mon Sep 17 00:00:00 2001 From: Marco Deicas <60855123+mdeicas@users.noreply.github.com> Date: Mon, 22 Aug 2022 11:22:18 -0400 Subject: [PATCH] External sources configuration (#1158) --- cmd/syft/cli/options/packages.go | 35 ++++++++-- internal/config/application.go | 4 +- internal/config/datasources.go | 11 ++++ syft/pkg/cataloger/cataloger.go | 2 - syft/pkg/cataloger/cataloger_test.go | 38 +++++++++-- .../pkg/cataloger/common/generic_cataloger.go | 5 ++ syft/pkg/cataloger/config.go | 5 +- syft/pkg/cataloger/golang/binary_cataloger.go | 66 +++++++++++++++++++ syft/pkg/cataloger/portage/cataloger.go | 5 ++ .../pkg/cataloger/python/package_cataloger.go | 5 ++ syft/pkg/cataloger/rpm/db_cataloger.go | 5 ++ .../cataloger/rust/audit_binary_cataloger.go | 5 ++ 12 files changed, 169 insertions(+), 17 deletions(-) create mode 100644 internal/config/datasources.go create mode 100644 syft/pkg/cataloger/golang/binary_cataloger.go diff --git a/cmd/syft/cli/options/packages.go b/cmd/syft/cli/options/packages.go index 08ff362b21a4..678100d30094 100644 --- a/cmd/syft/cli/options/packages.go +++ b/cmd/syft/cli/options/packages.go @@ -14,13 +14,20 @@ import ( ) type PackagesOptions struct { - Scope string - Output []string - OutputTemplatePath string - File string - Platform string - Exclude []string - Catalogers []string + Scope string + Output []string + OutputTemplatePath string + File string + Platform string + Host string + Username string + Password string + Dockerfile string + Exclude []string + OverwriteExistingImage bool + ImportTimeout uint + Catalogers []string + ExternalSourcesEnabled bool } var _ Interface = (*PackagesOptions)(nil) @@ -47,9 +54,19 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error { cmd.Flags().StringArrayVarP(&o.Catalogers, "catalogers", "", nil, "enable one or more package catalogers") + cmd.Flags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false, + "overwrite an existing image during the upload to Anchore Enterprise") + + cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30, + "set a timeout duration (in seconds) for the upload to Anchore Enterprise") + + cmd.Flags().BoolVarP(&o.ExternalSourcesEnabled, "external-sources-enabled", "", false, + "shut off any use of external sources during sbom generation (default false") + return bindPackageConfigOptions(cmd.Flags(), v) } +//nolint:funlen func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error { // Formatting & Input options ////////////////////////////////////////////// @@ -81,5 +98,9 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error { return err } + if err := v.BindPFlag("external_sources.external-sources-enabled", flags.Lookup("external-sources-enabled")); err != nil { + return err + } + return nil } diff --git a/internal/config/application.go b/internal/config/application.go index 627a6bc6a22c..54359dce2301 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -56,6 +56,7 @@ type Application struct { Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"` Attest attest `yaml:"attest" json:"attest" mapstructure:"attest"` Platform string `yaml:"platform" json:"platform" mapstructure:"platform"` + ExternalSources ExternalSources `yaml:"external_sources" json:"external_sources" mapstructure:"external_sources"` } func (cfg Application) ToCatalogerConfig() cataloger.Config { @@ -65,7 +66,8 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config { IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, Scope: cfg.Package.Cataloger.ScopeOpt, }, - Catalogers: cfg.Catalogers, + Catalogers: cfg.Catalogers, + ExternalSourcesEnabled: cfg.ExternalSources.ExternalSourcesEnabled, } } diff --git a/internal/config/datasources.go b/internal/config/datasources.go new file mode 100644 index 000000000000..cc0e507be4fa --- /dev/null +++ b/internal/config/datasources.go @@ -0,0 +1,11 @@ +package config + +import "github.com/spf13/viper" + +type ExternalSources struct { + ExternalSourcesEnabled bool `yaml:"external-sources-enabled" json:"external-sources-enabled" mapstructure:"external-sources-enabled"` +} + +func (e ExternalSources) loadDefaultValues(v *viper.Viper) { + v.SetDefault("external-sources-enabled", false) +} diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index 95de64534cf1..dd704695f4a5 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -23,7 +23,6 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/php" "github.com/anchore/syft/syft/pkg/cataloger/portage" "github.com/anchore/syft/syft/pkg/cataloger/python" - "github.com/anchore/syft/syft/pkg/cataloger/rekor" "github.com/anchore/syft/syft/pkg/cataloger/rpm" "github.com/anchore/syft/syft/pkg/cataloger/ruby" "github.com/anchore/syft/syft/pkg/cataloger/rust" @@ -105,7 +104,6 @@ func AllCatalogers(cfg Config) []pkg.Cataloger { cpp.NewConanCataloger(), portage.NewPortageCataloger(), haskell.NewHackageCataloger(), - rekor.NewRekorCataloger(), }, cfg.Catalogers) } diff --git a/syft/pkg/cataloger/cataloger_test.go b/syft/pkg/cataloger/cataloger_test.go index 5dfd5ccb26d2..70d7efa467e8 100644 --- a/syft/pkg/cataloger/cataloger_test.go +++ b/syft/pkg/cataloger/cataloger_test.go @@ -3,11 +3,18 @@ package cataloger import ( "testing" +<<<<<<< HEAD "github.com/stretchr/testify/assert" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" +======= + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" +>>>>>>> 1329688 (External sources configuration (#1158)) ) var _ pkg.Cataloger = (*dummy)(nil) @@ -24,12 +31,17 @@ func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relatio panic("not implemented") } +func (d dummy) UsesExternalSources() bool { + return false +} + func Test_filterCatalogers(t *testing.T) { tests := []struct { - name string - patterns []string - catalogers []string - want []string + name string + patterns []string + ExternalSourcesEnabled bool + catalogers []string + want []string }{ { name: "no filtering", @@ -144,6 +156,21 @@ func Test_filterCatalogers(t *testing.T) { "go-module-binary-cataloger", }, }, + { // Note: no catalogers with external sources are currently implemented + name: "external sources enabled", + patterns: []string{"all"}, + ExternalSourcesEnabled: true, + catalogers: []string{ + "ruby-gemspec-cataloger", + "python-package-cataloger", + "rekor-cataloger", + }, + want: []string{ + "ruby-gemspec-cataloger", + "python-package-cataloger", + "rekor-cataloger", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -151,7 +178,8 @@ func Test_filterCatalogers(t *testing.T) { for _, n := range tt.catalogers { catalogers = append(catalogers, dummy{name: n}) } - got := filterCatalogers(catalogers, tt.patterns) + cfg := Config{Catalogers: tt.patterns, ExternalSourcesEnabled: tt.ExternalSourcesEnabled} + got := filterCatalogers(catalogers, cfg) var gotNames []string for _, g := range got { gotNames = append(gotNames, g.Name()) diff --git a/syft/pkg/cataloger/common/generic_cataloger.go b/syft/pkg/cataloger/common/generic_cataloger.go index 5de3666bffab..beef2ab7409f 100644 --- a/syft/pkg/cataloger/common/generic_cataloger.go +++ b/syft/pkg/cataloger/common/generic_cataloger.go @@ -39,6 +39,11 @@ func (c *GenericCataloger) Name() string { return c.upstreamCataloger } +// UsesExternalSources indicates that any GenericCatalogor does not use external sources +func (c *GenericCataloger) UsesExternalSources() bool { + return false +} + // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var packages []pkg.Package diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go index 478fc292d111..20920b86ff5b 100644 --- a/syft/pkg/cataloger/config.go +++ b/syft/pkg/cataloger/config.go @@ -5,8 +5,9 @@ import ( ) type Config struct { - Search SearchConfig - Catalogers []string + Search SearchConfig + Catalogers []string + ExternalSourcesEnabled bool } func DefaultConfig() Config { diff --git a/syft/pkg/cataloger/golang/binary_cataloger.go b/syft/pkg/cataloger/golang/binary_cataloger.go new file mode 100644 index 000000000000..f6917e942d3e --- /dev/null +++ b/syft/pkg/cataloger/golang/binary_cataloger.go @@ -0,0 +1,66 @@ +/* +Package golang provides a concrete Cataloger implementation for go.mod files. +*/ +package golang + +import ( + "fmt" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/unionreader" + "github.com/anchore/syft/syft/source" +) + +const catalogerName = "go-module-binary-cataloger" + +type Cataloger struct{} + +// NewGoModuleBinaryCataloger returns a new Golang cataloger object. +func NewGoModuleBinaryCataloger() *Cataloger { + return &Cataloger{} +} + +// Name returns a string that uniquely describes a cataloger +func (c *Cataloger) Name() string { + return catalogerName +} + +// UsesExternalSources indicates that the golang binary cataloger does not use external sources +func (c *Cataloger) UsesExternalSources() bool { + return false +} + +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + + fileMatches, err := resolver.FilesByMIMEType(internal.ExecutableMIMETypeSet.List()...) + if err != nil { + return pkgs, nil, fmt.Errorf("failed to find bin by mime types: %w", err) + } + + for _, location := range fileMatches { + readerCloser, err := resolver.FileContentsByLocation(location) + if err != nil { + log.Warnf("golang cataloger: opening file: %v", err) + continue + } + + reader, err := unionreader.GetUnionReader(readerCloser) + if err != nil { + return nil, nil, err + } + + mods, archs := scanFile(reader, location.RealPath) + internal.CloseAndLogError(readerCloser, location.RealPath) + + for i, mod := range mods { + pkgs = append(pkgs, buildGoPkgInfo(location, mod, archs[i])...) + } + } + + return pkgs, nil, nil +} diff --git a/syft/pkg/cataloger/portage/cataloger.go b/syft/pkg/cataloger/portage/cataloger.go index a8cd5b796460..632e986e9ab7 100644 --- a/syft/pkg/cataloger/portage/cataloger.go +++ b/syft/pkg/cataloger/portage/cataloger.go @@ -37,6 +37,11 @@ func (c *Cataloger) Name() string { return "portage-cataloger" } +// UsesExternalSources indicates that the portage cataloger does not use external sources +func (c *Cataloger) UsesExternalSources() bool { + return false +} + // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files. func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob) diff --git a/syft/pkg/cataloger/python/package_cataloger.go b/syft/pkg/cataloger/python/package_cataloger.go index bdcb9c23a809..96d61834f81f 100644 --- a/syft/pkg/cataloger/python/package_cataloger.go +++ b/syft/pkg/cataloger/python/package_cataloger.go @@ -32,6 +32,11 @@ func (c *PackageCataloger) Name() string { return "python-package-cataloger" } +// UsesExternalSources indicates that the python package cataloger does not use external sources +func (c *PackageCataloger) UsesExternalSources() bool { + return false +} + // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var fileMatches []source.Location diff --git a/syft/pkg/cataloger/rpm/db_cataloger.go b/syft/pkg/cataloger/rpm/db_cataloger.go index 67169c4d88a4..884b96c87e4a 100644 --- a/syft/pkg/cataloger/rpm/db_cataloger.go +++ b/syft/pkg/cataloger/rpm/db_cataloger.go @@ -27,6 +27,11 @@ func (c *DBCataloger) Name() string { return dbCatalogerName } +// UsesExternalSources indicates that the rpmdb cataloger does not use external sources +func (c *Cataloger) UsesExternalSources() bool { + return false +} + // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. func (c *DBCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob) diff --git a/syft/pkg/cataloger/rust/audit_binary_cataloger.go b/syft/pkg/cataloger/rust/audit_binary_cataloger.go index 700b0f2ecb81..228ea62c05c5 100644 --- a/syft/pkg/cataloger/rust/audit_binary_cataloger.go +++ b/syft/pkg/cataloger/rust/audit_binary_cataloger.go @@ -28,6 +28,11 @@ func (c *Cataloger) Name() string { return catalogerName } +// UsesExternalSources indicates that the audit binary cataloger does not use external sources +func (c *Cataloger) UsesExternalSources() bool { + return false +} + // Catalog identifies executables then attempts to read Rust dependency information from them func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package