scribe patch - cataloger group, export lib

cmd/syft/cli/commands/lib_scribe.go

@@ -0,0 +1,61 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/anchore/clio"
+	"github.com/anchore/go-logger"
+	"github.com/anchore/stereoscope"
+	"github.com/anchore/syft/cmd/syft/cli/options"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/sbom"
+	"github.com/anchore/syft/syft/source"
+)
+
+func LibInitLoggingConfig(logWrapper logger.Logger) {
+	syft.SetLogger(logWrapper)
+	stereoscope.SetLogger(logWrapper)
+}
+
+func DefaultPackagesOptions() *packagesOptions {
+	return defaultPackagesOptions()
+}
+
+type PackagesOptions packagesOptions
+
+func GetSource(opts *options.Catalog, userInput string, filters ...func(*source.Detection) error) (source.Source, error) {
+	return getSource(opts, userInput, filters...)
+}
+
+func LibPackagesExec(id clio.Identification, opts *PackagesOptions, userInput string, l logger.Logger, enable_log bool) (*sbom.SBOM, error) {
+	if enable_log {
+		LibInitLoggingConfig(l)
+	}
+
+	src, err := getSource(&opts.Catalog, userInput)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if src != nil {
+			if err := src.Close(); err != nil {
+				log.Tracef("unable to close source: %+v", err)
+			}
+		}
+	}()
+
+	s, err := generateSBOM(id, src, &opts.Catalog)
+	if err != nil {
+		return nil, err
+	}
+
+	if s == nil {
+		return nil, fmt.Errorf("no SBOM produced for %q", userInput)
+	}
+
+	return s, nil
+
+}

cmd/syft/cli/eventloop/tasks.go

@@ -44,7 +44,7 @@ func generateCatalogPackagesTask(opts *options.Catalog) (Task, error) {
 	}
 
 	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
-		packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, opts.ToCatalogerConfig())
+		packageCatalog, relationships, theDistro, err := syft.CatalogPackagesScribe(src, opts.ToCatalogerConfig())
 
 		results.Packages = packageCatalog
 		results.LinuxDistribution = theDistro

cmd/syft/cli/options/catalog.go

@@ -143,6 +143,7 @@ func (cfg Catalog) ToCatalogerConfig() cataloger.Config {
 			GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
 		},
 		ExcludeBinaryOverlapByOwnership: cfg.ExcludeBinaryOverlapByOwnership,
+		CatalogerGroup:                  cfg.Package.CatalogerGroup,
 	}
 }
 

cmd/syft/cli/options/lib_scribe.go

@@ -0,0 +1,31 @@
+package options
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/anchore/syft/syft/sbom"
+)
+
+type SbomBuffer struct {
+	Format sbom.FormatEncoder
+	buf    *bytes.Buffer
+}
+
+func (w *SbomBuffer) Read() []byte {
+	if w.buf != nil {
+		return w.buf.Bytes()
+	}
+
+	return []byte{}
+}
+
+func (w *SbomBuffer) Write(s sbom.SBOM) error {
+	if w.buf == nil {
+		w.buf = &bytes.Buffer{}
+	}
+	if err := w.Format.Encode(w.buf, s); err != nil {
+		return fmt.Errorf("unable to encode SBOM: %w", err)
+	}
+	return nil
+}

cmd/syft/cli/options/pkg.go

@@ -5,9 +5,10 @@ import (
 )
 
 type pkg struct {
-	Cataloger               scope `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
-	SearchUnindexedArchives bool  `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
-	SearchIndexedArchives   bool  `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
+	Cataloger               scope           `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
+	SearchUnindexedArchives bool            `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
+	SearchIndexedArchives   bool            `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
+	CatalogerGroup          cataloger.Group `yaml:"cataloger-group" json:"cataloger-group" mapstructure:"cataloger-group"`
 }
 
 func defaultPkg() pkg {

syft/format/common/cyclonedxhelpers/lib_scribe.go

@@ -0,0 +1,10 @@
+package cyclonedxhelpers
+
+import (
+	"github.com/CycloneDX/cyclonedx-go"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func EncodeLicenses(p pkg.Package) *cyclonedx.Licenses {
+	return encodeLicenses(p)
+}

syft/format/syftjson/lib_scribe.go

@@ -0,0 +1,16 @@
+package syftjson
+
+import (
+	"github.com/anchore/syft/syft/format/syftjson/model"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func ToSourceModel(src source.Description) model.Source {
+	return toSourceModel(src)
+}
+
+func ToSyftPackage(p model.Package) pkg.Package {
+	m := make(map[string]string)
+	return toSyftPackage(p, m)
+}

syft/lib_scribe.go

@@ -0,0 +1,67 @@
+package syft
+
+import (
+	"fmt"
+
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/linux"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger"
+	"github.com/anchore/syft/syft/source"
+)
+
+// CatalogPackages takes an inventory of packages from the given image from a particular perspective
+// (e.g. squashed source, all-layers source). Returns the discovered  set of packages, the identified Linux
+// distribution, and the source object used to wrap the data source.
+func CatalogPackagesScribe(src source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
+	resolver, err := src.FileResolver(cfg.Search.Scope)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
+	}
+
+	// find the distro
+	release := linux.IdentifyRelease(resolver)
+	if release != nil {
+		log.Infof("identified distro: %s", release.String())
+	} else {
+		log.Info("could not identify distro")
+	}
+
+	if cfg.CatalogerGroup == "" {
+		switch t := src.(type) {
+		case *source.StereoscopeImageSource:
+			cfg.CatalogerGroup = cataloger.InstallationGroup
+		case *source.FileSource:
+			cfg.CatalogerGroup = cataloger.AllGroup
+		case *source.DirectorySource:
+			cfg.CatalogerGroup = cataloger.IndexGroup
+		default:
+			return nil, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", t)
+		}
+	}
+
+	groupCatalogers, err := cataloger.SelectGroup(cfg)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	enabledCatalogers := cataloger.FilterCatalogers(cfg, groupCatalogers)
+
+	catalog, relationships, err := cataloger.Catalog(resolver, release, cfg.Parallelism, enabledCatalogers...)
+
+	// apply exclusions to the package catalog
+	// default config value for this is true
+	// https://github.com/anchore/syft/issues/931
+	if cfg.ExcludeBinaryOverlapByOwnership {
+		for _, r := range relationships {
+			if cataloger.ExcludeBinaryByFileOwnershipOverlap(r, catalog) {
+				catalog.Delete(r.To.ID())
+				relationships = removeRelationshipsByID(relationships, r.To.ID())
+			}
+		}
+	}
+
+	// no need to consider source relationships for os -> binary exclusions
+	relationships = append(relationships, newSourceRelationshipsFromCatalog(src, catalog)...)
+	return catalog, relationships, release, err
+}

syft/pkg/cataloger/config.go

@@ -17,6 +17,7 @@ type Config struct {
 	Catalogers                      []string
 	Parallelism                     int
 	ExcludeBinaryOverlapByOwnership bool
+	CatalogerGroup                  Group
 }
 
 func DefaultConfig() Config {

syft/pkg/cataloger/haskell/parse_stack_lock.go

@@ -46,7 +46,6 @@ func parseStackLock(_ file.Resolver, _ *generic.Environment, reader file.Locatio
 	}
 
 	var lockFile stackLock
-
 	if err := yaml.Unmarshal(bytes, &lockFile); err != nil {
 		log.WithFields("error", err).Tracef("failed to parse stack.yaml.lock file %q", reader.RealPath)
 		return nil, nil, nil
@@ -63,6 +62,9 @@ func parseStackLock(_ file.Resolver, _ *generic.Environment, reader file.Locatio
 	}
 
 	for _, pack := range lockFile.Packages {
+		if pack.Completed.Hackage == "" {
+			continue
+		}
 		pkgName, pkgVersion, pkgHash := parseStackPackageEncoding(pack.Completed.Hackage)
 		pkgs = append(
 			pkgs,
@@ -86,7 +88,9 @@ func parseStackPackageEncoding(pkgEncoding string) (name, version, hash string)
 	remainingEncoding := pkgEncoding[lastDashIdx+1:]
 	encodingSplits := strings.Split(remainingEncoding, "@")
 	version = encodingSplits[0]
-	startHash, endHash := strings.Index(encodingSplits[1], ":")+1, strings.Index(encodingSplits[1], ",")
-	hash = encodingSplits[1][startHash:endHash]
+	if len(encodingSplits) > 1 {
+		startHash, endHash := strings.Index(encodingSplits[1], ":")+1, strings.Index(encodingSplits[1], ",")
+		hash = encodingSplits[1][startHash:endHash]
+	}
 	return
 }

syft/pkg/cataloger/lib_scribe.go

@@ -0,0 +1,42 @@
+package cataloger
+
+import (
+	"fmt"
+
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func FilterCatalogers(cfg Config, groupCatalogers []pkg.Cataloger) []pkg.Cataloger {
+	return filterCatalogers(groupCatalogers, cfg.Catalogers)
+}
+
+func SelectGroup(cfg Config) ([]pkg.Cataloger, error) {
+	switch cfg.CatalogerGroup {
+	case IndexGroup:
+		log.Info("cataloging index group")
+		return DirectoryCatalogers(cfg), nil
+	case InstallationGroup:
+		log.Info("cataloging installation group")
+		return ImageCatalogers(cfg), nil
+	case AllGroup:
+		log.Info("cataloging all group")
+		return AllCatalogers(cfg), nil
+	default:
+		return nil, fmt.Errorf("unknown cataloger group, Group: %s", cfg.CatalogerGroup)
+	}
+}
+
+type Group string
+
+const (
+	IndexGroup        Group = "index"
+	InstallationGroup Group = "install"
+	AllGroup          Group = "all"
+)
+
+var AllGroups = []Group{
+	IndexGroup,
+	InstallationGroup,
+	AllGroup,
+}