Add list-providers command

.github/scripts/aggregate-all-provider-cache.py

@@ -1,11 +1,24 @@
 #!/usr/bin/env python3
-import yaml
+import os
+import ast
 import sys
 import subprocess
 
-with open(".grype-db.yaml") as f:
-    providers = [x["name"] for x in yaml.safe_load(f.read()).get("provider", {}).get("configs", [])]
+output = os.environ.get("PROVIDERS_USED", None)
+
+if not output:
+    print(f"invoking grype-db to get list of providers to use")
+    output = subprocess.run("make show-providers", shell=True, check=True, stdout=subprocess.PIPE, stderr=sys.stderr).stdout
+else:
+    print("using values from $PROVIDERS_USED environment variable")
+
+print(f"output:   {output!r}")
+
+# why in the world would we use ast instead of JSON?!
+# short answer: python borks when there are strings with single quotes instead of double quotes
+providers = ast.literal_eval(output)
 
 print(f"providers: {providers}")
+
 for provider in providers:
     subprocess.run(f"make download-provider-cache provider={provider}", shell=True, check=True, stdout=sys.stdout, stderr=sys.stderr)

.github/workflows/daily-data-sync.yaml

@@ -19,6 +19,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+        with:
+          python: false
+
+      - name: Login to ghcr.io
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin
+
       - name: Read configured providers
         id: read-providers
         # TODO: honor CI overrides
@@ -59,9 +68,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload the provider workspace state
-        run: make upload-provider-cache provider=${{ matrix.provider }}
-
       - uses: 8398a7/action-slack@v3
         with:
           status: ${{ job.status }}
@@ -71,10 +77,18 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
         if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}
 
+      - name: Upload the provider workspace state
+        # even if the job fails, we want to upload yesterdays cache as todays cache to continue the DB build
+        if: ${{ always() }}
+        run: make upload-provider-cache provider=${{ matrix.provider }}
+
   aggregate-cache:
     name: "Aggregate provider cache"
     runs-on: ubuntu-20.04
-    needs: update-provider
+    if: ${{ always() }}
+    needs:
+      - update-provider
+      - discover-providers
     # set the permissions granted to the github token to read the pull cache from ghcr.io
     permissions:
       packages: write
@@ -93,17 +107,9 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin
 
       - name: Aggregate vulnerability data
-      # TODO: hook up to matrix override
         run: make aggregate-all-provider-cache
+        env:
+          PROVIDERS_USED: ${{ needs.discover-providers.outputs.providers }}
 
       - name: Upload vulnerability data cache image
         run: make upload-all-provider-cache
-
-      - uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: workflow,eventName
-          text: Daily Data Sync aggregation failed
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
-        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}

Makefile

@@ -5,7 +5,7 @@ TEMP_DIR = ./.tmp
 RESULTS_DIR = $(TEMP_DIR)/results
 
 DB_ARCHIVE = ./grype-db-cache.tar.gz
-GRYPE_DB = go run ./cmd/$(BIN)/main.go
+GRYPE_DB = go run ./cmd/$(BIN)/main.go -c publish/.grype-db.yaml
 GRYPE_DB_DATA_IMAGE_NAME = ghcr.io/anchore/$(BIN)/data
 date = $(shell date -u +"%y-%m-%d")
 
@@ -197,7 +197,7 @@ update-test-fixtures:
 .PHONY: show-providers
 show-providers:
 	@# this is used in CI to generate a job matrix, pulling data for each provider concurrently
-	@cat .grype-db.yaml | python -c 'import yaml; import json; import sys; print(json.dumps([x["name"] for x in yaml.safe_load(sys.stdin).get("provider",{}).get("configs",[])]));'
+	@$(GRYPE_DB) list-providers -q -o json
 
 .PHONY: download-provider-cache
 download-provider-cache:
@@ -214,6 +214,7 @@ upload-provider-cache: ci-check
 	$(call title,Uploading "$(provider)" existing provider data cache)
 
 	@rm -f $(DB_ARCHIVE)
+	$(GRYPE_DB) cache status -p $(provider)
 	$(GRYPE_DB) cache backup -v --path $(DB_ARCHIVE) -p $(provider)
 	oras push -v $(GRYPE_DB_DATA_IMAGE_NAME)/$(provider):$(date) $(DB_ARCHIVE) --annotation org.opencontainers.image.source=$(SOURCE_REPO_URL)
 	$(TEMP_DIR)/crane tag $(GRYPE_DB_DATA_IMAGE_NAME)/$(provider):$(date) latest
@@ -228,6 +229,7 @@ upload-all-provider-cache: ci-check
 	$(call title,Uploading existing provider data cache)
 
 	@rm -f $(DB_ARCHIVE)
+	$(GRYPE_DB) cache status
 	$(GRYPE_DB) cache backup -v --path $(DB_ARCHIVE)
 	oras push -v $(GRYPE_DB_DATA_IMAGE_NAME):$(date) $(DB_ARCHIVE) --annotation org.opencontainers.image.source=$(SOURCE_REPO_URL)
 	$(TEMP_DIR)/crane tag $(GRYPE_DB_DATA_IMAGE_NAME):$(date) latest

cmd/grype-db/cli/cli.go

@@ -40,6 +40,7 @@ func New(opts ...Option) *cobra.Command {
 	root.AddCommand(commands.Pull(app))
 	root.AddCommand(commands.Build(app))
 	root.AddCommand(commands.Package(app))
+	root.AddCommand(commands.ListProviders(app))
 	root.AddCommand(cache)
 
 	return root

cmd/grype-db/cli/commands/cache_status.go

@@ -6,24 +6,39 @@ import (
 	"time"
 
 	"github.com/gookit/color"
+	"github.com/scylladb/go-set/strset"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 
 	"github.com/anchore/grype-db/cmd/grype-db/application"
 	"github.com/anchore/grype-db/cmd/grype-db/cli/options"
+	"github.com/anchore/grype-db/internal/log"
 	"github.com/anchore/grype-db/pkg/provider"
 	"github.com/anchore/grype-db/pkg/provider/entry"
 )
 
 var _ options.Interface = &cacheStatusConfig{}
 
 type cacheStatusConfig struct {
-	options.Store `yaml:"provider" json:"provider" mapstructure:"provider"`
+	Provider struct {
+		options.Store     `yaml:",inline" mapstructure:",squash"`
+		options.Selection `yaml:",inline" mapstructure:",squash"`
+	} `yaml:"provider" json:"provider" mapstructure:"provider"`
+}
+
+func (o *cacheStatusConfig) AddFlags(flags *pflag.FlagSet) {
+	options.AddAllFlags(flags, &o.Provider.Store, &o.Provider.Selection)
+}
+
+func (o *cacheStatusConfig) BindFlags(flags *pflag.FlagSet, v *viper.Viper) error {
+	return options.BindAllFlags(flags, v, &o.Provider.Store, &o.Provider.Selection)
 }
 
 func CacheStatus(app *application.Application) *cobra.Command {
-	cfg := cacheStatusConfig{
-		Store: options.DefaultStore(),
-	}
+	cfg := cacheStatusConfig{}
+	cfg.Provider.Store = options.DefaultStore()
+	cfg.Provider.Selection = options.DefaultSelection()
 
 	cmd := &cobra.Command{
 		Use:     "status",
@@ -43,7 +58,7 @@ func CacheStatus(app *application.Application) *cobra.Command {
 }
 
 func cacheStatus(cfg cacheStatusConfig) error {
-	providerNames, err := readProviderNamesFromRoot(cfg.Store.Root)
+	providerNames, err := readProviderNamesFromRoot(cfg.Provider.Root)
 	if err != nil {
 		return err
 	}
@@ -56,8 +71,15 @@ func cacheStatus(cfg cacheStatusConfig) error {
 	var sds []*provider.State
 	var errs []error
 
+	allowableProviders := strset.New(cfg.Provider.IncludeFilter...)
+
 	for _, name := range providerNames {
-		workspace := provider.NewWorkspace(cfg.Store.Root, name)
+		if allowableProviders.Size() > 0 && !allowableProviders.Has(name) {
+			log.WithFields("provider", name).Trace("skipping...")
+			continue
+		}
+
+		workspace := provider.NewWorkspace(cfg.Provider.Root, name)
 		sd, err := workspace.ReadState()
 		if err != nil {
 			sds = append(sds, nil)
@@ -75,6 +97,8 @@ func cacheStatus(cfg cacheStatusConfig) error {
 		sds = append(sds, sd)
 	}
 
+	success := true
+
 	for idx, sd := range sds {
 		validMsg := "valid"
 		isValid := true
@@ -98,6 +122,8 @@ func cacheStatus(cfg cacheStatusConfig) error {
 			}
 		}
 
+		success = success && isValid
+
 		fmt.Printf("  • %s\n", name)
 		statusFmt := color.HiRed
 		if isValid {
@@ -108,6 +134,11 @@ func cacheStatus(cfg cacheStatusConfig) error {
 
 		fmt.Printf("    └── status:  %s\n", statusFmt.Sprint(validMsg))
 	}
+
+	if !success {
+		os.Exit(1)
+	}
+
 	return nil
 }
 

cmd/grype-db/cli/commands/list_providers.go

@@ -0,0 +1,104 @@
+package commands
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/scylladb/go-set/strset"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
+	"github.com/anchore/grype-db/cmd/grype-db/application"
+	"github.com/anchore/grype-db/cmd/grype-db/cli/options"
+	"github.com/anchore/grype-db/internal/log"
+	"github.com/anchore/grype-db/pkg/provider/providers"
+	"github.com/anchore/grype-db/pkg/provider/providers/vunnel"
+)
+
+var _ options.Interface = &listProvidersConfig{}
+
+type listProvidersConfig struct {
+	options.Format   `yaml:",inline" mapstructure:",squash"`
+	options.Provider `yaml:"provider" json:"provider" mapstructure:"provider"`
+}
+
+func (o *listProvidersConfig) AddFlags(flags *pflag.FlagSet) {
+	options.AddAllFlags(flags, &o.Format, &o.Provider)
+}
+
+func (o *listProvidersConfig) BindFlags(flags *pflag.FlagSet, v *viper.Viper) error {
+	return options.BindAllFlags(flags, v, &o.Format, &o.Provider)
+}
+
+func ListProviders(app *application.Application) *cobra.Command {
+	cfg := listProvidersConfig{
+		Provider: options.DefaultProvider(),
+		Format: options.Format{
+			Output:           "text",
+			AllowableFormats: []string{"text", "json"},
+		},
+	}
+
+	cmd := &cobra.Command{
+		Use:   "list-providers",
+		Short: "list all configured providers",
+		Args: chainArgs(
+			cobra.NoArgs,
+			func(cmd *cobra.Command, args []string) error {
+				allowableOutputs := strset.New(cfg.Format.AllowableFormats...)
+				if !allowableOutputs.Has(cfg.Format.Output) {
+					return fmt.Errorf("invalid output format: %s (allowable: %s)", cfg.Format.Output, strings.Join(cfg.Format.AllowableFormats, ", "))
+				}
+				return nil
+			},
+		),
+		PreRunE: app.Setup(&cfg),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return app.Run(cmd.Context(), async(func() error {
+				return runListProviders(cfg)
+			}))
+		},
+	}
+
+	commonConfiguration(app, cmd, &cfg)
+
+	return cmd
+}
+
+func runListProviders(cfg listProvidersConfig) error {
+	ps, err := providers.New(cfg.Root, vunnel.Config{
+		Executor:         cfg.Vunnel.Executor,
+		DockerTag:        cfg.Vunnel.DockerTag,
+		DockerImage:      cfg.Vunnel.DockerImage,
+		GenerateConfigs:  cfg.Vunnel.GenerateConfigs,
+		ExcludeProviders: cfg.Vunnel.ExcludeProviders,
+		Env:              cfg.Vunnel.Env,
+	}, cfg.Provider.Configs...)
+	if err != nil {
+		if errors.Is(err, providers.ErrNoProviders) {
+			log.Error("configure a provider via the application config or use -g to generate a list of configs from vunnel")
+		}
+		return err
+	}
+
+	if cfg.Format.Output == "text" {
+		for _, p := range ps {
+			fmt.Println(p.ID().Name)
+		}
+	} else if cfg.Format.Output == "json" {
+		names := make([]string, 0, len(ps))
+		for _, p := range ps {
+			names = append(names, p.ID().Name)
+		}
+		by, err := json.Marshal(names)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(by))
+	}
+
+	return nil
+}