Fixes to HTTP Header functionality for CLI commands (#1852)

* Simplify HTTP header transport creation - Since we only ever parse and store header values at command startup, we can just log fatally if there is a problem getting the values, instead of propagating an error. This makes the code to construct our http.Transport cleaner and simpler. Signed-off-by: Narsimham Chelluri (Narsa) <narsa@kusari.dev> * Move --header-file to guacone root command Signed-off-by: Narsimham Chelluri (Narsa) <narsa@kusari.dev> * Fix ingestor not using HTTP header file - also fix guacone collect github not using GQL address Signed-off-by: Narsimham Chelluri (Narsa) <narsa@kusari.dev> * Remove unused GQL addr field (dead code) Signed-off-by: Narsimham Chelluri (Narsa) <narsa@kusari.dev> --------- Signed-off-by: Narsimham Chelluri (Narsa) <narsa@kusari.dev>
guacsec · Apr 29, 2024 · 46e8893 · 46e8893
1 parent 8189495
commit 46e8893
Show file tree

Hide file tree

Showing 24 changed files with 197 additions and 189 deletions.
diff --git a/cmd/guaccollect/cmd/gcs.go b/cmd/guaccollect/cmd/gcs.go
@@ -20,7 +20,6 @@ import (
 type gcsOptions struct {
 	pubSubAddr        string
 	blobAddr          string
-	graphqlEndpoint   string
 	csubClientOptions csub_client.CsubClientOptions
 	bucket            string
 }
@@ -39,7 +38,6 @@ var gcsCmd = &cobra.Command{
 		opts, err := validateGCSFlags(
 			viper.GetString("pubsub-addr"),
 			viper.GetString("blob-addr"),
-			viper.GetString("gql-addr"),
 			viper.GetString("csub-addr"),
 			viper.GetString(gcsCredentialsPathFlag),
 			viper.GetBool("csub-tls"),
@@ -93,17 +91,15 @@ var gcsCmd = &cobra.Command{
 func validateGCSFlags(
 	pubSubAddr,
 	blobAddr,
-	gqlEndpoint,
 	csubAddr,
 	credentialsPath string,
 	csubTls,
 	csubTlsSkipVerify bool,
 	args []string,
 ) (gcsOptions, error) {
 	opts := gcsOptions{
-		pubSubAddr:      pubSubAddr,
-		blobAddr:        blobAddr,
-		graphqlEndpoint: gqlEndpoint,
+		pubSubAddr: pubSubAddr,
+		blobAddr:   blobAddr,
 	}
 
 	csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)

diff --git a/cmd/guaccollect/cmd/s3.go b/cmd/guaccollect/cmd/s3.go
@@ -28,7 +28,6 @@ type s3Options struct {
 	mp                string                        // message provider name (sqs or kafka, will default to kafka)
 	mpEndpoint        string                        // endpoint for the message provider (only for polling behaviour)
 	poll              bool                          // polling or non-polling behaviour? (defaults to non-polling)
-	graphqlEndpoint   string                        // endpoint for the graphql server
 	csubClientOptions csub_client.CsubClientOptions // options for the collectsub client
 }
 
@@ -64,7 +63,6 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll
 		s3Opts, err := validateS3Opts(
 			viper.GetString("pubsub-addr"),
 			viper.GetString("blob-addr"),
-			viper.GetString("gql-addr"),
 			viper.GetString("csub-addr"),
 			viper.GetString("s3-url"),
 			viper.GetString("s3-bucket"),
@@ -116,7 +114,6 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll
 func validateS3Opts(
 	pubSubAddr,
 	blobAddr,
-	graphqlEndpoint,
 	csubAddr,
 	s3url,
 	s3bucket,
@@ -162,7 +159,6 @@ func validateS3Opts(
 		mp:                mp,
 		mpEndpoint:        mpEndpoint,
 		poll:              poll,
-		graphqlEndpoint:   graphqlEndpoint,
 		csubClientOptions: csubClientOptions,
 	}
 

diff --git a/cmd/guacingest/cmd/ingest.go b/cmd/guacingest/cmd/ingest.go
@@ -18,13 +18,15 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
 	"strings"
 	"sync"
 	"syscall"
 
 	"github.com/guacsec/guac/pkg/blob"
+	"github.com/guacsec/guac/pkg/cli"
 	"github.com/guacsec/guac/pkg/collectsub/client"
 	csub_client "github.com/guacsec/guac/pkg/collectsub/client"
 	"github.com/guacsec/guac/pkg/emitter"
@@ -41,17 +43,18 @@ type options struct {
 	blobAddr          string
 	csubClientOptions client.CsubClientOptions
 	graphqlEndpoint   string
+	headerFile        string
 }
 
 func ingest(cmd *cobra.Command, args []string) {
-
 	opts, err := validateFlags(
 		viper.GetString("pubsub-addr"),
 		viper.GetString("blob-addr"),
 		viper.GetString("csub-addr"),
+		viper.GetString("gql-addr"),
+		viper.GetString("header-file"),
 		viper.GetBool("csub-tls"),
 		viper.GetBool("csub-tls-skip-verify"),
-		viper.GetString("gql-addr"),
 		args)
 	if err != nil {
 		fmt.Printf("unable to validate flags: %v\n", err)
@@ -61,6 +64,7 @@ func ingest(cmd *cobra.Command, args []string) {
 
 	ctx, cf := context.WithCancel(logging.WithLogger(context.Background()))
 	logger := logging.FromContext(ctx)
+	transport := cli.HTTPHeaderTransport(ctx, opts.headerFile, http.DefaultTransport)
 
 	if strings.HasPrefix(opts.pubsubAddr, "nats://") {
 		// initialize jetstream
@@ -90,7 +94,7 @@ func ingest(cmd *cobra.Command, args []string) {
 	defer csubClient.Close()
 
 	emit := func(d *processor.Document) error {
-		if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, csubClient); err != nil {
+		if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient); err != nil {
 			logger.Errorf("unable to ingest document %q : %v", d.SourceInformation.Source, err)
 		}
 		return nil
@@ -116,7 +120,7 @@ func ingest(cmd *cobra.Command, args []string) {
 	wg.Wait()
 }
 
-func validateFlags(pubsubAddr string, blobAddr string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, graphqlEndpoint string, args []string) (options, error) {
+func validateFlags(pubsubAddr, blobAddr, csubAddr, graphqlEndpoint, headerFile string, csubTls, csubTlsSkipVerify bool, args []string) (options, error) {
 	var opts options
 	opts.pubsubAddr = pubsubAddr
 	opts.blobAddr = blobAddr
@@ -126,6 +130,7 @@ func validateFlags(pubsubAddr string, blobAddr string, csubAddr string, csubTls
 	}
 	opts.csubClientOptions = csubOpts
 	opts.graphqlEndpoint = graphqlEndpoint
+	opts.headerFile = headerFile
 
 	return opts, nil
 }
diff --git a/cmd/guacingest/cmd/root.go b/cmd/guacingest/cmd/root.go
@@ -30,7 +30,7 @@ import (
 func init() {
 	cobra.OnInitialize(cli.InitConfig)
 
-	set, err := cli.BuildFlags([]string{"pubsub-addr", "blob-addr", "csub-addr", "gql-addr"})
+	set, err := cli.BuildFlags([]string{"pubsub-addr", "blob-addr", "csub-addr", "gql-addr", "header-file"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)
 		os.Exit(1)

diff --git a/cmd/guacone/cmd/bad.go b/cmd/guacone/cmd/bad.go
@@ -57,12 +57,7 @@ var queryBadCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		transport, err := cli.NewHTTPHeaderTransport(opts.headerFile, http.DefaultTransport)
-		if err != nil {
-			logger.Fatalf("unable to create HTTP transport: %v", err)
-		}
-
-		httpClient := http.Client{Transport: transport}
+		httpClient := http.Client{Transport: cli.HTTPHeaderTransport(ctx, opts.headerFile, http.DefaultTransport)}
 		gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient)
 
 		certifyBadResponse, err := model.CertifyBads(ctx, gqlclient, model.CertifyBadSpec{})
@@ -258,7 +253,7 @@ func validateQueryBadFlags(graphqlEndpoint, headerFile string, depth int) (query
 }
 
 func init() {
-	set, err := cli.BuildFlags([]string{"header-file", "search-depth"})
+	set, err := cli.BuildFlags([]string{"search-depth"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)
 		os.Exit(1)

diff --git a/cmd/guacone/cmd/certify.go b/cmd/guacone/cmd/certify.go
@@ -18,6 +18,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -35,6 +36,7 @@ import (
 type certifyOptions struct {
 	// gql endpoint
 	graphqlEndpoint string
+	headerFile      string
 	// // certifyBad/certifyGood
 	good          bool
 	certifyType   string
@@ -53,23 +55,24 @@ var certifyCmd = &cobra.Command{
   <subject> is in the form of "<purl>" for package, "<vcs_tool>+<transport>" for source, or "<algorithm>:<digest>" for artifact.`,
 	TraverseChildren: true,
 	Run: func(cmd *cobra.Command, args []string) {
-		ctx := logging.WithLogger(context.Background())
-		logger := logging.FromContext(ctx)
-
 		opts, err := validateCertifyFlags(
 			viper.GetString("gql-addr"),
+			viper.GetString("header-file"),
 			viper.GetBool("cert-good"),
 			viper.GetBool("package-name"),
 			args,
 		)
-
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
 			_ = cmd.Help()
 			os.Exit(1)
 		}
 
-		assemblerFunc := ingestor.GetAssembler(ctx, logger, opts.graphqlEndpoint)
+		ctx := logging.WithLogger(context.Background())
+		logger := logging.FromContext(ctx)
+		transport := cli.HTTPHeaderTransport(ctx, opts.headerFile, http.DefaultTransport)
+
+		assemblerFunc := ingestor.GetAssembler(ctx, logger, opts.graphqlEndpoint, transport)
 
 		preds := &assembler.IngestPredicates{}
 		var pkgInput *model.PkgInputSpec
@@ -156,9 +159,10 @@ var certifyCmd = &cobra.Command{
 	},
 }
 
-func validateCertifyFlags(graphqlEndpoint string, good, pkgName bool, args []string) (certifyOptions, error) {
+func validateCertifyFlags(graphqlEndpoint, headerFile string, good, pkgName bool, args []string) (certifyOptions, error) {
 	var opts certifyOptions
 	opts.graphqlEndpoint = graphqlEndpoint
+	opts.headerFile = headerFile
 	opts.good = good
 	opts.pkgName = pkgName
 	if len(args) != 3 {

diff --git a/cmd/guacone/cmd/deps_dev.go b/cmd/guacone/cmd/deps_dev.go
@@ -18,6 +18,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
 	"sync"
@@ -48,22 +49,24 @@ type depsDevOptions struct {
 	retrieveDependencies bool
 	// gql endpoint
 	graphqlEndpoint string
+	headerFile      string
 }
 
 var depsDevCmd = &cobra.Command{
 	Use:   "deps_dev [flags] <purl1> <purl2>...",
 	Short: "takes purls and queries them against deps.dev to find additional metadata to add to GUAC graph utilizing Nats pubsub and blob store",
 	Run: func(cmd *cobra.Command, args []string) {
-		ctx := logging.WithLogger(context.Background())
-		logger := logging.FromContext(ctx)
-
 		opts, csc, err := validateDepsDevFlags(args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
 			_ = cmd.Help()
 			os.Exit(1)
 		}
 
+		ctx := logging.WithLogger(context.Background())
+		logger := logging.FromContext(ctx)
+		transport := cli.HTTPHeaderTransport(ctx, opts.headerFile, http.DefaultTransport)
+
 		// Register collector
 		depsDevCollector, err := deps_dev.NewDepsCollector(ctx, opts.dataSource, opts.poll, opts.retrieveDependencies, 30*time.Second)
 		if err != nil {
@@ -80,7 +83,7 @@ var depsDevCmd = &cobra.Command{
 		emit := func(d *processor.Document) error {
 			totalNum += 1
 
-			if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, csc); err != nil {
+			if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csc); err != nil {
 				gotErr = true
 				return fmt.Errorf("unable to ingest document: %w", err)
 			}
@@ -135,6 +138,7 @@ func validateDepsDevFlags(args []string) (*depsDevOptions, client.Client, error)
 		poll:                 viper.GetBool("poll"),
 		retrieveDependencies: viper.GetBool("retrieve-dependencies"),
 		graphqlEndpoint:      viper.GetString("gql-addr"),
+		headerFile:           viper.GetString("header-file"),
 	}
 	useCsub := viper.GetBool("use-csub")
 	if useCsub {

diff --git a/cmd/guacone/cmd/files.go b/cmd/guacone/cmd/files.go
@@ -19,6 +19,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -48,6 +49,7 @@ type fileOptions struct {
 	path string
 	// gql endpoint
 	graphqlEndpoint string
+	headerFile      string
 	// csub client options for identifier strings
 	csubClientOptions client.CsubClientOptions
 }
@@ -56,13 +58,11 @@ var filesCmd = &cobra.Command{
 	Use:   "files [flags] file_path",
 	Short: "take a folder of files and create a GUAC graph, this command talks directly to the graphQL endpoint",
 	Run: func(cmd *cobra.Command, args []string) {
-		ctx := logging.WithLogger(context.Background())
-		logger := logging.FromContext(ctx)
-
 		opts, err := validateFilesFlags(
 			viper.GetString("verifier-key-path"),
 			viper.GetString("verifier-key-id"),
 			viper.GetString("gql-addr"),
+			viper.GetString("header-file"),
 			viper.GetString("csub-addr"),
 			viper.GetBool("csub-tls"),
 			viper.GetBool("csub-tls-skip-verify"),
@@ -73,6 +73,10 @@ var filesCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
+		ctx := logging.WithLogger(context.Background())
+		logger := logging.FromContext(ctx)
+		transport := cli.HTTPHeaderTransport(ctx, opts.headerFile, http.DefaultTransport)
+
 		// Register Keystore
 		inmemory := inmemory.NewInmemoryProvider()
 		err = key.RegisterKeyProvider(inmemory, inmemory.Type())
@@ -122,7 +126,7 @@ var filesCmd = &cobra.Command{
 
 		emit := func(d *processor.Document) error {
 			totalNum += 1
-			if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, csubClient); err != nil {
+			if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient); err != nil {
 				gotErr = true
 				filesWithErrors = append(filesWithErrors, d.SourceInformation.Source)
 				return fmt.Errorf("unable to ingest document: %w", err)
@@ -154,9 +158,10 @@ var filesCmd = &cobra.Command{
 	},
 }
 
-func validateFilesFlags(keyPath string, keyID string, graphqlEndpoint string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, args []string) (fileOptions, error) {
+func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool, args []string) (fileOptions, error) {
 	var opts fileOptions
 	opts.graphqlEndpoint = graphqlEndpoint
+	opts.headerFile = headerFile
 
 	if keyPath != "" {
 		if strings.HasSuffix(keyPath, "pem") {