add support for --e2e-docker-config-file flag

Signed-off-by: Divya Rani <drani@vmware.com>

cmd/sonobuoy/app/args.go

@@ -50,6 +50,7 @@ const (
 	e2eSkipFlag                  = "e2e-skip"
 	e2eParallelFlag              = "e2e-parallel"
 	e2eRegistryConfigFlag        = "e2e-repo-config"
+	e2eDockerConfigFileFlag      = "e2e-docker-config-file"
 	e2eRegistryFlag              = "e2e-repo"
 	pluginImageFlag              = "plugin-image"
 	filenameFlag                 = "filename"
@@ -224,7 +225,7 @@ func AddSonobuoyConfigFlag(cfg *SonobuoyConfig, flags *pflag.FlagSet) {
 // AddLegacyE2EFlags is a way to add flags which target the e2e plugin specifically
 // by leveraging the existing flags. They typically wrap other fields (like the env var
 // overrides) and modify those.
-func AddLegacyE2EFlags(env *PluginEnvVars, pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) {
+func AddLegacyE2EFlags(cfg *SonobuoyConfig, env *PluginEnvVars, pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) {
 	m := &Mode{
 		env:  env,
 		name: "",
@@ -289,6 +290,15 @@ func AddLegacyE2EFlags(env *PluginEnvVars, pluginTransforms *map[string][]func(*
 		&envVarModierFlag{plugin: e2ePlugin, field: "KUBE_TEST_REPO", PluginEnvVars: *env}, e2eRegistryFlag,
 		"Specify a registry to use as the default for pulling Kubernetes test images. Same as providing --e2e-repo-config but specifying the same repo repeatedly.",
 	)
+
+	fs.Var(
+		&e2eDockerConfigFlag{
+			plugin:     e2ePlugin,
+			config:     cfg,
+			transforms: *pluginTransforms,
+		}, e2eDockerConfigFileFlag,
+		"A docker credentials configuration file used which contains authorization token that can be used to pull images from certain private registries provided by the users",
+	)
 }
 
 // AddRBACModeFlags adds an E2E Argument with the provided default.
@@ -563,9 +573,11 @@ func (f *e2eRepoFlag) Set(str string) error {
 	}
 
 	f.transforms[f.plugin] = append(f.transforms[f.plugin], func(m *manifest.Manifest) error {
-		m.ConfigMap = map[string]string{
-			name: string(fData),
+		if m.ConfigMap == nil {
+			m.ConfigMap = map[string]string{}
 		}
+		m.ConfigMap[name] = string(fData)
+
 		m.Spec.Env = append(m.Spec.Env, corev1.EnvVar{
 			Name:  "KUBE_TEST_REPO_LIST",
 			Value: fmt.Sprintf("/tmp/sonobuoy/config/%v", name),
@@ -575,6 +587,40 @@ func (f *e2eRepoFlag) Set(str string) error {
 	return nil
 }
 
+type e2eDockerConfigFlag struct {
+	plugin   string
+	filename string
+	config   *SonobuoyConfig
+
+	transforms map[string][]func(*manifest.Manifest) error
+	// Value to put in the configmap as the filename. Defaults to filename.
+	filenameOverride string
+}
+
+func (f *e2eDockerConfigFlag) String() string { return f.filename }
+func (f *e2eDockerConfigFlag) Type() string   { return "json-filepath" }
+func (f *e2eDockerConfigFlag) Set(str string) error {
+	f.config.E2EDockerConfigFile = str
+	name := filepath.Base(str)
+	if len(f.filenameOverride) > 0 {
+		name = f.filenameOverride
+	}
+	fData, err := os.ReadFile(str)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read file %q", str)
+	}
+
+	f.transforms[e2ePlugin] = append(f.transforms[e2ePlugin], func(m *manifest.Manifest) error {
+		if m.ConfigMap == nil {
+			m.ConfigMap = map[string]string{}
+		}
+		m.ConfigMap[name] = string(fData)
+		return nil
+
+	})
+	return nil
+}
+
 // The ssh-key flag needs to store the path to the ssh key but also
 // wire up the e2e plugin for using it.
 type sshPathFlag struct {

cmd/sonobuoy/app/gen.go

@@ -21,6 +21,8 @@ package app
 import (
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/vmware-tanzu/sonobuoy/pkg/client"
@@ -94,7 +96,7 @@ func GenFlagSet(cfg *genFlags, rbac RBACMode) *pflag.FlagSet {
 
 	AddPluginSetFlag(&cfg.plugins, genset)
 	AddPluginEnvFlag(&cfg.pluginEnvs, genset)
-	AddLegacyE2EFlags(&cfg.pluginEnvs, &cfg.pluginTransforms, genset)
+	AddLegacyE2EFlags(&cfg.sonobuoyConfig, &cfg.pluginEnvs, &cfg.pluginTransforms, genset)
 
 	AddNodeSelectorsFlag(&cfg.nodeSelectors, genset)
 
@@ -165,6 +167,15 @@ func (g *genFlags) Config() (*client.GenConfig, error) {
 		k8sVersion = g.k8sVersion.String()
 	}
 
+	if g.sonobuoyConfig.E2EDockerConfigFile != "" {
+		if err := verifyKubernetesVersion(k8sVersion); err != nil {
+			return nil, err
+		}
+		if g.sonobuoyConfig.ImagePullSecrets == "" {
+			g.sonobuoyConfig.ImagePullSecrets = "auth-repo-cred"
+		}
+	}
+
 	return &client.GenConfig{
 		Config:             &g.sonobuoyConfig.Config,
 		EnableRBAC:         rbacEnabled,
@@ -207,6 +218,7 @@ func NewCmdGen() *cobra.Command {
 		Args:  cobra.ExactArgs(0),
 	}
 	GenCommand.Flags().AddFlagSet(GenFlagSet(&genflags, EnabledRBACMode))
+
 	return GenCommand
 }
 
@@ -261,3 +273,25 @@ func getClient(kubeconfig *Kubeconfig) (*kubernetes.Clientset, error) {
 
 	return client, kubeError
 }
+
+func verifyKubernetesVersion(k8sVersion string) error {
+	parts := versionMatchRE.FindStringSubmatch(k8sVersion)
+	if parts == nil {
+		return fmt.Errorf("could not parse %q as version", k8sVersion)
+	}
+	numbers, _ := parts[1], parts[2]
+
+	versionInfo := strings.Split(numbers, ".")
+	minorVersion := versionInfo[1]
+
+	minorVersionInt, err := strconv.ParseInt(minorVersion, 10, 0)
+	if err != nil {
+		return err
+	}
+
+	if minorVersionInt < 27 {
+		err = fmt.Errorf("e2e-docker-config-file is only supported for Kubernetes 1.27 or later")
+	}
+
+	return err
+}

cmd/sonobuoy/app/gen_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright the Sonobuoy contributors 2019
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+
+	 Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"testing"
+)
+
+func TestVerifyKubernetesVersion(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		k8sVersion string
+		expectErr  string
+	}{
+		{
+			desc:       "Usage of e2e-docker-config-file flag with Kubernetes versions below 1.27 should throw error",
+			k8sVersion: "1.26-alpha-3",
+			expectErr:  "e2e-docker-config-file is only supported for Kubernetes 1.27 or later",
+		},
+		{
+			desc:       "Usage of e2e-docker-config-file flag along with Kubernetes versions 1.27 or later should work as expected",
+			k8sVersion: "1.27",
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.desc, func(t *testing.T) {
+			err := verifyKubernetesVersion(testCase.k8sVersion)
+			if err != nil {
+				if len(testCase.expectErr) == 0 {
+					t.Fatalf("Expected nil error but got %v", err)
+				}
+				if err.Error() != testCase.expectErr {
+					t.Fatalf("Expected error %q but got %q", err, testCase.expectErr)
+				}
+			}
+		})
+
+	}
+}

cmd/sonobuoy/app/images.go

@@ -451,7 +451,6 @@ func translateRegistry(imageURL string, customRegistry string, customRegistryLis
 	if len(customRegistry) > 0 {
 		return fmt.Sprintf("%s/%s", customRegistry, parts[countParts-1])
 	}
-
 	// For now, if not given a customRegistry, assume they gave the customRegistryList as non-nil.
 	switch registryAndUser {
 	case "gcr.io/e2e-test-images":

cmd/sonobuoy/app/root.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"regexp"
 	"sync"
 
 	"github.com/sirupsen/logrus"
@@ -34,6 +35,10 @@ import (
 // and cause a panic otherwise.
 var once sync.Once
 
+// versionMatchRE splits a version string into numeric and "extra" parts
+// source: https://github.com/kubernetes/kubernetes/blob/af1bf4306709020ed2002618e099b3d0acf3c7b5/staging/src/k8s.io/apimachinery/pkg/util/version/version.go#L37
+var versionMatchRE = regexp.MustCompile(`^\s*v?([0-9]+(?:\.[0-9]+)*)(.*)*$`)
+
 func NewSonobuoyCommand() *cobra.Command {
 	cmds := &cobra.Command{
 		Use:               "sonobuoy",
@@ -106,8 +111,10 @@ func prerunChecks(cmd *cobra.Command, args []string) error {
 	// Getting a list of all flags provided by the user.
 	flagsSet := map[string]bool{}
 	flagsDebug := []string{}
+	flagArgs := map[string]string{}
 	cmd.Flags().Visit(func(f *pflag.Flag) {
 		flagsSet[f.Name] = true
+		flagArgs[f.Name] = f.Value.String()
 		flagsDebug = append(flagsDebug, fmt.Sprintf("%v=%v", f.Name, f.Value.String()))
 	})
 
@@ -135,6 +142,13 @@ func prerunChecks(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("%v and %v flags are both set and may collide", e2eRegistryConfigFlag, e2eRegistryFlag)
 	}
 
+	if flagsSet[e2eDockerConfigFileFlag] && flagsSet["kubernetes-version"] {
+		k8sVersion := flagArgs["kubernetes-version"]
+		if err := verifyKubernetesVersion(k8sVersion); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

pkg/client/gen.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -51,7 +52,8 @@ const (
 
 	aggregatorEnvOverrideKey = `sonobuoy`
 
-	envVarKeyExtraArgs = "E2E_EXTRA_ARGS"
+	envVarKeyExtraArgs         = "E2E_EXTRA_ARGS"
+	defaultImagePullSecretName = "auth-repo-cred"
 
 	// sonobuoyKey is just a true/false env to indicate that the container was launched/tagged by Sonobuoy.
 	sonobuoyKey                 = "SONOBUOY"
@@ -164,6 +166,7 @@ func (*SonobuoyClient) GenerateManifestAndPlugins(cfg *GenConfig) ([]byte, []*ma
 		if len(p.ConfigMap) == 0 {
 			continue
 		}
+
 		configs[p.SonobuoyConfig.PluginName] = p.ConfigMap
 		p.ExtraVolumes = append(p.ExtraVolumes,
 			manifest.Volume{
@@ -218,6 +221,11 @@ func generateYAMLComponents(w io.Writer, cfg *GenConfig, plugins []*manifest.Man
 	if err := generateServiceAcct(w, cfg); err != nil {
 		return err
 	}
+	if cfg.Config.E2EDockerConfigFile != "" {
+		if err := generateRegistrySecret(w, cfg); err != nil {
+			return err
+		}
+	}
 	if err := generateRBAC(w, cfg); err != nil {
 		return err
 	}
@@ -227,6 +235,7 @@ func generateYAMLComponents(w io.Writer, cfg *GenConfig, plugins []*manifest.Man
 	if err := generateSecret(w, cfg); err != nil {
 		return err
 	}
+
 	if err := generatePluginConfigmap(w, cfg, plugins); err != nil {
 		return err
 	}
@@ -261,7 +270,9 @@ func generateAdditionalConfigmaps(w io.Writer, cfg *GenConfig, configs map[strin
 		sort.Strings(filenames)
 		for _, filename := range filenames {
 			cm.Data[filename] = configs[pluginName][filename]
+
 		}
+
 		if err := appendAsYAML(w, cm); err != nil {
 			return err
 		}
@@ -282,6 +293,7 @@ func generatePluginConfigmap(w io.Writer, cfg *GenConfig, plugins []*manifest.Ma
 		}
 		cm.Data[fmt.Sprintf("plugin-%v.yaml", i)] = strings.TrimSpace(string(b))
 	}
+
 	return appendAsYAML(w, cm)
 }
 
@@ -309,6 +321,24 @@ func appendAsYAML(w io.Writer, o kuberuntime.Object) error {
 	return err
 }
 
+func generateRegistrySecret(w io.Writer, cfg *GenConfig) error {
+	contents, err := os.ReadFile(cfg.Config.E2EDockerConfigFile)
+	if err != nil {
+		return fmt.Errorf("error reading docker config file: %v", err)
+	}
+	s := &corev1.Secret{
+		Type: corev1.SecretTypeDockerConfigJson,
+		Data: map[string][]byte{corev1.DockerConfigJsonKey: contents},
+	}
+	s.Name = cfg.Config.ImagePullSecrets
+	s.Namespace = cfg.Config.Namespace
+
+	s.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"})
+
+	return appendAsYAML(w, s)
+
+}
+
 func generateSecret(w io.Writer, cfg *GenConfig) error {
 	if len(cfg.SSHKeyPath) == 0 {
 		return nil
@@ -790,14 +820,14 @@ func E2EManifest(cfg *GenConfig) *manifest.Manifest {
 	}
 	m.PodSpec.PodSpec.NodeSelector = map[string]string{"kubernetes.io/os": "linux"}
 
-	m.Spec.Env = updateExtraArgs(m.Spec.Env, cfg.Config.ProgressUpdatesPort)
+	m.Spec.Env = updateExtraArgs(m.Spec.Env, cfg.Config.ProgressUpdatesPort, cfg.Config.E2EDockerConfigFile)
 
 	return m
 }
 
 // updateExtraArgs adds the flag expected by the e2e plugin for the progress report URL.
 // If no port is given, the default "8099" is used.
-func updateExtraArgs(envs []corev1.EnvVar, port string) []corev1.EnvVar {
+func updateExtraArgs(envs []corev1.EnvVar, port, e2eDockerConfigFile string) []corev1.EnvVar {
 	for _, env := range envs {
 		// If set by user, just leave as-is.
 		if env.Name == envVarKeyExtraArgs {
@@ -808,6 +838,11 @@ func updateExtraArgs(envs []corev1.EnvVar, port string) []corev1.EnvVar {
 		port = config.DefaultProgressUpdatesPort
 	}
 	val := fmt.Sprintf("--progress-report-url=http://localhost:%v/progress", port)
+	if e2eDockerConfigFile != "" {
+		credFile := filepath.Base(e2eDockerConfigFile)
+		registryCredLocation := fmt.Sprintf("%s/%s", sonobuoyDefaultConfigDir, credFile)
+		val += fmt.Sprintf(" --e2e-docker-config-file=%s", registryCredLocation)
+	}
 	envs = append(envs, corev1.EnvVar{Name: envVarKeyExtraArgs, Value: val})
 	return envs
 }

pkg/client/interfaces.go

@@ -110,7 +110,7 @@ func (gc *GenConfig) Validate() error {
 		}
 
 		for key, value := range m.ConfigMap {
-			if strings.HasSuffix(key, ".yml") || strings.HasSuffix(key, ".yaml") {
+			if strings.HasSuffix(key, ".yml") || strings.HasSuffix(key, ".yaml") || strings.HasSuffix(key, ".json") {
 				var i interface{}
 				if err := yaml.Unmarshal([]byte(value), &i); err != nil {
 					return fmt.Errorf("failed to parse value of key %v in ConfigMap for plugin %v: %v", key, m.SonobuoyConfig.PluginName, err)

pkg/config/config.go

@@ -155,6 +155,7 @@ type Config struct {
 	AggregatorPermissions    string            `json:"AggregatorPermissions" mapstructure:"AggregatorPermissions"`
 	ServiceAccountName       string            `json:"ServiceAccountName" mapstructure:"ServiceAccountName"`
 	ExistingServiceAccount   bool              `json:"ExistingServiceAccount,omitempty" mapstructure:"ExistingServiceAccount,omitempty"`
+	E2EDockerConfigFile      string            `json:"E2EDockerConfigFile,omitempty" mapstructure:"E2EDockerConfigFile,omitempty"`
 	NamespacePSAEnforceLevel string            `json:"NamespacePSAEnforceLevel,omitempty" mapstructure:"NamespacePSAEnforceLevel,omitempty"`
 
 	// ProgressUpdatesPort is the port on which the Sonobuoy worker will listen for status updates from its plugin.