ROX-20268 Support running E2E tests on GKE (#1371)

Implement support for deploying e2e test suite to GKE (support
propagating tenant image pull secrets into tenant namespaces for pulling
images from a private Quay repository).
Version bump golangci-linter, because the old version was throwing
errors.

---------

Co-authored-by: Moritz Clasmeier <mclasmeier@redhat.com>

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
       - id: detect-secrets
         args: ["--baseline", ".secrets.baseline"]
   - repo: https://github.com/golangci/golangci-lint
-    rev: v1.53.2
+    rev: v1.54.2
     hooks:
       - id: golangci-lint
   - repo: https://github.com/pre-commit/pre-commit-hooks

dev/env/defaults/00-defaults.env

@@ -61,7 +61,7 @@ export FLEETSHARD_SYNC_RESOURCES_DEFAULT='{"requests":{"cpu":"200m","memory":"30
 export DB_RESOURCES_DEFAULT='{"requests":{"cpu":"100m","memory":"300Mi"},"limits":{"cpu":"100m","memory":"300Mi"}}'
 export RHACS_OPERATOR_RESOURCES_DEFAULTS='{"requests":{"cpu":"200m","memory":"300Mi"},"limits":{"cpu":"200m","memory":"300Mi"}}'
 
-export ENABLE_EXTERNAL_CONFIG_DEFAULT=false
+export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT=""
 export RHACS_TARGETED_OPERATOR_UPGRADES_DEFAULT="false"
 export RHACS_STANDALONE_MODE_DEFAULT="false"

dev/env/defaults/06-gke.env

@@ -0,0 +1,15 @@
+# shellcheck shell=bash
+
+is_gke_cluster() {
+    local serverName
+    serverName=$(try_kubectl config view --minify -o jsonpath='{.clusters[0].name}')
+    if [[ "$serverName" =~ ^gke_ ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+if is_gke_cluster; then
+    export CLUSTER_TYPE_DEFAULT="gke"
+fi

dev/env/defaults/cluster-type-colima/env

@@ -10,5 +10,4 @@ else
   export DOCKER_DEFAULT="colima nerdctl -- -n k8s.io" # When nerdctl and docker can be used
 fi
 
-export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/defaults/cluster-type-crc/env

@@ -1,3 +1,2 @@
 export ENABLE_CENTRAL_EXTERNAL_CERTIFICATE="true"
-export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/defaults/cluster-type-docker/env

@@ -3,5 +3,4 @@ export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
 export INSTALL_OLM_DEFAULT="true"
-export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/defaults/cluster-type-gke/env

@@ -0,0 +1,8 @@
+export OPENSHIFT_MARKETPLACE_DEFAULT="false"
+export KUBECTL_DEFAULT="kubectl"
+export OPERATOR_SOURCE_DEFAULT="quay"
+export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
+export INSTALL_OLM_DEFAULT="true"
+export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
+export INSTALL_OPENSHIFT_ROUTER_DEFAULT="true"
+export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/defaults/cluster-type-kind/env

@@ -4,5 +4,4 @@ export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
 export INSTALL_OLM_DEFAULT="true"
-export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/defaults/cluster-type-minikube/env

@@ -4,5 +4,4 @@ export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
 export INSTALL_OLM_DEFAULT="true"
-export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/defaults/cluster-type-openshift/env

@@ -16,3 +16,4 @@ if [[ -z "${CLUSTER_DNS:-}" ]]; then
 fi
 export CLUSTER_DNS_DEFAULT
 export INSTALL_OPENSHIFT_ROUTER_DEFAULT="false"
+export ENABLE_EXTERNAL_CONFIG_DEFAULT="false"

dev/env/defaults/cluster-type-rancher-desktop/env

@@ -7,5 +7,4 @@ export INSTALL_OLM_DEFAULT="true"
 export RANCHER_DESKTOP_BIN=${RANCHER_DESKTOP_BIN:-"${HOME}/.rd/bin"}
 export KUBECTL_DEFAULT="${RANCHER_DESKTOP_BIN}/kubectl"
 export DOCKER_DEFAULT="${RANCHER_DESKTOP_BIN}/docker"
-export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/manifests/fleetshard-sync/01-fleetshard-sync-secrets.yaml

@@ -11,3 +11,5 @@ stringData:
   aws-role-arn: "${AWS_ROLE_ARN}"
   aws-access-key-id: "${AWS_ACCESS_KEY_ID}"
   aws-secret-access-key: "${AWS_SECRET_ACCESS_KEY}"
+  tenant-image-pull-secret: |
+    ${TENANT_IMAGE_PULL_SECRET}

dev/env/manifests/fleetshard-sync/02-fleetshard-sync-deployment.yaml

@@ -67,6 +67,11 @@ spec:
                 secretKeyRef:
                   name: fleetshard-sync
                   key: "aws-secret-access-key"
+            - name: TENANT_IMAGE_PULL_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: fleetshard-sync
+                  key: "tenant-image-pull-secret"
           image: "${FLEET_MANAGER_IMAGE}"
           imagePullPolicy: IfNotPresent
           name: fleetshard-sync

dev/env/scripts/docker.sh

@@ -90,7 +90,10 @@ ensure_fleet_manager_image_exists() {
 }
 
 is_local_deploy() {
-    if [[ "$CLUSTER_TYPE" == "openshift-ci" || "$CLUSTER_TYPE" == "infra-openshift" ]]; then
+    if [[ "$CLUSTER_TYPE" == "openshift-ci" \
+        || "$CLUSTER_TYPE" == "infra-openshift" \
+        || "$CLUSTER_TYPE" == "gke" \
+        ]]; then
         return 1
     fi
     if is_running_inside_docker; then

dev/env/scripts/up.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 ## This script takes care of deploying Managed Service components.
-
 GITROOT="$(git rev-parse --show-toplevel)"
 export GITROOT
 # shellcheck source=/dev/null
@@ -76,6 +75,12 @@ if [[ "$RHACS_STANDALONE_MODE" == "true" ]]; then
 fi
 
 log "Deploying fleetshard-sync"
+TENANT_IMAGE_PULL_SECRET=""
+if [[ "$INHERIT_IMAGEPULLSECRETS" == "true" ]]; then # pragma: allowlist secret
+    TENANT_IMAGE_PULL_SECRET=$($KUBECTL -n "$ACSCS_NAMESPACE" get secret quay-ips -o jsonpath="{.data['\.dockerconfigjson']}" | base64 -d)
+fi
+export TENANT_IMAGE_PULL_SECRET
+
 exec_fleetshard_sync.sh apply "${MANIFESTS_DIR}/fleetshard-sync"
 inject_exported_env_vars "$ACSCS_NAMESPACE" "fleetshard-sync"
 

fleetshard/config/config.go

@@ -2,6 +2,7 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -35,11 +36,15 @@ type Config struct {
 	MetricsAddress       string        `env:"FLEETSHARD_METRICS_ADDRESS" envDefault:":8080"`
 	EgressProxyImage     string        `env:"EGRESS_PROXY_IMAGE"`
 	DefaultBaseCRDURL    string        `env:"DEFAULT_BASE_CRD_URL" envDefault:"https://raw.githubusercontent.com/stackrox/stackrox/%s/operator/bundle/manifests/"`
-
-	ManagedDB        ManagedDB
-	Telemetry        Telemetry
-	AuditLogging     AuditLogging
-	SecretEncryption SecretEncryption
+	// TenantImagePullSecret can be used to inject a Kubernetes image pull secret into tenant namespaces.
+	// If it is empty, nothing is injected (for example, it is not required when running on OpenShift).
+	// It is however required in some situations (such as remote GKE clusters) when central images need to fetched from a private Quay registry.
+	// It needs to given as Docker Config JSON object.
+	TenantImagePullSecret string `env:"TENANT_IMAGE_PULL_SECRET"`
+	ManagedDB             ManagedDB
+	Telemetry             Telemetry
+	AuditLogging          AuditLogging
+	SecretEncryption      SecretEncryption
 }
 
 // ManagedDB for configuring managed DB specific parameters
@@ -90,6 +95,7 @@ func GetConfig() (*Config, error) {
 	}
 	validateManagedDBConfig(c, &configErrors)
 	validateSecretEncryptionConfig(c, &configErrors)
+	validateTenantImagePullSecrets(c, &configErrors)
 
 	cfgErr := configErrors.ToError()
 	if cfgErr != nil {
@@ -124,6 +130,27 @@ func validateSecretEncryptionConfig(c Config, configErrors *errorhelpers.ErrorLi
 	}
 }
 
+func validateTenantImagePullSecrets(c Config, configErrors *errorhelpers.ErrorList) {
+	if c.TenantImagePullSecret == "" {
+		return
+	}
+
+	type dockerConfig struct {
+		Auths map[string]map[string]string `json:"auths,omitempty"`
+	}
+
+	var cfg dockerConfig
+
+	if err := json.Unmarshal([]byte(c.TenantImagePullSecret), &cfg); err != nil {
+		configErrors.AddError(errors.Wrapf(err, "invalid tenant image pull secret JSON"))
+		return
+	}
+
+	if cfg.Auths == nil || len(cfg.Auths) == 0 {
+		configErrors.AddError(errors.New("invalid tenant image pull secret"))
+	}
+}
+
 func isDevEnvironment(c Config) bool {
 	return c.Environment == EnvDev || c.Environment == ""
 }

fleetshard/main.go

@@ -40,11 +40,12 @@ func main() {
 	glog.Infof("ClusterID: %s", config.ClusterID)
 	glog.Infof("RuntimePollPeriod: %s", config.RuntimePollPeriod.String())
 	glog.Infof("AuthType: %s", config.AuthType)
-
 	glog.Infof("ManagedDB.Enabled: %t", config.ManagedDB.Enabled)
 	glog.Infof("ManagedDB.SecurityGroup: %s", config.ManagedDB.SecurityGroup)
 	glog.Infof("ManagedDB.SubnetGroup: %s", config.ManagedDB.SubnetGroup)
-
+	if len(config.TenantImagePullSecret) > 0 {
+		glog.Infof("Image pull secret configured, will be injected into tenant namespaces.")
+	}
 	glog.Info("Creating k8s client...")
 	k8sClient := k8s.CreateClientOrDie()
 	ctrl.SetLogger(logger.NewKubeAPILogger())

fleetshard/pkg/central/reconciler/reconciler.go

@@ -84,21 +84,24 @@ const (
 	manualDeclarativeConfigSecretName   = "cloud-service-manual-declarative-configs"   // pragma: allowlist secret
 
 	authProviderDeclarativeConfigKey = "default-sso-auth-provider"
+
+	tenantImagePullSecretName = "stackrox" // pragma: allowlist secret
 )
 
 type verifyAuthProviderExistsFunc func(ctx context.Context, central private.ManagedCentral,
 	client ctrlClient.Client) (bool, error)
 
 // CentralReconcilerOptions are the static options for creating a reconciler.
 type CentralReconcilerOptions struct {
-	UseRoutes         bool
-	WantsAuthProvider bool
-	EgressProxyImage  string
-	ManagedDBEnabled  bool
-	Telemetry         config.Telemetry
-	ClusterName       string
-	Environment       string
-	AuditLogging      config.AuditLogging
+	UseRoutes             bool
+	WantsAuthProvider     bool
+	EgressProxyImage      string
+	ManagedDBEnabled      bool
+	Telemetry             config.Telemetry
+	ClusterName           string
+	Environment           string
+	AuditLogging          config.AuditLogging
+	TenantImagePullSecret string
 }
 
 // CentralReconciler is a reconciler tied to a one Central instance. It installs, updates and deletes Central instances
@@ -130,6 +133,7 @@ type CentralReconciler struct {
 	wantsAuthProvider      bool
 	hasAuthProvider        bool
 	verifyAuthProviderFunc verifyAuthProviderExistsFunc
+	tenantImagePullSecret  []byte
 }
 
 // Reconcile takes a private.ManagedCentral and tries to install it into the cluster managed by the fleet-shard.
@@ -177,6 +181,13 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 		return nil, errors.Wrapf(err, "unable to ensure that namespace %s exists", remoteCentralNamespace)
 	}
 
+	if len(r.tenantImagePullSecret) > 0 {
+		err = r.ensureImagePullSecretConfigured(ctx, remoteCentralNamespace, tenantImagePullSecretName, r.tenantImagePullSecret)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	err = r.restoreCentralSecrets(ctx, remoteCentral)
 	if err != nil {
 		return nil, err
@@ -1146,6 +1157,38 @@ func (r *CentralReconciler) getNamespace(name string) (*corev1.Namespace, error)
 	return namespace, nil
 }
 
+func (r *CentralReconciler) getSecret(namespaceName string, secretName string) (*corev1.Secret, error) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespaceName,
+		},
+	}
+	err := r.client.Get(context.Background(), ctrlClient.ObjectKey{Namespace: namespaceName, Name: secretName}, secret)
+	if err != nil {
+		return nil, errors.Wrapf(err, "retrieving secret %s/%s", namespaceName, secretName)
+	}
+	return secret, nil
+}
+
+func (r *CentralReconciler) createImagePullSecret(ctx context.Context, namespaceName string, secretName string, imagePullSecretJSON []byte) error {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespaceName,
+			Name:      secretName,
+		},
+		Type: "kubernetes.io/dockerconfigjson",
+		Data: map[string][]byte{
+			".dockerconfigjson": imagePullSecretJSON,
+		},
+	}
+
+	if err := r.client.Create(ctx, secret); err != nil {
+		return errors.Wrapf(err, "creating image pull secret %s/%s", namespaceName, secretName)
+	}
+
+	return nil
+}
+
 func (r *CentralReconciler) createTenantNamespace(ctx context.Context, namespace *corev1.Namespace) error {
 	err := r.client.Create(ctx, namespace)
 	if err != nil {
@@ -1167,6 +1210,22 @@ func (r *CentralReconciler) ensureNamespaceExists(name string, labels map[string
 	return nil
 }
 
+func (r *CentralReconciler) ensureImagePullSecretConfigured(ctx context.Context, namespaceName string, secretName string, imagePullSecret []byte) error {
+	// Ensure that the secret exists.
+	_, err := r.getSecret(namespaceName, secretName)
+	if err == nil {
+		// Secret exists already.
+		return nil
+	}
+	if !apiErrors.IsNotFound(err) {
+		// Unexpected error.
+		return errors.Wrapf(err, "retrieving secret %s/%s", namespaceName, secretName)
+	}
+	// We have an IsNotFound error.
+	glog.Infof("Creating image pull secret %s/%s", namespaceName, secretName)
+	return r.createImagePullSecret(ctx, namespaceName, secretName, imagePullSecret)
+}
+
 func (r *CentralReconciler) ensureNamespaceDeleted(ctx context.Context, name string) (bool, error) {
 	namespace, err := r.getNamespace(name)
 	if err != nil {
@@ -1701,7 +1760,6 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, fleetmanagerClient *fleet
 	secretCipher cipher.Cipher, encryptionKeyGenerator cipher.KeyGenerator,
 	opts CentralReconcilerOptions,
 ) *CentralReconciler {
-
 	return &CentralReconciler{
 		client:                 k8sClient,
 		fleetmanagerClient:     fleetmanagerClient,
@@ -1724,6 +1782,7 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, fleetmanagerClient *fleet
 		managedDBInitFunc:           managedDBInitFunc,
 
 		verifyAuthProviderFunc: hasAuthProvider,
+		tenantImagePullSecret:  []byte(opts.TenantImagePullSecret),
 
 		resourcesChart: resourcesChart,
 	}

fleetshard/pkg/runtime/runtime.go

@@ -130,14 +130,15 @@ func (r *Runtime) Start() error {
 	routesAvailable := r.routesAvailable()
 
 	reconcilerOpts := centralReconciler.CentralReconcilerOptions{
-		UseRoutes:         routesAvailable,
-		WantsAuthProvider: r.config.CreateAuthProvider,
-		EgressProxyImage:  r.config.EgressProxyImage,
-		ManagedDBEnabled:  r.config.ManagedDB.Enabled,
-		Telemetry:         r.config.Telemetry,
-		ClusterName:       r.config.ClusterName,
-		Environment:       r.config.Environment,
-		AuditLogging:      r.config.AuditLogging,
+		UseRoutes:             routesAvailable,
+		WantsAuthProvider:     r.config.CreateAuthProvider,
+		EgressProxyImage:      r.config.EgressProxyImage,
+		ManagedDBEnabled:      r.config.ManagedDB.Enabled,
+		Telemetry:             r.config.Telemetry,
+		ClusterName:           r.config.ClusterName,
+		Environment:           r.config.Environment,
+		AuditLogging:          r.config.AuditLogging,
+		TenantImagePullSecret: r.config.TenantImagePullSecret, // pragma: allowlist secret
 	}
 
 	ticker := concurrency.NewRetryTicker(func(ctx context.Context) (timeToNextTick time.Duration, err error) {