Multi cloud E2E Updates (openshift#568)

* Adding changes from ci-multi-cloud branch

* Updating flags in makefile

* Updating flags in makefile

* Updating flags in makefile

* Update defaults

* Fix helpers

* Cleaning up makefile

* Update makefile

* Update kustomization

* Cleaning up Makefile

* Adding changes for AWS CredentialFile

* Handling no default backuplocations

* Removing comments

* Adding config check in helpers

* Fix AWS Test case

* Fix basic review comments

Makefile

@@ -1,13 +1,48 @@
 OADP_TEST_NAMESPACE ?= openshift-adp
-REGION ?= us-east-1
-PROVIDER ?= aws
-CLUSTER_PROFILE ?= aws
+CLUSTER_TYPE ?= aws
+
+# CONFIGS FOR CLOUD
+# bsl / blob storage cred dir
+OADP_CRED_DIR ?= /var/run/oadp-credentials
+# vsl / volume/cluster cred dir
+CLUSTER_PROFILE_DIR ?= /Users/drajds/.aws
+
+# bsl cred file
+OADP_CRED_FILE ?= ${OADP_CRED_DIR}/new-aws-credentials
+# vsl cred file
+CI_CRED_FILE ?= ${CLUSTER_PROFILE_DIR}/.awscred
+
+# aws configs - default
+BSL_REGION ?= us-east-1
+VSL_REGION ?= ${LEASED_RESOURCE}
+# BSL_AWS_PROFILE ?= default
+BSL_AWS_PROFILE ?= migration-engineering
+
+# vsl secret
 CREDS_SECRET_REF ?= cloud-credentials
-OADP_AWS_CRED_FILE ?= /var/run/oadp-credentials/aws-credentials
-OADP_S3_BUCKET ?= /var/run/oadp-credentials/velero-bucket-name
+# bucket file
+OADP_BUCKET_FILE ?= ${OADP_CRED_DIR}/new-velero-bucket-name
+# azure cluster resource file - only in CI
+AZURE_RESOURCE_FILE ?= /var/run/secrets/ci.openshift.io/multi-stage/metadata.json
+
+# Misc
+OPENSHIFT_CI ?= true
 VELERO_INSTANCE_NAME ?= velero-sample
 E2E_TIMEOUT_MULTIPLIER ?= 1
 
+ifeq ($(CLUSTER_TYPE), gcp)
+	CI_CRED_FILE = ${CLUSTER_PROFILE_DIR}/gce.json
+	OADP_CRED_FILE = ${OADP_CRED_DIR}/gcp-credentials
+	CREDS_SECRET_REF = cloud-credentials-gcp
+	OADP_BUCKET_FILE = ${OADP_CRED_DIR}/gcp-velero-bucket-name
+else ifeq ($(CLUSTER_TYPE), azure4)
+	CLUSTER_TYPE = azure
+	CI_CRED_FILE = ${CLUSTER_PROFILE_DIR}/osServicePrincipal.json
+	OADP_CRED_FILE = ${OADP_CRED_DIR}/azure-credentials
+	CREDS_SECRET_REF = cloud-credentials-azure
+	OADP_BUCKET_FILE = ${OADP_CRED_DIR}/azure-velero-bucket-name
+endif
+
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.21
 
@@ -290,16 +325,28 @@ catalog-build: opm ## Build a catalog image.
 catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
 
-S3_BUCKET := $(shell cat $(OADP_S3_BUCKET) | awk '/velero-bucket-name/  {gsub(/"/, "", $$2);gsub(/}/,""); print $$2}')
+OADP_BUCKET := $(shell cat $(OADP_BUCKET_FILE))
+TEST_FILTER := $(shell echo '! aws && ! gcp && ! azure' | sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//")
 SETTINGS_TMP=/tmp/test-settings
-test-e2e:
+
+test-e2e-setup:
 	mkdir -p $(SETTINGS_TMP)
-	PROVIDER="$(PROVIDER)" BUCKET="$(S3_BUCKET)" REGION="$(REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) /bin/bash tests/e2e/scripts/aws_settings.sh
-	ginkgo -mod=mod tests/e2e/ -- -cloud=$(OADP_AWS_CRED_FILE) \
+	PROVIDER="$(CLUSTER_TYPE)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \
+	VSL_REGION="$(VSL_REGION)" BSL_AWS_PROFILE="$(BSL_AWS_PROFILE)" BSL_REGION="$(BSL_REGION)" /bin/bash "tests/e2e/scripts/$(CLUSTER_TYPE)_settings.sh"
+
+test-e2e: test-e2e-setup
+	ginkgo run -mod=mod tests/e2e/ -- -credentials=$(OADP_CRED_FILE) \
 	-velero_namespace=$(OADP_TEST_NAMESPACE) \
-	-settings=$(SETTINGS_TMP)/awscreds \
+	-settings=$(SETTINGS_TMP)/oadpcreds \
 	-velero_instance_name=$(VELERO_INSTANCE_NAME) \
 	-timeout_multiplier=$(E2E_TIMEOUT_MULTIPLIER) \
-	-cluster_profile=$(CLUSTER_PROFILE)
+	-cluster_profile=$(CLUSTER_TYPE) \
+	--ginkgo.label-filter="$(TEST_FILTER)" \
+	-openshift_ci=$(OPENSHIFT_CI) \
+	-ci_cred_file=$(CI_CRED_FILE) \
+	-azure_resource_file=$(AZURE_RESOURCE_FILE) \
+	-provider=$(CLUSTER_TYPE) \
+	-creds_secret_ref=$(CREDS_SECRET_REF)
+
 test-e2e-cleanup:
 	rm -rf $(SETTINGS_TMP)

controllers/bsl.go

@@ -301,7 +301,7 @@ func (r *DPAReconciler) validateProviderPluginAndSecret(bslSpec velerov1.BackupS
 		r.Log.Info(fmt.Sprintf("%s backupstoragelocation is configured but velero plugin for %s is not present", bslSpec.Provider, bslSpec.Provider))
 		//TODO: set warning condition on Velero CR
 	}
-	secretName, _ := r.getSecretNameAndKey(bslSpec.Credential, oadpv1alpha1.DefaultPlugin(bslSpec.Provider))
+	secretName, _ := r.getSecretNameAndKey(&bslSpec, oadpv1alpha1.DefaultPlugin(bslSpec.Provider))
 
 	_, err := r.getProviderSecret(secretName)
 

controllers/registry.go

@@ -283,16 +283,29 @@ func (r *DPAReconciler) buildRegistryDeployment(registryDeployment *appsv1.Deplo
 		},
 	}
 
-	// attach gcp secret volume if provider is gcp
-	if bsl.Spec.Provider == GCPProvider {
-		// check for secret name
-		secretName, _ := r.getSecretNameAndKey(bsl.Spec.Credential, oadpv1alpha1.DefaultPluginGCP)
+	// attach secret volume for cloud providers
+	if _, ok := bsl.Spec.Config["credentialsFile"]; ok {
+		if cloudProviderMap, bslCredOk := credentials.PluginSpecificFields[oadpv1alpha1.DefaultPlugin(bsl.Spec.Provider)]; bslCredOk {
+			registryDeployment.Spec.Template.Spec.Volumes = append(
+				registryDeployment.Spec.Template.Spec.Volumes,
+				corev1.Volume{
+					Name: cloudProviderMap.SecretName,
+					VolumeSource: corev1.VolumeSource{
+						Secret: &corev1.SecretVolumeSource{
+							SecretName: cloudProviderMap.BslSecretName,
+						},
+					},
+				},
+			)
+		}
+	} else if bsl.Spec.Provider == GCPProvider {
+		cloudProviderMap := credentials.PluginSpecificFields[oadpv1alpha1.DefaultPluginGCP]
 		registryDeployment.Spec.Template.Spec.Volumes = []corev1.Volume{
 			{
-				Name: secretName,
+				Name: cloudProviderMap.SecretName,
 				VolumeSource: corev1.VolumeSource{
 					Secret: &corev1.SecretVolumeSource{
-						SecretName: secretName,
+						SecretName: cloudProviderMap.SecretName,
 					},
 				},
 			},
@@ -379,7 +392,7 @@ func (r *DPAReconciler) buildRegistryContainer(bsl *velerov1.BackupStorageLocati
 	// append secret volumes if the BSL provider is GCP
 	if bsl.Spec.Provider == GCPProvider {
 		// check for secret name
-		secretName, _ := r.getSecretNameAndKey(bsl.Spec.Credential, oadpv1alpha1.DefaultPluginGCP)
+		secretName, _ := r.getSecretNameAndKey(&bsl.Spec, oadpv1alpha1.DefaultPluginGCP)
 		containers[0].VolumeMounts = []corev1.VolumeMount{
 			{
 				Name:      secretName,
@@ -516,7 +529,7 @@ func (r *DPAReconciler) getGCPRegistryEnvVars(bsl *velerov1.BackupStorageLocatio
 
 		if gcpEnvVars[i].Name == RegistryStorageGCSKeyfile {
 			// check for secret key
-			_, secretKey := r.getSecretNameAndKey(bsl.Spec.Credential, oadpv1alpha1.DefaultPluginGCP)
+			_, secretKey := r.getSecretNameAndKey(&bsl.Spec, oadpv1alpha1.DefaultPluginGCP)
 			gcpEnvVars[i].Value = credentials.PluginSpecificFields[oadpv1alpha1.DefaultPluginGCP].MountPath + "/" + secretKey
 		}
 	}
@@ -546,18 +559,24 @@ func (r *DPAReconciler) getSecretNameAndKeyforBackupLocation(bslspec oadpv1alpha
 		}
 	}
 	if bslspec.Velero != nil {
-		return r.getSecretNameAndKey(bslspec.Velero.Credential, oadpv1alpha1.DefaultPlugin(bslspec.Velero.Provider))
+		return r.getSecretNameAndKey(bslspec.Velero, oadpv1alpha1.DefaultPlugin(bslspec.Velero.Provider))
 	}
 
 	return "", ""
 }
 
-func (r *DPAReconciler) getSecretNameAndKey(credential *corev1.SecretKeySelector, plugin oadpv1alpha1.DefaultPlugin) (string, string) {
+func (r *DPAReconciler) getSecretNameAndKey(bslSpec *velerov1.BackupStorageLocationSpec, plugin oadpv1alpha1.DefaultPlugin) (string, string) {
 	// Assume default values unless user has overriden them
 	secretName := credentials.PluginSpecificFields[plugin].SecretName
 	secretKey := credentials.PluginSpecificFields[plugin].PluginSecretKey
-
+	if _, ok := bslSpec.Config["credentialsFile"]; ok {
+		secretName = credentials.PluginSpecificFields[plugin].BslSecretName
+		secretKey = credentials.PluginSpecificFields[plugin].PluginSecretKey
+	}
+	r.Log.Info(fmt.Sprintf("secret: %s", secretName))
+	r.Log.Info(fmt.Sprintf("key: %s", secretKey))
 	// check if user specified the Credential Name and Key
+	credential := bslSpec.Credential
 	if credential != nil {
 		if len(credential.Name) > 0 {
 			secretName = credential.Name
@@ -1152,7 +1171,7 @@ func (r *DPAReconciler) updateRegistrySecret(secret *corev1.Secret, bsl *velerov
 
 func (r *DPAReconciler) populateAWSRegistrySecret(bsl *velerov1.BackupStorageLocation, registrySecret *corev1.Secret) error {
 	// Check for secret name
-	secretName, secretKey := r.getSecretNameAndKey(bsl.Spec.Credential, oadpv1alpha1.DefaultPluginAWS)
+	secretName, secretKey := r.getSecretNameAndKey(&bsl.Spec, oadpv1alpha1.DefaultPluginAWS)
 
 	// fetch secret and error
 	secret, err := r.getProviderSecret(secretName)
@@ -1181,7 +1200,7 @@ func (r *DPAReconciler) populateAWSRegistrySecret(bsl *velerov1.BackupStorageLoc
 
 func (r *DPAReconciler) populateAzureRegistrySecret(bsl *velerov1.BackupStorageLocation, registrySecret *corev1.Secret) error {
 	// Check for secret name
-	secretName, secretKey := r.getSecretNameAndKey(bsl.Spec.Credential, oadpv1alpha1.DefaultPluginMicrosoftAzure)
+	secretName, secretKey := r.getSecretNameAndKey(&bsl.Spec, oadpv1alpha1.DefaultPluginMicrosoftAzure)
 	r.Log.Info(fmt.Sprintf("Azure secret name: %s and secret key: %s", secretName, secretKey))
 
 	// fetch secret and error

pkg/credentials/credentials.go

@@ -15,6 +15,8 @@ type DefaultPluginFields struct {
 	IsCloudProvider    bool
 	SecretName         string
 	MountPath          string
+	BslSecretName      string
+	BSlMountPath       string
 	EnvCredentialsFile string
 	PluginImage        string
 	PluginSecretKey    string
@@ -31,6 +33,8 @@ var (
 			IsCloudProvider:    true,
 			SecretName:         "cloud-credentials",
 			MountPath:          "/credentials",
+			BslSecretName:      "bsl-cloud-credentials-aws",
+			BSlMountPath:       "/bsl-cloud-credentials-aws",
 			EnvCredentialsFile: common.AWSSharedCredentialsFileEnvKey,
 			PluginName:         common.VeleroPluginForAWS,
 			PluginSecretKey:    "cloud",
@@ -39,6 +43,8 @@ var (
 			IsCloudProvider:    true,
 			SecretName:         "cloud-credentials-gcp",
 			MountPath:          "/credentials-gcp",
+			BslSecretName:      "bsl-cloud-credentials-gcp",
+			BSlMountPath:       "/bsl-cloud-credentials-gcp",
 			EnvCredentialsFile: common.GCPCredentialsEnvKey,
 			PluginName:         common.VeleroPluginForGCP,
 			PluginSecretKey:    "cloud",
@@ -47,6 +53,8 @@ var (
 			IsCloudProvider:    true,
 			SecretName:         "cloud-credentials-azure",
 			MountPath:          "/credentials-azure",
+			BslSecretName:      "bsl-cloud-credentials-azure",
+			BSlMountPath:       "/bsl-cloud-credentials-azure",
 			EnvCredentialsFile: common.AzureCredentialsFileEnvKey,
 			PluginName:         common.VeleroPluginForAzure,
 			PluginSecretKey:    "cloud",
@@ -215,6 +223,23 @@ func AppendCloudProviderVolumes(dpa *oadpv1alpha1.DataProtectionApplication, ds
 
 		}
 	}
+	for _, bslSpec := range dpa.Spec.BackupLocations {
+		if _, ok := bslSpec.Velero.Config["credentialsFile"]; ok {
+			if cloudProviderMap, bslCredOk := PluginSpecificFields[oadpv1alpha1.DefaultPlugin(bslSpec.Velero.Provider)]; bslCredOk {
+				ds.Spec.Template.Spec.Volumes = append(
+					ds.Spec.Template.Spec.Volumes,
+					corev1.Volume{
+						Name: cloudProviderMap.BslSecretName,
+						VolumeSource: corev1.VolumeSource{
+							Secret: &corev1.SecretVolumeSource{
+								SecretName: cloudProviderMap.BslSecretName,
+							},
+						},
+					},
+				)
+			}
+		}
+	}
 	return nil
 }
 
@@ -280,6 +305,30 @@ func AppendPluginSpecificSpecs(dpa *oadpv1alpha1.DataProtectionApplication, vele
 					},
 				})
 
+			// append bsl volume secret
+			for _, bslSpec := range dpa.Spec.BackupLocations {
+				if _, ok := bslSpec.Velero.Config["credentialsFile"]; ok {
+					if cloudProviderMap, bslCredOk := PluginSpecificFields[oadpv1alpha1.DefaultPlugin(bslSpec.Velero.Provider)]; bslCredOk {
+						veleroContainer.VolumeMounts = append(
+							veleroContainer.VolumeMounts,
+							corev1.VolumeMount{
+								Name:      cloudProviderMap.BslSecretName,
+								MountPath: pluginSpecificMap.BSlMountPath,
+							})
+						veleroDeployment.Spec.Template.Spec.Volumes = append(
+							veleroDeployment.Spec.Template.Spec.Volumes,
+							corev1.Volume{
+								Name: cloudProviderMap.BslSecretName,
+								VolumeSource: corev1.VolumeSource{
+									Secret: &corev1.SecretVolumeSource{
+										SecretName: cloudProviderMap.BslSecretName,
+									},
+								},
+							},
+						)
+					}
+				}
+			}
 		}
 	}
 	// append custom plugin init containers

tests/e2e/backup_restore_suite_test.go

@@ -23,7 +23,7 @@ var _ = Describe("AWS backup restore tests", func() {
 		testSuiteInstanceName := "ts-" + instanceName
 		dpaCR.Name = testSuiteInstanceName
 
-		credData, err := utils.ReadFile(cloud)
+		credData, err := utils.ReadFile(credFile)
 		Expect(err).NotTo(HaveOccurred())
 		err = CreateCredentialsSecret(credData, namespace, GetSecretRef(credSecretRef))
 		Expect(err).NotTo(HaveOccurred())