Skip to content

Commit

Permalink
Handling Azure Parameters in CI Env using templates. (#582)
Browse files Browse the repository at this point in the history 
* Adding azure parameters

* Adding Openshift CI params

* Update Test Suite and Helpers

* Adding cred ref

* Adding cred ref

* Changing azure oadp cred dir to tmp

* Fixing duplicate error

* Removing OpenShift CI

* Changing test instance name

* Changing the AWS BSL Profile to default

* Changing BslMountPath variable in registry controller
  • Loading branch information
@deepakraj1997
deepakraj1997 authored Apr 29, 2022
1 parent 499d9dd commit 69ce063
Show file tree
Hide file tree
Showing 9 changed files with 70 additions and 252 deletions.
17 changes: 9 additions & 8 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,21 @@ CI_CRED_FILE ?= ${CLUSTER_PROFILE_DIR}/.awscred
# aws configs - default
BSL_REGION ?= us-east-1
VSL_REGION ?= ${LEASED_RESOURCE}
# BSL_AWS_PROFILE ?= default
BSL_AWS_PROFILE ?= migration-engineering
BSL_AWS_PROFILE ?= default
# BSL_AWS_PROFILE ?= migration-engineering

# vsl secret
CREDS_SECRET_REF ?= cloud-credentials
# bucket file
OADP_BUCKET_FILE ?= ${OADP_CRED_DIR}/new-velero-bucket-name
# azure cluster resource file - only in CI
AZURE_RESOURCE_FILE ?= /var/run/secrets/ci.openshift.io/multi-stage/metadata.json
AZURE_CI_JSON_CRED_FILE ?= ${CLUSTER_PROFILE_DIR}/osServicePrincipal.json
AZURE_OADP_JSON_CRED_FILE ?= ${OADP_CRED_DIR}/azure-credentials

# Misc
OPENSHIFT_CI ?= true
VELERO_INSTANCE_NAME ?= velero-sample
VELERO_INSTANCE_NAME ?= velero-test
E2E_TIMEOUT_MULTIPLIER ?= 1
ARTIFACT_DIR ?= /tmp
OC_CLI = $(shell which oc)
Expand All @@ -43,8 +45,8 @@ ifeq ($(CLUSTER_TYPE), gcp)
OADP_BUCKET_FILE = ${OADP_CRED_DIR}/gcp-velero-bucket-name
else ifeq ($(CLUSTER_TYPE), azure4)
CLUSTER_TYPE = azure
CI_CRED_FILE = ${CLUSTER_PROFILE_DIR}/osServicePrincipal.json
OADP_CRED_FILE = ${OADP_CRED_DIR}/azure-credentials
CI_CRED_FILE = /tmp/ci-azure-credentials
OADP_CRED_FILE = /tmp/oadp-azure-credentials
CREDS_SECRET_REF = cloud-credentials-azure
OADP_BUCKET_FILE = ${OADP_CRED_DIR}/azure-velero-bucket-name
endif
Expand Down Expand Up @@ -346,6 +348,8 @@ SETTINGS_TMP=/tmp/test-settings

test-e2e-setup:
mkdir -p $(SETTINGS_TMP)
TARGET_CI_CRED_FILE="$(CI_CRED_FILE)" AZURE_RESOURCE_FILE="$(AZURE_RESOURCE_FILE)" CI_JSON_CRED_FILE="$(AZURE_CI_JSON_CRED_FILE)" \
OADP_JSON_CRED_FILE="$(AZURE_OADP_JSON_CRED_FILE)" OADP_CRED_FILE="$(OADP_CRED_FILE)" OPENSHIFT_CI="$(OPENSHIFT_CI)" \
PROVIDER="$(CLUSTER_TYPE)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \
VSL_REGION="$(VSL_REGION)" BSL_AWS_PROFILE="$(BSL_AWS_PROFILE)" BSL_REGION="$(BSL_REGION)" /bin/bash "tests/e2e/scripts/$(CLUSTER_TYPE)_settings.sh"

Expand All @@ -355,11 +359,8 @@ test-e2e: test-e2e-setup
-settings=$(SETTINGS_TMP)/oadpcreds \
-velero_instance_name=$(VELERO_INSTANCE_NAME) \
-timeout_multiplier=$(E2E_TIMEOUT_MULTIPLIER) \
-cluster_profile=$(CLUSTER_TYPE) \
--ginkgo.label-filter="$(TEST_FILTER)" \
-openshift_ci=$(OPENSHIFT_CI) \
-ci_cred_file=$(CI_CRED_FILE) \
-azure_resource_file=$(AZURE_RESOURCE_FILE) \
-provider=$(CLUSTER_TYPE) \
-creds_secret_ref=$(CREDS_SECRET_REF) \
-artifact_dir=$(ARTIFACT_DIR) \
Expand Down
4 changes: 2 additions & 2 deletions controllers/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ func (r *DPAReconciler) buildRegistryContainer(bsl *velerov1.BackupStorageLocati
containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: credentials.PluginSpecificFields[oadpv1alpha1.DefaultPlugin(bsl.Spec.Provider)].BslSecretName,
MountPath: credentials.PluginSpecificFields[oadpv1alpha1.DefaultPlugin(bsl.Spec.Provider)].BSlMountPath,
MountPath: credentials.PluginSpecificFields[oadpv1alpha1.DefaultPlugin(bsl.Spec.Provider)].BslMountPath,
},
}
}
Expand Down Expand Up @@ -536,7 +536,7 @@ func (r *DPAReconciler) getGCPRegistryEnvVars(bsl *velerov1.BackupStorageLocatio
// check for secret name
_, secretKey := r.getSecretNameAndKey(&bsl.Spec, oadpv1alpha1.DefaultPluginGCP)
if _, ok := bsl.Spec.Config["credentialsFile"]; ok {
gcpEnvVars[i].Value = credentials.PluginSpecificFields[oadpv1alpha1.DefaultPluginGCP].BSlMountPath + "/" + secretKey
gcpEnvVars[i].Value = credentials.PluginSpecificFields[oadpv1alpha1.DefaultPluginGCP].BslMountPath + "/" + secretKey
} else {
gcpEnvVars[i].Value = credentials.PluginSpecificFields[oadpv1alpha1.DefaultPluginGCP].MountPath + "/" + secretKey
}
Expand Down
10 changes: 5 additions & 5 deletions pkg/credentials/credentials.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ type DefaultPluginFields struct {
SecretName string
MountPath string
BslSecretName string
BSlMountPath string
BslMountPath string
EnvCredentialsFile string
PluginImage string
PluginSecretKey string
Expand All @@ -34,7 +34,7 @@ var (
SecretName: "cloud-credentials",
MountPath: "/credentials",
BslSecretName: "bsl-cloud-credentials-aws",
BSlMountPath: "/bsl-cloud-credentials-aws",
BslMountPath: "/bsl-cloud-credentials-aws",
EnvCredentialsFile: common.AWSSharedCredentialsFileEnvKey,
PluginName: common.VeleroPluginForAWS,
PluginSecretKey: "cloud",
Expand All @@ -44,7 +44,7 @@ var (
SecretName: "cloud-credentials-gcp",
MountPath: "/credentials-gcp",
BslSecretName: "bsl-cloud-credentials-gcp",
BSlMountPath: "/bsl-cloud-credentials-gcp",
BslMountPath: "/bsl-cloud-credentials-gcp",
EnvCredentialsFile: common.GCPCredentialsEnvKey,
PluginName: common.VeleroPluginForGCP,
PluginSecretKey: "cloud",
Expand All @@ -54,7 +54,7 @@ var (
SecretName: "cloud-credentials-azure",
MountPath: "/credentials-azure",
BslSecretName: "bsl-cloud-credentials-azure",
BSlMountPath: "/bsl-cloud-credentials-azure",
BslMountPath: "/bsl-cloud-credentials-azure",
EnvCredentialsFile: common.AzureCredentialsFileEnvKey,
PluginName: common.VeleroPluginForAzure,
PluginSecretKey: "cloud",
Expand Down Expand Up @@ -313,7 +313,7 @@ func AppendPluginSpecificSpecs(dpa *oadpv1alpha1.DataProtectionApplication, vele
veleroContainer.VolumeMounts,
corev1.VolumeMount{
Name: cloudProviderMap.BslSecretName,
MountPath: pluginSpecificMap.BSlMountPath,
MountPath: pluginSpecificMap.BslMountPath,
})
veleroDeployment.Spec.Template.Spec.Volumes = append(
veleroDeployment.Spec.Template.Spec.Volumes,
Expand Down
29 changes: 2 additions & 27 deletions tests/e2e/dpa_deployment_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,33 +494,8 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
err := dpaCR.Build(installCase.BRestoreType)
Expect(err).NotTo(HaveOccurred())
if len(installCase.DpaSpec.BackupLocations) > 0 {
switch dpaCR.Provider {
case "aws":
if installCase.DpaSpec.BackupLocations[0].Velero.Config != nil {
installCase.DpaSpec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-aws/cloud"
}
case "gcp":
if installCase.DpaSpec.BackupLocations[0].Velero.Config != nil {
installCase.DpaSpec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-gcp/cloud"
}
case "azure":
installCase.DpaSpec.BackupLocations[0].Velero.Config = map[string]string{
"credentialsFile": "bsl-cloud-credentials-azure/cloud",
"subscriptionId": dpaCR.DpaAzureConfig.BslSubscriptionId,
"storageAccount": dpaCR.DpaAzureConfig.BslStorageAccount,
"resourceGroup": dpaCR.DpaAzureConfig.BslResourceGroup,
"storageAccountKeyEnvVar": dpaCR.DpaAzureConfig.BslStorageAccountKeyEnvVar,
}
installCase.DpaSpec.SnapshotLocations = []oadpv1alpha1.SnapshotLocation{
{
Velero: &velero.VolumeSnapshotLocationSpec{
Provider: dpaCR.Provider,
Config: map[string]string{
"subscriptionId": dpaCR.DpaAzureConfig.VslSubscriptionId,
},
},
},
}
if installCase.DpaSpec.BackupLocations[0].Velero.Config != nil {
installCase.DpaSpec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-" + dpaCR.Provider + "/cloud"
}
}
err = dpaCR.CreateOrUpdate(installCase.DpaSpec)
Expand Down
79 changes: 7 additions & 72 deletions tests/e2e/e2e_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package e2e_test
import (
"errors"
"flag"
"fmt"
"log"
"strconv"
"testing"
"time"

Expand All @@ -16,20 +14,17 @@ import (
)

// Common vars obtained from flags passed in ginkgo.
var credFile, namespace, credSecretRef, instanceName, provider, azure_resource_file, openshift_ci, ci_cred_file, settings, bsl_profile, artifact_dir, oc_cli string
var credFile, namespace, credSecretRef, instanceName, provider, ci_cred_file, settings, artifact_dir, oc_cli string
var timeoutMultiplier time.Duration

func init() {
flag.StringVar(&credFile, "credentials", "", "Cloud Credentials file path location")
flag.StringVar(&namespace, "velero_namespace", "velero", "Velero Namespace")
flag.StringVar(&settings, "settings", "./templates/default_settings.json", "Settings of the velero instance")
flag.StringVar(&instanceName, "velero_instance_name", "example-velero", "Velero Instance Name")
flag.StringVar(&bsl_profile, "cluster_profile", "aws", "Cluster profile")
flag.StringVar(&credSecretRef, "creds_secret_ref", "cloud-credentials", "Credential secret ref for backup storage location")
flag.StringVar(&provider, "provider", "aws", "BSL provider")
flag.StringVar(&azure_resource_file, "azure_resource_file", "azure resource file", "Resource Group Dir for azure")
flag.StringVar(&provider, "provider", "aws", "Cloud provider")
flag.StringVar(&ci_cred_file, "ci_cred_file", credFile, "CI Cloud Cred File")
flag.StringVar(&openshift_ci, "openshift_ci", "false", "ENV for tests")
flag.StringVar(&artifact_dir, "artifact_dir", "/tmp", "Directory for storing must gather")
flag.StringVar(&oc_cli, "oc_cli", "oc", "OC CLI Client")

Expand Down Expand Up @@ -70,73 +65,13 @@ var _ = BeforeSuite(func() {
dpaCR.CustomResource = Dpa
testSuiteInstanceName := "ts-" + instanceName
dpaCR.Name = testSuiteInstanceName
openshift_ci_bool, _ := strconv.ParseBool(openshift_ci)
dpaCR.OpenshiftCi = openshift_ci_bool

if openshift_ci_bool == true {
switch dpaCR.Provider {
case "aws":
cloudCredData, err := utils.ReadFile(dpaCR.Credentials)
Expect(err).NotTo(HaveOccurred())
err = CreateCredentialsSecret(cloudCredData, namespace, "bsl-cloud-credentials-aws")
Expect(err).NotTo(HaveOccurred())
dpaCR.Credentials = ci_cred_file
case "gcp":
cloudCredData, err := utils.ReadFile(dpaCR.Credentials)
Expect(err).NotTo(HaveOccurred())
err = CreateCredentialsSecret(cloudCredData, namespace, "bsl-cloud-credentials-gcp")
Expect(err).NotTo(HaveOccurred())
dpaCR.Credentials = ci_cred_file
case "azure":
cloudCredData, err := utils.GetJsonData(dpaCR.Credentials) // azure credentials need to be in json - can be changed
Expect(err).NotTo(HaveOccurred())
dpaCR.DpaAzureConfig = DpaAzureConfig{
BslSubscriptionId: fmt.Sprintf("%v", cloudCredData["subscriptionId"]),
BslResourceGroup: fmt.Sprintf("%v", cloudCredData["resourceGroup"]),
BslStorageAccount: fmt.Sprintf("%v", cloudCredData["storageAccount"]),
BslStorageAccountKeyEnvVar: "AZURE_STORAGE_ACCOUNT_ACCESS_KEY",
VslSubscriptionId: fmt.Sprintf("%v", cloudCredData["subscriptionId"]),
VslResourceGroup: fmt.Sprintf("%v", cloudCredData["resourceGroup"]),
}

// bsl cloud
cloudCreds := GetAzureCreds(cloudCredData)
err = CreateCredentialsSecret(cloudCreds, namespace, "bsl-cloud-credentials-azure")
Expect(err).NotTo(HaveOccurred())
// ci cloud
ciJsonData, err := utils.GetJsonData(ci_cred_file)
Expect(err).NotTo(HaveOccurred())
cloudCredData, err := utils.ReadFile(dpaCR.Credentials)
Expect(err).NotTo(HaveOccurred())
err = CreateCredentialsSecret(cloudCredData, namespace, "bsl-cloud-credentials-"+provider)
Expect(err).NotTo(HaveOccurred())
dpaCR.Credentials = ci_cred_file

if _, ok := ciJsonData["resourceGroup"]; !ok {
resourceGroup, err := GetAzureResource(azure_resource_file)
Expect(err).NotTo(HaveOccurred())
ciJsonData["resourceGroup"] = resourceGroup
}
dpaCR.DpaAzureConfig.VslSubscriptionId = fmt.Sprintf("%v", ciJsonData["subscriptionId"])
dpaCR.DpaAzureConfig.VslResourceGroup = fmt.Sprintf("%v", ciJsonData["resourceGroup"])
ciCreds := GetAzureCreds(ciJsonData)
dpaCR.Credentials = "/tmp/azure-credentials"
err = utils.WriteFile(dpaCR.Credentials, ciCreds)
Expect(err).NotTo(HaveOccurred())
}
} else {
if dpaCR.Provider == "azure" {
cloudCredData, err := utils.GetJsonData(dpaCR.Credentials) // azure credentials need to be in json - can be changed
Expect(err).NotTo(HaveOccurred())
dpaCR.DpaAzureConfig = DpaAzureConfig{
BslSubscriptionId: fmt.Sprintf("%v", cloudCredData["subscriptionId"]),
BslResourceGroup: fmt.Sprintf("%v", cloudCredData["resourceGroup"]),
BslStorageAccount: fmt.Sprintf("%v", cloudCredData["storageAccount"]),
BslStorageAccountKeyEnvVar: "AZURE_STORAGE_ACCOUNT_ACCESS_KEY",
VslSubscriptionId: fmt.Sprintf("%v", cloudCredData["subscriptionId"]),
VslResourceGroup: fmt.Sprintf("%v", cloudCredData["resourceGroup"]),
}
ciCreds := GetAzureCreds(cloudCredData)
dpaCR.Credentials = "/tmp/azure-credentials"
err = utils.WriteFile(dpaCR.Credentials, ciCreds)
Expect(err).NotTo(HaveOccurred())
}
}
credData, err := utils.ReadFile(dpaCR.Credentials)
Expect(err).NotTo(HaveOccurred())
err = CreateCredentialsSecret(credData, namespace, credSecretRef)
Expand Down
79 changes: 2 additions & 77 deletions tests/e2e/lib/dpa_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,27 +37,16 @@ const (
RESTIC BackupRestoreType = "restic"
)

type DpaAzureConfig struct {
VslSubscriptionId string
VslResourceGroup string
BslSubscriptionId string
BslResourceGroup string
BslStorageAccount string
BslStorageAccountKeyEnvVar string
}

type DpaCustomResource struct {
Name string
Namespace string
SecretName string
backupRestoreType BackupRestoreType
CustomResource *oadpv1alpha1.DataProtectionApplication
Client client.Client
DpaAzureConfig DpaAzureConfig
Credentials string
CredSecretRef string
Provider string
OpenshiftCi bool
}

var VeleroPrefix = "velero-e2e-" + string(uuid.NewUUID())
Expand Down Expand Up @@ -98,72 +87,8 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error {
},
},
}
switch v.Provider {
case "aws":
if v.OpenshiftCi {
if dpaInstance.Spec.BackupLocations[0].Velero.Config != nil {
dpaInstance.Spec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-aws/cloud"
}
} else {
dpaInstance.Spec.BackupLocations[0].Velero.Credential = &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: v.CredSecretRef,
},
Key: "cloud",
}
}
case "gcp":
if v.OpenshiftCi {
if dpaInstance.Spec.BackupLocations[0].Velero.Config != nil {
dpaInstance.Spec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-gcp/cloud"
}
} else {
dpaInstance.Spec.BackupLocations[0].Velero.Credential = &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: v.CredSecretRef,
},
Key: "cloud",
}
}
// dpaInstance.Spec.Configuration.Velero.DefaultPlugins = append(dpaInstance.Spec.Configuration.Velero.DefaultPlugins, oadpv1alpha1.DefaultPluginGCP)
// dpaInstance.Spec.SnapshotLocations = []oadpv1alpha1.SnapshotLocation{
// {
// Velero: &velero.VolumeSnapshotLocationSpec{
// Provider: v.Provider,
// Config: map[string]string{
// "snapshotLocation": v.gcpConfig.VslRegion,
// },
// },
// },
// }
case "azure":
dpaInstance.Spec.BackupLocations[0].Velero.Config = map[string]string{
"subscriptionId": v.DpaAzureConfig.BslSubscriptionId,
"storageAccount": v.DpaAzureConfig.BslStorageAccount,
"resourceGroup": v.DpaAzureConfig.BslResourceGroup,
"storageAccountKeyEnvVar": v.DpaAzureConfig.BslStorageAccountKeyEnvVar,
}
if v.OpenshiftCi {
dpaInstance.Spec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-azure/cloud" // <secret_name>/<key>
} else {
dpaInstance.Spec.BackupLocations[0].Velero.Credential = &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: v.CredSecretRef,
},
Key: "cloud",
}
}
dpaInstance.Spec.SnapshotLocations = []oadpv1alpha1.SnapshotLocation{
{
Velero: &velero.VolumeSnapshotLocationSpec{
Provider: v.Provider,
Config: map[string]string{
"subscriptionId": v.DpaAzureConfig.VslSubscriptionId,
"resourceGroup": v.DpaAzureConfig.VslResourceGroup,
},
},
},
}
if dpaInstance.Spec.BackupLocations[0].Velero.Config != nil {
dpaInstance.Spec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-" + v.Provider + "/cloud"
}
v.backupRestoreType = backupRestoreType
switch backupRestoreType {
Expand Down
Loading

0 comments on commit 69ce063

Please sign in to comment.