fix: Nutanix CSI credentials Secret creation (#34)

The existing code created a ClusterResourceSet with the user provided Secret.
However, that won't work unless that Secret has an embedded Secret in it.

charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml

@@ -8,5 +8,7 @@ metadata:
   name: '{{ .Values.hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name }}'
 data:
   values.yaml: |-
+    # The Secret containing the credentials will be created by the handler.
     createSecret: false
+    secretName: nutanix-csi-credentials
 {{- end -}}

pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go

@@ -21,6 +21,11 @@ import (
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
 )
 
+var defaultStorageClassParams = map[string]string{
+	"csi.storage.k8s.io/fstype": "ext4",
+	"type":                      "gp3",
+}
+
 type AWSEBSConfig struct {
 	*options.GlobalOptions
 	defaultAWSEBSConfigMapName string
@@ -81,14 +86,14 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context,
 	defaultStorageConfig *v1alpha1.DefaultStorage,
 ) error {
 	allStorageClasses := make([]runtime.Object, 0, len(configs))
-	for _, c := range configs {
-		setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName &&
+	for _, config := range configs {
+		setAsDefault := config.Name == defaultStorageConfig.StorageClassConfigName &&
 			v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName
 		allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass(
-			c,
-			a.config.GlobalOptions.DefaultsNamespace(),
+			config,
 			v1alpha1.AWSEBSProvisioner,
 			setAsDefault,
+			defaultStorageClassParams,
 		))
 	}
 	cm, err := lifecycleutils.CreateConfigMapForCRS(

pkg/handlers/generic/lifecycle/csi/handler.go

@@ -118,7 +118,7 @@ func (c *CSIHandler) AfterControlPlaneInitialized(
 			)
 			continue
 		}
-		log.Info(fmt.Sprintf("Creating csi provider %s", provider.Name))
+		log.Info(fmt.Sprintf("Creating CSI provider %s", provider.Name))
 		err = handler.Apply(
 			ctx,
 			provider,
@@ -129,11 +129,17 @@ func (c *CSIHandler) AfterControlPlaneInitialized(
 			log.Error(
 				err,
 				fmt.Sprintf(
-					"failed to create %s csi driver object.",
+					"failed to delpoy %s CSI driver",
 					provider.Name,
 				),
 			)
 			resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
+			resp.SetMessage(
+				fmt.Sprintf(
+					"failed to deploy CSI driver: %v",
+					err,
+				),
+			)
 		}
 	}
 }

pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 
 	"github.com/spf13/pflag"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -24,16 +23,32 @@ import (
 )
 
 const (
-	defaultHelmRepositoryURL              = "https://nutanix.github.io/helm/"
-	defaultStorageHelmChartVersion        = "v2.6.6"
-	defaultStorageHelmChartName           = "nutanix-csi-storage"
-	defaultStorageHelmReleaseNameTemplate = "nutanix-csi-storage-%s"
-
-	defaultSnapshotHelmChartVersion        = "v6.3.2"
-	defaultSnapshotHelmChartName           = "nutanix-csi-snapshot"
-	defaultSnapshotHelmReleaseNameTemplate = "nutanix-csi-snapshot-%s"
+	defaultHelmRepositoryURL           = "https://nutanix.github.io/helm/"
+	defaultStorageHelmChartVersion     = "v2.6.6"
+	defaultStorageHelmChartName        = "nutanix-csi-storage"
+	defaultStorageHelmReleaseName      = "nutanix-csi-storage"
+	defaultStorageHelmReleaseNamespace = "ntnx-system"
+
+	defaultSnapshotHelmChartVersion     = "v6.3.2"
+	defaultSnapshotHelmChartName        = "nutanix-csi-snapshot"
+	defaultSnapshotHelmReleaseName      = "nutanix-csi-snapshot"
+	defaultSnapshotHelmReleaseNamespace = "ntnx-system"
+
+	//nolint:gosec // Does not contain hard coded credentials.
+	defaultCredentialsSecretName = "nutanix-csi-credentials"
 )
 
+var defaultStorageClassParameters = map[string]string{
+	"storageType":                                           "NutanixVolumes",
+	"csi.storage.k8s.io/fstype":                             "xfs",
+	"csi.storage.k8s.io/provisioner-secret-name":            defaultCredentialsSecretName,
+	"csi.storage.k8s.io/provisioner-secret-namespace":       defaultStorageHelmReleaseNamespace,
+	"csi.storage.k8s.io/node-publish-secret-name":           defaultCredentialsSecretName,
+	"csi.storage.k8s.io/node-publish-secret-namespace":      defaultStorageHelmReleaseNamespace,
+	"csi.storage.k8s.io/controller-expand-secret-name":      defaultCredentialsSecretName,
+	"csi.storage.k8s.io/controller-expand-secret-namespace": defaultStorageHelmReleaseNamespace,
+}
+
 type NutanixCSIConfig struct {
 	*options.GlobalOptions
 	defaultValuesTemplateConfigMapName string
@@ -80,42 +95,38 @@ func (n *NutanixCSI) Apply(
 	default:
 		return fmt.Errorf("stategy %s not implemented", strategy)
 	}
+
 	if provider.Credentials != nil {
-		sec := &corev1.Secret{
-			TypeMeta: metav1.TypeMeta{
-				APIVersion: corev1.SchemeGroupVersion.String(),
-				Kind:       "Secret",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      provider.Credentials.Name,
-				Namespace: req.Cluster.Namespace,
-			},
+		key := ctrlclient.ObjectKey{
+			Name:      defaultCredentialsSecretName,
+			Namespace: defaultStorageHelmReleaseNamespace,
 		}
-		err := n.client.Get(
+		err := lifecycleutils.CopySecretToRemoteCluster(
 			ctx,
-			ctrlclient.ObjectKeyFromObject(sec),
-			sec,
-		)
-		if err != nil {
-			return err
-		}
-		err = lifecycleutils.EnsureCRSForClusterFromObjects(
-			ctx,
-			fmt.Sprintf("nutanix-csi-credentials-crs-%s", req.Cluster.Name),
 			n.client,
+			provider.Credentials.Name,
+			key,
 			&req.Cluster,
-			sec,
 		)
 		if err != nil {
-			return err
+			return fmt.Errorf(
+				"error creating credentials Secret for the Nutanix CSI driver: %w",
+				err,
+			)
 		}
 	}
-	return n.createStorageClasses(
+
+	err := n.createStorageClasses(
 		ctx,
 		provider.StorageClassConfig,
 		&req.Cluster,
 		defaultStorageConfig,
 	)
+	if err != nil {
+		return fmt.Errorf("error creating StorageClasses for the Nutanix CSI driver: %w", err)
+	}
+
+	return nil
 }
 
 func (n *NutanixCSI) handleHelmAddonApply(
@@ -149,8 +160,8 @@ func (n *NutanixCSI) handleHelmAddonApply(
 			ClusterSelector: metav1.LabelSelector{
 				MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name},
 			},
-			ReleaseNamespace: req.Cluster.Namespace,
-			ReleaseName:      fmt.Sprintf(defaultStorageHelmReleaseNameTemplate, req.Cluster.Name),
+			ReleaseNamespace: defaultStorageHelmReleaseNamespace,
+			ReleaseName:      defaultStorageHelmReleaseName,
 			Version:          defaultStorageHelmChartVersion,
 			ValuesTemplate:   values,
 		},
@@ -174,16 +185,16 @@ func (n *NutanixCSI) handleHelmAddonApply(
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: req.Cluster.Namespace,
-			Name:      "nutanix-csi-snapshot" + req.Cluster.Name,
+			Name:      "nutanix-csi-snapshot-" + req.Cluster.Name,
 		},
 		Spec: caaphv1.HelmChartProxySpec{
 			RepoURL:   defaultHelmRepositoryURL,
 			ChartName: defaultSnapshotHelmChartName,
 			ClusterSelector: metav1.LabelSelector{
 				MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name},
 			},
-			ReleaseNamespace: req.Cluster.Namespace,
-			ReleaseName:      fmt.Sprintf(defaultSnapshotHelmReleaseNameTemplate, req.Cluster.Name),
+			ReleaseNamespace: defaultSnapshotHelmReleaseNamespace,
+			ReleaseName:      defaultSnapshotHelmReleaseName,
 			Version:          defaultSnapshotHelmChartVersion,
 		},
 	}
@@ -205,20 +216,21 @@ func (n *NutanixCSI) handleHelmAddonApply(
 	return nil
 }
 
-func (n *NutanixCSI) createStorageClasses(ctx context.Context,
+func (n *NutanixCSI) createStorageClasses(
+	ctx context.Context,
 	configs []v1alpha1.StorageClassConfig,
 	cluster *clusterv1.Cluster,
 	defaultStorageConfig *v1alpha1.DefaultStorage,
 ) error {
 	allStorageClasses := make([]runtime.Object, 0, len(configs))
-	for _, c := range configs {
-		setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName &&
+	for _, config := range configs {
+		setAsDefault := config.Name == defaultStorageConfig.StorageClassConfigName &&
 			v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName
 		allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass(
-			c,
-			n.config.GlobalOptions.DefaultsNamespace(),
+			config,
 			v1alpha1.NutanixProvisioner,
 			setAsDefault,
+			defaultStorageClassParameters,
 		))
 	}
 	cm, err := lifecycleutils.CreateConfigMapForCRS(

pkg/handlers/generic/lifecycle/utils/scs.go

@@ -0,0 +1,52 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+)
+
+const (
+	kindStorageClass = "StorageClass"
+)
+
+func CreateStorageClass(
+	storageConfig v1alpha1.StorageClassConfig,
+	provisionerName v1alpha1.StorageProvisioner,
+	isDefault bool,
+	defaultParameters map[string]string,
+) *storagev1.StorageClass {
+	parameters := make(map[string]string)
+	// set the defaults first so that user provided parameters can override them
+	for k, v := range defaultParameters {
+		parameters[k] = v
+	}
+	// set user provided parameters, overriding any defaults with the same key
+	for k, v := range storageConfig.Parameters {
+		parameters[k] = v
+	}
+
+	sc := storagev1.StorageClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       kindStorageClass,
+			APIVersion: storagev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: storageConfig.Name,
+		},
+		Provisioner:          string(provisionerName),
+		Parameters:           parameters,
+		VolumeBindingMode:    ptr.To(storageConfig.VolumeBindingMode),
+		ReclaimPolicy:        ptr.To(storageConfig.ReclaimPolicy),
+		AllowVolumeExpansion: ptr.To(storageConfig.AllowExpansion),
+	}
+	if isDefault {
+		sc.ObjectMeta.Annotations = defaultStorageClassMap
+	}
+	return &sc
+}