rbd: create token and use it for vault SA

create the token if kubernetes version in
1.24+ and use it for vault sa.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
Signed-off-by: Rakshith R <rar@redhat.com>

charts/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml

@@ -31,4 +31,7 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
     verbs: ["list", "get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts/token"]
+    verbs: ["create"]
 {{- end -}}

charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml

@@ -69,6 +69,9 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts/token"]
+    verbs: ["create"]
 {{- end }}
 
 {{- end -}}

deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml

@@ -30,6 +30,9 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
     verbs: ["list", "get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts/token"]
+    verbs: ["create"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

deploy/rbd/kubernetes/csi-provisioner-rbac.yaml

@@ -63,6 +63,9 @@ rules:
   - apiGroups: [""]
     resources: ["serviceaccounts"]
     verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts/token"]
+    verbs: ["create"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

internal/kms/vault_sa.go

@@ -24,8 +24,10 @@ import (
 	"os"
 
 	"github.com/libopenstorage/secrets/vault"
+	authenticationv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 )
 
 const (
@@ -35,6 +37,9 @@ const (
 	// should be available in the Tenants namespace. This ServiceAccount
 	// will be used to connect to Hashicorp Vault.
 	vaultTenantSAName = "ceph-csi-vault-sa"
+	// Kubernetes version which requires ServiceAccount token creation.
+	kubeMinMajorVersionForCreateToken = "1"
+	kubeMinMinorVersionForCreateToken = "24"
 )
 
 /*
@@ -292,9 +297,9 @@ func (kms *vaultTenantSA) getToken() (string, error) {
 	}
 
 	for _, secretRef := range sa.Secrets {
-		secret, err := c.CoreV1().Secrets(kms.Tenant).Get(context.TODO(), secretRef.Name, metav1.GetOptions{})
-		if err != nil {
-			return "", fmt.Errorf("failed to get Secret %s/%s: %w", kms.Tenant, secretRef.Name, err)
+		secret, sErr := c.CoreV1().Secrets(kms.Tenant).Get(context.TODO(), secretRef.Name, metav1.GetOptions{})
+		if sErr != nil {
+			return "", fmt.Errorf("failed to get Secret %s/%s: %w", kms.Tenant, secretRef.Name, sErr)
 		}
 
 		token, ok := secret.Data["token"]
@@ -303,7 +308,7 @@ func (kms *vaultTenantSA) getToken() (string, error) {
 		}
 	}
 
-	return "", fmt.Errorf("failed to find token in ServiceAccount %s/%s", kms.Tenant, kms.tenantSAName)
+	return kms.createToken(sa, c)
 }
 
 // getTokenPath creates a temporary directory structure that contains the token
@@ -327,3 +332,34 @@ func (kms *vaultTenantSA) getTokenPath() (string, error) {
 
 	return dir + "/token", nil
 }
+
+// createToken creates required service account token for kubernetes 1.24+,
+// else returns error.
+// From kubernetes v1.24+, secret for service account tokens are not
+// automatically created. Hence, use the create token api to fetch it.
+// refer: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md \
+// #no-really-you-must-read-this-before-you-upgrade-1
+func (kms *vaultTenantSA) createToken(sa *corev1.ServiceAccount, client *kubernetes.Clientset) (string, error) {
+	version, err := client.ServerVersion()
+	if err != nil {
+		return "", fmt.Errorf("failed to get ServerVersion: %w", err)
+	}
+
+	if version.Major >= kubeMinMajorVersionForCreateToken &&
+		version.Minor >= kubeMinMinorVersionForCreateToken {
+		tokenRequest := &authenticationv1.TokenRequest{}
+		token, err := client.CoreV1().ServiceAccounts(kms.Tenant).CreateToken(
+			context.TODO(),
+			sa.Name,
+			tokenRequest,
+			metav1.CreateOptions{},
+		)
+		if err != nil {
+			return "", fmt.Errorf("failed to create token for service account %s/%s: %w", kms.Tenant, sa.Name, err)
+		}
+
+		return token.Status.Token, nil
+	}
+
+	return "", fmt.Errorf("failed to find token in ServiceAccount %s/%s", kms.Tenant, kms.tenantSAName)
+}