test(e2e): refactor e2e (#188)

* test(e2e): refactor e2e

* test(e2e): fix e2e err

* test(e2e): remove useless test

* test(e2e): remove csi assert

* test(e2e): fix tls-pkcs12 not found

* test(e2e): enable parallel

* test(e2e): fix e2e err

* test(e2e): increase assert timeout and pod resource

* fix: listener-volume scope missing listener address

* test(e2e): catch log when test err

* test(e2e): fix catch syntex err

* test(e2e): change catch order

* test(e2e): fix catch syntx err

* test(e2e): cut down resource limit

* test(e2e): enable all case

* test(e2e): cut down resource limit

* test(e2e): fix k8s search pod restart

* pref: fix typo

* pref: fmt code

.chainsaw.yaml

@@ -5,11 +5,11 @@ metadata:
 spec:
   timeouts:
     apply: 120s
-    assert: 180s
+    assert: 200s
     cleanup: 120s
     delete: 120s
     error: 10s
     exec: 45s
+  forceTerminationGracePeriod: 10s
   # skipDelete: true
   failFast: true
-  parallel: 1 # use 1 concurrent to test, to voide multiple csi driver conflict

Makefile

@@ -339,9 +339,8 @@ $(CHAINSAW): $(LOCALBIN)
 
 .PHONY: chainsaw-setup
 chainsaw-setup: ## Run the chainsaw setup
-	make docker-build
 	make csi-docker-build
-	$(KIND) --name $(KIND_CLUSTER_NAME) load docker-image $(IMG) $(CSIDRIVER_IMG)
+	$(KIND) --name $(KIND_CLUSTER_NAME) load docker-image $(CSIDRIVER_IMG)
 	KUBECONFIG=$(KIND_KUBECONFIG) make helm-install-depends
 	KUBECONFIG=$(KIND_KUBECONFIG) make deploy
 

cmd/csi_driver/main.go

@@ -29,6 +29,7 @@ import (
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	listenerv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/listeners/v1alpha1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -55,6 +56,7 @@ var (
 
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(listenerv1alpha1.AddToScheme(scheme))
 
 	utilruntime.Must(secretv1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme

config/csi/daemonset.yaml

@@ -27,19 +27,19 @@ spec:
       securityContext:
         {}
       containers:
-        - name: secret-operator
+        - name: secret-csi-driver
           securityContext:
             privileged: true
             runAsUser: 0
           image: "csi-driver:latest"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
-              cpu: 100m
+              cpu: 512m
               memory: 128Mi
             requests:
-              cpu: 100m
-              memory: 128Mi
+              cpu: 50m
+              memory: 50Mi
           env:
             - name: ADDRESS
               value: unix:///csi/csi.sock

config/rbac/role.yaml

@@ -38,6 +38,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - listeners.kubedoop.dev
+  resources:
+  - listeners
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - secrets.kubedoop.dev
   resources:

internal/controller/secretclass_controller.go

@@ -44,6 +44,7 @@ type SecretClassReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=listeners.kubedoop.dev,resources=listeners,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

internal/csi/backend/ca/ca_manager.go

@@ -22,10 +22,10 @@ var (
 )
 
 type CertificateManager struct {
-	client               client.Client
-	caCertficateLifetime time.Duration
-	auto                 bool
-	name, namespace      string
+	client                client.Client
+	caCertificateLifetime time.Duration
+	auto                  bool
+	name, namespace       string
 
 	secret *corev1.Secret
 	cas    []*CertificateAuthority
@@ -39,16 +39,16 @@ type CertificateManager struct {
 // Now, pem key supports only RSA 256.
 func NewCertificateManager(
 	client client.Client,
-	caCertficateLifetime time.Duration,
+	caCertificateLifetime time.Duration,
 	auto bool,
 	name, namespace string,
 ) *CertificateManager {
 	obj := &CertificateManager{
-		client:               client,
-		caCertficateLifetime: caCertficateLifetime,
-		auto:                 auto,
-		name:                 name,
-		namespace:            namespace,
+		client:                client,
+		caCertificateLifetime: caCertificateLifetime,
+		auto:                  auto,
+		name:                  name,
+		namespace:             namespace,
 
 		secret: &corev1.Secret{
 			ObjectMeta: ctrl.ObjectMeta{
@@ -95,7 +95,7 @@ func (c *CertificateManager) secretCreateIfDoesNotExist(ctx context.Context) err
 		return err
 	}
 
-	logger.V(1).Info("Created a new secret", "name", c.name, "namespace", c.namespace, "auto", c.auto)
+	logger.V(1).Info("created a new secret", "name", c.name, "namespace", c.namespace, "auto", c.auto)
 	return nil
 
 }
@@ -115,7 +115,7 @@ func (c CertificateManager) getPEMKeyPairsFromSecret(ctx context.Context) ([]PEM
 		}
 	}
 
-	logger.V(0).Info("Get certificate authorities PEM key pairs from secret", "name", c.name, "namespace", c.namespace, "len", len(keyPairs))
+	logger.V(0).Info("got certificate authorities PEM key pairs from secret", "name", c.name, "namespace", c.namespace, "len", len(keyPairs))
 	return keyPairs, nil
 }
 
@@ -171,7 +171,7 @@ func (c *CertificateManager) getCertificateAuthorities(pemKeyPairs []PEMkeyPair)
 			return nil, err
 		}
 		if ca.Certificate.NotAfter.Before(time.Now()) {
-			logger.V(0).Info("Certificate authority is expired, skip it.", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
+			logger.V(0).Info("certificate authority is expired, skip it.", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
 			continue
 		}
 		cas = append(cas, ca)
@@ -187,7 +187,7 @@ func (c *CertificateManager) getCertificateAuthorities(pemKeyPairs []PEMkeyPair)
 			)
 		}
 
-		logger.V(0).Info("Could not find any certificate authorities, created a new self-signed certificate authority", "name", c.name, "namespace", c.namespace, "auto", c.auto)
+		logger.V(0).Info("could not find any certificate authorities, created a new self-signed certificate authority", "name", c.name, "namespace", c.namespace, "auto", c.auto)
 		ca, err := c.createSelfSignedCertificateAuthority()
 		if err != nil {
 			return nil, err
@@ -208,12 +208,12 @@ func (c *CertificateManager) getCertificateAuthorities(pemKeyPairs []PEMkeyPair)
 
 // create a new self-signed certificate authority only no certificate authority is found
 func (c *CertificateManager) createSelfSignedCertificateAuthority() (*CertificateAuthority, error) {
-	notAfter := time.Now().Add(c.caCertficateLifetime)
+	notAfter := time.Now().Add(c.caCertificateLifetime)
 	ca, err := NewSelfSignedCertificateAuthority(notAfter, nil, nil)
 	if err != nil {
 		return nil, err
 	}
-	logger.V(0).Info("Created new self-signed certificate authority", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
+	logger.V(0).Info("created new self-signed certificate authority", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
 	return ca, nil
 }
 
@@ -242,25 +242,25 @@ func (c *CertificateManager) rotateCertificateAuthority(cas []*CertificateAuthor
 
 	newestCA := cas[len(cas)-1]
 
-	if time.Now().Add(c.caCertficateLifetime / 2).After(newestCA.Certificate.NotAfter) {
+	if time.Now().Add(c.caCertificateLifetime / 2).After(newestCA.Certificate.NotAfter) {
 		if c.auto {
-			newCA, err := newestCA.Rotate(time.Now().Add(c.caCertficateLifetime))
+			newCA, err := newestCA.Rotate(time.Now().Add(c.caCertificateLifetime))
 			if err != nil {
 				return nil, err
 			}
-			logger.V(0).Info("Rotated certificate authority, because the old ca is about to expire",
+			logger.V(0).Info("rotated certificate authority, because the old ca is about to expire",
 				"serialNumber", newestCA.SerialNumber(),
 				"notAfter", newCA.Certificate.NotAfter,
 			)
 			cas = append(cas, newCA)
 		} else {
-			logger.V(0).Info("Certificate authority is about to expire, but auto-generate is disabled, please rotate manually.",
+			logger.V(0).Info("certificate authority is about to expire, but auto-generate is disabled, please rotate manually.",
 				"serialNumber", newestCA.SerialNumber(),
 				"notAfter", newestCA.Certificate.NotAfter,
 			)
 		}
 	} else {
-		logger.V(0).Info("Certificate authority is still valid, no need to rotate",
+		logger.V(0).Info("certificate authority is still valid, no need to rotate",
 			"serialNumber", newestCA.SerialNumber(),
 			"notAfter", newestCA.Certificate.NotAfter,
 		)
@@ -283,7 +283,7 @@ func (c *CertificateManager) getAliveCertificateAuthority(atAfter time.Time, cas
 		}
 		return 0
 	})
-	logger.V(0).Info("Get alive certificate authority", "serialNumber", oldestCA.SerialNumber(), "notAfter", oldestCA.Certificate.NotAfter)
+	logger.V(0).Info("got alive certificate authority", "serialNumber", oldestCA.SerialNumber(), "notAfter", oldestCA.Certificate.NotAfter)
 
 	return oldestCA
 }

internal/csi/controller.go

@@ -56,7 +56,7 @@ func (c *ControllerServer) CreateVolume(ctx context.Context, request *csi.Create
 	requiredCap := request.CapacityRange.GetRequiredBytes()
 
 	if request.Parameters["secretFinalizer"] == "true" {
-		logger.V(1).Info("Finalizer is true")
+		logger.V(1).Info("finalizer is true")
 	}
 
 	// requests.parameters is StorageClass.Parameters, which is set by user when creating PVC.
@@ -179,7 +179,7 @@ func (c *ControllerServer) DeleteVolume(ctx context.Context, request *csi.Delete
 	}
 
 	if !dynamic {
-		logger.V(5).Info("Volume is not dynamic, skip delete volume")
+		logger.V(5).Info("volume is not dynamic, skip delete volume")
 		return &csi.DeleteVolumeResponse{}, nil
 	}
 

internal/csi/driver.go

@@ -47,7 +47,7 @@ func NewDriver(
 
 func (d *Driver) Run(ctx context.Context, testMode bool) error {
 
-	logger.V(1).Info("Driver information", "versionInfo", version.GetVersion(d.name))
+	logger.V(1).Info("driver information", "versionInfo", version.GetVersion(d.name))
 
 	// check node id
 	if d.nodeID == "" {
@@ -56,7 +56,7 @@ func (d *Driver) Run(ctx context.Context, testMode bool) error {
 
 	ns := NewNodeServer(
 		d.nodeID,
-		mount.New(""),
+		mount.New("secret-csi"),
 		d.client,
 	)
 
@@ -72,7 +72,7 @@ func (d *Driver) Run(ctx context.Context, testMode bool) error {
 	}()
 
 	d.server.Wait()
-	logger.Info("Server stopped")
+	logger.Info("csi driver stopped")
 	return nil
 }
 

internal/csi/node.go

@@ -144,7 +144,7 @@ func (n *NodeServer) updatePod(ctx context.Context, pod *corev1.Pod, volumeID st
 	}
 	patch := client.MergeFrom(pod.DeepCopy())
 	if expiresTime == nil {
-		logger.V(5).Info("Expiration time is nil, skip update pod annotation", "pod", pod.Name)
+		logger.V(5).Info("expiration time is nil, skip update pod annotation", "pod", pod.Name)
 		return nil
 	}
 
@@ -157,14 +157,14 @@ func (n *NodeServer) updatePod(ctx context.Context, pod *corev1.Pod, volumeID st
 
 	annotationExpiresName := constants.PrefixLabelRestarterExpiresAt + hex.EncodeToString(volumeTag)
 	expiresTimeStr := expiresTime.Format(time.RFC3339)
-	logger.V(5).Info("Update pod annotation", "pod", pod.Name, "key", annotationExpiresName, "value", expiresTimeStr)
+	logger.V(5).Info("update pod annotation", "pod", pod.Name, "key", annotationExpiresName, "value", expiresTimeStr)
 
 	pod.Annotations[annotationExpiresName] = expiresTimeStr
 
 	if err := n.client.Patch(ctx, pod, patch); err != nil {
 		return err
 	}
-	logger.V(5).Info("Pod patched", "pod", pod.Name)
+	logger.V(5).Info("pod patched", "pod", pod.Name)
 	return nil
 }
 
@@ -177,9 +177,9 @@ func (n *NodeServer) writeData(targetPath string, data map[string]string) error
 		if err := os.WriteFile(fileName, []byte(content), fs.FileMode(0644)); err != nil {
 			return err
 		}
-		logger.V(5).Info("File written", "file", fileName)
+		logger.V(5).Info("file written", "file", fileName)
 	}
-	logger.V(5).Info("Data written", "target", targetPath)
+	logger.V(5).Info("data written", "target", targetPath)
 	return nil
 }
 
@@ -218,7 +218,7 @@ func (n *NodeServer) mount(targetPath string) error {
 	if err := n.mounter.Mount("tmpfs", targetPath, "tmpfs", opts); err != nil {
 		return status.Error(codes.Internal, err.Error())
 	}
-	logger.V(1).Info("Volume mounted", "source", "tmpfs", "target", targetPath, "fsType", "tmpfs", "options", opts)
+	logger.V(1).Info("volume mounted", "source", "tmpfs", "target", targetPath, "fsType", "tmpfs", "options", opts)
 	return nil
 }
 
@@ -239,7 +239,7 @@ func (n *NodeServer) NodeUnpublishVolume(ctx context.Context, request *csi.NodeU
 	if err := n.mounter.Unmount(targetPath); err != nil {
 		// FIXME: use status.Error to return error
 		// return nil, status.Error(codes.Internal, err.Error())
-		logger.V(0).Info("Volume not found, skip delete volume")
+		logger.V(0).Info("volume not found, skip delete volume")
 	}
 
 	// remove the target path

internal/csi/server.go

@@ -72,7 +72,7 @@ func (s *nonBlockingServer) serveGrpc(endpoint string, ids csi.IdentityServer, c
 	if proto == "unix" {
 		addr = "/" + addr
 		if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
-			logger.V(0).Info("Failed to remove", "addr", addr, "error", err.Error())
+			logger.V(0).Info("failed to remove", "addr", addr, "error", err.Error())
 		}
 	}
 

pkg/kerberos/kadmin.go

@@ -118,7 +118,7 @@ func (k *Kadmin) Query(query string) (result string, err error) {
 		kadminLogger.Error(err, "Failed to execute kadmin query", "cmd", cmd.String(), "output", result)
 		return "", err
 	}
-	kadminLogger.Info("Executed kadmin query", "cmd", cmd.String(), "output", result)
+	kadminLogger.Info("executed kadmin query", "cmd", cmd.String(), "output", result)
 
 	return result, nil
 
@@ -149,7 +149,7 @@ func (k *Kadmin) Ktadd(principals ...string) ([]byte, error) {
 		return nil, err
 	}
 
-	kadminLogger.Info("Saved keytab", "principal", principals, "keytab", keytab, "output", output)
+	kadminLogger.Info("saved keytab", "principal", principals, "keytab", keytab, "output", output)
 
 	return os.ReadFile(keytab)
 }
@@ -186,7 +186,7 @@ func (k *Kadmin) AddPrincipal(principal string) error {
 		return err
 	}
 
-	kadminLogger.Info("Added principal", "principal", principal, "output", output)
+	kadminLogger.Info("created a new principal", "principal", principal, "output", output)
 
 	return nil
 }

pkg/pod_info/pod_info.go

@@ -76,7 +76,7 @@ func (p *PodInfo) getNodeAddresses(ctx context.Context) ([]Address, error) {
 		}
 	}
 
-	logger.V(1).Info("get node ip filter by internal and external", "pod", p.getPodName(),
+	logger.V(1).Info("got node ip filter by internal and external", "pod", p.getPodName(),
 		"namespace", p.getPodNamespace(), "addresses", addresses)
 	return addresses, nil
 }
@@ -333,6 +333,12 @@ func (p *PodInfo) getListenerAddresses(ctx context.Context) ([]Address, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		// check listener status
+		if len(listener.Status.IngressAddresses) == 0 {
+			return nil, fmt.Errorf("listener %s/%s status not ready", listener.Namespace, listener.Name)
+		}
+
 		for _, ingressAddress := range listener.Status.IngressAddresses {
 			if ingressAddress.AddressType == operatorlistenersv1alpha1.AddressTypeHostname {
 				addresses = append(addresses, Address{