*: use client-go to list nodes

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
aws · May 25, 2020 · bd8a46b · bd8a46b
1 parent ee1b465
commit bd8a46b
Show file tree

Hide file tree

Showing 6 changed files with 141 additions and 118 deletions.
diff --git a/CHANGELOG/CHANGELOG-1.2.md b/CHANGELOG/CHANGELOG-1.2.md
@@ -40,6 +40,7 @@ See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.2.5...v1.2.6
 
 ### `eks`
 
+- Use [`client-go` to list nodes and CSRs](https://github.com/aws/aws-k8s-tester/commit/).
 - Disable [`eks/hollow-nodes` CSI driver](https://github.com/aws/aws-k8s-tester/commit/876d0a6650f333076ee8137d416adbe5477a2fc7).
   - `"E | csi_plugin.go:271] Failed to initialize CSINodeInfo: error updating CSINode annotation: timed out waiting for the condition; caused by: the server could not find the requested resource"`
   - `"F | csi_plugin.go:285] Failed to initialize CSINodeInfo after retrying"`

diff --git a/eks/hollow-nodes/hollow-nodes.go b/eks/hollow-nodes/hollow-nodes.go
@@ -232,7 +232,7 @@ func (ng *nodeGroup) checkNodes() (readyNodes []string, createdNodes []string, e
 				if cond.Type != v1.NodeReady {
 					continue
 				}
-				ng.cfg.Logger.Info("checked node readiness",
+				ng.cfg.Logger.Info("node is ready!",
 					zap.String("name", nodeName),
 					zap.String("type", fmt.Sprintf("%s", cond.Type)),
 					zap.String("status", fmt.Sprintf("%s", cond.Status)),

diff --git a/eks/hollow-nodes/remote/hollow-nodes.go b/eks/hollow-nodes/remote/hollow-nodes.go
@@ -1,4 +1,5 @@
-// Package remote implements remote Hollow Nodes.
+// Package remote implements remote hollow nodes.
+//
 // ref. https://github.com/kubernetes/kubernetes/blob/master/pkg/kubemark/hollow_kubelet.go
 //
 // The purpose is to make it easy to run on EKS.
@@ -750,15 +751,6 @@ func (ts *tester) checkNodes() error {
 	}
 	cmdLogs := strings.Join(argsLogs, " ")
 
-	argsGetNodes := []string{
-		ts.cfg.EKSConfig.KubectlPath,
-		"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
-		"get",
-		"nodes",
-		"-o=wide",
-	}
-	cmdGetNodes := strings.Join(argsGetNodes, " ")
-
 	expectedNodes := ts.cfg.EKSConfig.AddOnHollowNodesRemote.Nodes * int(ts.cfg.EKSConfig.AddOnHollowNodesRemote.DeploymentReplicas)
 
 	// TODO: :some" hollow nodes may fail from resource quota
@@ -801,7 +793,7 @@ func (ts *tester) checkNodes() error {
 				if cond.Type != v1.NodeReady {
 					continue
 				}
-				ts.cfg.Logger.Info("checked node readiness",
+				ts.cfg.Logger.Info("node is ready!",
 					zap.String("name", nodeName),
 					zap.String("type", fmt.Sprintf("%s", cond.Type)),
 					zap.String("status", fmt.Sprintf("%s", cond.Status)),
@@ -825,15 +817,6 @@ func (ts *tester) checkNodes() error {
 		}
 		fmt.Printf("\n\n\"%s\":\n%s\n", cmdLogs, out)
 
-		ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
-		output, err = exec.New().CommandContext(ctx, argsGetNodes[0], argsGetNodes[1:]...).CombinedOutput()
-		cancel()
-		out = string(output)
-		if err != nil {
-			ts.cfg.Logger.Warn("'kubectl get nodes' failed", zap.Error(err))
-		}
-		fmt.Printf("\n\"%s\":\n%s\n", cmdGetNodes, out)
-
 		ts.cfg.EKSConfig.AddOnHollowNodesRemote.CreatedNodeNames = createdNodeNames
 		ts.cfg.EKSConfig.Sync()
 		if readies > 0 && readies >= expectedNodes {

diff --git a/eks/mng/nodes.go b/eks/mng/nodes.go
@@ -26,9 +26,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/eks/eksiface"
 	"github.com/dustin/go-humanize"
 	"go.uber.org/zap"
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/utils/exec"
 )
 
 // TemplateMNG is the CloudFormation template for EKS managed node group.
@@ -875,27 +874,7 @@ func (ts *tester) waitForNodes(mngName string) error {
 		ec2PrivateDNS[strings.Split(v.PrivateDNSName, ".")[0]] = struct{}{}
 	}
 
-	argsGetCSRs := []string{
-		ts.cfg.EKSConfig.KubectlPath,
-		"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
-		"get",
-		"csr",
-		"-o=wide",
-	}
-	cmdGetCSRs := strings.Join(argsGetCSRs, " ")
-
-	argsGetNodes := []string{
-		ts.cfg.EKSConfig.KubectlPath,
-		"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
-		"get",
-		"nodes",
-		"--show-labels",
-		"-o=wide",
-	}
-	cmdGetNodes := strings.Join(argsGetNodes, " ")
-
 	ts.cfg.Logger.Info("checking nodes readiness")
-	var items []v1.Node
 	retryStart := time.Now()
 	ready := false
 	for time.Now().Sub(retryStart) < waitDur {
@@ -905,17 +884,14 @@ func (ts *tester) waitForNodes(mngName string) error {
 		case <-time.After(5 * time.Second):
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-		nodes, err := ts.cfg.K8SClient.KubernetesClientSet().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
-		cancel()
+		nodes, err := ts.cfg.K8SClient.ListNodes(150, 5*time.Second)
 		if err != nil {
 			ts.cfg.Logger.Warn("get nodes failed", zap.Error(err))
 			continue
 		}
-		items = nodes.Items
 
 		readies := 0
-		for _, node := range items {
+		for _, node := range nodes {
 			labels := node.GetLabels()
 			if labels["NGName"] != mngName {
 				continue
@@ -977,7 +953,7 @@ func (ts *tester) waitForNodes(mngName string) error {
 				if cond.Type != v1.NodeReady {
 					continue
 				}
-				ts.cfg.Logger.Info("checked node readiness",
+				ts.cfg.Logger.Info("node is ready!",
 					zap.String("name", nodeName),
 					zap.String("type", fmt.Sprintf("%s", cond.Type)),
 					zap.String("status", fmt.Sprintf("%s", cond.Status)),
@@ -991,23 +967,25 @@ func (ts *tester) waitForNodes(mngName string) error {
 			zap.Int("desired-ready-nodes", cur.ASGDesiredCapacity),
 		)
 
-		ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
-		output, err := exec.New().CommandContext(ctx, argsGetCSRs[0], argsGetCSRs[1:]...).CombinedOutput()
-		cancel()
-		out := string(output)
-		if err != nil {
-			ts.cfg.Logger.Warn("'kubectl get csr' failed", zap.Error(err))
-		}
-		fmt.Printf("\n\n\"%s\":\n%s\n", cmdGetCSRs, out)
-
-		ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
-		output, err = exec.New().CommandContext(ctx, argsGetNodes[0], argsGetNodes[1:]...).CombinedOutput()
-		cancel()
-		out = string(output)
+		/*
+			e.g.
+			"/tmp/kubectl-test-v1.16.9 --kubeconfig=/tmp/leegyuho-test-eks.kubeconfig.yaml get csr -o=wide":
+			NAME        AGE   REQUESTOR                                                   CONDITION
+			csr-4msk5   58s   system:node:ip-192-168-65-124.us-west-2.compute.internal    Approved,Issued
+			csr-9dbs8   57s   system:node:ip-192-168-208-6.us-west-2.compute.internal     Approved,Issued
+		*/
+		output, err := ts.cfg.K8SClient.ListCSRs(150, 5*time.Second)
 		if err != nil {
-			ts.cfg.Logger.Warn("'kubectl get nodes' failed", zap.Error(err))
+			ts.cfg.Logger.Warn("list CSRs failed", zap.Error(err))
+		} else {
+			for _, cv := range output {
+				ts.cfg.Logger.Info("current CSR",
+					zap.String("name", cv.GetName()),
+					zap.String("requester", cv.Spec.Username),
+					zap.String("status", extractCSRStatus(cv)),
+				)
+			}
 		}
-		fmt.Printf("\n\"%s\":\n%s\n", cmdGetNodes, out)
 
 		if readies >= cur.ASGDesiredCapacity {
 			ready = true
@@ -1018,11 +996,6 @@ func (ts *tester) waitForNodes(mngName string) error {
 		return fmt.Errorf("MNG %q not ready", mngName)
 	}
 
-	fmt.Printf("%q nodes are ready!\n", mngName)
-	for _, v := range items {
-		fmt.Printf("node %q address: %+v\n", v.GetName(), v.Status.Addresses)
-	}
-	println()
 	return ts.cfg.EKSConfig.Sync()
 }
 
@@ -1040,3 +1013,31 @@ func IsDeleted(err error) bool {
 	// ResourceNotFoundException: nodeGroup eks-2019120505-pdx-us-west-2-tqy2d-managed-node-group not found for cluster eks-2019120505-pdx-us-west-2-tqy2d\n\tstatus code: 404, request id: 330998c1-22e9-4a8b-b180-420dadade090
 	return strings.Contains(err.Error(), " not found for cluster ")
 }
+
+// "pkg/printers/internalversion/printers.go"
+func extractCSRStatus(csr certificatesv1beta1.CertificateSigningRequest) string {
+	var approved, denied bool
+	for _, c := range csr.Status.Conditions {
+		switch c.Type {
+		case certificatesv1beta1.CertificateApproved:
+			approved = true
+		case certificatesv1beta1.CertificateDenied:
+			denied = true
+		default:
+			return ""
+		}
+	}
+	var status string
+	// must be in order of presidence
+	if denied {
+		status += "Denied"
+	} else if approved {
+		status += "Approved"
+	} else {
+		status += "Pending"
+	}
+	if len(csr.Status.Certificate) > 0 {
+		status += ",Issued"
+	}
+	return status
+}
diff --git a/eks/ng/nodes.go b/eks/ng/nodes.go
@@ -21,9 +21,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
 	"go.uber.org/zap"
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/utils/exec"
 )
 
 /*
@@ -768,27 +767,7 @@ func (ts *tester) waitForNodes(asgName string) error {
 		ec2PrivateDNS[strings.Split(v.PrivateDNSName, ".")[0]] = struct{}{}
 	}
 
-	argsGetCSRs := []string{
-		ts.cfg.EKSConfig.KubectlPath,
-		"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
-		"get",
-		"csr",
-		"-o=wide",
-	}
-	cmdGetCSRs := strings.Join(argsGetCSRs, " ")
-
-	argsGetNodes := []string{
-		ts.cfg.EKSConfig.KubectlPath,
-		"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
-		"get",
-		"nodes",
-		"--show-labels",
-		"-o=wide",
-	}
-	cmdGetNodes := strings.Join(argsGetNodes, " ")
-
 	ts.cfg.Logger.Info("checking nodes readiness", zap.Duration("wait", waitDur))
-	var items []v1.Node
 	retryStart := time.Now()
 	ready := false
 	for time.Now().Sub(retryStart) < waitDur {
@@ -798,17 +777,14 @@ func (ts *tester) waitForNodes(asgName string) error {
 		case <-time.After(5 * time.Second):
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-		nodes, err := ts.cfg.K8SClient.KubernetesClientSet().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
-		cancel()
+		nodes, err := ts.cfg.K8SClient.ListNodes(150, 5*time.Second)
 		if err != nil {
 			ts.cfg.Logger.Warn("get nodes failed", zap.Error(err))
 			continue
 		}
-		items = nodes.Items
 
 		readies := 0
-		for _, node := range items {
+		for _, node := range nodes {
 			labels := node.GetLabels()
 			if labels["NGName"] != asgName {
 				continue
@@ -870,7 +846,7 @@ func (ts *tester) waitForNodes(asgName string) error {
 				if cond.Type != v1.NodeReady {
 					continue
 				}
-				ts.cfg.Logger.Info("checked node readiness",
+				ts.cfg.Logger.Info("node is ready!",
 					zap.String("name", nodeName),
 					zap.String("type", fmt.Sprintf("%s", cond.Type)),
 					zap.String("status", fmt.Sprintf("%s", cond.Status)),
@@ -884,23 +860,25 @@ func (ts *tester) waitForNodes(asgName string) error {
 			zap.Int64("desired-ready-nodes", cur.ASGDesiredCapacity),
 		)
 
-		ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
-		output, err := exec.New().CommandContext(ctx, argsGetCSRs[0], argsGetCSRs[1:]...).CombinedOutput()
-		cancel()
-		out := string(output)
-		if err != nil {
-			ts.cfg.Logger.Warn("'kubectl get csr' failed", zap.Error(err))
-		}
-		fmt.Printf("\n\n\"%s\":\n%s\n", cmdGetCSRs, out)
-
-		ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
-		output, err = exec.New().CommandContext(ctx, argsGetNodes[0], argsGetNodes[1:]...).CombinedOutput()
-		cancel()
-		out = string(output)
+		/*
+			e.g.
+			"/tmp/kubectl-test-v1.16.9 --kubeconfig=/tmp/leegyuho-test-eks.kubeconfig.yaml get csr -o=wide":
+			NAME        AGE   REQUESTOR                                                   CONDITION
+			csr-4msk5   58s   system:node:ip-192-168-65-124.us-west-2.compute.internal    Approved,Issued
+			csr-9dbs8   57s   system:node:ip-192-168-208-6.us-west-2.compute.internal     Approved,Issued
+		*/
+		output, err := ts.cfg.K8SClient.ListCSRs(150, 5*time.Second)
 		if err != nil {
-			ts.cfg.Logger.Warn("'kubectl get nodes' failed", zap.Error(err))
+			ts.cfg.Logger.Warn("list CSRs failed", zap.Error(err))
+		} else {
+			for _, cv := range output {
+				ts.cfg.Logger.Info("current CSR",
+					zap.String("name", cv.GetName()),
+					zap.String("requester", cv.Spec.Username),
+					zap.String("status", extractCSRStatus(cv)),
+				)
+			}
 		}
-		fmt.Printf("\n\"%s\":\n%s\n", cmdGetNodes, out)
 
 		if int64(readies) >= cur.ASGDesiredCapacity {
 			ready = true
@@ -911,10 +889,33 @@ func (ts *tester) waitForNodes(asgName string) error {
 		return fmt.Errorf("NG %q not ready", asgName)
 	}
 
-	fmt.Printf("%q nodes are ready!\n", asgName)
-	for _, v := range items {
-		fmt.Printf("node %q address: %+v\n", v.GetName(), v.Status.Addresses)
-	}
-	println()
 	return ts.cfg.EKSConfig.Sync()
 }
+
+// "pkg/printers/internalversion/printers.go"
+func extractCSRStatus(csr certificatesv1beta1.CertificateSigningRequest) string {
+	var approved, denied bool
+	for _, c := range csr.Status.Conditions {
+		switch c.Type {
+		case certificatesv1beta1.CertificateApproved:
+			approved = true
+		case certificatesv1beta1.CertificateDenied:
+			denied = true
+		default:
+			return ""
+		}
+	}
+	var status string
+	// must be in order of presidence
+	if denied {
+		status += "Denied"
+	} else if approved {
+		status += "Approved"
+	} else {
+		status += "Pending"
+	}
+	if len(csr.Status.Certificate) > 0 {
+		status += ",Issued"
+	}
+	return status
+}