eks: handle NotFound errors in delete

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
aws · Jun 29, 2020 · 2d7b30d · 2d7b30d
1 parent 9d5179e
commit 2d7b30d
Show file tree

Hide file tree

Showing 21 changed files with 85 additions and 75 deletions.
diff --git a/eks/alb-2048/alb-2048.go b/eks/alb-2048/alb-2048.go
@@ -27,7 +27,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	v1beta1 "k8s.io/api/extensions/v1beta1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/exec"
@@ -297,7 +297,7 @@ func (ts *tester) deleteALBServiceAccount() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB Ingress Controller ServiceAccount (%v)", err)
 	}
@@ -400,7 +400,7 @@ func (ts *tester) deleteALBRBACClusterRole() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB Ingress Controller RBAC ClusterRole (%v)", err)
 	}
@@ -474,7 +474,7 @@ func (ts *tester) deleteALBRBACClusterRoleBinding() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB Ingress Controller RBAC ClusterRoleBinding (%v)", err)
 	}
@@ -585,7 +585,7 @@ func (ts *tester) deleteALBDeployment() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB Ingress Controller Deployment (%v)", err)
 	}
@@ -755,7 +755,7 @@ func (ts *tester) delete2048Deployment() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB 2048 Deployment (%v)", err)
 	}
@@ -900,7 +900,7 @@ func (ts *tester) delete2048Service() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB 2048 Service (%v)", err)
 	}
@@ -1148,7 +1148,7 @@ func (ts *tester) delete2048Ingress() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete ALB 2048 Ingress (%v)", err)
 	}

diff --git a/eks/app-mesh/app-mesh.go b/eks/app-mesh/app-mesh.go
@@ -25,7 +25,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/apps/v1"
-	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/exec"
 )
@@ -410,7 +410,7 @@ func (ts *tester) deleteController() (err error) {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 	} else {
 		ts.cfg.Logger.Info("deleted AppMesh controller deployment")
@@ -444,7 +444,7 @@ func (ts *tester) deleteController() (err error) {
 					},
 				)
 			cancel()
-			if err != nil && !api_errors.IsNotFound(err) {
+			if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 				ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 			} else {
 				ts.cfg.Logger.Info("deleted AppMesh injector replicaset", zap.String("name", name))
@@ -529,7 +529,7 @@ func (ts *tester) deleteInjector() (err error) {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 	} else {
 		ts.cfg.Logger.Info("deleted AppMesh injector Service")
@@ -550,7 +550,7 @@ func (ts *tester) deleteInjector() (err error) {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 	} else {
 		ts.cfg.Logger.Info("deleted AppMesh injector deployment")
@@ -584,7 +584,7 @@ func (ts *tester) deleteInjector() (err error) {
 					},
 				)
 			cancel()
-			if err != nil && !api_errors.IsNotFound(err) {
+			if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 				ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 			} else {
 				ts.cfg.Logger.Info("deleted AppMesh injector replicaset", zap.String("name", name))

diff --git a/eks/cluster-loader/remote/cluster-loader.go b/eks/cluster-loader/remote/cluster-loader.go
@@ -32,7 +32,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -241,7 +241,7 @@ func (ts *tester) createServiceAccount() error {
 			metav1.CreateOptions{},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to create cluster loader ServiceAccount (%v)", err)
 	}
@@ -268,7 +268,7 @@ func (ts *tester) deleteServiceAccount() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete cluster loader ServiceAccount (%v)", err)
 	}
@@ -345,7 +345,7 @@ func (ts *tester) deleteALBRBACClusterRole() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete cluster loader RBAC ClusterRole (%v)", err)
 	}
@@ -424,7 +424,7 @@ func (ts *tester) deleteALBRBACClusterRoleBinding() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete cluster loader RBAC ClusterRoleBinding (%v)", err)
 	}
@@ -489,7 +489,7 @@ func (ts *tester) deleteConfigMap() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return err
 	}

diff --git a/eks/configmaps/local/configmaps.go b/eks/configmaps/local/configmaps.go
@@ -27,6 +27,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"go.uber.org/zap"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 )
 
 // Config defines configmaps local tester configuration.
@@ -148,8 +149,8 @@ func (ts *tester) Delete() error {
 		k8s_client.DefaultNamespaceDeletionInterval,
 		k8s_client.DefaultNamespaceDeletionTimeout,
 		k8s_client.WithForceDelete(true),
-	); err != nil {
-		return fmt.Errorf("failed to delete configmaps local tester namespace (%v)", err)
+	); err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
+		errs = append(errs, fmt.Sprintf("failed to delete configmaps local tester namespace (%v)", err))
 	}
 
 	if len(errs) > 0 {

diff --git a/eks/configmaps/remote/configmaps.go b/eks/configmaps/remote/configmaps.go
@@ -36,7 +36,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -272,7 +272,7 @@ func (ts *tester) deleteServiceAccount() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete configmaps ServiceAccount (%v)", err)
 	}
@@ -351,7 +351,7 @@ func (ts *tester) deleteALBRBACClusterRole() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete configmaps RBAC ClusterRole (%v)", err)
 	}
@@ -430,7 +430,7 @@ func (ts *tester) deleteALBRBACClusterRoleBinding() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete configmaps RBAC ClusterRoleBinding (%v)", err)
 	}

diff --git a/eks/conformance/conformance_test.go b/eks/conformance/conformance_test.go
@@ -27,7 +27,7 @@ func Test_results(t *testing.T) {
 		t.Skip(err)
 	}
 
-	if err = readResults(zap.NewExample(), s, "test-data/test.tar.gz"); err != nil {
+	if err = readResults(zap.NewExample(), os.Stderr, s, "test-data/test.tar.gz"); err != nil {
 		t.Fatal(err)
 	}
 

diff --git a/eks/csrs/remote/csrs.go b/eks/csrs/remote/csrs.go
@@ -36,7 +36,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -272,7 +272,7 @@ func (ts *tester) deleteServiceAccount() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete csrs ServiceAccount (%v)", err)
 	}
@@ -352,7 +352,7 @@ func (ts *tester) deleteALBRBACClusterRole() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete csrs RBAC ClusterRole (%v)", err)
 	}
@@ -431,7 +431,7 @@ func (ts *tester) deleteALBRBACClusterRoleBinding() error {
 			},
 		)
 	cancel()
-	if err != nil && !api_errors.IsNotFound(err) {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete csrs RBAC ClusterRoleBinding (%v)", err)
 	}

diff --git a/eks/cuda-vector-add/cuda-vector-add.go b/eks/cuda-vector-add/cuda-vector-add.go
@@ -16,6 +16,7 @@ import (
 	"github.com/aws/aws-k8s-tester/pkg/timeutil"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/exec"
@@ -283,10 +284,11 @@ func (ts *tester) Delete() error {
 			metav1.DeleteOptions{},
 		)
 	cancel()
-	if err != nil {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
+		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete Pod (%v)", err)
 	}
-	ts.cfg.Logger.Info("deleted Pod", zap.String("pod-name", podName))
+	ts.cfg.Logger.Info("deleted Pod", zap.String("pod-name", podName), zap.Error(err))
 
 	if err := k8s_client.DeleteNamespaceAndWait(
 		ts.cfg.Logger,

diff --git a/eks/fargate/fargate.go b/eks/fargate/fargate.go
@@ -31,6 +31,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/exec"
 )
@@ -442,7 +443,8 @@ func (ts *tester) deleteSecret() error {
 			},
 		)
 	cancel()
-	if err != nil {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
+		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete Secret %q (%v)", ts.cfg.EKSConfig.AddOnFargate.SecretName, err)
 	}
 	ts.cfg.Logger.Info("deleted Secret", zap.String("name", ts.cfg.EKSConfig.AddOnFargate.SecretName))
@@ -641,7 +643,8 @@ func (ts *tester) deletePod() error {
 			},
 		)
 	cancel()
-	if err != nil {
+	if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
+		ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
 		return fmt.Errorf("failed to delete Pod %q (%v)", fargatePodName, err)
 	}
 	ts.cfg.Logger.Info("deleted Pod", zap.String("name", fargatePodName))

diff --git a/eks/hollow-nodes/local/hollow-nodes.go b/eks/hollow-nodes/local/hollow-nodes.go
@@ -21,7 +21,7 @@ import (
 	"github.com/aws/aws-k8s-tester/pkg/timeutil"
 	"github.com/aws/aws-sdk-go/aws"
 	"go.uber.org/zap"
-	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -139,7 +139,7 @@ func (ts *tester) deleteCreatedNodes() error {
 			},
 		)
 		cancel()
-		if err != nil && !api_errors.IsNotFound(err) {
+		if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
 			ts.cfg.Logger.Warn("failed to delete node", zap.Int("index", i), zap.String("name", nodeName), zap.Error(err))
 			errs = append(errs, err.Error())
 		} else {