Skip to content

Commit

Permalink
Fix helm not deleting CRD issue
Browse files Browse the repository at this point in the history 
Signed-off-by: Niladri Halder <niladri.halder26@gmail.com>
  • Loading branch information
@niladrih
niladrih committed Apr 29, 2022
1 parent 695b096 commit 5e075f0
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 40 deletions.
23 changes: 23 additions & 0 deletions e2etest/controlplane/controlplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,17 @@ func NewHandler(t *testing.T, config HandlerConfig) *Handler {
//NOTE: This does not remove CRDs.
// Ref: https://helm.sh/docs/topics/charts/#limitations-on-crds
_, err := action.NewUninstall(config.helmConfig).Run(config.helmRelease.Name)
if err != nil {
return err
}

//Deleting CRDs manually
err = utils.DeleteCRDs(
ctx,
t,
config.TestEnvConfig.Client().RESTConfig(),
[]string{"imagecaches.kubefledged.io"},
)
return err
},
IsDeleted: func(ctx context.Context) bool {
Expand Down Expand Up @@ -300,6 +311,18 @@ func NewHandler(t *testing.T, config HandlerConfig) *Handler {
t.Logf("Waiting for 5 seconds while the Operator cleans up the control plane in %s namespace...", config.Namespace)
time.Sleep(5 * time.Second)

//Deleting CRD manually, because helm does not delete CRD
// Ref: https://helm.sh/docs/topics/charts/#limitations-on-crds
err = utils.DeleteCRDs(
ctx,
t,
config.TestEnvConfig.Client().RESTConfig(),
[]string{"imagecaches.kubefledged.io"},
)
if err != nil {
return err
}

const (
namespacePlaceholder = "{{KUBEFLEDGED_NAMESPACE}}"
)
Expand Down
3 changes: 0 additions & 3 deletions e2etest/install_using_helm_chart_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,6 @@ func TestInstallUsingHelmChart(t *testing.T) {
).
Teardown(
func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {

//NOTE: This does not remove CRDs.
// Ref: https://helm.sh/docs/topics/charts/#limitations-on-crds
err := ctrlPlaneHandler.Delete(ctx, t)
if err != nil {
t.Fatalf("Control Plane delete using helm chart did not succeed, all pods were not deleted: %s", err.Error())
Expand Down
3 changes: 0 additions & 3 deletions e2etest/install_using_operator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,6 @@ func TestInstallUsingOperator(t *testing.T) {
).
Teardown(
func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {

//NOTE: This does not remove CRDs.
// Ref: https://helm.sh/docs/topics/charts/#limitations-on-crds
err := ctrlPlaneHandler.Delete(ctx, t)
if err != nil {
t.Fatalf("Control Plane delete using helm operator did not succeed, all pods were not deleted: %s", err.Error())
Expand Down
25 changes: 25 additions & 0 deletions e2etest/utils/helm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@ limitations under the License.
package utils

import (
"context"
"testing"

"github.com/pkg/errors"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/kube"
apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/deprecated/typed/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
)

const HELM_DRIVER = "memory"
Expand All @@ -44,3 +50,22 @@ func NewHelmDefaultConfig(t *testing.T, releaseName, releaseNamespace, kubeconfi

return actionConfig
}

func DeleteCRDs(ctx context.Context, t *testing.T, cfg *rest.Config, crdNames []string) error {
t.Helper()

client, err := apiextensionsv1client.NewForConfig(cfg)
if err != nil {
return errors.Wrap(err, "failed to delete CRDs")
}

for _, crdName := range crdNames {
err = client.CustomResourceDefinitions().Delete(crdName, &metav1.DeleteOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return errors.Wrapf(err, "failed to delete CRD %s", crdName)
}
}

t.Logf("Successfully deleted CRDs %v", crdNames)
return nil
}
59 changes: 28 additions & 31 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ go 1.17
require (
github.com/golang/glog v1.0.0
k8s.io/api v0.23.6
k8s.io/apiextensions-apiserver v0.23.6
k8s.io/apimachinery v0.23.6
k8s.io/apiserver v0.23.5
k8s.io/apiserver v0.23.6
k8s.io/client-go v0.23.6
sigs.k8s.io/e2e-framework v0.0.6
)
Expand All @@ -27,30 +28,41 @@ require (
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/containerd/containerd v1.6.1 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.11+incompatible // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/go-errors/errors v1.0.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmoiron/sqlx v1.3.4 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
Expand All @@ -67,11 +79,14 @@ require (
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
Expand All @@ -83,60 +98,42 @@ require (
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/vladimirvivien/gexe v0.1.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.43.0 // indirect
gopkg.in/gorp.v1 v1.7.2 // indirect
k8s.io/apiextensions-apiserver v0.23.5 // indirect
k8s.io/cli-runtime v0.23.5 // indirect
k8s.io/component-base v0.23.5 // indirect
k8s.io/kubectl v0.23.5 // indirect
oras.land/oras-go v1.1.1 // indirect
sigs.k8s.io/kustomize/api v0.10.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pkg/errors v0.9.1
github.com/spf13/pflag v1.0.5 // indirect
github.com/vladimirvivien/gexe v0.1.1 // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/gorp.v1 v1.7.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
helm.sh/helm/v3 v3.8.2
k8s.io/cli-runtime v0.23.5 // indirect
k8s.io/component-base v0.23.6 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/kube-openapi v0.0.0-20220310132336-3f90b8c54bbb // indirect
k8s.io/kubectl v0.23.5 // indirect
k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
oras.land/oras-go v1.1.1 // indirect
sigs.k8s.io/controller-runtime v0.11.0 // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/kustomize/api v0.10.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
10 changes: 7 additions & 3 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1771,8 +1771,9 @@ k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
k8s.io/api v0.23.6 h1:yOK34wbYECH4RsJbQ9sfkFK3O7f/DUHRlzFehkqZyVw=
k8s.io/api v0.23.6/go.mod h1:1kFaYxGCFHYp3qd6a85DAj/yW8aVD6XLZMqJclkoi9g=
k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4=
k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ=
k8s.io/apiextensions-apiserver v0.23.6 h1:v58cQ6Z0/GK1IXYr+oW0fnYl52o9LTY0WgoWvI8uv5Q=
k8s.io/apiextensions-apiserver v0.23.6/go.mod h1:YVh17Mphv183THQJA5spNFp9XfoidFyL3WoDgZxQIZU=
k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
Expand All @@ -1787,8 +1788,9 @@ k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ=
k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4=
k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE=
k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw=
k8s.io/apiserver v0.23.6 h1:p94LiXcsSnpSDIl4cv98liBuFKcaygSCNopFNfMg/Ac=
k8s.io/apiserver v0.23.6/go.mod h1:5PU32F82tfErXPmf7FXhd/UcuLfh97tGepjKUgJ2atg=
k8s.io/cli-runtime v0.23.5 h1:Z7XUpGoJZYZB2uNjQfJjMbyDKyVkoBGye62Ap0sWQHY=
k8s.io/cli-runtime v0.23.5/go.mod h1:oY6QDF2qo9xndSq32tqcmRp2UyXssdGrLfjAVymgbx4=
k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
Expand All @@ -1802,13 +1804,15 @@ k8s.io/client-go v0.23.6/go.mod h1:Umt5icFOMLV/+qbtZ3PR0D+JA6lvvb3syzodv4irpK4=
k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0=
k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
k8s.io/code-generator v0.23.6/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE=
k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0=
k8s.io/component-base v0.23.6 h1:8dhVZ4VrRcNdV2EGjl8tj8YOHwX6ysgCGMJ2Oyy0NW8=
k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0=
k8s.io/component-helpers v0.23.5/go.mod h1:5riXJgjTIs+ZB8xnf5M2anZ8iQuq37a0B/0BgoPQuSM=
k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
Expand Down

0 comments on commit 5e075f0

Please sign in to comment.