From 4efe57b3a5c7fa15597bdf94ff5708176f846140 Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Thu, 19 Oct 2023 17:44:37 +0400 Subject: [PATCH 1/9] invalidate --- client/client.go | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/client/client.go b/client/client.go index 611d398..d21fd5e 100644 --- a/client/client.go +++ b/client/client.go @@ -2,7 +2,10 @@ package client import ( "fmt" + "github.com/pkg/errors" + "k8s.io/client-go/discovery/cached/memory" "os" + "reflect" "strings" "time" @@ -326,6 +329,21 @@ func getInClusterConfig() (config *rest.Config, defaultNs string, err error) { // NOTE that fetching all preferred resources can give errors if there are non-working // api controllers in cluster. func (c *Client) APIResourceList(apiVersion string) (lists []*metav1.APIResourceList, err error) { + lists, err = c.apiResourceList(apiVersion) + if err != nil { + fmt.Println("KUBEERR2", err, reflect.TypeOf(err)) + if errors.Is(err, memory.ErrCacheNotFound) { + c.cachedDiscovery.Invalidate() + return c.apiResourceList(apiVersion) + } + + return nil, err + } + + return lists, nil +} + +func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResourceList, err error) { if apiVersion == "" { // Get all preferred resources. // Can return errors if api controllers are not available. @@ -348,21 +366,12 @@ func (c *Client) APIResourceList(apiVersion string) (lists []*metav1.APIResource list, err := c.discovery().ServerResourcesForGroupVersion(gv.String()) if err != nil { - return nil, fmt.Errorf("apiVersion '%s' has no supported resources in cluster: %v", apiVersion, err) + fmt.Println("KUBEERR1", err, reflect.TypeOf(err)) + return nil, errors.Wrapf(err, "apiVersion '%s' has no supported resources in cluster", apiVersion) } lists = []*metav1.APIResourceList{list} } - // TODO should it copy group and version into each resource? - - // TODO create debug command to output this from cli - // Debug mode will list all available CRDs for apiVersion - // for _, r := range list.APIResources { - // log.Debugf("GVR: %30s %30s %30s", list.GroupVersion, r.Kind, - // fmt.Sprintf("%+v", append([]string{r.Name}, r.ShortNames...)), - // ) - // } - return } @@ -382,10 +391,6 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, return resource, nil } - if c.cachedDiscovery != nil { - c.cachedDiscovery.Invalidate() - } - resource = getApiResourceFromResourceLists(kind, lists) if resource != nil { return resource, nil @@ -419,6 +424,7 @@ func (c *Client) GroupVersionResource(apiVersion, kind string) (gvr schema.Group func (c *Client) discovery() discovery.DiscoveryInterface { if c.cachedDiscovery != nil { + fmt.Println("HAS CACHE") return c.cachedDiscovery } return c.Discovery() From 00942dd194f804b56213f6494e1f6893428a6625 Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Thu, 19 Oct 2023 20:05:44 +0400 Subject: [PATCH 2/9] +++ --- client/client.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client/client.go b/client/client.go index d21fd5e..b453e1f 100644 --- a/client/client.go +++ b/client/client.go @@ -369,6 +369,7 @@ func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResource fmt.Println("KUBEERR1", err, reflect.TypeOf(err)) return nil, errors.Wrapf(err, "apiVersion '%s' has no supported resources in cluster", apiVersion) } + fmt.Println("AFTER LIST SEARCH") lists = []*metav1.APIResourceList{list} } @@ -386,16 +387,26 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, return nil, err } + fmt.Println("LISTS", lists, err) + + fmt.Println("KIND", kind) + resource := getApiResourceFromResourceLists(kind, lists) if resource != nil { return resource, nil } + fmt.Println("AFTER1", resource) + + fmt.Println("INVALIDATE") + c.cachedDiscovery.Invalidate() resource = getApiResourceFromResourceLists(kind, lists) if resource != nil { return resource, nil } + fmt.Println("AFTER2", resource) + // If resource is not found, append additional error, may be the custom API of the resource is not available. additionalErr := "" if err != nil { From 6f888badbc4c269170255d6d94168126da5f5811 Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Thu, 19 Oct 2023 23:40:33 +0400 Subject: [PATCH 3/9] +++2 --- client/client.go | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/client/client.go b/client/client.go index b453e1f..77280b6 100644 --- a/client/client.go +++ b/client/client.go @@ -2,15 +2,15 @@ package client import ( "fmt" - "github.com/pkg/errors" - "k8s.io/client-go/discovery/cached/memory" "os" "reflect" "strings" "time" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" apixv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -331,10 +331,13 @@ func getInClusterConfig() (config *rest.Config, defaultNs string, err error) { func (c *Client) APIResourceList(apiVersion string) (lists []*metav1.APIResourceList, err error) { lists, err = c.apiResourceList(apiVersion) if err != nil { - fmt.Println("KUBEERR2", err, reflect.TypeOf(err)) - if errors.Is(err, memory.ErrCacheNotFound) { + fmt.Println("KUBEERR2", err, reflect.TypeOf(err), reflect.TypeOf(errors.Unwrap(err))) + if apierrors.IsNotFound(errors.Unwrap(err)) { + fmt.Println("INVALIDATE LIST") c.cachedDiscovery.Invalidate() return c.apiResourceList(apiVersion) + } else { + fmt.Println("Unknown error type", reflect.TypeOf(err), reflect.TypeOf(errors.Unwrap(err))) } return nil, err @@ -366,10 +369,8 @@ func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResource list, err := c.discovery().ServerResourcesForGroupVersion(gv.String()) if err != nil { - fmt.Println("KUBEERR1", err, reflect.TypeOf(err)) return nil, errors.Wrapf(err, "apiVersion '%s' has no supported resources in cluster", apiVersion) } - fmt.Println("AFTER LIST SEARCH") lists = []*metav1.APIResourceList{list} } @@ -387,8 +388,6 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, return nil, err } - fmt.Println("LISTS", lists, err) - fmt.Println("KIND", kind) resource := getApiResourceFromResourceLists(kind, lists) @@ -397,15 +396,15 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, } fmt.Println("AFTER1", resource) - fmt.Println("INVALIDATE") - c.cachedDiscovery.Invalidate() - - resource = getApiResourceFromResourceLists(kind, lists) - if resource != nil { - return resource, nil - } - - fmt.Println("AFTER2", resource) + //fmt.Println("INVALIDATE") + //c.cachedDiscovery.Invalidate() + // + //resource = getApiResourceFromResourceLists(kind, lists) + //if resource != nil { + // return resource, nil + //} + // + //fmt.Println("AFTER2", resource) // If resource is not found, append additional error, may be the custom API of the resource is not available. additionalErr := "" @@ -435,12 +434,19 @@ func (c *Client) GroupVersionResource(apiVersion, kind string) (gvr schema.Group func (c *Client) discovery() discovery.DiscoveryInterface { if c.cachedDiscovery != nil { - fmt.Println("HAS CACHE") return c.cachedDiscovery } return c.Discovery() } +// InvalidateDiscoveryCache allows you to invalidate cache manually, for example, when you are deploying CRD +// KubeClient tries to invalidate cache automatically when needed, but you can save a few resources to call this manually +func (c *Client) InvalidateDiscoveryCache() { + if c.cachedDiscovery != nil { + c.cachedDiscovery.Invalidate() + } +} + func equalLowerCasedToOneOf(term string, choices ...string) bool { if len(choices) == 0 { return false From 82b67358b9e692a2f65cecc5c0ba60a5e4e34f87 Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Thu, 19 Oct 2023 23:55:27 +0400 Subject: [PATCH 4/9] +++3 --- client/client.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/client.go b/client/client.go index 77280b6..0d25f8f 100644 --- a/client/client.go +++ b/client/client.go @@ -331,13 +331,14 @@ func getInClusterConfig() (config *rest.Config, defaultNs string, err error) { func (c *Client) APIResourceList(apiVersion string) (lists []*metav1.APIResourceList, err error) { lists, err = c.apiResourceList(apiVersion) if err != nil { - fmt.Println("KUBEERR2", err, reflect.TypeOf(err), reflect.TypeOf(errors.Unwrap(err))) - if apierrors.IsNotFound(errors.Unwrap(err)) { + fmt.Println("KUBEERR2", err, reflect.TypeOf(err), errors.Cause(err), reflect.TypeOf(errors.Unwrap(err))) + + if apierrors.IsNotFound(errors.Cause(err)) { fmt.Println("INVALIDATE LIST") c.cachedDiscovery.Invalidate() return c.apiResourceList(apiVersion) } else { - fmt.Println("Unknown error type", reflect.TypeOf(err), reflect.TypeOf(errors.Unwrap(err))) + fmt.Println("Unknown error type", reflect.TypeOf(err), reflect.TypeOf(errors.Cause(err))) } return nil, err @@ -369,6 +370,7 @@ func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResource list, err := c.discovery().ServerResourcesForGroupVersion(gv.String()) if err != nil { + fmt.Println("KUBEERR1", reflect.TypeOf(err)) return nil, errors.Wrapf(err, "apiVersion '%s' has no supported resources in cluster", apiVersion) } lists = []*metav1.APIResourceList{list} From b93cba3745fdf7ace9e8fc259a577e2520999eb5 Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Fri, 20 Oct 2023 00:17:36 +0400 Subject: [PATCH 5/9] +++4 --- client/client.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/client/client.go b/client/client.go index 0d25f8f..e5a98e8 100644 --- a/client/client.go +++ b/client/client.go @@ -10,7 +10,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" apixv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -332,13 +331,13 @@ func (c *Client) APIResourceList(apiVersion string) (lists []*metav1.APIResource lists, err = c.apiResourceList(apiVersion) if err != nil { fmt.Println("KUBEERR2", err, reflect.TypeOf(err), errors.Cause(err), reflect.TypeOf(errors.Unwrap(err))) - - if apierrors.IsNotFound(errors.Cause(err)) { + if errors.Cause(err).Error() == "not found" { + // *errors.errorString type is here, we can't check it another way fmt.Println("INVALIDATE LIST") c.cachedDiscovery.Invalidate() return c.apiResourceList(apiVersion) } else { - fmt.Println("Unknown error type", reflect.TypeOf(err), reflect.TypeOf(errors.Cause(err))) + fmt.Println("Unknown apiResourceList error:", errors.Cause(err)) } return nil, err @@ -370,7 +369,7 @@ func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResource list, err := c.discovery().ServerResourcesForGroupVersion(gv.String()) if err != nil { - fmt.Println("KUBEERR1", reflect.TypeOf(err)) + // if not found, err has type *errors.errorString here return nil, errors.Wrapf(err, "apiVersion '%s' has no supported resources in cluster", apiVersion) } lists = []*metav1.APIResourceList{list} @@ -390,13 +389,11 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, return nil, err } - fmt.Println("KIND", kind) - resource := getApiResourceFromResourceLists(kind, lists) if resource != nil { return resource, nil } - fmt.Println("AFTER1", resource) + fmt.Println("AFTER1: KIND NOT FOUND", kind, resource, lists) //fmt.Println("INVALIDATE") //c.cachedDiscovery.Invalidate() From 288469f08a18bc5342322bae258e60d9401ebdfd Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Fri, 20 Oct 2023 00:46:16 +0400 Subject: [PATCH 6/9] +++5 --- client/client.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/client/client.go b/client/client.go index e5a98e8..6d5d071 100644 --- a/client/client.go +++ b/client/client.go @@ -393,17 +393,18 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, if resource != nil { return resource, nil } - fmt.Println("AFTER1: KIND NOT FOUND", kind, resource, lists) + fmt.Println("AFTER1: KIND NOT FOUND", kind, resource) - //fmt.Println("INVALIDATE") - //c.cachedDiscovery.Invalidate() - // - //resource = getApiResourceFromResourceLists(kind, lists) - //if resource != nil { - // return resource, nil - //} - // - //fmt.Println("AFTER2", resource) + fmt.Println("INVALIDATE") + c.cachedDiscovery.Invalidate() + + resource = getApiResourceFromResourceLists(kind, lists) + if resource != nil { + fmt.Println("RESOURCE FOUND FOR KIND", kind) + return resource, nil + } + + fmt.Println("AFTER2", resource) // If resource is not found, append additional error, may be the custom API of the resource is not available. additionalErr := "" From a70cd6641913df762c42980633883f9cd1f6127c Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Fri, 20 Oct 2023 01:24:15 +0400 Subject: [PATCH 7/9] +++6 --- client/client.go | 50 +++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/client/client.go b/client/client.go index 6d5d071..c3515d2 100644 --- a/client/client.go +++ b/client/client.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" apixv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -382,7 +383,29 @@ func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResource // // NOTE that fetching with empty apiVersion can give errors if there are non-working // api controllers in cluster. -func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, err error) { +func (c *Client) APIResource(apiVersion, kind string) (*metav1.APIResource, error) { + resource, err := c.apiResource(apiVersion, kind) + if err != nil { + fmt.Println("TRY 1 failed", err) + if apiErrors.IsNotFound(err) { + fmt.Println("TRY 1 NOT FOUND. Invalidate") + c.cachedDiscovery.Invalidate() + resource, err = c.apiResource(apiVersion, kind) + fmt.Println("TRY 2", resource, err) + } else { + return nil, fmt.Errorf("apiVersion '%s', kind '%s' is not supported by cluster: %w", apiVersion, kind, err) + } + } + fmt.Println("AFTER TRY 2", resource, err) + if err != nil { + return nil, fmt.Errorf("apiVersion '%s', kind '%s' is not supported by cluster: %w", apiVersion, kind, err) + } + + return resource, nil + +} + +func (c *Client) apiResource(apiVersion, kind string) (res *metav1.APIResource, err error) { lists, err := c.APIResourceList(apiVersion) if err != nil && len(lists) == 0 { // apiVersion is defined and there is a ServerResourcesForGroupVersion error @@ -390,29 +413,12 @@ func (c *Client) APIResource(apiVersion, kind string) (res *metav1.APIResource, } resource := getApiResourceFromResourceLists(kind, lists) - if resource != nil { - return resource, nil - } - fmt.Println("AFTER1: KIND NOT FOUND", kind, resource) - - fmt.Println("INVALIDATE") - c.cachedDiscovery.Invalidate() - - resource = getApiResourceFromResourceLists(kind, lists) - if resource != nil { - fmt.Println("RESOURCE FOUND FOR KIND", kind) - return resource, nil + if resource == nil { + gv, _ := schema.ParseGroupVersion(apiVersion) + return nil, apiErrors.NewNotFound(schema.GroupResource{Group: gv.Group, Resource: kind}, "") } - fmt.Println("AFTER2", resource) - - // If resource is not found, append additional error, may be the custom API of the resource is not available. - additionalErr := "" - if err != nil { - additionalErr = fmt.Sprintf(", additional error: %s", err.Error()) - } - err = fmt.Errorf("apiVersion '%s', kind '%s' is not supported by cluster%s", apiVersion, kind, additionalErr) - return nil, err + return resource, nil } // GroupVersionResource returns a GroupVersionResource object to use with dynamic informer. From 51c8c90869a8aaa3e32ee463d46b720327a59ebb Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Fri, 20 Oct 2023 11:38:33 +0400 Subject: [PATCH 8/9] remove fmt --- client/client.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/client/client.go b/client/client.go index c3515d2..fca6486 100644 --- a/client/client.go +++ b/client/client.go @@ -3,7 +3,6 @@ package client import ( "fmt" "os" - "reflect" "strings" "time" @@ -331,14 +330,10 @@ func getInClusterConfig() (config *rest.Config, defaultNs string, err error) { func (c *Client) APIResourceList(apiVersion string) (lists []*metav1.APIResourceList, err error) { lists, err = c.apiResourceList(apiVersion) if err != nil { - fmt.Println("KUBEERR2", err, reflect.TypeOf(err), errors.Cause(err), reflect.TypeOf(errors.Unwrap(err))) if errors.Cause(err).Error() == "not found" { // *errors.errorString type is here, we can't check it another way - fmt.Println("INVALIDATE LIST") c.cachedDiscovery.Invalidate() return c.apiResourceList(apiVersion) - } else { - fmt.Println("Unknown apiResourceList error:", errors.Cause(err)) } return nil, err @@ -380,23 +375,20 @@ func (c *Client) apiResourceList(apiVersion string) (lists []*metav1.APIResource } // APIResource fetches APIResource object from cluster that specifies the name of a resource and whether it is namespaced. +// if resource not found, we try to invalidate cache and // // NOTE that fetching with empty apiVersion can give errors if there are non-working // api controllers in cluster. func (c *Client) APIResource(apiVersion, kind string) (*metav1.APIResource, error) { resource, err := c.apiResource(apiVersion, kind) if err != nil { - fmt.Println("TRY 1 failed", err) if apiErrors.IsNotFound(err) { - fmt.Println("TRY 1 NOT FOUND. Invalidate") c.cachedDiscovery.Invalidate() resource, err = c.apiResource(apiVersion, kind) - fmt.Println("TRY 2", resource, err) } else { return nil, fmt.Errorf("apiVersion '%s', kind '%s' is not supported by cluster: %w", apiVersion, kind, err) } } - fmt.Println("AFTER TRY 2", resource, err) if err != nil { return nil, fmt.Errorf("apiVersion '%s', kind '%s' is not supported by cluster: %w", apiVersion, kind, err) } From a157bce35f8bb6892b256f6e34ca572d2cf64ee8 Mon Sep 17 00:00:00 2001 From: Yuriy Losev Date: Fri, 20 Oct 2023 13:26:15 +0400 Subject: [PATCH 9/9] lints --- .github/workflows/code-checks.yaml | 4 ++-- .github/workflows/lint.yaml | 6 +++--- .github/workflows/tests.yaml | 4 ++-- .golangci.yaml | 13 +++++++------ client/client.go | 1 - 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index 9ac5485..9611f43 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -10,10 +10,10 @@ jobs: name: Go code style runs-on: ubuntu-latest steps: - - name: Set up Go 1.19 + - name: Set up Go 1.20 uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: '1.20' - uses: actions/checkout@v3 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a71930f..c34e0d2 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,10 +11,10 @@ jobs: name: Run Go linters runs-on: ubuntu-latest steps: - - name: Set up Go 1.19 + - name: Set up Go 1.20 uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: '1.20' id: go - name: Check out addon-operator code @@ -38,7 +38,7 @@ jobs: - name: Run golangci-lint run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b . v1.52.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b . v1.54.2 ./golangci-lint run --sort-results codespell: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f34a524..a7ed34e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -10,10 +10,10 @@ jobs: name: Run unit tests runs-on: ubuntu-latest steps: - - name: Set up Go 1.19 + - name: Set up Go 1.20 uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: '1.20' id: go - name: Check out addon-operator code diff --git a/.golangci.yaml b/.golangci.yaml index 02c9e4c..4fd7a1a 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -37,12 +37,13 @@ linters-settings: goimports: local-prefixes: github.com/flant/ depguard: - list-type: blacklist - include-go-root: true - packages: - - github.com/evanphx/json-patch - packages-with-error-message: - - github.com/evanphx/json-patch: "The 'github.com/evanphx/json-patch' package is superseded. Use pkg/utils/jsonpatch.go instead." + rules: + Main: + files: + - $all + deny: + - pkg: github.com/evanphx/json-patch + desc: "The 'github.com/evanphx/json-patch' package is superseded. Use pkg/utils/jsonpatch.go instead." issues: exclude: # Using underscores is a common practice, refactor in the future diff --git a/client/client.go b/client/client.go index fca6486..fcd95ee 100644 --- a/client/client.go +++ b/client/client.go @@ -394,7 +394,6 @@ func (c *Client) APIResource(apiVersion, kind string) (*metav1.APIResource, erro } return resource, nil - } func (c *Client) apiResource(apiVersion, kind string) (res *metav1.APIResource, err error) {