diff --git a/pkg/client/apiutil/restmapper.go b/pkg/client/apiutil/restmapper.go
index 24af330c8b..927be22b4e 100644
--- a/pkg/client/apiutil/restmapper.go
+++ b/pkg/client/apiutil/restmapper.go
@@ -53,7 +53,7 @@ func NewDynamicRESTMapper(cfg *rest.Config, httpClient *http.Client) (meta.RESTM
 // client for discovery information to do REST mappings.
 type mapper struct {
 	mapper      meta.RESTMapper
-	client      *discovery.DiscoveryClient
+	client      discovery.DiscoveryInterface
 	knownGroups map[string]*restmapper.APIGroupResources
 	apiGroups   map[string]*metav1.APIGroup
 
@@ -280,11 +280,15 @@ func (m *mapper) fetchGroupVersionResourcesLocked(groupName string, versions ...
 		groupVersion := schema.GroupVersion{Group: groupName, Version: version}
 
 		apiResourceList, err := m.client.ServerResourcesForGroupVersion(groupVersion.String())
-		if apierrors.IsNotFound(err) && m.isGroupVersionCached(groupVersion) {
+		if apierrors.IsNotFound(err) {
 			// If the version is not found, we remove the group from the cache
 			// so it gets refreshed on the next call.
-			delete(m.apiGroups, groupName)
-			delete(m.knownGroups, groupName)
+			if m.isAPIGroupCached(groupVersion) {
+				delete(m.apiGroups, groupName)
+			}
+			if m.isGroupVersionCached(groupVersion) {
+				delete(m.knownGroups, groupName)
+			}
 			continue
 		} else if err != nil {
 			failedGroups[groupVersion] = err
@@ -313,3 +317,19 @@ func (m *mapper) isGroupVersionCached(gv schema.GroupVersion) bool {
 
 	return false
 }
+
+// isAPIGroupCached checks if a version for a group is cached in the api groups cache.
+func (m *mapper) isAPIGroupCached(gv schema.GroupVersion) bool {
+	cachedGroup, ok := m.apiGroups[gv.Group]
+	if !ok {
+		return false
+	}
+
+	for _, version := range cachedGroup.Versions {
+		if version.Version == gv.Version {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/pkg/client/apiutil/restmapper_wb_test.go b/pkg/client/apiutil/restmapper_wb_test.go
new file mode 100644
index 0000000000..96dbe79e77
--- /dev/null
+++ b/pkg/client/apiutil/restmapper_wb_test.go
@@ -0,0 +1,203 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiutil
+
+import (
+	"testing"
+
+	gmg "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/restmapper"
+)
+
+func TestLazyRestMapper_fetchGroupVersionResourcesLocked_CacheInvalidation(t *testing.T) {
+	tests := []struct {
+		name                                   string
+		groupName                              string
+		versions                               []string
+		cachedAPIGroups, expectedAPIGroups     map[string]*metav1.APIGroup
+		cachedKnownGroups, expectedKnownGroups map[string]*restmapper.APIGroupResources
+	}{
+		{
+			name:      "Not found version for cached groupVersion in apiGroups and knownGroups",
+			groupName: "group1",
+			versions:  []string{"v1", "v2"},
+			cachedAPIGroups: map[string]*metav1.APIGroup{
+				"group1": {
+					Name: "group1",
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: "v1",
+						},
+					},
+				},
+			},
+			cachedKnownGroups: map[string]*restmapper.APIGroupResources{
+				"group1": {
+					VersionedResources: map[string][]metav1.APIResource{
+						"v1": {
+							{
+								Name: "resource1",
+							},
+						},
+					},
+				},
+			},
+			expectedAPIGroups:   map[string]*metav1.APIGroup{},
+			expectedKnownGroups: map[string]*restmapper.APIGroupResources{},
+		},
+		{
+			name:      "Not found version for cached groupVersion only in apiGroups",
+			groupName: "group1",
+			versions:  []string{"v1", "v2"},
+			cachedAPIGroups: map[string]*metav1.APIGroup{
+				"group1": {
+					Name: "group1",
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: "v1",
+						},
+					},
+				},
+			},
+			cachedKnownGroups: map[string]*restmapper.APIGroupResources{
+				"group1": {
+					VersionedResources: map[string][]metav1.APIResource{
+						"v3": {
+							{
+								Name: "resource1",
+							},
+						},
+					},
+				},
+			},
+			expectedAPIGroups: map[string]*metav1.APIGroup{},
+			expectedKnownGroups: map[string]*restmapper.APIGroupResources{
+				"group1": {
+					VersionedResources: map[string][]metav1.APIResource{
+						"v3": {
+							{
+								Name: "resource1",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:      "Not found version for cached groupVersion only in knownGroups",
+			groupName: "group1",
+			versions:  []string{"v1", "v2"},
+			cachedAPIGroups: map[string]*metav1.APIGroup{
+				"group1": {
+					Name: "group1",
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: "v3",
+						},
+					},
+				},
+			},
+			cachedKnownGroups: map[string]*restmapper.APIGroupResources{
+				"group1": {
+					VersionedResources: map[string][]metav1.APIResource{
+						"v2": {
+							{
+								Name: "resource1",
+							},
+						},
+					},
+				},
+			},
+			expectedAPIGroups: map[string]*metav1.APIGroup{
+				"group1": {
+					Name: "group1",
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: "v3",
+						},
+					},
+				},
+			},
+			expectedKnownGroups: map[string]*restmapper.APIGroupResources{},
+		},
+		{
+			name:      "Not found version for non cached groupVersion",
+			groupName: "group1",
+			versions:  []string{"v1", "v2"},
+			cachedAPIGroups: map[string]*metav1.APIGroup{
+				"group1": {
+					Name: "group1",
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: "v3",
+						},
+					},
+				},
+			},
+			cachedKnownGroups: map[string]*restmapper.APIGroupResources{
+				"group1": {
+					VersionedResources: map[string][]metav1.APIResource{
+						"v3": {
+							{
+								Name: "resource1",
+							},
+						},
+					},
+				},
+			},
+			expectedAPIGroups: map[string]*metav1.APIGroup{
+				"group1": {
+					Name: "group1",
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: "v3",
+						},
+					},
+				},
+			},
+			expectedKnownGroups: map[string]*restmapper.APIGroupResources{
+				"group1": {
+					VersionedResources: map[string][]metav1.APIResource{
+						"v3": {
+							{
+								Name: "resource1",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := gmg.NewWithT(t)
+			m := &mapper{
+				mapper:      restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}),
+				client:      fake.NewSimpleClientset().Discovery(),
+				apiGroups:   tt.cachedAPIGroups,
+				knownGroups: tt.cachedKnownGroups,
+			}
+			_, err := m.fetchGroupVersionResourcesLocked(tt.groupName, tt.versions...)
+			g.Expect(err).NotTo(gmg.HaveOccurred())
+			g.Expect(m.apiGroups).To(gmg.BeComparableTo(tt.expectedAPIGroups))
+			g.Expect(m.knownGroups).To(gmg.BeComparableTo(tt.expectedKnownGroups))
+		})
+	}
+}