diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
index e7f2945f1d..73ad68fe43 100644
--- a/pkg/cache/cache.go
+++ b/pkg/cache/cache.go
@@ -419,19 +419,6 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
 		}
 	}
 
-	for namespace, cfg := range opts.DefaultNamespaces {
-		cfg = defaultConfig(cfg, optionDefaultsToConfig(&opts))
-		if namespace == metav1.NamespaceAll {
-			cfg.FieldSelector = fields.AndSelectors(
-				appendIfNotNil(
-					namespaceAllSelector(maps.Keys(opts.DefaultNamespaces)),
-					cfg.FieldSelector,
-				)...,
-			)
-		}
-		opts.DefaultNamespaces[namespace] = cfg
-	}
-
 	for obj, byObject := range opts.ByObject {
 		isNamespaced, err := apiutil.IsObjectNamespaced(obj, opts.Scheme, opts.Mapper)
 		if err != nil {
@@ -485,6 +472,22 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
 		opts.ByObject[obj] = byObject
 	}
 
+	// Default namespaces after byObject has been defaulted, otherwise a namespace without selectors
+	// will get the `Default` selectors, then get copied to byObject and then not get defaulted from
+	// byObject, as it already has selectors.
+	for namespace, cfg := range opts.DefaultNamespaces {
+		cfg = defaultConfig(cfg, optionDefaultsToConfig(&opts))
+		if namespace == metav1.NamespaceAll {
+			cfg.FieldSelector = fields.AndSelectors(
+				appendIfNotNil(
+					namespaceAllSelector(maps.Keys(opts.DefaultNamespaces)),
+					cfg.FieldSelector,
+				)...,
+			)
+		}
+		opts.DefaultNamespaces[namespace] = cfg
+	}
+
 	// Default the resync period to 10 hours if unset
 	if opts.SyncPeriod == nil {
 		opts.SyncPeriod = &defaultSyncPeriod
diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go
index cfe0856a1e..c83b2ddcf9 100644
--- a/pkg/cache/cache_test.go
+++ b/pkg/cache/cache_test.go
@@ -1630,6 +1630,26 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca
 					}},
 					expectedPods: []string{"test-pod-4"},
 				}),
+				Entry("namespaces configured, type-level label selector matches everything, overrides global selector", selectorsTestCase{
+					options: cache.Options{
+						DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}},
+						ByObject: map[client.Object]cache.ByObject{
+							&corev1.Pod{}: {Label: labels.Everything()},
+						},
+						DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"does-not": "match-anything"}),
+					},
+					expectedPods: []string{"test-pod-1", "test-pod-5"},
+				}),
+				Entry("namespaces configured, global selector is used", selectorsTestCase{
+					options: cache.Options{
+						DefaultNamespaces: map[string]cache.Config{testNamespaceTwo: {}},
+						ByObject: map[client.Object]cache.ByObject{
+							&corev1.Pod{}: {},
+						},
+						DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"common-label": "common"}),
+					},
+					expectedPods: []string{"test-pod-3"},
+				}),
 				Entry("global label selector matches one pod", selectorsTestCase{
 					options: cache.Options{
 						DefaultLabelSelector: labels.SelectorFromSet(map[string]string{