Add multi-namespace cache support

Makefile

@@ -82,8 +82,8 @@ SKIP_DOCKER_COMMAND ?= false
 GLOBAL_OPERATOR_NAMESPACE ?= elastic-system
 # namespace in which the namespace operator is deployed (see config/namespace-operator)
 NAMESPACE_OPERATOR_NAMESPACE ?= elastic-namespace-operators
-# namespace in which the namespace operator should watch resources
-MANAGED_NAMESPACE ?= default
+# comma separated list of namespaces in which the namespace operator should watch resources
+MANAGED_NAMESPACES ?=
 
 ## -- Security
 
@@ -170,7 +170,8 @@ go-run:
 				--development --operator-roles=global,namespace \
 				--log-verbosity=$(LOG_VERBOSITY) \
 				--ca-cert-validity=10h --ca-cert-rotate-before=1h \
-				--operator-namespace=default --namespace= \
+				--operator-namespace=default \
+				--namespace-list=$(MANAGED_NAMESPACES) \
 				--auto-install-webhooks=false
 
 go-debug:
@@ -184,7 +185,7 @@ go-debug:
 		--ca-cert-validity=10h \
 		--ca-cert-rotate-before=1h \
 		--operator-namespace=default \
-		--namespace= \
+		--namespace-list=$(MANAGED_NAMESPACES) \
 		--auto-install-webhooks=false)
 
 build-operator-image:

cmd/manager/main.go

@@ -39,6 +39,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
@@ -49,6 +50,7 @@ const (
 
 	AutoPortForwardFlagName = "auto-port-forward"
 	NamespaceFlagName       = "namespace"
+	NamespaceListFlagName   = "namespace-list"
 
 	CACertValidityFlag     = "ca-cert-validity"
 	CACertRotateBeforeFlag = "ca-cert-rotate-before"
@@ -85,6 +87,11 @@ func init() {
 		"",
 		"namespace in which this operator should manage resources (defaults to all namespaces)",
 	)
+	Cmd.Flags().StringSlice(
+		NamespaceListFlagName,
+		nil,
+		"List of namespaces to be managed (overrides the namespace flag)",
+	)
 	Cmd.Flags().Bool(
 		AutoPortForwardFlagName,
 		false,
@@ -213,8 +220,23 @@ func execute() {
 	log.Info("Setting up manager")
 	opts := ctrl.Options{
 		Scheme: clientgoscheme.Scheme,
-		// restrict the operator to watch resources within a single namespace, unless empty
-		Namespace: viper.GetString(NamespaceFlagName),
+	}
+
+	// figure out the managed namespaces (namespace-list flag overrides the namespace flag)
+	managedNamespace := viper.GetString(NamespaceFlagName)
+	managedNamespaceList := viper.GetStringSlice(NamespaceListFlagName)
+	switch {
+	case len(managedNamespaceList) == 1:
+		log.Info("Operator configured to manage a single namespace", "namespace", managedNamespaceList[0])
+		opts.Namespace = managedNamespaceList[0]
+	case len(managedNamespaceList) > 1:
+		log.Info("Operator configured to manage multiple namespaces", "namespaces", managedNamespaceList)
+		opts.NewCache = cache.MultiNamespacedCacheBuilder(managedNamespaceList)
+	case managedNamespace == "":
+		log.Info("Operator configured to manage all namespaces")
+	default:
+		log.Info("Operator configured to manage a single namespace", "namespace", managedNamespace)
+		opts.Namespace = managedNamespace
 	}
 
 	// only expose prometheus metrics if provided a non-zero port

config/e2e/batch_job.yaml

@@ -33,7 +33,7 @@ spec:
       containers:
         - name: e2e
           image: {{ .E2EImage }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           args: [{{- if .TestRegex -}}"-run", "{{ .TestRegex }}",{{- end -}}"-args", "-testContextPath","/etc/e2e/testcontext.json"]
           volumeMounts:
             - name: test-config

config/e2e/managed_namespaces.yaml

@@ -1,10 +1,10 @@
 {{- $testRun := .TestRun -}}
-{{- range .NamespaceOperators }}
+{{- range .NamespaceOperator.ManagedNamespaces }}
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: {{ .ManagedNamespace }}
+  name: {{ . }}
   labels:
     test-run: {{ $testRun }}
 {{- end }}

config/e2e/namespace_operator.yaml

@@ -142,23 +142,21 @@ rules:
   - get
   - update
   - patch
-
-{{- range .NamespaceOperators }}
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ .Name }}
-  namespace: {{ .Namespace }}
+  name: {{ .NamespaceOperator.Name }}
+  namespace: {{ .NamespaceOperator.Namespace }}
   labels:
     test-run: {{ $testRun }}
 ---
 # allow operator to manage resources in its namespace
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ .Name }}
-  namespace: {{ .Namespace }}
+  name: {{ .NamespaceOperator.Name }}
+  namespace: {{ .NamespaceOperator.Namespace }}
   labels:
     test-run: {{ $testRun }}
 roleRef:
@@ -167,15 +165,18 @@ roleRef:
   name: elastic-operator
 subjects:
 - kind: ServiceAccount
-  name: {{ .Name }}
-  namespace: {{ .Namespace }}
+  name: {{ .NamespaceOperator.Name }}
+  namespace: {{ .NamespaceOperator.Namespace }}
+
+{{- $nsOperator := .NamespaceOperator -}}
+{{ range $nsOperator.ManagedNamespaces }}
 ---
 # allow operator to manage resources in the managed namespace
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ .Name }}
-  namespace: {{ .ManagedNamespace }}
+  name: {{ $nsOperator.Name }}
+  namespace: {{ . }}
   labels:
     test-run: {{ $testRun }}
 roleRef:
@@ -184,34 +185,36 @@ roleRef:
   name: elastic-namespace-operator
 subjects:
 - kind: ServiceAccount
-  name: {{ .Name }}
-  namespace: {{ .Namespace }}
+  name: {{ $nsOperator.Name }}
+  namespace: {{ $nsOperator.Namespace }}
+{{- end }}
+
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: {{ .Name }}
-  namespace: {{ .Namespace }}
+  name: {{ .NamespaceOperator.Name }}
+  namespace: {{ .NamespaceOperator.Namespace }}
   labels:
-    control-plane: {{ .Name }}
+    control-plane: {{ .NamespaceOperator.Name }}
     test-run: {{ $testRun }}
 spec:
   selector:
     matchLabels:
-      control-plane: {{ .Name }}
-  serviceName: {{ .Name }}
+      control-plane: {{ .NamespaceOperator.Name }}
+  serviceName: {{ .NamespaceOperator.Name }}
   template:
     metadata:
       labels:
-        control-plane: {{ .Name }}
+        control-plane: {{ .NamespaceOperator.Name }}
         test-run: {{ $testRun }}
     spec:
-      serviceAccountName: {{ .Name }}
+      serviceAccountName: {{ .NamespaceOperator.Name }}
       containers:
       - image: {{ $operatorImage }}
         imagePullPolicy: IfNotPresent
         name: manager
-        args: ["manager", "--namespace", "{{ .ManagedNamespace }}", "--operator-roles", "namespace"]
+        args: ["manager", "--namespace-list", "{{ .NamespaceOperator.ManagedNamespaces | join "," }}", "--operator-roles", "namespace"]
         env:
           - name: OPERATOR_NAMESPACE
             valueFrom:
@@ -227,4 +230,3 @@ spec:
             cpu: 100m
             memory: 20Mi
       terminationGracePeriodSeconds: 10
-{{- end }}

config/e2e/operator_namespaces.yaml

@@ -6,13 +6,10 @@ metadata:
   name: {{ .GlobalOperator.Namespace }}
   labels:
     test-run: {{ $testRun }}
-
-{{- range .OperatorNamespaces }}
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: {{ . }}
+  name: {{ .NamespaceOperator.Namespace }}
   labels:
     test-run: {{ $testRun }}
-{{- end }}

config/e2e/rbac.yaml

@@ -59,11 +59,9 @@ subjects:
   - kind: ServiceAccount
     name: {{ .GlobalOperator.Name }}
     namespace: {{ .GlobalOperator.Namespace }}
-{{- range .NamespaceOperators }}
   - kind: ServiceAccount
-    name: {{ .Name }}
-    namespace: {{ .Namespace }}
-{{- end }}
+    name: {{ .NamespaceOperator.Name }}
+    namespace: {{ .NamespaceOperator.Namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

config/operator/Makefile

@@ -34,14 +34,23 @@ generate-global:
 		-e "s|<NAMESPACE>|$$NAMESPACE|g"
 
 generate-namespace:
-ifndef MANAGED_NAMESPACE
-	$(error MANAGED_NAMESPACE not set)
+ifndef MANAGED_NAMESPACES
+	$(error MANAGED_NAMESPACES not set)
 endif
-	@ for yaml in $$(LC_COLLATE=C ls namespace/*yaml); do \
+	@ for yaml in $(shell find namespace -type f -name '*.yaml' ! -name 'managed_ns_role_binding.template.yaml'); do \
 		printf "\n---\n" && cat $$yaml ; \
 	done | \
 	sed \
 		-e "/# /d" \
 		-e "s|<OPERATOR_IMAGE>|$$OPERATOR_IMAGE|g" \
 		-e "s|<NAMESPACE>|$$NAMESPACE|g" \
-		-e "s|<MANAGED_NAMESPACE>|$$MANAGED_NAMESPACE|g"
+		-e "s|<MANAGED_NAMESPACE_LIST>|$$MANAGED_NAMESPACES|g"
+
+	@ (MANAGED_NS_LIST=$${MANAGED_NAMESPACES//,/ }; \
+	  for managed_ns in $$MANAGED_NS_LIST; do \
+		printf "\n---\n" && sed \
+		-e "/# /d" \
+		-e "s|<NAMESPACE>|$$NAMESPACE|g" \
+		-e "s|<MANAGED_NAMESPACE>|$$managed_ns|g" \
+		namespace/managed_ns_role_binding.template.yaml ;\
+	  done)

config/operator/namespace/role_bindings.template.yaml → config/operator/namespace/managed_ns_role_binding.template.yaml

@@ -1,18 +1,3 @@
-# allow operator to manage resources in its namespace
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: elastic-operator
-  namespace: <NAMESPACE>
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: elastic-operator
-subjects:
-- kind: ServiceAccount
-  name: elastic-namespace-operator
-  namespace: <NAMESPACE>
----
 # allow operator to manage resources in the managed namespace
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

config/operator/namespace/operator.template.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - image: <OPERATOR_IMAGE>
         name: manager
-        args: ["manager", "--namespace", "<MANAGED_NAMESPACE>", "--operator-roles", "namespace"]
+        args: ["manager", "--namespace-list", "<MANAGED_NAMESPACE_LIST>", "--operator-roles", "namespace"]
         env:
           - name: OPERATOR_NAMESPACE
             valueFrom:

config/operator/namespace/operator_ns_role_binding.template.yaml

@@ -0,0 +1,14 @@
+# allow operator to manage resources in its namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: elastic-operator
+  namespace: <NAMESPACE>
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: elastic-operator
+subjects:
+- kind: ServiceAccount
+  name: elastic-namespace-operator
+  namespace: <NAMESPACE>

test/e2e/apm/association_test.go

@@ -22,12 +22,6 @@ import (
 
 // TestCrossNSAssociation tests associating Elasticsearch and an APM Server running in different namespaces.
 func TestCrossNSAssociation(t *testing.T) {
-	// This test currently does not work in the E2E environment because each namespace has a dedicated
-	// controller (see https://github.com/elastic/cloud-on-k8s/issues/1438)
-	if !(test.Ctx().Local) {
-		t.SkipNow()
-	}
-
 	esNamespace := test.Ctx().ManagedNamespace(0)
 	apmNamespace := test.Ctx().ManagedNamespace(1)
 	name := "test-cross-ns-assoc"
@@ -46,8 +40,7 @@ func TestCrossNSAssociation(t *testing.T) {
 			"setup.template.settings.index.number_of_replicas": 0, // avoid ES yellow state on a 1 node ES cluster
 		})
 
-	test.Sequence(nil, test.EmptySteps, esBuilder, apmBuilder).
-		RunSequential(t)
+	test.Sequence(nil, test.EmptySteps, esBuilder, apmBuilder).RunSequential(t)
 }
 
 func TestAPMAssociationWithNonExistentES(t *testing.T) {

test/e2e/cmd/run/eventlog.go

@@ -67,9 +67,9 @@ func newEventLogger(client *kubernetes.Clientset, testCtx test.Context, logFileP
 	s := struct{}{}
 	el.interestingNamespaces[testCtx.E2ENamespace] = s
 	el.interestingNamespaces[testCtx.GlobalOperator.Namespace] = s
-	for _, ns := range testCtx.NamespaceOperators {
-		el.interestingNamespaces[ns.Namespace] = s
-		el.interestingNamespaces[ns.ManagedNamespace] = s
+	el.interestingNamespaces[testCtx.NamespaceOperator.Namespace] = s
+	for _, ns := range testCtx.NamespaceOperator.ManagedNamespaces {
+		el.interestingNamespaces[ns] = s
 	}
 
 	return el