Add an option which enables the memory optimization inside of spod daemon

api/spod/v1alpha1/spod_types.go

@@ -258,6 +258,11 @@ type SPODSpec struct {
 	// EnableProfiling tells the operator whether or not to enable profiling
 	// support for this SPOD instance.
 	EnableProfiling bool `json:"enableProfiling,omitempty"`
+	// EnableMemoryOptimization enables memory optimization in the controller
+	// running inside of SPOD instance and watching for pods in the cluster.
+	// This will make the controller loading in the cache memory only the pods
+	// labelled explicitly for profile recording with 'spo.x-k8s.io/enable-recording=true'.
+	EnableMemoryOptimization bool `json:"enableMemoryOptimization,omitempty"`
 	// tells the operator whether or not to enable SELinux support for this
 	// SPOD instance.
 	EnableSelinux *bool `json:"enableSelinux,omitempty"`

bundle/manifests/security-profiles-operator.x-k8s.io_securityprofilesoperatordaemons.yaml

@@ -923,6 +923,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

cmd/security-profiles-operator/main.go

@@ -32,6 +32,8 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/urfave/cli/v2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -56,6 +58,7 @@ import (
 	nodestatus "sigs.k8s.io/security-profiles-operator/internal/pkg/manager/nodestatus"
 	"sigs.k8s.io/security-profiles-operator/internal/pkg/manager/recordingmerger"
 	"sigs.k8s.io/security-profiles-operator/internal/pkg/manager/spod"
+	"sigs.k8s.io/security-profiles-operator/internal/pkg/manager/spod/bindata"
 	"sigs.k8s.io/security-profiles-operator/internal/pkg/manager/workloadannotator"
 	"sigs.k8s.io/security-profiles-operator/internal/pkg/nonrootenabler"
 	"sigs.k8s.io/security-profiles-operator/internal/pkg/util"
@@ -70,6 +73,7 @@ const (
 	selinuxFlag        string = "with-selinux"
 	apparmorFlag       string = "with-apparmor"
 	webhookFlag        string = "webhook"
+	memOptimFlag       string = "with-mem-optim"
 	defaultWebhookPort int    = 9443
 )
 
@@ -158,6 +162,11 @@ func main() {
 					Value:   false,
 					EnvVars: []string{config.EnableRecordingEnvKey},
 				},
+				&cli.BoolFlag{
+					Name:  memOptimFlag,
+					Usage: "Enable memory optimization by watching only labeled pods",
+					Value: false,
+				},
 			},
 		},
 		&cli.Command{
@@ -419,6 +428,24 @@ func getEnabledControllers(ctx *cli.Context) []controller.Controller {
 	return controllers
 }
 
+// newMemoryOptimizedCache creates a memory optimized cache for daemon controller.
+// This will load into cache memory only the pods objects which are labeled for recording.
+func newMemoryOptimizedCache(ctx *cli.Context) cache.NewCacheFunc {
+	if ctx.Bool(memOptimFlag) {
+		return cache.BuilderWithOptions(cache.Options{
+			Resync: &sync,
+			SelectorsByObject: cache.SelectorsByObject{
+				&corev1.Pod{}: {
+					Label: labels.SelectorFromSet(labels.Set{
+						bindata.EnableRecordingLabel: "true",
+					}),
+				},
+			},
+		})
+	}
+	return nil
+}
+
 func runDaemon(ctx *cli.Context, info *version.Info) error {
 	// security-profiles-operator-daemon
 	printInfo("spod", info)
@@ -438,6 +465,7 @@ func runDaemon(ctx *cli.Context, info *version.Info) error {
 	ctrlOpts := ctrl.Options{
 		SyncPeriod:             &sync,
 		HealthProbeBindAddress: fmt.Sprintf(":%d", config.HealthProbePort),
+		NewCache:               newMemoryOptimizedCache(ctx),
 	}
 
 	setControllerOptionsForNamespaces(&ctrlOpts)

deploy/base-crds/crds/securityprofilesoperatordaemon.yaml

@@ -921,6 +921,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

deploy/helm/crds/crds.yaml

@@ -1448,6 +1448,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

deploy/namespace-operator.yaml

@@ -1448,6 +1448,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

deploy/openshift-dev.yaml

@@ -1448,6 +1448,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

deploy/openshift-downstream.yaml

@@ -1448,6 +1448,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

deploy/operator.yaml

@@ -1448,6 +1448,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

deploy/webhook-operator.yaml

@@ -1624,6 +1624,13 @@ spec:
                 description: tells the operator whether or not to enable log enrichment
                   support for this SPOD instance.
                 type: boolean
+              enableMemoryOptimization:
+                description: EnableMemoryOptimization enables memory optimization
+                  in the controller running inside of SPOD instance and watching for
+                  pods in the cluster. This will make the controller loading in the
+                  cache memory only the pods labelled explicitly for profile recording
+                  with 'spo.x-k8s.io/enable-recording=true'.
+                type: boolean
               enableProfiling:
                 description: EnableProfiling tells the operator whether or not to
                   enable profiling support for this SPOD instance.

installation-usage.md

@@ -16,6 +16,7 @@
 - [Customise the daemon resource requirements](#customise-the-daemon-resource-requirements)
 - [Restrict the allowed syscalls in seccomp profiles](#restrict-the-allowed-syscalls-in-seccomp-profiles)
 - [Constrain spod scheduling](#constrain-spod-scheduling)
+- [Enable memory optimization in spod](#enable-memory-optimization-in-spod)
 - [Create a seccomp profile](#create-a-seccomp-profile)
   - [Apply a seccomp profile to a pod](#apply-a-seccomp-profile-to-a-pod)
   - [Base syscalls for a container runtime](#base-syscalls-for-a-container-runtime)
@@ -278,6 +279,34 @@ kubectl -n security-profiles-operator patch spod spod --type merge -p
 kubectl -n security-profiles-operator patch spod spod --type merge -p
 '{"spec":{"affinity": {...}}}'
 ```
+
+## Enable memory optimization in spod
+
+The controller running inside of spod daemon process is watching all pods available in the cluster when profile recording
+is enabled. It will perform some pre-filtering before the reconciliation to select only the pods running on local
+node as well as pods annotated for recording, but this operation takes place after all pods objects are loaded
+into the cache memory of the informer. This can lead to very high memory usage in large clusters with 1000s of pods, resulting
+in spod daemon running out of memory or crashing.
+
+In order to prevent this situation, the spod daemon can be configured to only load into the cache memory the pods explicitly
+labeled for profile recording. This can be achieved by enabling memory optimization as follows:
+
+```
+kubectl -n security-profiles-operator patch spod spod --type=merge -p '{"spec":{"enableMemoryOptimization":true}}'
+```
+
+If you want now to record a security profile for a pod, this pod needs to be explicitly labeled with `spo.x-k8s.io/enable-recording`,
+as follows:
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  name: my-recording-pod
+  labels:
+    spo.x-k8s.io/enable-recording: "true"
+```
+
 ## Create a seccomp profile
 
 Use the `SeccompProfile` kind to create profiles. Example:

internal/pkg/manager/spod/bindata/webhook.go

@@ -69,15 +69,22 @@ var (
 )
 
 const (
-	webhookName                 = config.OperatorName + "-webhook"
-	webhookConfigName           = "spo-mutating-webhook-configuration"
-	serviceAccountName          = "spo-webhook"
-	certsMountPath              = "/tmp/k8s-webhook-server/serving-certs"
-	containerPort               = 9443
-	serviceName                 = "webhook-service"
-	webhookServerCert           = "webhook-server-cert"
-	webhookEnableBindingLabel   = "spo.x-k8s.io/enable-binding"
-	webhookEnableRecordingLabel = "spo.x-k8s.io/enable-recording"
+	// EnableRecordingLabel this label can be applied to a namespace or a pod
+	// in order to enable profile recording.
+	EnableRecordingLabel = "spo.x-k8s.io/enable-recording"
+	// EnableBindingLabel this label can be applied to a namespace in order to
+	// enable profile binding.
+	EnableBindingLabel = "spo.x-k8s.io/enable-binding"
+)
+
+const (
+	webhookName        = config.OperatorName + "-webhook"
+	webhookConfigName  = "spo-mutating-webhook-configuration"
+	serviceAccountName = "spo-webhook"
+	certsMountPath     = "/tmp/k8s-webhook-server/serving-certs"
+	containerPort      = 9443
+	serviceName        = "webhook-service"
+	webhookServerCert  = "webhook-server-cert"
 )
 
 type Webhook struct {
@@ -397,7 +404,7 @@ var webhookConfig = &admissionregv1.MutatingWebhookConfiguration{
 			NamespaceSelector: &metav1.LabelSelector{
 				MatchExpressions: []metav1.LabelSelectorRequirement{
 					{
-						Key:      webhookEnableBindingLabel,
+						Key:      EnableBindingLabel,
 						Operator: metav1.LabelSelectorOpExists,
 					},
 				},
@@ -420,7 +427,7 @@ var webhookConfig = &admissionregv1.MutatingWebhookConfiguration{
 			NamespaceSelector: &metav1.LabelSelector{
 				MatchExpressions: []metav1.LabelSelectorRequirement{
 					{
-						Key:      webhookEnableRecordingLabel,
+						Key:      EnableRecordingLabel,
 						Operator: metav1.LabelSelectorOpExists,
 					},
 				},

internal/pkg/manager/spod/spod_controller.go

@@ -592,6 +592,13 @@ func (r *ReconcileSPOd) getConfiguredSPOd(
 		templateSpec.HostPID = true
 	}
 
+	// Enable memory optimization for spod controller
+	if cfg.Spec.EnableMemoryOptimization {
+		templateSpec.Containers[bindata.ContainerIDDaemon].Args = append(
+			templateSpec.Containers[bindata.ContainerIDDaemon].Args,
+			"--with-mem-optim=true")
+	}
+
 	// Metrics parameters
 	templateSpec.Containers = append(
 		templateSpec.Containers,

test/e2e_flaky_test.go

@@ -68,6 +68,7 @@ func (e *e2e) TestSecurityProfilesOperator_Flaky() {
 		e.testCaseProfileRecordingDeploymentLogs()
 		e.testCaseRecordingFinalizers()
 		e.testCaseProfileRecordingDeploymentScaleUpDownLogs()
+		e.testCaseProfileRecordingWithMemoryOptimization()
 	})
 
 	e.Run("cluster-wide: Seccomp: Verify profile recording bpf", func() {
@@ -77,6 +78,7 @@ func (e *e2e) TestSecurityProfilesOperator_Flaky() {
 		e.testCaseBpfRecorderDeployment()
 		e.testCaseBpfRecorderParallel()
 		e.testCaseBpfRecorderSelectContainer()
+		e.testCaseBpfRecorderWithMemoryOptimization()
 	})
 
 	// Clean up cluster-wide deployment to prepare for namespace deployment

test/e2e_test.go

@@ -112,6 +112,10 @@ func (e *e2e) TestSecurityProfilesOperator() {
 			"SPOD: Change profiling",
 			e.testCaseProfilingChange,
 		},
+		{
+			"SPOD: Enable memory optimiztaion",
+			e.testCaseMemOptmEnable,
+		},
 		{
 			"SPOD: Change resource requirements",
 			e.testCaseResourceRequirementsChange,

test/suite_test.go

@@ -721,6 +721,24 @@ func (e *e2e) enableBpfRecorderInSpod() {
 	e.kubectlOperatorNS("rollout", "status", "ds", "spod", "--timeout", defaultBpfRecorderOpTimeout)
 }
 
+func (e *e2e) enableMemoryOptimization() {
+	e.logf("Enable memory optimization in SPOD")
+	e.kubectlOperatorNS("patch", "spod", "spod", "-p", `{"spec":{"enableMemoryOptimization": true}}`, "--type=merge")
+	time.Sleep(defaultWaitTime)
+
+	e.waitInOperatorNSFor("condition=ready", "spod", "spod")
+	e.kubectlOperatorNS("rollout", "status", "ds", "spod", "--timeout", defaultBpfRecorderOpTimeout)
+}
+
+func (e *e2e) disableMemoryOptimization() {
+	e.logf("Enable memory optimization in SPOD")
+	e.kubectlOperatorNS("patch", "spod", "spod", "-p", `{"spec":{"enableMemoryOptimization": false}}`, "--type=merge")
+	time.Sleep(defaultWaitTime)
+
+	e.waitInOperatorNSFor("condition=ready", "spod", "spod")
+	e.kubectlOperatorNS("rollout", "status", "ds", "spod", "--timeout", defaultBpfRecorderOpTimeout)
+}
+
 func (e *e2e) deployRecordingSa(namespace string) {
 	saTemplate := `
     apiVersion: v1

test/tc_bpf_recorder_test.go

@@ -297,3 +297,40 @@ func (e *e2e) testCaseBpfRecorderSelectContainer() {
 	e.kubectl("delete", "-f", exampleRecordingBpfSpecificContainerPath)
 	e.kubectl("delete", "sp", profileNameNginx)
 }
+
+func (e *e2e) testCaseBpfRecorderWithMemoryOptimization() {
+	e.bpfRecorderOnlyTestCase()
+
+	e.enableMemoryOptimization()
+	defer e.disableMemoryOptimization()
+
+	restoreNs := e.switchToRecordingNs(nsRecordingEnabled)
+	defer restoreNs()
+
+	e.logf("Creating bpf recording for static pod test")
+	e.kubectl("create", "-f", exampleRecordingBpfPath)
+
+	since, podName := e.createRecordingTestPod()
+
+	resourceName := recordingName + "-nginx"
+	e.waitForBpfRecorderLogs(since, resourceName)
+
+	e.kubectl("delete", "pod", podName)
+
+	profile := e.retryGetSeccompProfile(resourceName)
+	e.Contains(profile, "listen")
+
+	e.kubectl("delete", "-f", exampleRecordingBpfPath)
+	e.kubectl("delete", "sp", resourceName)
+
+	metrics := e.getSpodMetrics()
+	// we don't use resource name here, because the metrics are tracked by the annotation name which contains
+	// underscores instead of dashes
+	metricName := recordingName + "_nginx"
+	e.Regexp(fmt.Sprintf(`(?m)security_profiles_operator_seccomp_profile_bpf_total{`+
+		`mount_namespace=".*",`+
+		`node=".*",`+
+		`profile="%s_.*"} \d+`,
+		metricName,
+	), metrics)
+}