Add annotation to exclude containers from injection. (#2587)

* Add annotation to exclude containers from injection.

* Update pkg/webhook/mutation/pod_mutator/oneagent_mutation/containers.go

Co-authored-by: Andrii Soldatenko <andrii.soldatenko@dynatrace.com>

* fixup! Add annotation to exclude containers from injection.

---------

Co-authored-by: Andrii Soldatenko <andrii.soldatenko@dynatrace.com>

pkg/webhook/config.go

@@ -43,6 +43,8 @@ const (
 	// "fail", the init container will exit with error code 1. Defaults to "silent".
 	AnnotationFailurePolicy = "oneagent.dynatrace.com/failure-policy"
 
+	AnnotationContainerInjection = "container.inject.dynatrace.com"
+
 	// DefaultInstallPath is the default directory to install the app-only OneAgent package.
 	DefaultInstallPath = "/opt/dynatrace/oneagent-paas"
 

pkg/webhook/mutation/pod_mutator/dataingest_mutation/containers.go

@@ -1,18 +1,31 @@
 package dataingest_mutation
 
-import corev1 "k8s.io/api/core/v1"
+import (
+	dtwebhook "github.com/Dynatrace/dynatrace-operator/pkg/webhook"
+	dtwebhookutil "github.com/Dynatrace/dynatrace-operator/pkg/webhook/util"
+	corev1 "k8s.io/api/core/v1"
+)
 
-func mutateUserContainers(pod *corev1.Pod) {
-	for i := range pod.Spec.Containers {
-		container := &pod.Spec.Containers[i]
+func mutateUserContainers(request *dtwebhook.BaseRequest) {
+	for i := range request.Pod.Spec.Containers {
+		container := &request.Pod.Spec.Containers[i]
+
+		if dtwebhookutil.IsContainerExcludedFromInjection(request, container.Name) {
+			log.Info("Container excluded from data ingest injection", "container", container.Name)
+			continue
+		}
 		setupVolumeMountsForUserContainer(container)
 	}
 }
 
-func reinvokeUserContainers(pod *corev1.Pod) bool {
+func reinvokeUserContainers(request *dtwebhook.BaseRequest) bool {
 	var updated bool
-	for i := range pod.Spec.Containers {
-		container := &pod.Spec.Containers[i]
+	for i := range request.Pod.Spec.Containers {
+		container := &request.Pod.Spec.Containers[i]
+		if dtwebhookutil.IsContainerExcludedFromInjection(request, container.Name) {
+			log.Info("Container excluded from data ingest injection", "container", container.Name)
+			continue
+		}
 		if containerIsInjected(container) {
 			continue
 		}

pkg/webhook/mutation/pod_mutator/dataingest_mutation/containers_test.go

@@ -8,26 +8,74 @@ import (
 )
 
 func TestMutateUserContainers(t *testing.T) {
+	dynakube := getTestDynakube()
+	annotations := map[string]string{"container.inject.dyantrace/container": "false"}
+
 	t.Run("Add volume mounts to containers", func(t *testing.T) {
-		pod := getTestPod(nil)
+		request := createTestMutationRequest(getTestDynakube(), nil)
+		mutateUserContainers(request.BaseRequest)
+
+		for _, container := range request.Pod.Spec.Containers {
+			require.GreaterOrEqual(t, len(container.VolumeMounts), 2)
+		}
+	})
+
+	t.Run("Do not inject container if excluded in dynkube", func(t *testing.T) {
+		dynakube.Annotations = annotations
+
+		request := createTestMutationRequest(dynakube, nil)
+		mutateUserContainers(request.BaseRequest)
+
+		for _, container := range request.Pod.Spec.Containers {
+			require.GreaterOrEqual(t, len(container.VolumeMounts), 2)
+		}
+	})
 
-		mutateUserContainers(pod)
+	t.Run("Do not inject container if excluded in pod", func(t *testing.T) {
+		request := createTestMutationRequest(dynakube, annotations)
+		mutateUserContainers(request.BaseRequest)
 
-		for _, container := range pod.Spec.Containers {
+		for _, container := range request.Pod.Spec.Containers {
 			require.GreaterOrEqual(t, len(container.VolumeMounts), 2)
 		}
 	})
 }
 
 func TestReinvokeUserContainers(t *testing.T) {
+	dynakube := getTestDynakube()
+	annotations := map[string]string{"container.inject.dyantrace/container": "false"}
+
 	t.Run("Add volume mounts to containers", func(t *testing.T) {
-		pod := getTestPod(nil)
+		request := createTestReinvocationRequest(dynakube, nil)
+		reinvokeUserContainers(request.BaseRequest)
+		request.Pod.Spec.Containers = append(request.Pod.Spec.Containers, corev1.Container{})
+		reinvokeUserContainers(request.BaseRequest)
+
+		for _, container := range request.Pod.Spec.Containers {
+			require.GreaterOrEqual(t, len(container.VolumeMounts), 2)
+		}
+	})
+
+	t.Run("Do not inject container if excluded in dynkube", func(t *testing.T) {
+		dynakube.Annotations = annotations
+
+		request := createTestReinvocationRequest(dynakube, nil)
+		reinvokeUserContainers(request.BaseRequest)
+		request.Pod.Spec.Containers = append(request.Pod.Spec.Containers, corev1.Container{})
+		reinvokeUserContainers(request.BaseRequest)
+
+		for _, container := range request.Pod.Spec.Containers {
+			require.GreaterOrEqual(t, len(container.VolumeMounts), 2)
+		}
+	})
 
-		reinvokeUserContainers(pod)
-		pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{})
-		reinvokeUserContainers(pod)
+	t.Run("Do not inject container if excluded in pod", func(t *testing.T) {
+		request := createTestReinvocationRequest(dynakube, annotations)
+		reinvokeUserContainers(request.BaseRequest)
+		request.Pod.Spec.Containers = append(request.Pod.Spec.Containers, corev1.Container{})
+		reinvokeUserContainers(request.BaseRequest)
 
-		for _, container := range pod.Spec.Containers {
+		for _, container := range request.Pod.Spec.Containers {
 			require.GreaterOrEqual(t, len(container.VolumeMounts), 2)
 		}
 	})

pkg/webhook/mutation/pod_mutator/dataingest_mutation/mutator.go

@@ -58,7 +58,7 @@ func (mutator *DataIngestPodMutator) Mutate(ctx context.Context, request *dtwebh
 		return err
 	}
 	setupVolumes(request.Pod)
-	mutateUserContainers(request.Pod)
+	mutateUserContainers(request.BaseRequest)
 	updateInstallContainer(request.InstallContainer, workload)
 	setInjectedAnnotation(request.Pod)
 	return nil
@@ -69,7 +69,7 @@ func (mutator *DataIngestPodMutator) Reinvoke(request *dtwebhook.ReinvocationReq
 		return false
 	}
 	log.Info("reinvoking", "podName", request.PodName())
-	return reinvokeUserContainers(request.Pod)
+	return reinvokeUserContainers(request.BaseRequest)
 }
 
 func (mutator *DataIngestPodMutator) ensureDataIngestSecret(request *dtwebhook.MutationRequest) error {

pkg/webhook/mutation/pod_mutator/oneagent_mutation/containers.go

@@ -7,6 +7,7 @@ import (
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/kubeobjects/env"
 	maputils "github.com/Dynatrace/dynatrace-operator/pkg/util/map"
 	dtwebhook "github.com/Dynatrace/dynatrace-operator/pkg/webhook"
+	dtwebhookutil "github.com/Dynatrace/dynatrace-operator/pkg/webhook/util"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -20,12 +21,22 @@ func (mutator *OneAgentPodMutator) setContainerCount(initContainer *corev1.Conta
 	initContainer.Env = env.AddOrUpdate(initContainer.Env, corev1.EnvVar{Name: consts.AgentContainerCountEnv, Value: desiredContainerCountEnvVarValue})
 }
 
-func (mutator *OneAgentPodMutator) mutateUserContainers(request *dtwebhook.MutationRequest) {
+func (mutator *OneAgentPodMutator) mutateUserContainers(request *dtwebhook.MutationRequest) int {
+	injectedContainers := 0
 	for i := range request.Pod.Spec.Containers {
 		container := &request.Pod.Spec.Containers[i]
+
+		if dtwebhookutil.IsContainerExcludedFromInjection(request.BaseRequest, container.Name) {
+			log.Info("Container excluded from code modules ingest injection", "container", container.Name)
+			continue
+		}
+
 		addContainerInfoInitEnv(request.InstallContainer, i+1, container.Name, container.Image)
 		mutator.addOneAgentToContainer(request.ToReinvocationRequest(), container)
+		injectedContainers++
 	}
+
+	return injectedContainers
 }
 
 // reinvokeUserContainers mutates each user container that hasn't been injected yet.
@@ -36,26 +47,32 @@ func (mutator *OneAgentPodMutator) reinvokeUserContainers(request *dtwebhook.Rei
 	oneAgentInstallContainer := findOneAgentInstallContainer(pod.Spec.InitContainers)
 	newContainers := []*corev1.Container{}
 
+	injectedContainers := 0
+
 	for i := range pod.Spec.Containers {
 		currentContainer := &pod.Spec.Containers[i]
+		if dtwebhookutil.IsContainerExcludedFromInjection(request.BaseRequest, currentContainer.Name) {
+			log.Info("Container excluded from code modules ingest injection", "container", currentContainer.Name)
+			continue
+		}
 		if containerIsInjected(currentContainer) {
+			injectedContainers++
 			continue
 		}
 		newContainers = append(newContainers, currentContainer)
 	}
 
-	oldContainersLen := len(pod.Spec.Containers) - len(newContainers)
+	if len(newContainers) == 0 {
+		return false
+	}
+
 	for i := range newContainers {
 		currentContainer := newContainers[i]
-		addContainerInfoInitEnv(oneAgentInstallContainer, oldContainersLen+i+1, currentContainer.Name, currentContainer.Image)
+		addContainerInfoInitEnv(oneAgentInstallContainer, injectedContainers+i+1, currentContainer.Name, currentContainer.Image)
 		mutator.addOneAgentToContainer(request, currentContainer)
 	}
 
-	if len(newContainers) == 0 {
-		return false
-	}
-
-	mutator.setContainerCount(oneAgentInstallContainer, len(request.Pod.Spec.Containers))
+	mutator.setContainerCount(oneAgentInstallContainer, injectedContainers+len(newContainers))
 	return true
 }
 

pkg/webhook/mutation/pod_mutator/oneagent_mutation/containers_test.go

@@ -1,6 +1,7 @@
 package oneagent_mutation
 
 import (
+	"maps"
 	"testing"
 
 	dynatracev1beta1 "github.com/Dynatrace/dynatrace-operator/pkg/api/v1beta1/dynakube"
@@ -145,6 +146,81 @@ func TestReinvokeUserContainers(t *testing.T) {
 	}
 }
 
+func TestContainerExclusion(t *testing.T) {
+	testCases := []struct {
+		name                               string
+		dynakube                           dynatracev1beta1.DynaKube
+		expectedAdditionalEnvCount         int
+		expectedAdditionalVolumeMountCount int
+		expectedInitContainerEnvCount      int
+		dynakubeAnnotations                map[string]string
+		podAnnotations                     map[string]string
+	}{
+		{
+			name:                               "container exclusion on dynakube level",
+			dynakube:                           *getTestDynakubeWithContainerExclusion(),
+			expectedAdditionalEnvCount:         0,
+			expectedAdditionalVolumeMountCount: 0,
+			expectedInitContainerEnvCount:      3,
+			dynakubeAnnotations: map[string]string{
+				dtwebhook.AnnotationContainerInjection + "/sidecar-container": "false",
+			},
+		},
+		{
+			name:                               "container exclusion on dynakube level, do not exclude",
+			dynakube:                           *getTestDynakubeWithContainerExclusion(),
+			expectedAdditionalEnvCount:         2,
+			expectedAdditionalVolumeMountCount: 3,
+			expectedInitContainerEnvCount:      5,
+			dynakubeAnnotations: map[string]string{
+				dtwebhook.AnnotationContainerInjection + "/sidecar-container": "true",
+			},
+		},
+		{
+			name:                               "container exclusion on pod level",
+			dynakube:                           *getTestDynakube(),
+			expectedAdditionalEnvCount:         0,
+			expectedAdditionalVolumeMountCount: 0,
+			expectedInitContainerEnvCount:      3,
+			podAnnotations: map[string]string{
+				dtwebhook.AnnotationContainerInjection + "/sidecar-container": "false",
+			},
+		},
+		{
+			name:                               "container exclusion on pod level, do not exclude",
+			dynakube:                           *getTestDynakube(),
+			expectedAdditionalEnvCount:         2,
+			expectedAdditionalVolumeMountCount: 3,
+			expectedInitContainerEnvCount:      5,
+			podAnnotations: map[string]string{
+				dtwebhook.AnnotationContainerInjection + "/sidecar-container": "true",
+			},
+		},
+	}
+
+	for index, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			mutator := createTestPodMutator([]client.Object{getTestInitSecret()})
+			request := createTestMutationRequest(&testCases[index].dynakube, testCase.podAnnotations, getTestNamespace(nil)).ToReinvocationRequest()
+
+			maps.Copy(request.DynaKube.Annotations, testCase.dynakubeAnnotations)
+
+			initialNumberOfContainerEnvsLen := len(request.Pod.Spec.Containers[0].Env)
+			initialContainerVolumeMountsLen := len(request.Pod.Spec.Containers[0].VolumeMounts)
+			request.Pod.Spec.InitContainers = append(request.Pod.Spec.InitContainers, corev1.Container{
+				Name: dtwebhook.InstallContainerName,
+			})
+			installContainer := &request.Pod.Spec.InitContainers[1]
+
+			mutator.reinvokeUserContainers(request)
+
+			require.Len(t, installContainer.Env, testCase.expectedInitContainerEnvCount) // CONTAINERS_COUNT + N*(CONTAINER_x_IMAGE, CONTAINER_x_NAME)
+			assert.Len(t, request.Pod.Spec.Containers[1].VolumeMounts, initialContainerVolumeMountsLen+testCase.expectedAdditionalVolumeMountCount)
+			assert.Len(t, request.Pod.Spec.Containers[1].Env, initialNumberOfContainerEnvsLen+testCase.expectedAdditionalEnvCount)
+		})
+	}
+}
+
 func assertContainersNamesAndImages(t *testing.T, request *dtwebhook.ReinvocationRequest, installContainer *corev1.Container, containersNumber int) {
 	for containerIdx := 0; containerIdx < containersNumber; containerIdx++ {
 		internalContainerIndex := 1 + containerIdx // starting from 1
@@ -173,13 +249,13 @@ func TestVersionDetectionMappingDrivenByNamespaceAnnotations(t *testing.T) {
 		customProductAnnotationName      = "custom-product"
 		customStageAnnotationName        = "custom-stage"
 		customBuildVersionAnnotationName = "custom-build-version"
-		customVersionFieldPath           = "metadata.annotations['" + customVersionAnnotationName + "']"
-		customProductFieldPath           = "metadata.annotations['" + customProductAnnotationName + "']"
-		customStageFieldPath             = "metadata.annotations['" + customStageAnnotationName + "']"
-		customBuildVersionFieldPath      = "metadata.annotations['" + customBuildVersionAnnotationName + "']"
+		customVersionFieldPath           = "metadata.podAnnotations['" + customVersionAnnotationName + "']"
+		customProductFieldPath           = "metadata.podAnnotations['" + customProductAnnotationName + "']"
+		customStageFieldPath             = "metadata.podAnnotations['" + customStageAnnotationName + "']"
+		customBuildVersionFieldPath      = "metadata.podAnnotations['" + customBuildVersionAnnotationName + "']"
 	)
 
-	t.Run("version and product env vars are set using values referenced in namespace annotations", func(t *testing.T) {
+	t.Run("version and product env vars are set using values referenced in namespace podAnnotations", func(t *testing.T) {
 		podAnnotations := map[string]string{
 			customVersionAnnotationName: customVersionValue,
 			customProductAnnotationName: customProductValue,
@@ -196,7 +272,7 @@ func TestVersionDetectionMappingDrivenByNamespaceAnnotations(t *testing.T) {
 
 		doTestMappings(t, podAnnotations, namespaceAnnotations, expectedMappings, unexpectedMappingsKeys)
 	})
-	t.Run("only version env vars is set using value referenced in namespace annotations, product is default", func(t *testing.T) {
+	t.Run("only version env vars is set using value referenced in namespace podAnnotations, product is default", func(t *testing.T) {
 		podAnnotations := map[string]string{
 			customVersionAnnotationName: customVersionValue,
 		}
@@ -211,7 +287,7 @@ func TestVersionDetectionMappingDrivenByNamespaceAnnotations(t *testing.T) {
 
 		doTestMappings(t, podAnnotations, namespaceAnnotations, expectedMappings, unexpectedMappingsKeys)
 	})
-	t.Run("optional env vars (stage, build-version) are set using values referenced in namespace annotations, default ones remain default", func(t *testing.T) {
+	t.Run("optional env vars (stage, build-version) are set using values referenced in namespace podAnnotations, default ones remain default", func(t *testing.T) {
 		podAnnotations := map[string]string{
 			customStageAnnotationName:        customReleaseStageValue,
 			customBuildVersionAnnotationName: customBuildVersionValue,
@@ -229,7 +305,7 @@ func TestVersionDetectionMappingDrivenByNamespaceAnnotations(t *testing.T) {
 
 		doTestMappings(t, podAnnotations, namespaceAnnotations, expectedMappings, nil)
 	})
-	t.Run("all env vars are namespace-annotations driven", func(t *testing.T) {
+	t.Run("all env vars are namespace-podAnnotations driven", func(t *testing.T) {
 		podAnnotations := map[string]string{
 			customVersionAnnotationName:      customVersionValue,
 			customProductAnnotationName:      customProductValue,

pkg/webhook/mutation/pod_mutator/oneagent_mutation/env.go

@@ -15,12 +15,12 @@ import (
 func addPreloadEnv(container *corev1.Container, installPath string) {
 	preloadPath := filepath.Join(installPath, consts.LibAgentProcPath)
 
-	env := env.FindEnvVar(container.Env, preloadEnv)
-	if env != nil {
-		if strings.Contains(env.Value, installPath) {
+	ldPreloadEnv := env.FindEnvVar(container.Env, preloadEnv)
+	if ldPreloadEnv != nil {
+		if strings.Contains(ldPreloadEnv.Value, installPath) {
 			return
 		}
-		env.Value = concatPreloadPaths(env.Value, preloadPath)
+		ldPreloadEnv.Value = concatPreloadPaths(ldPreloadEnv.Value, preloadPath)
 	} else {
 		container.Env = append(container.Env,
 			corev1.EnvVar{

pkg/webhook/mutation/pod_mutator/oneagent_mutation/mutator.go

@@ -62,8 +62,8 @@ func (mutator *OneAgentPodMutator) Mutate(ctx context.Context, request *dtwebhoo
 	installerInfo := getInstallerInfo(request.Pod, request.DynaKube)
 	mutator.addVolumes(request.Pod, request.DynaKube)
 	mutator.configureInitContainer(request, installerInfo)
-	mutator.setContainerCount(request.InstallContainer, len(request.Pod.Spec.Containers))
-	mutator.mutateUserContainers(request)
+	injecteContainers := mutator.mutateUserContainers(request)
+	mutator.setContainerCount(request.InstallContainer, injecteContainers)
 	addInjectionConfigVolumeMount(request.InstallContainer)
 	setInjectedAnnotation(request.Pod)
 	return nil