Add e2e tests for the remote config updater (#1243)

* Add basic E2E test of kind on EC2 * add minimal dda tests * update gitlab config * update gitlab config * fix gitlab and update docs * build e2e image * use public-images trigger * add image pull secret * fix imgpullsecret syntax * configure ecr creds * use kube provider * fix imagePullSecret transformation * add imagePullSecret to manager deployment.yaml * fix gitlab syntax * disable webhook * use terratest * use unique stack names and refactor * refactor * remove transformation func * fix make command * increase retry * fix img env var, increase retry * fix image edit * increase retry, get pod logs * agent pod logs * debug * remove waituntilpodavailable assertion * add go.work * update image env var * fix go.work version * remove go.work * fix capitlization * Add rc-updater e2e tests Signed-off-by: Momar TOURÉ <momar.toure@datadoghq.com> * changes after review * use patches in kustomize * Update test * add sleep time to allow for the agent to have the time to be deployed * fix bug * fix bug * debug * fix bugs * debug * update kustomization * update after rebase --------- Signed-off-by: Momar TOURÉ <momar.toure@datadoghq.com> Co-authored-by: Fanny Jiang <fanny.jiang@datadoghq.com>
DataDog · Oct 3, 2024 · 0d97b8a · 0d97b8a
1 parent 7984f26
commit 0d97b8a
Show file tree

Hide file tree

Showing 12 changed files with 789 additions and 17 deletions.
diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml
@@ -1,11 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
 images:
 - name: controller
   newName: gcr.io/datadoghq/operator
-  newTag: 1.7.0
-apiVersion: kustomize.config.k8s.io/v1beta1
+  newTag: latest
 kind: Kustomization
-namespace: system
 namePrefix: datadog-operator-e2e-
+namespace: system
+patches:
+- path: rc-e2e-manager.yaml
+  target:
+    kind: Deployment
+    name: manager
 resources:
 - ../crd
 - ../rbac

diff --git a/config/e2e/rc-e2e-manager.yaml b/config/e2e/rc-e2e-manager.yaml
@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: manager
+  namespace: system
+  labels:
+    app.kubernetes.io/name: datadog-operator
+    control-plane: controller-manager
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: datadog-operator
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: datadog-operator
+    spec:
+      containers:
+      - command:
+        - /manager
+        args:
+        - --enable-leader-election
+        - --pprof
+        - --remoteConfigEnabled
+        image: controller:latest
+        imagePullPolicy: IfNotPresent
+        name: manager
+        env:
+        - name: DD_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: datadog-secret
+              key: api-key
+        - name: DD_APP_KEY
+          valueFrom:
+            secretKeyRef:
+              name: datadog-secret
+              key: app-key
+        - name: DD_CLUSTER_NAME
+          value: rc-updater-e2e-test-cluster
+        - name: DD_SITE
+          value: datadoghq.com
+        resources:
+          limits:
+            cpu: 100m
+            memory: 250Mi
+          requests:
+            cpu: 100m
+            memory: 250Mi
+        ports:
+          - name: metrics
+            containerPort: 8080
+            protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /healthz/
+            port: 8081
+          periodSeconds: 10
+      imagePullSecrets:
+        - name: registry-credentials
+      terminationGracePeriodSeconds: 10
+      serviceAccountName: controller-manager
diff --git a/docs/how-to-contribute.md b/docs/how-to-contribute.md
@@ -126,6 +126,14 @@ $ K8S_VERSION=1.25 IMG=your-dockerhub/operator:tag aws-vault exec sso-agent-sand
 # Run E2E tests with K8S_VERSION, IMG, and IMAGE_PULL_PASSWORD environment variables (for pulling operator image from a private registry).
 $ K8S_VERSION=1.25 IMG=669783387624.dkr.ecr.us-east-1.amazonaws.com/operator:PIPELINE_ID-COMMIT_HASH IMAGE_PULL_PASSWORD=$(aws-vault exec sso-agent-qa-read-only -- aws ecr get-login-password) aws-vault exec sso-agent-sandbox-account-admin -- make e2e-tests
 ```
+> **NOTE:**  The remote configuration updater test requires the owner of the API Key to have the permission `Fleet Policies Write`. 
+> To get the permission:
+>- Log in the ddev org
+>- Go to the [roles page](https://dddev.datadoghq.com/organization-settings/roles)
+>- Search for `Fleet Policies Write`, Click on it
+>- Click on `request role`
+>- It should be added after few minutes.
+
 
 
 [pulumi]:https://www.pulumi.com/

diff --git a/pkg/remoteconfig/updater.go b/pkg/remoteconfig/updater.go
@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff"
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -241,7 +242,7 @@ func (r *RemoteConfigUpdater) agentConfigUpdateCallback(updates map[string]state
 		r.logger.Info("Merged", "update", mergedUpdate)
 	}
 
-	dda, err := r.getDatadogAgentInstance(ctx)
+	dda, err := r.getDatadogAgentWithRetry(ctx)
 	if err != nil {
 		r.logger.Error(err, "Failed to get updatable agents")
 		return
@@ -394,6 +395,34 @@ func (r *RemoteConfigUpdater) getDatadogAgentInstance(ctx context.Context) (v2al
 	return ddaList.Items[0], nil
 }
 
+func (r *RemoteConfigUpdater) getDatadogAgentWithRetry(ctx context.Context) (v2alpha1.DatadogAgent, error) {
+	var dda v2alpha1.DatadogAgent
+	var err error
+
+	operation := func() error {
+		dda, err = r.getDatadogAgentInstance(ctx)
+		if err != nil {
+			r.logger.Error(err, "Failed to get updatable agents, retrying...")
+			return err
+		}
+		return nil
+	}
+
+	// Create a backoff strategy
+	expBackoff := backoff.NewExponentialBackOff()
+	expBackoff.InitialInterval = 1 * time.Second
+	expBackoff.MaxInterval = 10 * time.Second
+	expBackoff.MaxElapsedTime = 3 * time.Minute
+
+	err = backoff.Retry(operation, expBackoff)
+	if err != nil {
+		r.logger.Error(err, "Failed to get updatable agents after retries")
+		return dda, err
+	}
+
+	return dda, nil
+}
+
 func (r *RemoteConfigUpdater) updateInstanceStatus(dda v2alpha1.DatadogAgent, cfg DatadogAgentRemoteConfig) error {
 
 	newddaStatus := dda.Status.DeepCopy()

diff --git a/test/e2e/common.go b/test/e2e/common.go
@@ -17,6 +17,7 @@ import (
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/resid"
 	"sigs.k8s.io/yaml"
 
 	"github.com/DataDog/datadog-operator/pkg/plugin/common"
@@ -173,16 +174,16 @@ func updateKustomization(kustomizeDirPath string, kustomizeResourcePaths []strin
 	}
 
 	// Update resources with target e2e-manager resource yaml
-	for _, res := range kustomizeResourcePaths {
-		exists := false
-		for _, r := range k.Resources {
-			if r == res {
-				exists = true
-				break
-			}
-		}
-		if !exists {
-			k.Resources = append(k.Resources, res)
+	if kustomizeResourcePaths != nil {
+		// We empty slice to avoid accumulating patches from previous tests
+		k.Patches = k.Patches[:0]
+		for _, res := range kustomizeResourcePaths {
+			k.Patches = append(k.Patches, types.Patch{
+				Path: res,
+				Target: &types.Selector{
+					ResId: resid.NewResIdKindOnly("Deployment", "manager"),
+				},
+			})
 		}
 	}
 

diff --git a/test/e2e/kind_test.go b/test/e2e/kind_test.go
@@ -138,9 +138,6 @@ func kindProvisioner(k8sVersion string, extraKustomizeResources []string) e2e.Pr
 			return err
 		}
 
-		if extraKustomizeResources == nil {
-			extraKustomizeResources = []string{defaultMgrFileName}
-		}
 		updateKustomization(kustomizeDirPath, extraKustomizeResources)
 
 		e2eKustomize, err := kustomize.NewDirectory(ctx, "e2e-manager",

diff --git a/test/e2e/manifests/datadog-agent-rc-updater.yaml b/test/e2e/manifests/datadog-agent-rc-updater.yaml
@@ -0,0 +1,58 @@
+apiVersion: datadoghq.com/v2alpha1
+kind: DatadogAgent
+metadata:
+  name: datadog
+spec:
+  features:
+    clusterChecks:
+      enabled: true
+      useClusterChecksRunners: true
+    liveContainerCollection:
+      enabled: true
+    cws:
+      enabled: false
+    usm:
+      enabled: false
+    cspm:
+      enabled: false
+    sbom:
+      enabled: false
+  global:
+    credentials:
+      apiSecret:
+        secretName: datadog-secret
+        keyName: api-key
+      appSecret:
+        secretName: datadog-secret
+        keyName: app-key
+    clusterName: rc-updater-e2e-test-cluster
+    site: datadoghq.com
+    kubelet:
+        tlsVerify: false
+  override:
+    nodeAgent:
+      labels:
+        agent.datadoghq.com/e2e-test: datadog-agent-rc
+      containers:
+        agent:
+          env:
+            - name: DD_SKIP_SSL_VALIDATION
+              value: "false"
+    clusterAgent:
+      labels:
+        agent.datadoghq.com/e2e-test: datadog-agent-minimum
+      env:
+        - name: DD_CLUSTER_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: datadog-cluster-name
+              key: DD_CLUSTER_NAME
+    clusterChecksRunner:
+      labels:
+        agent.datadoghq.com/e2e-test: datadog-agent-minimum
+      env:
+        - name: DD_CLUSTER_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: datadog-cluster-name
+              key: DD_CLUSTER_NAME
diff --git a/test/e2e/rc-updater/api/client.go b/test/e2e/rc-updater/api/client.go
@@ -0,0 +1,73 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+// Package api provides test helpers to interact with the Datadog API
+package api
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
+
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
+)
+
+// Client represents the datadog API context
+type Client struct {
+	api    *datadog.APIClient
+	ctx    context.Context
+	http   http.Client
+	apiKey string
+	appKey string
+}
+
+// NewClient initialise a client with the API and APP keys
+func NewClient() *Client {
+	apiKey, _ := runner.GetProfile().SecretStore().Get(parameters.APIKey)
+	appKey, _ := runner.GetProfile().SecretStore().Get(parameters.APPKey)
+	ctx := context.WithValue(
+		context.Background(),
+		datadog.ContextAPIKeys,
+		map[string]datadog.APIKey{
+			"apiKeyAuth": {
+				Key: apiKey,
+			},
+			"appKeyAuth": {
+				Key: appKey,
+			},
+		},
+	)
+
+	cfg := datadog.NewConfiguration()
+
+	return &Client{
+		api:    datadog.NewAPIClient(cfg),
+		ctx:    ctx,
+		http:   http.Client{},
+		apiKey: apiKey,
+		appKey: appKey,
+	}
+}
+
+// GetAPIKey returns the APIKey
+func GetAPIKey() (string, error) {
+	apiKey, err := runner.GetProfile().SecretStore().Get(parameters.APIKey)
+	if err != nil {
+		return "", fmt.Errorf("could not get APIKey")
+	}
+	return apiKey, nil
+}
+
+// GetAPPKey returns the APPKey
+func GetAPPKey() (string, error) {
+	appKey, err := runner.GetProfile().SecretStore().Get(parameters.APPKey)
+	if err != nil {
+		return "", fmt.Errorf("could not get APPKey")
+	}
+	return appKey, nil
+}