From e005fb522bb4a7ea411c3bd1c24da8c7c8e130ac Mon Sep 17 00:00:00 2001 From: Karl Isenberg Date: Wed, 19 Jul 2023 14:53:19 -0700 Subject: [PATCH] Rewrite some e2e tests to handle resource defaults --- e2e/nomostest/testpredicates/predicates.go | 100 +++++- e2e/testcases/helm_sync_test.go | 24 +- .../override_resource_limits_test.go | 317 +++++++++++------- e2e/testcases/reconciler_manager_test.go | 56 +--- e2e/testcases/stress_test.go | 4 +- .../reconciler_container_resources.go | 14 +- 6 files changed, 316 insertions(+), 199 deletions(-) diff --git a/e2e/nomostest/testpredicates/predicates.go b/e2e/nomostest/testpredicates/predicates.go index d0ab40707e..c6372a2c55 100644 --- a/e2e/nomostest/testpredicates/predicates.go +++ b/e2e/nomostest/testpredicates/predicates.go @@ -26,7 +26,6 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -251,8 +250,9 @@ func HasExactlyImage(containerName, expectImageName, expectImageTag, expectImage } } -// HasCorrectResourceRequestsLimits verify a root/namespace reconciler container has the correct resource requests and limits. -func HasCorrectResourceRequestsLimits(containerName string, cpuRequest, cpuLimit, memoryRequest, memoryLimit resource.Quantity) Predicate { +// DeploymentContainerResourcesEqual verify a reconciler deployment container +// has the expected resource requests and limits. +func DeploymentContainerResourcesEqual(expectedSpec v1beta1.ContainerResourcesSpec) Predicate { return func(o client.Object) error { if o == nil { return ErrObjectNotFound @@ -272,24 +272,98 @@ func HasCorrectResourceRequestsLimits(containerName string, cpuRequest, cpuLimit return WrongTypeErr(dep, &appsv1.Deployment{}) } for _, container := range dep.Spec.Template.Spec.Containers { - if containerName == container.Name { - if !equality.Semantic.DeepEqual(container.Resources.Requests[corev1.ResourceCPU], cpuRequest) { - return errors.Errorf("The CPU request of the %q container should be %v, got %v", container.Name, cpuRequest, container.Resources.Requests[corev1.ResourceCPU]) + if expectedSpec.ContainerName == container.Name { + + expected := expectedSpec.CPURequest + found := container.Resources.Requests.Cpu() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected CPU request of the %q container: %s, got: %s", + container.Name, &expected, found) } - if !equality.Semantic.DeepEqual(container.Resources.Limits[corev1.ResourceCPU], cpuLimit) { - return errors.Errorf("The CPU limit of the %q container should be %v, got %v", container.Name, cpuLimit, container.Resources.Limits[corev1.ResourceCPU]) + + expected = expectedSpec.MemoryRequest + found = container.Resources.Requests.Memory() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected Memory request of the %q container: %s, got: %s", + container.Name, &expected, found) } - if !equality.Semantic.DeepEqual(container.Resources.Requests[corev1.ResourceMemory], memoryRequest) { - return errors.Errorf("The memory request of the %q container should be %v, got %v", container.Name, memoryRequest, container.Resources.Requests[corev1.ResourceMemory]) + + expected = expectedSpec.CPULimit + found = container.Resources.Limits.Cpu() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected CPU limit of the %q container: %s, got: %s", + container.Name, &expected, found) } - if !equality.Semantic.DeepEqual(container.Resources.Limits[corev1.ResourceMemory], memoryLimit) { - return errors.Errorf("The memory limit of the %q container should be %v, got %v", container.Name, memoryLimit, container.Resources.Limits[corev1.ResourceMemory]) + + expected = expectedSpec.MemoryLimit + found = container.Resources.Limits.Memory() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected Memory limit of the %q container: %s, got: %s", + container.Name, &expected, found) } return nil } } - return errors.Errorf("Container %q not found", containerName) + return errors.Errorf("Container %q not found", expectedSpec.ContainerName) + } +} + +func DeploymentContainerResourcesAllEqual(expectedByName map[string]v1beta1.ContainerResourcesSpec) Predicate { + return func(o client.Object) error { + if o == nil { + return ErrObjectNotFound + } + if uObj, ok := o.(*unstructured.Unstructured); ok { + rObj, err := kinds.ToTypedObject(uObj, core.Scheme) + if err != nil { + return err + } + o, err = kinds.ObjectAsClientObject(rObj) + if err != nil { + return err + } + } + d, ok := o.(*appsv1.Deployment) + if !ok { + return WrongTypeErr(d, &appsv1.Deployment{}) + } + for _, container := range d.Spec.Template.Spec.Containers { + expectedSpec, ok := expectedByName[container.Name] + if !ok { + return fmt.Errorf("found unexpected container: %q", + container.Name) + } + + expected := expectedSpec.CPURequest + found := container.Resources.Requests.Cpu() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected CPU request of the %q container: %s, got: %s", + container.Name, &expected, found) + } + + expected = expectedSpec.MemoryRequest + found = container.Resources.Requests.Memory() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected Memory request of the %q container: %s, got: %s", + container.Name, &expected, found) + } + + expected = expectedSpec.CPULimit + found = container.Resources.Limits.Cpu() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected CPU limit of the %q container: %s, got: %s", + container.Name, &expected, found) + } + + expected = expectedSpec.MemoryLimit + found = container.Resources.Limits.Memory() + if found.Cmp(expected) != 0 { + return fmt.Errorf("expected Memory limit of the %q container: %s, got: %s", + container.Name, &expected, found) + } + } + return nil } } diff --git a/e2e/testcases/helm_sync_test.go b/e2e/testcases/helm_sync_test.go index e0fab5aa74..a0c0c8c5d4 100644 --- a/e2e/testcases/helm_sync_test.go +++ b/e2e/testcases/helm_sync_test.go @@ -81,11 +81,13 @@ func TestPublicHelm(t *testing.T) { expectedMemoryLimit = "300Mi" } if err := nt.Validate("my-wordpress", "wordpress", &appsv1.Deployment{}, containerImagePullPolicy("Always"), - testpredicates.HasCorrectResourceRequestsLimits("wordpress", - resource.MustParse(expectedCPURequest), - resource.MustParse(expectedCPULimit), - resource.MustParse(expectedMemoryRequest), - resource.MustParse(expectedMemoryLimit)), + testpredicates.DeploymentContainerResourcesEqual(v1beta1.ContainerResourcesSpec{ + ContainerName: "wordpress", + CPURequest: resource.MustParse(expectedCPURequest), + CPULimit: resource.MustParse(expectedCPULimit), + MemoryRequest: resource.MustParse(expectedMemoryRequest), + MemoryLimit: resource.MustParse(expectedMemoryLimit), + }), testpredicates.HasExactlyImage("wordpress", "bitnami/wordpress", "", "sha256:362cb642db481ebf6f14eb0244fbfb17d531a84ecfe099cd3bba6810db56694e"), testpredicates.DeploymentHasEnvVar("wordpress", "WORDPRESS_USERNAME", "test-user"), testpredicates.DeploymentHasEnvVar("wordpress", "WORDPRESS_EMAIL", "test-user@example.com"), @@ -273,11 +275,13 @@ service: expectedMemoryLimit = "300Mi" } if err := nt.Validate("my-wordpress", "wordpress", &appsv1.Deployment{}, containerImagePullPolicy("Always"), - testpredicates.HasCorrectResourceRequestsLimits("wordpress", - resource.MustParse(expectedCPURequest), - resource.MustParse(expectedCPULimit), - resource.MustParse(expectedMemoryRequest), - resource.MustParse(expectedMemoryLimit)), + testpredicates.DeploymentContainerResourcesEqual(v1beta1.ContainerResourcesSpec{ + ContainerName: "wordpress", + CPURequest: resource.MustParse(expectedCPURequest), + CPULimit: resource.MustParse(expectedCPULimit), + MemoryRequest: resource.MustParse(expectedMemoryRequest), + MemoryLimit: resource.MustParse(expectedMemoryLimit), + }), testpredicates.HasExactlyImage("wordpress", "bitnami/wordpress", "", "sha256:362cb642db481ebf6f14eb0244fbfb17d531a84ecfe099cd3bba6810db56694e"), testpredicates.DeploymentHasEnvVar("wordpress", "WORDPRESS_USERNAME", "test-user-1"), testpredicates.DeploymentHasEnvVar("wordpress", "WORDPRESS_EMAIL", "test-user@example.com"), diff --git a/e2e/testcases/override_resource_limits_test.go b/e2e/testcases/override_resource_limits_test.go index 642a9350ed..17ce287dce 100644 --- a/e2e/testcases/override_resource_limits_test.go +++ b/e2e/testcases/override_resource_limits_test.go @@ -20,7 +20,7 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/types" + "kpt.dev/configsync/e2e" "kpt.dev/configsync/e2e/nomostest" "kpt.dev/configsync/e2e/nomostest/ntopts" nomostesting "kpt.dev/configsync/e2e/nomostest/testing" @@ -41,35 +41,33 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { nt := nomostest.New(t, nomostesting.OverrideAPI, ntopts.SkipAutopilotCluster, ntopts.NamespaceRepo(backendNamespace, configsync.RepoSyncName), ntopts.NamespaceRepo(frontendNamespace, configsync.RepoSyncName)) - if err := nt.WatchForAllSyncs(); err != nil { - nt.T.Fatal(err) - } - rootReconcilerNN := types.NamespacedName{ - Name: nomostest.DefaultRootReconcilerName, - Namespace: v1.NSConfigManagementSystem, - } - backendReconcilerNN := types.NamespacedName{ - Name: core.NsReconcilerName(backendNamespace, configsync.RepoSyncName), - Namespace: v1.NSConfigManagementSystem, - } - frontendReconcilerNN := types.NamespacedName{ - Name: core.NsReconcilerName(frontendNamespace, configsync.RepoSyncName), - Namespace: v1.NSConfigManagementSystem, + rootSyncNN := nomostest.RootSyncNN(configsync.RootSyncName) + rootReconcilerNN := core.RootReconcilerObjectKey(rootSyncNN.Name) + backendReconcilerNN := core.NsReconcilerObjectKey(backendNamespace, configsync.RepoSyncName) + frontendReconcilerNN := core.NsReconcilerObjectKey(frontendNamespace, configsync.RepoSyncName) + + // Get RootSync + rootSyncObj := &v1alpha1.RootSync{} + err := nt.Validate(rootSyncNN.Name, rootSyncNN.Namespace, rootSyncObj) + if err != nil { + nt.T.Fatal(err) } // Get the default CPU/memory requests and limits of the reconciler container and the git-sync container - defaultResources := controllers.ReconcilerContainerResourceDefaults() - defaultReconcilerCPURequest, defaultReconcilerMemRequest := defaultResources[reconcilermanager.Reconciler].CPURequest, defaultResources[reconcilermanager.Reconciler].MemoryRequest - defaultReconcilerCPULimits, defaultReconcilerMemLimits := defaultResources[reconcilermanager.Reconciler].CPULimit, defaultResources[reconcilermanager.Reconciler].MemoryLimit - defaultGitSyncCPURequest, defaultGitSyncMemRequest := defaultResources[reconcilermanager.GitSync].CPURequest, defaultResources[reconcilermanager.GitSync].MemoryRequest - defaultGitSyncCPULimits, defaultGitSyncMemLimits := defaultResources[reconcilermanager.GitSync].CPULimit, defaultResources[reconcilermanager.GitSync].MemoryLimit + var defaultResources map[string]v1beta1.ContainerResourcesSpec + if *e2e.VPA && nomostest.IsReconcilerAutoscalingEnabled(rootSyncObj) { + defaultResources = controllers.ReconcilerContainerResourceAutoscaleDefaults() + } else { + defaultResources = controllers.ReconcilerContainerResourceDefaults() + } // Verify root-reconciler uses the default resource requests and limits rootReconcilerDeployment := &appsv1.Deployment{} - err := nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, rootReconcilerDeployment, - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + err = nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, rootReconcilerDeployment, + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -77,8 +75,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify ns-reconciler-backend uses the default resource requests and limits nsReconcilerBackendDeployment := &appsv1.Deployment{} err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, nsReconcilerBackendDeployment, - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -86,8 +85,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify ns-reconciler-frontend uses the default resource requests and limits nsReconcilerFrontendDeployment := &appsv1.Deployment{} err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, nsReconcilerFrontendDeployment, - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -108,15 +108,24 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { nt.MustMergePatch(rootSync, `{"spec": {"override": {"resources": [{"containerName": "reconciler", "cpuRequest": "500m", "cpuLimit": "800m", "memoryRequest": "400Mi", "memoryLimit": "411Mi"}]}}}`) rootReconcilerDeploymentGeneration++ + updatedRootReconcilerResources := v1beta1.ContainerResourcesSpec{ + ContainerName: reconcilermanager.Reconciler, + CPURequest: resource.MustParse("500m"), + CPULimit: resource.MustParse("800m"), + MemoryRequest: resource.MustParse("400Mi"), + MemoryLimit: resource.MustParse("411Mi"), + } + // Verify the reconciler container of root-reconciler uses the new resource request and limits, and the git-sync container uses the default resource requests and limits. err = nt.Watcher.WatchObject(kinds.Deployment(), rootReconcilerNN.Name, rootReconcilerNN.Namespace, []testpredicates.Predicate{ hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("800m"), resource.MustParse("400Mi"), resource.MustParse("411Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits), + testpredicates.DeploymentContainerResourcesEqual(updatedRootReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), }, - testwatcher.WatchTimeout(30*time.Second)) + testwatcher.WatchTimeout(30*time.Second), + ) if err != nil { nt.T.Fatal(err) } @@ -124,8 +133,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify ns-reconciler-backend uses the default resource requests and limits err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -133,8 +143,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify ns-reconciler-frontend uses the default resource requests and limits err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -191,26 +202,39 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify the resource requests and limits of root-reconciler are not affected by the resource changes of ns-reconciler-backend and ns-reconciler-fronend err = nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("800m"), resource.MustParse("400Mi"), resource.MustParse("411Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(updatedRootReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } + updatedBackendReconcilerResources := v1beta1.ContainerResourcesSpec{} + v1alpha1.Convert_v1alpha1_ContainerResourcesSpec_To_v1beta1_ContainerResourcesSpec(&repoSyncBackend.Spec.Override.Resources[0], &updatedBackendReconcilerResources, nil) + updatedBackendGitSyncResources := v1beta1.ContainerResourcesSpec{} + v1alpha1.Convert_v1alpha1_ContainerResourcesSpec_To_v1beta1_ContainerResourcesSpec(&repoSyncBackend.Spec.Override.Resources[1], &updatedBackendGitSyncResources, nil) + // Verify ns-reconciler-backend uses the new resource requests and limits err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("1000m"), resource.MustParse("500Mi"), resource.MustParse("555Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("600m"), resource.MustParse("1"), resource.MustParse("600Mi"), resource.MustParse("666Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedBackendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedBackendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } + updatedFrontendReconcilerResources := v1beta1.ContainerResourcesSpec{} + v1alpha1.Convert_v1alpha1_ContainerResourcesSpec_To_v1beta1_ContainerResourcesSpec(&repoSyncFrontend.Spec.Override.Resources[0], &updatedFrontendReconcilerResources, nil) + updatedFrontendGitSyncResources := v1beta1.ContainerResourcesSpec{} + v1alpha1.Convert_v1alpha1_ContainerResourcesSpec_To_v1beta1_ContainerResourcesSpec(&repoSyncFrontend.Spec.Override.Resources[1], &updatedFrontendGitSyncResources, nil) + // Verify ns-reconciler-frontend uses the new resource requests and limits err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -219,15 +243,24 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { nt.MustMergePatch(rootSync, `{"spec": {"override": {"resources": [{"containerName": "git-sync", "cpuLimit": "333m"}]}}}`) rootReconcilerDeploymentGeneration++ + updatedRootReconcilerGitSyncResources := v1beta1.ContainerResourcesSpec{ + ContainerName: reconcilermanager.GitSync, + CPURequest: defaultResources[reconcilermanager.GitSync].CPURequest, + CPULimit: resource.MustParse("333m"), + MemoryRequest: defaultResources[reconcilermanager.GitSync].MemoryRequest, + MemoryLimit: defaultResources[reconcilermanager.GitSync].MemoryLimit, + } + // Verify the reconciler container root-reconciler uses the default resource requests and limits, and the git-sync container uses the new resource limits. err = nt.Watcher.WatchObject(kinds.Deployment(), rootReconcilerNN.Name, rootReconcilerNN.Namespace, []testpredicates.Predicate{ hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, resource.MustParse("333m"), defaultGitSyncMemRequest, defaultGitSyncMemLimits), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(updatedRootReconcilerGitSyncResources), }, - testwatcher.WatchTimeout(30*time.Second)) + testwatcher.WatchTimeout(30*time.Second), + ) if err != nil { nt.T.Fatal(err) } @@ -235,8 +268,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify the resource limits of ns-reconciler-backend are not affected by the resource limit change of root-reconciler err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("1000m"), resource.MustParse("500Mi"), resource.MustParse("555Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("600m"), resource.MustParse("1"), resource.MustParse("600Mi"), resource.MustParse("666Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedBackendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedBackendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -244,8 +278,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify the resource limits of ns-reconciler-frontend are not affected by the resource limit change of root-reconciler err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -259,8 +294,8 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { rootReconcilerNN.Name, rootReconcilerNN.Namespace, []testpredicates.Predicate{ hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), }, testwatcher.WatchTimeout(30*time.Second)) if err != nil { @@ -270,8 +305,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify the resource requests and limits of ns-reconciler-backend are not affected by the resource limit change of root-reconciler err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("1000m"), resource.MustParse("500Mi"), resource.MustParse("555Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("600m"), resource.MustParse("1"), resource.MustParse("600Mi"), resource.MustParse("666Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedBackendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedBackendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -279,8 +315,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify the resource requests and limits of ns-reconciler-frontend are not affected by the resource limit change of root-reconciler err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -297,8 +334,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify ns-reconciler-backend uses the default resource requests and limits err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -306,8 +344,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify root-reconciler uses the default resource requests and limits err = nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -315,8 +354,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify the resource requests and limits of ns-reconciler-frontend are not affected by the resource limit change of ns-reconciler-backend err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -333,8 +373,9 @@ func TestOverrideReconcilerResourcesV1Alpha1(t *testing.T) { // Verify ns-reconciler-frontend uses the default resource requests and limits err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -344,35 +385,33 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { nt := nomostest.New(t, nomostesting.OverrideAPI, ntopts.SkipAutopilotCluster, ntopts.NamespaceRepo(backendNamespace, configsync.RepoSyncName), ntopts.NamespaceRepo(frontendNamespace, configsync.RepoSyncName)) - if err := nt.WatchForAllSyncs(); err != nil { - nt.T.Fatal(err) - } - rootReconcilerNN := types.NamespacedName{ - Name: nomostest.DefaultRootReconcilerName, - Namespace: v1.NSConfigManagementSystem, - } - backendReconcilerNN := types.NamespacedName{ - Name: core.NsReconcilerName(backendNamespace, configsync.RepoSyncName), - Namespace: v1.NSConfigManagementSystem, - } - frontendReconcilerNN := types.NamespacedName{ - Name: core.NsReconcilerName(frontendNamespace, configsync.RepoSyncName), - Namespace: v1.NSConfigManagementSystem, + rootSyncNN := nomostest.RootSyncNN(configsync.RootSyncName) + rootReconcilerNN := core.RootReconcilerObjectKey(rootSyncNN.Name) + backendReconcilerNN := core.NsReconcilerObjectKey(backendNamespace, configsync.RepoSyncName) + frontendReconcilerNN := core.NsReconcilerObjectKey(frontendNamespace, configsync.RepoSyncName) + + // Get RootSync + rootSyncObj := &v1beta1.RootSync{} + err := nt.Validate(rootSyncNN.Name, rootSyncNN.Namespace, rootSyncObj) + if err != nil { + nt.T.Fatal(err) } // Get the default CPU/memory requests and limits of the reconciler container and the git-sync container - defaultResources := controllers.ReconcilerContainerResourceDefaults() - defaultReconcilerCPURequest, defaultReconcilerMemRequest := defaultResources[reconcilermanager.Reconciler].CPURequest, defaultResources[reconcilermanager.Reconciler].MemoryRequest - defaultReconcilerCPULimits, defaultReconcilerMemLimits := defaultResources[reconcilermanager.Reconciler].CPULimit, defaultResources[reconcilermanager.Reconciler].MemoryLimit - defaultGitSyncCPURequest, defaultGitSyncMemRequest := defaultResources[reconcilermanager.GitSync].CPURequest, defaultResources[reconcilermanager.GitSync].MemoryRequest - defaultGitSyncCPULimits, defaultGitSyncMemLimits := defaultResources[reconcilermanager.GitSync].CPULimit, defaultResources[reconcilermanager.GitSync].MemoryLimit + var defaultResources map[string]v1beta1.ContainerResourcesSpec + if *e2e.VPA && nomostest.IsReconcilerAutoscalingEnabled(rootSyncObj) { + defaultResources = controllers.ReconcilerContainerResourceAutoscaleDefaults() + } else { + defaultResources = controllers.ReconcilerContainerResourceDefaults() + } // Verify root-reconciler uses the default resource requests and limits rootReconcilerDeployment := &appsv1.Deployment{} - err := nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, rootReconcilerDeployment, - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + err = nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, rootReconcilerDeployment, + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -380,8 +419,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify ns-reconciler-backend uses the default resource requests and limits nsReconcilerBackendDeployment := &appsv1.Deployment{} err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, nsReconcilerBackendDeployment, - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -389,8 +429,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify ns-reconciler-frontend uses the default resource requests and limits nsReconcilerFrontendDeployment := &appsv1.Deployment{} err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, nsReconcilerFrontendDeployment, - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -411,13 +452,21 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { nt.MustMergePatch(rootSync, `{"spec": {"override": {"resources": [{"containerName": "reconciler", "cpuRequest": "500m", "cpuLimit": "800m", "memoryRequest": "400Mi", "memoryLimit": "411Mi"}]}}}`) rootReconcilerDeploymentGeneration++ + updatedRootReconcilerResources := v1beta1.ContainerResourcesSpec{ + ContainerName: reconcilermanager.Reconciler, + CPURequest: resource.MustParse("500m"), + CPULimit: resource.MustParse("800m"), + MemoryRequest: resource.MustParse("400Mi"), + MemoryLimit: resource.MustParse("411Mi"), + } + // Verify the reconciler container of root-reconciler uses the new resource request and limits, and the git-sync container uses the default resource requests and limits. err = nt.Watcher.WatchObject(kinds.Deployment(), rootReconcilerNN.Name, rootReconcilerNN.Namespace, []testpredicates.Predicate{ hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("800m"), resource.MustParse("400Mi"), resource.MustParse("411Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits), + testpredicates.DeploymentContainerResourcesEqual(updatedRootReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), }, testwatcher.WatchTimeout(30*time.Second)) if err != nil { @@ -427,8 +476,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify ns-reconciler-backend uses the default resource requests and limits err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -436,8 +486,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify ns-reconciler-frontend uses the default resource requests and limits err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -494,26 +545,35 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify the resource requests and limits of root-reconciler are not affected by the resource changes of ns-reconciler-backend and ns-reconciler-fronend err = nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("800m"), resource.MustParse("400Mi"), resource.MustParse("411Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(updatedRootReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } + updatedBackendReconcilerResources := repoSyncBackend.Spec.Override.Resources[0] + updatedBackendGitSyncResources := repoSyncBackend.Spec.Override.Resources[1] + // Verify ns-reconciler-backend uses the new resource requests and limits err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("1000m"), resource.MustParse("500Mi"), resource.MustParse("555Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("600m"), resource.MustParse("1"), resource.MustParse("600Mi"), resource.MustParse("666Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedBackendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedBackendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } + updatedFrontendReconcilerResources := repoSyncFrontend.Spec.Override.Resources[0] + updatedFrontendGitSyncResources := repoSyncFrontend.Spec.Override.Resources[1] + // Verify ns-reconciler-frontend uses the new resource requests and limits err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -522,15 +582,24 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { nt.MustMergePatch(rootSync, `{"spec": {"override": {"resources": [{"containerName": "git-sync", "cpuLimit": "333m"}]}}}`) rootReconcilerDeploymentGeneration++ + updatedRootReconcilerGitSyncResources := v1beta1.ContainerResourcesSpec{ + ContainerName: reconcilermanager.GitSync, + CPURequest: defaultResources[reconcilermanager.GitSync].CPURequest, + CPULimit: resource.MustParse("333m"), + MemoryRequest: defaultResources[reconcilermanager.GitSync].MemoryRequest, + MemoryLimit: defaultResources[reconcilermanager.GitSync].MemoryLimit, + } + // Verify the reconciler container root-reconciler uses the default resource requests and limits, and the git-sync container uses the new resource limits. err = nt.Watcher.WatchObject(kinds.Deployment(), rootReconcilerNN.Name, rootReconcilerNN.Namespace, []testpredicates.Predicate{ hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, resource.MustParse("333m"), defaultGitSyncMemRequest, defaultGitSyncMemLimits), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(updatedRootReconcilerGitSyncResources), }, - testwatcher.WatchTimeout(30*time.Second)) + testwatcher.WatchTimeout(30*time.Second), + ) if err != nil { nt.T.Fatal(err) } @@ -538,8 +607,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify the resource limits of ns-reconciler-backend are not affected by the resource limit change of root-reconciler err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("1000m"), resource.MustParse("500Mi"), resource.MustParse("555Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("600m"), resource.MustParse("1"), resource.MustParse("600Mi"), resource.MustParse("666Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedBackendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedBackendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -547,8 +617,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify the resource limits of ns-reconciler-frontend are not affected by the resource limit change of root-reconciler err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -562,8 +633,8 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { rootReconcilerNN.Name, rootReconcilerNN.Namespace, []testpredicates.Predicate{ hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), }, testwatcher.WatchTimeout(30*time.Second)) if err != nil { @@ -573,8 +644,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify the resource requests and limits of ns-reconciler-backend are not affected by the resource limit change of root-reconciler err = nt.Validate(backendReconcilerNN.Name, backendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("500m"), resource.MustParse("1000m"), resource.MustParse("500Mi"), resource.MustParse("555Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("600m"), resource.MustParse("1"), resource.MustParse("600Mi"), resource.MustParse("666Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedBackendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedBackendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -582,8 +654,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify the resource requests and limits of ns-reconciler-frontend are not affected by the resource limit change of root-reconciler err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -600,8 +673,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify ns-reconciler-backend uses the default resource requests and limits err = nt.Validate(core.NsReconcilerName(backendNamespace, configsync.RepoSyncName), v1.NSConfigManagementSystem, &appsv1.Deployment{}, hasGeneration(nsReconcilerBackendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -609,8 +683,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify root-reconciler uses the default resource requests and limits err = nt.Validate(rootReconcilerNN.Name, rootReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(rootReconcilerDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } @@ -618,8 +693,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify the resource requests and limits of ns-reconciler-frontend are not affected by the resource limit change of ns-reconciler-backend err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, resource.MustParse("511m"), resource.MustParse("2000m"), resource.MustParse("511Mi"), resource.MustParse("544Mi")), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, resource.MustParse("611m"), resource.MustParse("2"), resource.MustParse("611Mi"), resource.MustParse("644Mi"))) + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendReconcilerResources), + testpredicates.DeploymentContainerResourcesEqual(updatedFrontendGitSyncResources), + ) if err != nil { nt.T.Fatal(err) } @@ -636,8 +712,9 @@ func TestOverrideReconcilerResourcesV1Beta1(t *testing.T) { // Verify ns-reconciler-frontend uses the default resource requests and limits err = nt.Validate(frontendReconcilerNN.Name, frontendReconcilerNN.Namespace, &appsv1.Deployment{}, hasGeneration(nsReconcilerFrontendDeploymentGeneration), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.Reconciler, defaultReconcilerCPURequest, defaultReconcilerCPULimits, defaultReconcilerMemRequest, defaultReconcilerMemLimits), - testpredicates.HasCorrectResourceRequestsLimits(reconcilermanager.GitSync, defaultGitSyncCPURequest, defaultGitSyncCPULimits, defaultGitSyncMemRequest, defaultGitSyncMemLimits)) + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.Reconciler]), + testpredicates.DeploymentContainerResourcesEqual(defaultResources[reconcilermanager.GitSync]), + ) if err != nil { nt.T.Fatal(err) } diff --git a/e2e/testcases/reconciler_manager_test.go b/e2e/testcases/reconciler_manager_test.go index 79dc85e946..e790099b8f 100644 --- a/e2e/testcases/reconciler_manager_test.go +++ b/e2e/testcases/reconciler_manager_test.go @@ -662,54 +662,6 @@ func firstContainerNameIs(expected string) testpredicates.Predicate { } } -func hasExpectedContainerResources(expectedByName map[string]v1beta1.ContainerResourcesSpec) testpredicates.Predicate { - return func(o client.Object) error { - if o == nil { - return testpredicates.ErrObjectNotFound - } - d, ok := o.(*appsv1.Deployment) - if !ok { - return testpredicates.WrongTypeErr(d, &appsv1.Deployment{}) - } - for _, container := range d.Spec.Template.Spec.Containers { - expectedSpec, ok := expectedByName[container.Name] - if !ok { - return fmt.Errorf("found unexpected container: %q", - container.Name) - } - - expected := expectedSpec.CPURequest - found := container.Resources.Requests.Cpu() - if found.Cmp(expected) != 0 { - return fmt.Errorf("expected CPU request of the %q container: %s, got: %s", - container.Name, &expected, found) - } - - expected = expectedSpec.MemoryRequest - found = container.Resources.Requests.Memory() - if found.Cmp(expected) != 0 { - return fmt.Errorf("expected Memory request of the %q container: %s, got: %s", - container.Name, &expected, found) - } - - expected = expectedSpec.CPULimit - found = container.Resources.Limits.Cpu() - if found.Cmp(expected) != 0 { - return fmt.Errorf("expected CPU limit of the %q container: %s, got: %s", - container.Name, &expected, found) - } - - expected = expectedSpec.MemoryLimit - found = container.Resources.Limits.Memory() - if found.Cmp(expected) != 0 { - return fmt.Errorf("expected Memory limit of the %q container: %s, got: %s", - container.Name, &expected, found) - } - } - return nil - } -} - func totalContainerCPURequests(containers []corev1.Container) resource.Quantity { total := resource.MustParse("0m") for _, container := range containers { @@ -806,7 +758,7 @@ func TestAutopilotReconcilerAdjustment(t *testing.T) { totalContainerCPURequestIs(expectedTotalResources.CPURequest), totalContainerMemoryRequestIs(expectedTotalResources.MemoryRequest), firstContainerNameIs(firstContainerName), - hasExpectedContainerResources(expectedResources), + testpredicates.DeploymentContainerResourcesAllEqual(expectedResources), ) if err != nil { nt.T.Log("Reconciler container specs (expected):") @@ -865,7 +817,7 @@ func TestAutopilotReconcilerAdjustment(t *testing.T) { totalContainerCPURequestIs(expectedTotalResources.CPURequest), totalContainerMemoryRequestIs(expectedTotalResources.MemoryRequest), firstContainerNameIs(firstContainerName), - hasExpectedContainerResources(expectedResources), + testpredicates.DeploymentContainerResourcesAllEqual(expectedResources), ) if err != nil { nt.T.Log("Reconciler container specs (expected):") @@ -909,7 +861,7 @@ func TestAutopilotReconcilerAdjustment(t *testing.T) { totalContainerCPURequestIs(expectedTotalResources.CPURequest), totalContainerMemoryRequestIs(expectedTotalResources.MemoryRequest), firstContainerNameIs(firstContainerName), - hasExpectedContainerResources(expectedResources), + testpredicates.DeploymentContainerResourcesAllEqual(expectedResources), ) if err != nil { nt.T.Log("Reconciler container specs (expected):") @@ -965,7 +917,7 @@ func TestAutopilotReconcilerAdjustment(t *testing.T) { totalContainerCPURequestIs(expectedTotalResources.CPURequest), totalContainerMemoryRequestIs(expectedTotalResources.MemoryRequest), firstContainerNameIs(firstContainerName), - hasExpectedContainerResources(expectedResources), + testpredicates.DeploymentContainerResourcesAllEqual(expectedResources), ) if err != nil { nt.T.Log("Reconciler container specs (expected):") diff --git a/e2e/testcases/stress_test.go b/e2e/testcases/stress_test.go index 00e87f6e03..ccfa464552 100644 --- a/e2e/testcases/stress_test.go +++ b/e2e/testcases/stress_test.go @@ -209,9 +209,7 @@ func TestStressLargeNamespaceAutoscaling(t *testing.T) { // Wait for the reconciler Deployment to reflect the RootSync changes reconcilerKey := core.RootReconcilerObjectKey(configsync.RootSyncName) err := nt.Watcher.WatchObject(kinds.Deployment(), reconcilerKey.Name, reconcilerKey.Namespace, []testpredicates.Predicate{ - testpredicates.HasCorrectResourceRequestsLimits(reconcilerResourceSpec.ContainerName, - reconcilerResourceSpec.CPURequest, reconcilerResourceSpec.CPULimit, - reconcilerResourceSpec.MemoryRequest, reconcilerResourceSpec.MemoryLimit), + testpredicates.DeploymentContainerResourcesEqual(reconcilerResourceSpec), testpredicates.StatusEquals(nt.Scheme, kstatus.CurrentStatus), }) if err != nil { diff --git a/pkg/reconcilermanager/controllers/reconciler_container_resources.go b/pkg/reconcilermanager/controllers/reconciler_container_resources.go index 783ea303ea..6767419cea 100644 --- a/pkg/reconcilermanager/controllers/reconciler_container_resources.go +++ b/pkg/reconcilermanager/controllers/reconciler_container_resources.go @@ -15,27 +15,33 @@ import ( func ReconcilerContainerResourceDefaults() map[string]v1beta1.ContainerResourcesSpec { return map[string]v1beta1.ContainerResourcesSpec{ reconcilermanager.Reconciler: { + ContainerName: reconcilermanager.Reconciler, CPURequest: resource.MustParse("50m"), MemoryRequest: resource.MustParse("200Mi"), }, reconcilermanager.HydrationController: { + ContainerName: reconcilermanager.HydrationController, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("100Mi"), }, reconcilermanager.OciSync: { + ContainerName: reconcilermanager.OciSync, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("200Mi"), }, reconcilermanager.HelmSync: { + ContainerName: reconcilermanager.HelmSync, CPURequest: resource.MustParse("50m"), MemoryRequest: resource.MustParse("200Mi"), }, reconcilermanager.GitSync: { + ContainerName: reconcilermanager.GitSync, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("200Mi"), }, metrics.OtelAgentName: { - CPURequest: resource.MustParse("10m"), + ContainerName: metrics.OtelAgentName, + CPURequest: resource.MustParse("10m"), // TODO: CPU limits considered harmful. // A throttled otel-agent is also much more likely to drop metrics. CPULimit: resource.MustParse("1000m"), @@ -54,31 +60,37 @@ func ReconcilerContainerResourceDefaults() map[string]v1beta1.ContainerResources func ReconcilerContainerResourceAutoscaleDefaults() map[string]v1beta1.ContainerResourcesSpec { return map[string]v1beta1.ContainerResourcesSpec{ reconcilermanager.Reconciler: { + ContainerName: reconcilermanager.Reconciler, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("10Mi"), MemoryLimit: resource.MustParse("20Mi"), }, reconcilermanager.HydrationController: { + ContainerName: reconcilermanager.HydrationController, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("10Mi"), MemoryLimit: resource.MustParse("20Mi"), }, reconcilermanager.OciSync: { + ContainerName: reconcilermanager.OciSync, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("10Mi"), MemoryLimit: resource.MustParse("20Mi"), }, reconcilermanager.HelmSync: { + ContainerName: reconcilermanager.HelmSync, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("10Mi"), MemoryLimit: resource.MustParse("20Mi"), }, reconcilermanager.GitSync: { + ContainerName: reconcilermanager.GitSync, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("10Mi"), MemoryLimit: resource.MustParse("20Mi"), }, metrics.OtelAgentName: { + ContainerName: metrics.OtelAgentName, CPURequest: resource.MustParse("10m"), MemoryRequest: resource.MustParse("10Mi"), MemoryLimit: resource.MustParse("20Mi"),