[WIP] Configure VPA for reconciler, if enabled

Makefile.e2e

@@ -68,7 +68,7 @@ test-e2e-kind-multi-repo: config-sync-manifest-local
 		--share-test-env \
 		--timeout $(KIND_E2E_TIMEOUT) \
 		--test.v -v \
-		--num-clusters 15 \
+		--num-clusters 10 \
 		$(E2E_ARGS)
 
 # This target runs the first group of e2e tests with the multi-repo mode.

e2e/flags.go

@@ -65,6 +65,10 @@ var Stress = flag.Bool("stress", false,
 var KCC = flag.Bool("kcc", false,
 	"If true, run kcc tests.")
 
+// VPA enables running the e2e tests for vertical pod autoscaling.
+var VPA = flag.Bool("vpa", true,
+	"If true, run VPA tests.")
+
 // GceNode enables running the e2e tests for 'gcenode' auth type
 var GceNode = flag.Bool("gcenode", false,
 	"If true, run test with 'gcenode' auth type.")

e2e/nomostest/autoscaling.go

@@ -0,0 +1,79 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nomostest
+
+import (
+	"kpt.dev/configsync/pkg/core"
+	"kpt.dev/configsync/pkg/metadata"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// EnableReconcilerAutoscaling enables reconciler autoscaling on a
+// RootSync or RepoSync. The object is annotated locally, but not applied.
+// Returns true if a change was made, false if already enabled.
+func EnableReconcilerAutoscaling(rs client.Object) bool {
+	return SetReconcilerAutoscalingStrategy(rs, metadata.ReconcilerAutoscalingStrategyAuto)
+}
+
+// DisableReconcilerAutoscaling disables reconciler autoscaling on a
+// RootSync or RepoSync. The object annotated is removed locally, but not applied.
+// Returns true if a change was made, false if already enabled.
+func DisableReconcilerAutoscaling(rs client.Object) bool {
+	return RemoveReconcilerAutoscalingStrategy(rs)
+}
+
+// IsReconcilerAutoscalingEnabled returns true if reconciler-autoscaling-strategy
+// annotation is set to Foreground.
+func IsReconcilerAutoscalingEnabled(rs client.Object) bool {
+	return HasReconcilerAutoscalingStrategy(rs, metadata.ReconcilerAutoscalingStrategyAuto)
+}
+
+// HasReconcilerAutoscalingStrategy returns true if reconciler-autoscaling-strategy
+// annotation is set to the specified policy. Returns false if not set.
+func HasReconcilerAutoscalingStrategy(obj client.Object, policy metadata.ReconcilerAutoscalingStrategy) bool {
+	annotations := obj.GetAnnotations()
+	// don't panic if nil
+	if len(annotations) == 0 {
+		return false
+	}
+	foundPolicy, found := annotations[metadata.ReconcilerAutoscalingStrategyAnnotationKey]
+	return found && foundPolicy == string(policy)
+}
+
+// SetReconcilerAutoscalingStrategy sets the value of the reconciler-autoscaling-strategy
+// annotation locally (does not apply). Returns true if the object was modified.
+func SetReconcilerAutoscalingStrategy(obj client.Object, policy metadata.ReconcilerAutoscalingStrategy) bool {
+	if HasReconcilerAutoscalingStrategy(obj, policy) {
+		return false
+	}
+	core.SetAnnotation(obj, metadata.ReconcilerAutoscalingStrategyAnnotationKey, string(policy))
+	return true
+}
+
+// RemoveReconcilerAutoscalingStrategy removes the reconciler-autoscaling-strategy
+// annotation locally (does not apply). Returns true if the object was modified.
+func RemoveReconcilerAutoscalingStrategy(obj client.Object) bool {
+	annotations := obj.GetAnnotations()
+	// don't panic if nil
+	if len(annotations) == 0 {
+		return false
+	}
+	if _, found := annotations[metadata.ReconcilerAutoscalingStrategyAnnotationKey]; !found {
+		return false
+	}
+	delete(annotations, metadata.ReconcilerAutoscalingStrategyAnnotationKey)
+	obj.SetAnnotations(annotations)
+	return true
+}

e2e/nomostest/clean.go

@@ -111,6 +111,14 @@ func Clean(nt *NT) error {
 	if err := deleteResourceGroupController(nt); err != nil {
 		return err
 	}
+	if *e2e.VPA && *e2e.TestCluster == e2e.Kind {
+		if err := nt.uninstallVerticalPrivateAutoscaler(); err != nil {
+			return err
+		}
+		if err := nt.uninstallMetricsServer(); err != nil {
+			return err
+		}
+	}
 	// Reset any modified system namespaces.
 	if err := resetSystemNamespaces(nt); err != nil {
 		return err

e2e/nomostest/client.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 	admissionv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
@@ -30,6 +31,7 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime"
+	autoscalingv1vpa "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 	"kpt.dev/configsync/e2e"
 	"kpt.dev/configsync/e2e/nomostest/clusters"
@@ -82,6 +84,7 @@ func newScheme(t testing.NTB) *runtime.Scheme {
 		apiextensionsv1.SchemeBuilder,
 		appsv1.SchemeBuilder,
 		corev1.SchemeBuilder,
+		batchv1.SchemeBuilder,
 		configmanagementv1.SchemeBuilder,
 		configsyncv1alpha1.SchemeBuilder,
 		configsyncv1beta1.SchemeBuilder,
@@ -91,6 +94,7 @@ func newScheme(t testing.NTB) *runtime.Scheme {
 		rbacv1beta1.SchemeBuilder,
 		resourcegroupv1alpha1.SchemeBuilder.SchemeBuilder,
 		apiregistrationv1.SchemeBuilder,
+		autoscalingv1vpa.SchemeBuilder,
 	}
 	for _, b := range builders {
 		err := b.AddToScheme(s)

e2e/nomostest/clusters/gke.go

@@ -242,6 +242,9 @@ func createGKECluster(t testing.NTB, name string) error {
 		if len(addons) > 0 {
 			args = append(args, "--addons", strings.Join(addons, ","))
 		}
+		if *e2e.VPA {
+			args = append(args, "--enable-vertical-pod-autoscaling")
+		}
 	}
 	if len(scopes) > 0 {
 		args = append(args, "--scopes", strings.Join(scopes, ","))

e2e/nomostest/config_sync.go

@@ -59,6 +59,7 @@ import (
 	kstatus "sigs.k8s.io/cli-utils/pkg/kstatus/status"
 	"sigs.k8s.io/cli-utils/pkg/object/dependson"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -114,6 +115,14 @@ func isOtelCollectorDeployment(obj client.Object) bool {
 		obj.GetObjectKind().GroupVersionKind() == kinds.Deployment()
 }
 
+// isOtelAgentConfigMap returns true if passed obj is the
+// otel-agent ConfigMap in the config-management-monitoring namespace.
+func isOtelAgentConfigMap(obj client.Object) bool {
+	return obj.GetName() == ocmetrics.OtelAgentName &&
+		obj.GetNamespace() == configmanagement.ControllerNamespace &&
+		obj.GetObjectKind().GroupVersionKind() == kinds.ConfigMap()
+}
+
 // ResetReconcilerManagerConfigMap resets the reconciler manager config map
 // to what is defined in the manifest
 func ResetReconcilerManagerConfigMap(nt *NT) error {
@@ -151,6 +160,7 @@ func parseConfigSyncManifests(nt *NT) ([]client.Object, error) {
 	objs, err = multiRepoObjects(objs,
 		setReconcilerDebugMode,
 		setPollingPeriods,
+		setOtelAgentBatchDisabled,
 		setOtelCollectorPrometheusAnnotations)
 	if err != nil {
 		return nil, err
@@ -279,9 +289,12 @@ func parseManifestDir(dirPath string) ([]client.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	paths := make([]cmpath.Absolute, len(files))
-	for i, f := range files {
-		paths[i] = readPath.Join(cmpath.RelativeSlash(f.Name()))
+	paths := make([]cmpath.Absolute, 0, len(files))
+	for _, f := range files {
+		switch filepath.Ext(f.Name()) {
+		case ".yaml", ".yml", ".json":
+			paths = append(paths, readPath.Join(cmpath.RelativeSlash(f.Name())))
+		}
 	}
 	// Read the manifests cached in the tmpdir.
 	r := reader.File{}
@@ -576,6 +589,44 @@ func setPollingPeriods(obj client.Object) error {
 	return nil
 }
 
+// setOtelAgentBatch updates the otel-agent ConfigMap config to disable metrics
+// batching.
+func setOtelAgentBatchDisabled(obj client.Object) error {
+	if obj == nil {
+		return testpredicates.ErrObjectNotFound
+	}
+	if !isOtelAgentConfigMap(obj) {
+		return nil
+	}
+	cm, ok := obj.(*corev1.ConfigMap)
+	if !ok {
+		return fmt.Errorf("failed to cast %T to *corev1.ConfigMap", obj)
+	}
+	yamlString := cm.Data["otel-agent-config.yaml"]
+	var dataMap map[string]interface{}
+	if err := yaml.Unmarshal([]byte(yamlString), &dataMap); err != nil {
+		return errors.Wrapf(err, "unmarshaling yaml data from ConfigMap %s",
+			client.ObjectKeyFromObject(obj))
+	}
+	processors, found, err := unstructured.NestedMap(dataMap, "processors")
+	if err != nil {
+		return errors.Wrapf(err, "ConfigMap %s missing processors field",
+			client.ObjectKeyFromObject(obj))
+	}
+	if !found {
+		return errors.Errorf("ConfigMap %s missing processors field",
+			client.ObjectKeyFromObject(obj))
+	}
+	delete(processors, "batch")
+	yamlBytes, err := yaml.Marshal(dataMap)
+	if err != nil {
+		return errors.Wrapf(err, "marshaling yaml data for ConfigMap %s",
+			client.ObjectKeyFromObject(obj))
+	}
+	cm.Data["otel-agent-config.yaml"] = string(yamlBytes)
+	return nil
+}
+
 // setOtelCollectorPrometheusAnnotations updates the otel-collector Deployment
 // to add pod annotations to enable metrics scraping.
 func setOtelCollectorPrometheusAnnotations(obj client.Object) error {
@@ -666,6 +717,9 @@ func RootSyncObjectV1Alpha1(name, repoURL string, sourceFormat filesystem.Source
 	// Enable automatic deletion of managed objects by default.
 	// This helps ensure that test artifacts are cleaned up.
 	EnableDeletionPropagation(rs)
+	// Enable autoscaling by default.
+	// This helps validate VPA works for all test cases.
+	EnableReconcilerAutoscaling(rs)
 	return rs
 }
 
@@ -684,6 +738,11 @@ func RootSyncObjectV1Alpha1FromRootRepo(nt *NT, name string) *v1alpha1.RootSync
 	} else if rs.Spec.Override != nil {
 		rs.Spec.Override.ReconcileTimeout = nil
 	}
+	if nt.DefaultReconcilerAutoscalingStrategy != nil {
+		SetReconcilerAutoscalingStrategy(rs, *nt.DefaultReconcilerAutoscalingStrategy)
+	} else {
+		DisableReconcilerAutoscaling(rs)
+	}
 	return rs
 }
 
@@ -704,6 +763,9 @@ func RootSyncObjectV1Beta1(name, repoURL string, sourceFormat filesystem.SourceF
 	// Enable automatic deletion of managed objects by default.
 	// This helps ensure that test artifacts are cleaned up.
 	EnableDeletionPropagation(rs)
+	// Enable autoscaling by default.
+	// This helps validate VPA works for all test cases.
+	EnableReconcilerAutoscaling(rs)
 	return rs
 }
 
@@ -722,6 +784,11 @@ func RootSyncObjectV1Beta1FromRootRepo(nt *NT, name string) *v1beta1.RootSync {
 	} else if rs.Spec.Override != nil {
 		rs.Spec.Override.ReconcileTimeout = nil
 	}
+	if nt.DefaultReconcilerAutoscalingStrategy != nil {
+		SetReconcilerAutoscalingStrategy(rs, *nt.DefaultReconcilerAutoscalingStrategy)
+	} else {
+		DisableReconcilerAutoscaling(rs)
+	}
 	return rs
 }
 
@@ -740,6 +807,11 @@ func RootSyncObjectV1Beta1FromOtherRootRepo(nt *NT, syncName, repoName string) *
 	} else if rs.Spec.Override != nil {
 		rs.Spec.Override.ReconcileTimeout = nil
 	}
+	if nt.DefaultReconcilerAutoscalingStrategy != nil {
+		SetReconcilerAutoscalingStrategy(rs, *nt.DefaultReconcilerAutoscalingStrategy)
+	} else {
+		DisableReconcilerAutoscaling(rs)
+	}
 	return rs
 }
 
@@ -765,6 +837,9 @@ func RepoSyncObjectV1Alpha1(nn types.NamespacedName, repoURL string) *v1alpha1.R
 	// Enable automatic deletion of managed objects by default.
 	// This helps ensure that test artifacts are cleaned up.
 	EnableDeletionPropagation(rs)
+	// Enable autoscaling by default.
+	// This helps validate VPA works for all test cases.
+	EnableReconcilerAutoscaling(rs)
 	return rs
 }
 
@@ -783,6 +858,11 @@ func RepoSyncObjectV1Alpha1FromNonRootRepo(nt *NT, nn types.NamespacedName) *v1a
 	} else if rs.Spec.Override != nil {
 		rs.Spec.Override.ReconcileTimeout = nil
 	}
+	if nt.DefaultReconcilerAutoscalingStrategy != nil {
+		SetReconcilerAutoscalingStrategy(rs, *nt.DefaultReconcilerAutoscalingStrategy)
+	} else {
+		DisableReconcilerAutoscaling(rs)
+	}
 	// Enable automatic deletion of managed objects by default.
 	// This helps ensure that test artifacts are cleaned up.
 	EnableDeletionPropagation(rs)
@@ -811,6 +891,9 @@ func RepoSyncObjectV1Beta1(nn types.NamespacedName, repoURL string, sourceFormat
 	// Enable automatic deletion of managed objects by default.
 	// This helps ensure that test artifacts are cleaned up.
 	EnableDeletionPropagation(rs)
+	// Enable autoscaling by default.
+	// This helps validate VPA works for all test cases.
+	EnableReconcilerAutoscaling(rs)
 	return rs
 }
 
@@ -829,6 +912,11 @@ func RepoSyncObjectV1Beta1FromNonRootRepo(nt *NT, nn types.NamespacedName) *v1be
 	} else if rs.Spec.Override != nil {
 		rs.Spec.Override.ReconcileTimeout = nil
 	}
+	if nt.DefaultReconcilerAutoscalingStrategy != nil {
+		SetReconcilerAutoscalingStrategy(rs, *nt.DefaultReconcilerAutoscalingStrategy)
+	} else {
+		DisableReconcilerAutoscaling(rs)
+	}
 	// Add dependencies to ensure managed objects can be deleted.
 	if err := SetRepoSyncDependencies(nt, rs); err != nil {
 		nt.T.Fatal(err)
@@ -851,6 +939,11 @@ func RepoSyncObjectV1Beta1FromOtherRootRepo(nt *NT, nn types.NamespacedName, rep
 	} else if rs.Spec.Override != nil {
 		rs.Spec.Override.ReconcileTimeout = nil
 	}
+	if nt.DefaultReconcilerAutoscalingStrategy != nil {
+		SetReconcilerAutoscalingStrategy(rs, *nt.DefaultReconcilerAutoscalingStrategy)
+	} else {
+		DisableReconcilerAutoscaling(rs)
+	}
 	// Add dependencies to ensure managed objects can be deleted.
 	if err := SetRepoSyncDependencies(nt, rs); err != nil {
 		nt.T.Fatal(err)