feat: move Prometheus flags to config

charts/values.global.yaml

@@ -25,9 +25,8 @@ images:
     image: gke.gcr.io/prometheus-engine/alertmanager
     tag: v0.25.1-gmp.2-gke.0
   prometheus:
-    # TODO(bwplotka): Change to "v2.45.3-gmp.6-gke.0" once tags are cloned.
-    image: gke.gcr.io/prometheus-engine/prometheus@sha256
-    tag: 01fd523f6dfa54b229b04974195632c8268fbd3d51e66ad076b5d31366210c54
+    image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus
+    tag: latest
   configReloader:
     image: gke.gcr.io/prometheus-engine/config-reloader
     tag: v0.9.0-gke.1

e2e/collector_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/api/iterator"
 	"google.golang.org/protobuf/types/known/timestamppb"
+	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -196,6 +197,12 @@ func testCollectorDeployed(ctx context.Context, restConfig *rest.Config, kubeCli
 	}
 }
 
+// TODO(TheSpiritXIII): https://github.com/go-yaml/yaml/issues/125
+type PrometheusConfig struct {
+	// Google Cloud configuration. Matches our fork's configuration.
+	GoogleCloud *operator.GoogleCloudConfig `yaml:"google_cloud,omitempty"`
+}
+
 func testCollectorOperatorConfig(ctx context.Context, kubeClient client.Client) func(*testing.T) {
 	return func(t *testing.T) {
 		t.Log("checking collector is configured")
@@ -225,46 +232,42 @@ func testCollectorOperatorConfig(ctx context.Context, kubeClient client.Client)
 			t.Fatalf("update operatorconfig: %s", err)
 		}
 
-		// Keep checking the state of the collectors until they're running.
-		err := wait.PollUntilContextCancel(ctx, pollDuration, false, func(ctx context.Context) (bool, error) {
-			ds := appsv1.DaemonSet{
+		var err error
+		pollErr := wait.PollUntilContextCancel(ctx, pollDuration, false, func(ctx context.Context) (bool, error) {
+			configMap := corev1.ConfigMap{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      operator.NameCollector,
 					Namespace: operator.DefaultOperatorNamespace,
+					Name:      operator.NameCollector,
 				},
 			}
-			if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(&ds), &ds); err != nil {
+			if err = kubeClient.Get(ctx, client.ObjectKeyFromObject(&configMap), &configMap); err != nil {
 				if apierrors.IsNotFound(err) {
 					return false, nil
 				}
-				return false, fmt.Errorf("getting collector DaemonSet failed: %w", err)
+				return false, fmt.Errorf("getting collector ConfigMap failed: %w", err)
 			}
 
-			// Ensure prometheus container has expected args.
-			for _, c := range ds.Spec.Template.Spec.Containers {
-				if c.Name != operator.CollectorPrometheusContainerName {
-					continue
-				}
-
-				// We're mainly interested in the dynamic flags but checking the entire set including
-				// the static ones is ultimately simpler.
-				wantArgs := []string{
-					fmt.Sprintf("--export.match=%q", projectFilter),
-					fmt.Sprintf("--export.match=%q", locationFilter),
-					fmt.Sprintf("--export.match=%q", kubeletFilter)}
-				gotArgs := getEnvVar(c.Env, "EXTRA_ARGS")
-				for _, arg := range wantArgs {
-					if !strings.Contains(gotArgs, arg) {
-						return false, fmt.Errorf("expected arg %q not found in EXTRA_ARGS: %q", arg, gotArgs)
-					}
-				}
-
-				return true, nil
+			config := PrometheusConfig{}
+			data := configMap.Data["config.yaml"]
+			if err = yaml.Unmarshal([]byte(data), &config); err != nil {
+				return false, err
+			}
+			if config.GoogleCloud == nil || config.GoogleCloud.Export == nil {
+				err = fmt.Errorf("unable to find Google cloud config:\n%s", data)
+				return false, nil
+			}
+			if !cmp.Equal([]string{projectFilter, locationFilter, kubeletFilter}, config.GoogleCloud.Export.Match) {
+				err = errors.New("unable to find export matchers")
+				return false, nil
 			}
-			return false, errors.New("no container with name prometheus found")
+
+			return true, nil
 		})
-		if err != nil {
-			t.Fatalf("waiting for collector configuration failed: %s", err)
+		if pollErr != nil {
+			if wait.Interrupted(pollErr) {
+				pollErr = err
+			}
+			t.Fatalf("waiting for collector configuration failed: %s", pollErr)
 		}
 	}
 }

examples/collector-max-throughput.yaml

@@ -91,7 +91,7 @@ spec:
             privileged: false
             readOnlyRootFilesystem: true
         - name: prometheus
-          image: gke.gcr.io/prometheus-engine/prometheus@sha256:01fd523f6dfa54b229b04974195632c8268fbd3d51e66ad076b5d31366210c54
+          image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus:latest
           args:
             - --config.file=/prometheus/config_out/config.yaml
             - --enable-feature=exemplar-storage

manifests/operator.yaml

@@ -403,7 +403,7 @@ spec:
           privileged: false
           readOnlyRootFilesystem: true
       - name: prometheus
-        image: gke.gcr.io/prometheus-engine/prometheus@sha256:01fd523f6dfa54b229b04974195632c8268fbd3d51e66ad076b5d31366210c54
+        image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus:latest
         args:
         - --config.file=/prometheus/config_out/config.yaml
         - --enable-feature=exemplar-storage

pkg/operator/collection.go

@@ -24,7 +24,6 @@ import (
 	"net/url"
 	"path"
 	"sort"
-	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/prometheus/common/config"
@@ -173,7 +172,7 @@ func (r *collectionReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		return reconcile.Result{}, fmt.Errorf("ensure collector secrets: %w", err)
 	}
 	// Deploy Prometheus collector as a node agent.
-	if err := r.ensureCollectorDaemonSet(ctx, &config.Collection); err != nil {
+	if err := r.ensureCollectorDaemonSet(ctx); err != nil {
 		return reconcile.Result{}, fmt.Errorf("ensure collector daemon set: %w", err)
 	}
 
@@ -218,7 +217,7 @@ func (r *collectionReconciler) ensureCollectorSecrets(ctx context.Context, spec
 }
 
 // ensureCollectorDaemonSet populates the collector DaemonSet with operator-provided values.
-func (r *collectionReconciler) ensureCollectorDaemonSet(ctx context.Context, spec *monitoringv1.CollectionSpec) error {
+func (r *collectionReconciler) ensureCollectorDaemonSet(ctx context.Context) error {
 	logger, _ := logr.FromContext(ctx)
 
 	var ds appsv1.DaemonSet
@@ -233,20 +232,7 @@ func (r *collectionReconciler) ensureCollectorDaemonSet(ctx context.Context, spe
 		return err
 	}
 
-	var flags []string
-	// Populate export flags for collector if necessary.
-	for _, matcher := range spec.Filter.MatchOneOf {
-		flags = append(flags, fmt.Sprintf("--export.match=%q", matcher))
-	}
-	if spec.Credentials != nil {
-		p := path.Join(secretsDir, pathForSelector(r.opts.PublicNamespace, &monitoringv1.SecretOrConfigMap{Secret: spec.Credentials}))
-		flags = append(flags, fmt.Sprintf("--export.credentials-file=%q", p))
-	}
-
-	if len(spec.Compression) > 0 && spec.Compression != monitoringv1.CompressionNone {
-		flags = append(flags, fmt.Sprintf("--export.compression=%s", spec.Compression))
-	}
-	setContainerExtraArgs(ds.Spec.Template.Spec.Containers, CollectorPrometheusContainerName, strings.Join(flags, " "))
+	setContainerExtraArgs(ds.Spec.Template.Spec.Containers, CollectorPrometheusContainerName, "")
 
 	return r.client.Update(ctx, &ds)
 }
@@ -294,6 +280,20 @@ func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, spec *
 	if err != nil {
 		return fmt.Errorf("generate Prometheus config: %w", err)
 	}
+
+	var credentialsFile string
+	if spec.Credentials != nil {
+		credentialsFile = path.Join(secretsDir, pathForSelector(r.opts.PublicNamespace, &monitoringv1.SecretOrConfigMap{Secret: spec.Credentials}))
+	}
+
+	cfg.GoogleCloud = &GoogleCloudConfig{
+		Export: &GoogleCloudExportConfig{
+			Compression:     string(compression),
+			CredentialsFile: credentialsFile,
+			Match:           spec.Filter.MatchOneOf,
+		},
+	}
+
 	cfgEncoded, err := yaml.Marshal(cfg)
 	if err != nil {
 		return fmt.Errorf("marshal Prometheus config: %w", err)
@@ -345,6 +345,9 @@ type prometheusConfig struct {
 
 	// Secret management. Matches our fork's configuration.
 	SecretConfigs []secrets.SecretConfig `yaml:"kubernetes_secrets,omitempty"`
+
+	// Google Cloud configuration. Matches our fork's configuration.
+	GoogleCloud *GoogleCloudConfig `yaml:"google_cloud,omitempty"`
 }
 
 type update struct {
@@ -353,6 +356,16 @@ type update struct {
 	status bool
 }
 
+type GoogleCloudConfig struct {
+	Export *GoogleCloudExportConfig `yaml:"export,omitempty"`
+}
+
+type GoogleCloudExportConfig struct {
+	Match           []string `yaml:"match,omitempty"`
+	Compression     string   `yaml:"compression,omitempty"`
+	CredentialsFile string   `yaml:"credentials,omitempty"`
+}
+
 // makeCollectorConfig returns the Prometheus configuration based on the scrape configurations, the
 // list of objects to update and any error.
 func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, exports []monitoringv1.ExportSpec) (*prometheusConfig, []update, error) {