diff --git a/charts/values.global.yaml b/charts/values.global.yaml index 7d87d300f7..a0a4dd3beb 100644 --- a/charts/values.global.yaml +++ b/charts/values.global.yaml @@ -26,8 +26,8 @@ images: tag: v0.25.1-gmp.2-gke.0 prometheus: # TODO(bwplotka): Change to "v2.45.3-gmp.4-gke.0" once tags are cloned. - image: gke.gcr.io/prometheus-engine/prometheus@sha256 - tag: 7473d52f4a3e563e6377f8a6183091f25192b1e0705dd0933903e800bd69b7b2 + image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus + tag: latest configReloader: image: gke.gcr.io/prometheus-engine/config-reloader tag: v0.9.0-gke.1 diff --git a/e2e/collector_test.go b/e2e/collector_test.go index ae4b27b8ef..37e9d84482 100644 --- a/e2e/collector_test.go +++ b/e2e/collector_test.go @@ -29,6 +29,7 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/api/iterator" "google.golang.org/protobuf/types/known/timestamppb" + "gopkg.in/yaml.v3" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -196,6 +197,12 @@ func testCollectorDeployed(ctx context.Context, restConfig *rest.Config, kubeCli } } +// TODO(TheSpiritXIII): https://github.com/go-yaml/yaml/issues/125 +type PrometheusConfig struct { + // Google Cloud configuration. Matches our fork's configuration. + GoogleCloud *operator.GoogleCloudConfig `yaml:"google_cloud,omitempty"` +} + func testCollectorOperatorConfig(ctx context.Context, kubeClient client.Client) func(*testing.T) { return func(t *testing.T) { t.Log("checking collector is configured") @@ -225,49 +232,42 @@ func testCollectorOperatorConfig(ctx context.Context, kubeClient client.Client) t.Fatalf("update operatorconfig: %s", err) } - // Keep checking the state of the collectors until they're running. - err := wait.PollUntilContextCancel(ctx, pollDuration, false, func(ctx context.Context) (bool, error) { - ds := appsv1.DaemonSet{ + var err error + pollErr := wait.PollUntilContextCancel(ctx, pollDuration, false, func(ctx context.Context) (bool, error) { + configMap := corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: operator.NameCollector, Namespace: operator.DefaultOperatorNamespace, + Name: operator.NameCollector, }, } - if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(&ds), &ds); err != nil { + if err = kubeClient.Get(ctx, client.ObjectKeyFromObject(&configMap), &configMap); err != nil { if apierrors.IsNotFound(err) { return false, nil } - return false, fmt.Errorf("getting collector DaemonSet failed: %w", err) + return false, fmt.Errorf("getting collector ConfigMap failed: %w", err) } - // Ensure prometheus container has expected args. - for _, c := range ds.Spec.Template.Spec.Containers { - if c.Name != operator.CollectorPrometheusContainerName { - continue - } - - // We're mainly interested in the dynamic flags but checking the entire set including - // the static ones is ultimately simpler. - wantArgs := []string{ - fmt.Sprintf("--export.label.project-id=%q", projectID), - fmt.Sprintf("--export.label.location=%q", location), - fmt.Sprintf("--export.label.cluster=%q", cluster), - fmt.Sprintf("--export.match=%q", projectFilter), - fmt.Sprintf("--export.match=%q", locationFilter), - fmt.Sprintf("--export.match=%q", kubeletFilter)} - gotArgs := getEnvVar(c.Env, "EXTRA_ARGS") - for _, arg := range wantArgs { - if !strings.Contains(gotArgs, arg) { - return false, fmt.Errorf("expected arg %q not found in EXTRA_ARGS: %q", arg, gotArgs) - } - } - - return true, nil + config := PrometheusConfig{} + data := configMap.Data["config.yaml"] + if err = yaml.Unmarshal([]byte(data), &config); err != nil { + return false, err + } + if config.GoogleCloud == nil || config.GoogleCloud.Export == nil { + err = fmt.Errorf("unable to find Google cloud config:\n%s", data) + return false, nil + } + if !cmp.Equal([]string{projectFilter, locationFilter, kubeletFilter}, config.GoogleCloud.Export.Match) { + err = errors.New("unable to find export matchers") + return false, nil } - return false, errors.New("no container with name prometheus found") + + return true, nil }) - if err != nil { - t.Fatalf("waiting for collector configuration failed: %s", err) + if pollErr != nil { + if wait.Interrupted(pollErr) { + pollErr = err + } + t.Fatalf("waiting for collector configuration failed: %s", pollErr) } } } diff --git a/manifests/operator.yaml b/manifests/operator.yaml index 68612bd7ab..1b18046008 100644 --- a/manifests/operator.yaml +++ b/manifests/operator.yaml @@ -393,7 +393,7 @@ spec: privileged: false readOnlyRootFilesystem: true - name: prometheus - image: gke.gcr.io/prometheus-engine/prometheus@sha256:7473d52f4a3e563e6377f8a6183091f25192b1e0705dd0933903e800bd69b7b2 + image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus:latest args: - --config.file=/prometheus/config_out/config.yaml - --enable-feature=exemplar-storage diff --git a/pkg/operator/collection.go b/pkg/operator/collection.go index 83b2bd678a..d16ca8e708 100644 --- a/pkg/operator/collection.go +++ b/pkg/operator/collection.go @@ -178,7 +178,7 @@ func (r *collectionReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, fmt.Errorf("ensure collector daemon set: %w", err) } - if err := r.ensureCollectorConfig(ctx, &config.Collection, config.Features.Config.Compression, config.Exports); err != nil { + if err := r.ensureCollectorConfig(ctx, &config); err != nil { return reconcile.Result{}, fmt.Errorf("ensure collector config: %w", err) } @@ -250,18 +250,6 @@ func (r *collectionReconciler) ensureCollectorDaemonSet(ctx context.Context, spe fmt.Sprintf("--export.label.location=%q", location), fmt.Sprintf("--export.label.cluster=%q", cluster), } - // Populate export filtering from OperatorConfig. - for _, matcher := range spec.Filter.MatchOneOf { - flags = append(flags, fmt.Sprintf("--export.match=%q", matcher)) - } - if spec.Credentials != nil { - p := path.Join(secretsDir, pathForSelector(r.opts.PublicNamespace, &monitoringv1.SecretOrConfigMap{Secret: spec.Credentials})) - flags = append(flags, fmt.Sprintf("--export.credentials-file=%q", p)) - } - - if len(spec.Compression) > 0 && spec.Compression != monitoringv1.CompressionNone { - flags = append(flags, fmt.Sprintf("--export.compression=%s", spec.Compression)) - } setContainerExtraArgs(ds.Spec.Template.Spec.Containers, CollectorPrometheusContainerName, strings.Join(flags, " ")) return r.client.Update(ctx, &ds) @@ -327,11 +315,26 @@ func setConfigMapData(cm *corev1.ConfigMap, c monitoringv1.CompressionType, key } // ensureCollectorConfig generates the collector config and creates or updates it. -func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, compression monitoringv1.CompressionType, exports []monitoringv1.ExportSpec) error { - cfg, err := r.makeCollectorConfig(ctx, spec, exports) +func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, config *monitoringv1.OperatorConfig) error { + cfg, err := r.makeCollectorConfig(ctx, &config.Collection, config.Exports) if err != nil { return fmt.Errorf("generate Prometheus config: %w", err) } + + var credentialsFile string + if config.Collection.Credentials != nil { + credentialsFile = path.Join(secretsDir, pathForSelector(r.opts.PublicNamespace, &monitoringv1.SecretOrConfigMap{Secret: config.Collection.Credentials})) + } + compression := config.Features.Config.Compression + + cfg.GoogleCloud = &GoogleCloudConfig{ + Export: &GoogleCloudExportConfig{ + Compression: string(compression), + CredentialsFile: credentialsFile, + Match: config.Collection.Filter.MatchOneOf, + }, + } + cfgEncoded, err := yaml.Marshal(cfg) if err != nil { return fmt.Errorf("marshal Prometheus config: %w", err) @@ -357,14 +360,27 @@ func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, spec * return nil } -type prometheusConfig struct { +type PrometheusConfig struct { promconfig.Config `yaml:",inline"` // Secret management. Matches our fork's configuration. SecretConfigs []secrets.SecretConfig `yaml:"kubernetes_secrets,omitempty"` + + // Google Cloud configuration. Matches our fork's configuration. + GoogleCloud *GoogleCloudConfig `yaml:"google_cloud,omitempty"` +} + +type GoogleCloudConfig struct { + Export *GoogleCloudExportConfig `yaml:"export,omitempty"` +} + +type GoogleCloudExportConfig struct { + Match []string `yaml:"match,omitempty"` + Compression string `yaml:"compression,omitempty"` + CredentialsFile string `yaml:"credentials,omitempty"` } -func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, exports []monitoringv1.ExportSpec) (*prometheusConfig, error) { +func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, exports []monitoringv1.ExportSpec) (*PrometheusConfig, error) { logger, _ := logr.FromContext(ctx) cfg := &promconfig.Config{ @@ -495,7 +511,7 @@ func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *mo return cfg.ScrapeConfigs[i].JobName < cfg.ScrapeConfigs[j].JobName }) - return &prometheusConfig{ + return &PrometheusConfig{ Config: *cfg, SecretConfigs: secretConfigs, }, nil