Skip to content

Commit

Permalink
feat: move Prometheus flags to config
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheSpiritXIII
TheSpiritXIII committed Jun 11, 2024
1 parent 494e1a7 commit a9a57d5
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 53 deletions.
4 changes: 2 additions & 2 deletions charts/values.global.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ images:
tag: v0.25.1-gmp.2-gke.0
prometheus:
# TODO(bwplotka): Change to "v2.45.3-gmp.4-gke.0" once tags are cloned.
image: gke.gcr.io/prometheus-engine/prometheus@sha256
tag: 7473d52f4a3e563e6377f8a6183091f25192b1e0705dd0933903e800bd69b7b2
image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus
tag: latest
configReloader:
image: gke.gcr.io/prometheus-engine/config-reloader
tag: v0.9.0-gke.1
Expand Down
64 changes: 32 additions & 32 deletions e2e/collector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/google/go-cmp/cmp"
"google.golang.org/api/iterator"
"google.golang.org/protobuf/types/known/timestamppb"
"gopkg.in/yaml.v3"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -196,6 +197,12 @@ func testCollectorDeployed(ctx context.Context, restConfig *rest.Config, kubeCli
}
}

// TODO(TheSpiritXIII): https://github.com/go-yaml/yaml/issues/125
type PrometheusConfig struct {
// Google Cloud configuration. Matches our fork's configuration.
GoogleCloud *operator.GoogleCloudConfig `yaml:"google_cloud,omitempty"`
}

func testCollectorOperatorConfig(ctx context.Context, kubeClient client.Client) func(*testing.T) {
return func(t *testing.T) {
t.Log("checking collector is configured")
Expand Down Expand Up @@ -225,49 +232,42 @@ func testCollectorOperatorConfig(ctx context.Context, kubeClient client.Client)
t.Fatalf("update operatorconfig: %s", err)
}

// Keep checking the state of the collectors until they're running.
err := wait.PollUntilContextCancel(ctx, pollDuration, false, func(ctx context.Context) (bool, error) {
ds := appsv1.DaemonSet{
var err error
pollErr := wait.PollUntilContextCancel(ctx, pollDuration, false, func(ctx context.Context) (bool, error) {
configMap := corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: operator.NameCollector,
Namespace: operator.DefaultOperatorNamespace,
Name: operator.NameCollector,
},
}
if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(&ds), &ds); err != nil {
if err = kubeClient.Get(ctx, client.ObjectKeyFromObject(&configMap), &configMap); err != nil {
if apierrors.IsNotFound(err) {
return false, nil
}
return false, fmt.Errorf("getting collector DaemonSet failed: %w", err)
return false, fmt.Errorf("getting collector ConfigMap failed: %w", err)
}

// Ensure prometheus container has expected args.
for _, c := range ds.Spec.Template.Spec.Containers {
if c.Name != operator.CollectorPrometheusContainerName {
continue
}

// We're mainly interested in the dynamic flags but checking the entire set including
// the static ones is ultimately simpler.
wantArgs := []string{
fmt.Sprintf("--export.label.project-id=%q", projectID),
fmt.Sprintf("--export.label.location=%q", location),
fmt.Sprintf("--export.label.cluster=%q", cluster),
fmt.Sprintf("--export.match=%q", projectFilter),
fmt.Sprintf("--export.match=%q", locationFilter),
fmt.Sprintf("--export.match=%q", kubeletFilter)}
gotArgs := getEnvVar(c.Env, "EXTRA_ARGS")
for _, arg := range wantArgs {
if !strings.Contains(gotArgs, arg) {
return false, fmt.Errorf("expected arg %q not found in EXTRA_ARGS: %q", arg, gotArgs)
}
}

return true, nil
config := PrometheusConfig{}
data := configMap.Data["config.yaml"]
if err = yaml.Unmarshal([]byte(data), &config); err != nil {
return false, err
}
if config.GoogleCloud == nil || config.GoogleCloud.Export == nil {
err = fmt.Errorf("unable to find Google cloud config:\n%s", data)
return false, nil
}
if !cmp.Equal([]string{projectFilter, locationFilter, kubeletFilter}, config.GoogleCloud.Export.Match) {
err = errors.New("unable to find export matchers")
return false, nil
}
return false, errors.New("no container with name prometheus found")

return true, nil
})
if err != nil {
t.Fatalf("waiting for collector configuration failed: %s", err)
if pollErr != nil {
if wait.Interrupted(pollErr) {
pollErr = err
}
t.Fatalf("waiting for collector configuration failed: %s", pollErr)
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion manifests/operator.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ spec:
privileged: false
readOnlyRootFilesystem: true
- name: prometheus
image: gke.gcr.io/prometheus-engine/prometheus@sha256:7473d52f4a3e563e6377f8a6183091f25192b1e0705dd0933903e800bd69b7b2
image: us-central1-docker.pkg.dev/gpe-test-1/dhrabovcak/prometheus:latest
args:
- --config.file=/prometheus/config_out/config.yaml
- --enable-feature=exemplar-storage
Expand Down
52 changes: 34 additions & 18 deletions pkg/operator/collection.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ func (r *collectionReconciler) Reconcile(ctx context.Context, req reconcile.Requ
return reconcile.Result{}, fmt.Errorf("ensure collector daemon set: %w", err)
}

if err := r.ensureCollectorConfig(ctx, &config.Collection, config.Features.Config.Compression, config.Exports); err != nil {
if err := r.ensureCollectorConfig(ctx, &config); err != nil {
return reconcile.Result{}, fmt.Errorf("ensure collector config: %w", err)
}

Expand Down Expand Up @@ -250,18 +250,6 @@ func (r *collectionReconciler) ensureCollectorDaemonSet(ctx context.Context, spe
fmt.Sprintf("--export.label.location=%q", location),
fmt.Sprintf("--export.label.cluster=%q", cluster),
}
// Populate export filtering from OperatorConfig.
for _, matcher := range spec.Filter.MatchOneOf {
flags = append(flags, fmt.Sprintf("--export.match=%q", matcher))
}
if spec.Credentials != nil {
p := path.Join(secretsDir, pathForSelector(r.opts.PublicNamespace, &monitoringv1.SecretOrConfigMap{Secret: spec.Credentials}))
flags = append(flags, fmt.Sprintf("--export.credentials-file=%q", p))
}

if len(spec.Compression) > 0 && spec.Compression != monitoringv1.CompressionNone {
flags = append(flags, fmt.Sprintf("--export.compression=%s", spec.Compression))
}
setContainerExtraArgs(ds.Spec.Template.Spec.Containers, CollectorPrometheusContainerName, strings.Join(flags, " "))

return r.client.Update(ctx, &ds)
Expand Down Expand Up @@ -327,11 +315,26 @@ func setConfigMapData(cm *corev1.ConfigMap, c monitoringv1.CompressionType, key
}

// ensureCollectorConfig generates the collector config and creates or updates it.
func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, compression monitoringv1.CompressionType, exports []monitoringv1.ExportSpec) error {
cfg, err := r.makeCollectorConfig(ctx, spec, exports)
func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, config *monitoringv1.OperatorConfig) error {
cfg, err := r.makeCollectorConfig(ctx, &config.Collection, config.Exports)
if err != nil {
return fmt.Errorf("generate Prometheus config: %w", err)
}

var credentialsFile string
if config.Collection.Credentials != nil {
credentialsFile = path.Join(secretsDir, pathForSelector(r.opts.PublicNamespace, &monitoringv1.SecretOrConfigMap{Secret: config.Collection.Credentials}))
}
compression := config.Features.Config.Compression

cfg.GoogleCloud = &GoogleCloudConfig{
Export: &GoogleCloudExportConfig{
Compression: string(compression),
CredentialsFile: credentialsFile,
Match: config.Collection.Filter.MatchOneOf,
},
}

cfgEncoded, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshal Prometheus config: %w", err)
Expand All @@ -357,14 +360,27 @@ func (r *collectionReconciler) ensureCollectorConfig(ctx context.Context, spec *
return nil
}

type prometheusConfig struct {
type PrometheusConfig struct {
promconfig.Config `yaml:",inline"`

// Secret management. Matches our fork's configuration.
SecretConfigs []secrets.SecretConfig `yaml:"kubernetes_secrets,omitempty"`

// Google Cloud configuration. Matches our fork's configuration.
GoogleCloud *GoogleCloudConfig `yaml:"google_cloud,omitempty"`
}

type GoogleCloudConfig struct {
Export *GoogleCloudExportConfig `yaml:"export,omitempty"`
}

type GoogleCloudExportConfig struct {
Match []string `yaml:"match,omitempty"`
Compression string `yaml:"compression,omitempty"`
CredentialsFile string `yaml:"credentials,omitempty"`
}

func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, exports []monitoringv1.ExportSpec) (*prometheusConfig, error) {
func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *monitoringv1.CollectionSpec, exports []monitoringv1.ExportSpec) (*PrometheusConfig, error) {
logger, _ := logr.FromContext(ctx)

cfg := &promconfig.Config{
Expand Down Expand Up @@ -495,7 +511,7 @@ func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *mo
return cfg.ScrapeConfigs[i].JobName < cfg.ScrapeConfigs[j].JobName
})

return &prometheusConfig{
return &PrometheusConfig{
Config: *cfg,
SecretConfigs: secretConfigs,
}, nil
Expand Down

0 comments on commit a9a57d5

Please sign in to comment.