*: use a bearer token file

For security concerns, it's better to pass the bearer token via a Secret
rather than sticking it in the Prometheus custom resource.

Signed-off-by: Simon Pasquier <spasquie@redhat.com>

assets/prometheus-k8s/telemetry-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/name: prometheus-k8s
+  name: telemetry-server
+  namespace: openshift-monitoring
+type: Opaque

hack/local-cmo.sh

@@ -106,6 +106,7 @@ main(){
     -kubeconfig "${KUBECONFIG}" \
     -namespace=openshift-monitoring \
     -configmap=cluster-monitoring-config \
+    -enabled-remote-write \
     -logtostderr=true -v=4 2>&1 | tee operator.log
 }
 

jsonnet/components/prometheus.libsonnet

@@ -217,6 +217,19 @@ function(params)
 
     kubeRbacProxySecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-rbac-proxy'),
 
+    // Secret holding the token to authenticate against the Telemetry server when using native remote-write.
+    telemetrySecret: {
+      apiVersion: 'v1',
+      kind: 'Secret',
+      metadata: {
+        name: 'telemetry-server',
+        namespace: cfg.namespace,
+        labels: { 'app.kubernetes.io/name': 'prometheus-k8s' },
+      },
+      type: 'Opaque',
+      data: {},
+    },
+
     // This changes the Prometheuses to be scraped with TLS, authN and
     // authZ, which are not present in kube-prometheus.
 

pkg/manifests/manifests.go

@@ -147,6 +147,7 @@ var (
 	PrometheusK8sThanosSidecarServiceMonitor      = "prometheus-k8s/service-monitor-thanos-sidecar.yaml"
 	PrometheusK8sTAlertmanagerRoleBinding         = "prometheus-k8s/alertmanager-role-binding.yaml"
 	PrometheusK8sPodDisruptionBudget              = "prometheus-k8s/pod-disruption-budget.yaml"
+	PrometheusK8sTelemetry                        = "prometheus-k8s/telemetry-secret.yaml"
 
 	PrometheusUserWorkloadServingCertsCABundle        = "prometheus-user-workload/serving-certs-ca-bundle.yaml"
 	PrometheusUserWorkloadServiceAccount              = "prometheus-user-workload/service-account.yaml"
@@ -290,6 +291,8 @@ var (
 	ControlPlanePrometheusRule        = "control-plane/prometheus-rule.yaml"
 	ControlPlaneKubeletServiceMonitor = "control-plane/service-monitor-kubelet.yaml"
 	ControlPlaneEtcdServiceMonitor    = "control-plane/service-monitor-etcd.yaml"
+
+	telemetryTokenSecretKey = "token"
 )
 
 var (
@@ -1619,7 +1622,29 @@ func (f *Factory) PrometheusK8sTrustedCABundle() (*v1.ConfigMap, error) {
 	return cm, nil
 }
 
-func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap) (*monv1.Prometheus, error) {
+func (f *Factory) PrometheusK8sTelemetrySecret() (*v1.Secret, error) {
+	s, err := f.NewSecret(f.assets.MustNewAssetReader(PrometheusK8sTelemetry))
+	if err != nil {
+		return nil, err
+	}
+	compositeToken, err := json.Marshal(map[string]string{
+		"cluster_id":          f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID,
+		"authorization_token": f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.Token,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	b := make([]byte, base64.StdEncoding.EncodedLen(len(compositeToken)))
+	base64.StdEncoding.Encode(b, compositeToken)
+	s.Data = map[string][]byte{
+		telemetryTokenSecretKey: b,
+	}
+
+	return s, nil
+}
+
+func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap, telemetrySecret *v1.Secret) (*monv1.Prometheus, error) {
 	p, err := f.NewPrometheus(f.assets.MustNewAssetReader(PrometheusK8s))
 	if err != nil {
 		return nil, err
@@ -1674,23 +1699,18 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.Config
 		return nil, err
 	}
 
-	telemetryEnabled := f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.IsEnabled()
 	clusterID := f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID
-	if telemetryEnabled && f.config.RemoteWrite {
-
+	if telemetrySecret != nil {
 		selectorRelabelConfig, err := promqlgen.LabelSelectorsToRelabelConfig(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.TelemetryMatches)
 		if err != nil {
 			return nil, errors.Wrap(err, "generate label selector relabel config")
 		}
 
-		compositeToken, err := json.Marshal(map[string]string{
-			"cluster_id":          clusterID,
-			"authorization_token": f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.Token,
-		})
+		p.Spec.Secrets = append(p.Spec.Secrets, telemetrySecret.GetName())
 
 		spec := monv1.RemoteWriteSpec{
-			URL:         f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.TelemeterServerURL,
-			BearerToken: base64.StdEncoding.EncodeToString(compositeToken),
+			URL:             f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.TelemeterServerURL,
+			BearerTokenFile: fmt.Sprintf("/etc/prometheus/secrets/%s/%s", telemetrySecret.GetName(), telemetryTokenSecretKey),
 			QueueConfig: &monv1.QueueConfig{
 				// Amount of samples to load from the WAL into the in-memory
 				// buffer before waiting for samples to be sent successfully
@@ -1733,10 +1753,6 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.Config
 		}
 
 		p.Spec.RemoteWrite = []monv1.RemoteWriteSpec{spec}
-
-	}
-	if !telemetryEnabled {
-		p.Spec.RemoteWrite = nil
 	}
 
 	if len(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite) > 0 {

pkg/manifests/manifests_test.go

@@ -379,7 +379,7 @@ func TestUnconfiguredManifests(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = f.PrometheusK8s(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil)
+	_, err = f.PrometheusK8s(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1128,6 +1128,7 @@ func TestPrometheusK8sRemoteWriteClusterIDRelabel(t *testing.T) {
 			p, err := f.PrometheusK8s(
 				&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+				nil,
 			)
 			if err != nil {
 				t.Fatal(err)
@@ -1146,30 +1147,23 @@ func TestPrometheusK8sRemoteWriteClusterIDRelabel(t *testing.T) {
 }
 
 func TestPrometheusK8sRemoteWriteURLs(t *testing.T) {
+	telemetrySecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "telemetry",
+			Namespace: "openshift-monitoring",
+		},
+	}
 	for _, tc := range []struct {
 		name                    string
 		config                  func() *Config
+		telemetrySecret         *v1.Secret
 		expectedRemoteWriteURLs []string
 	}{
 		{
 			name: "default config",
 
 			config: func() *Config {
-				c := NewDefaultConfig()
-				return c
-			},
-
-			expectedRemoteWriteURLs: nil,
-		},
-		{
-			name: "legacy telemetry",
-
-			config: func() *Config {
-				c := NewDefaultConfig()
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID = "123"
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token = "secret"
-
-				return c
+				return NewDefaultConfig()
 			},
 
 			expectedRemoteWriteURLs: nil,
@@ -1179,8 +1173,6 @@ func TestPrometheusK8sRemoteWriteURLs(t *testing.T) {
 
 			config: func() *Config {
 				c := NewDefaultConfig()
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID = "123"
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token = "secret"
 				c.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite = []RemoteWriteSpec{{URL: "http://custom"}}
 
 				return c
@@ -1196,11 +1188,10 @@ func TestPrometheusK8sRemoteWriteURLs(t *testing.T) {
 			config: func() *Config {
 				c := NewDefaultConfig()
 				c.SetRemoteWrite(true)
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID = "123"
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token = "secret"
 
 				return c
 			},
+			telemetrySecret: telemetrySecret,
 
 			expectedRemoteWriteURLs: []string{
 				"https://infogw.api.openshift.com/metrics/v1/receive",
@@ -1212,12 +1203,11 @@ func TestPrometheusK8sRemoteWriteURLs(t *testing.T) {
 			config: func() *Config {
 				c := NewDefaultConfig()
 				c.SetRemoteWrite(true)
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID = "123"
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token = "secret"
 				c.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite = []RemoteWriteSpec{{URL: "http://custom"}}
 
 				return c
 			},
+			telemetrySecret: telemetrySecret,
 
 			expectedRemoteWriteURLs: []string{
 				"http://custom",
@@ -1231,12 +1221,11 @@ func TestPrometheusK8sRemoteWriteURLs(t *testing.T) {
 				c := NewDefaultConfig()
 				c.SetRemoteWrite(true)
 				c.ClusterMonitoringConfiguration.TelemeterClientConfig.TelemeterServerURL = "http://custom-telemeter"
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID = "123"
-				c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token = "secret"
 				c.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite = []RemoteWriteSpec{{URL: "http://custom-remote-write"}}
 
 				return c
 			},
+			telemetrySecret: telemetrySecret,
 
 			expectedRemoteWriteURLs: []string{
 				"http://custom-remote-write",
@@ -1251,6 +1240,7 @@ func TestPrometheusK8sRemoteWriteURLs(t *testing.T) {
 			p, err := f.PrometheusK8s(
 				&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+				tc.telemetrySecret,
 			)
 			if err != nil {
 				t.Fatal(err)
@@ -1321,6 +1311,7 @@ func TestPrometheusK8sRemoteWriteOauth2(t *testing.T) {
 	p, err := f.PrometheusK8s(
 		&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 		&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+		nil,
 	)
 	if err != nil {
 		t.Fatal(err)
@@ -1463,6 +1454,7 @@ func TestRemoteWriteAuthorizationConfig(t *testing.T) {
 			p, err := f.PrometheusK8s(
 				&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+				nil,
 			)
 			if err != nil {
 				t.Fatal(err)
@@ -1529,6 +1521,7 @@ ingress:
 	p, err := f.PrometheusK8s(
 		&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 		&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+		nil,
 	)
 	if err != nil {
 		t.Fatal(err)
@@ -1727,6 +1720,7 @@ func TestPrometheusQueryLogFileConfig(t *testing.T) {
 			p, err := f.PrometheusK8s(
 				&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+				nil,
 			)
 			if err != nil {
 				if !tc.errExpected {
@@ -1806,6 +1800,7 @@ func TestPrometheusRetentionConfigs(t *testing.T) {
 			p, err := f.PrometheusK8s(
 				&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+				nil,
 			)
 
 			if err != nil {
@@ -1857,6 +1852,7 @@ prometheusK8s:
 	p, err := f.PrometheusK8s(
 		&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 		&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+		nil,
 	)
 	if err != nil {
 		t.Fatal(err)
@@ -2063,6 +2059,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
 			p, err := f.PrometheusK8s(
 				&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+				nil,
 			)
 			if err != nil {
 				t.Fatal(err)
@@ -3401,6 +3398,7 @@ func TestNonHighlyAvailableInfrastructure(t *testing.T) {
 				p, err := f.PrometheusK8s(
 					&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 					&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+					nil,
 				)
 				if err != nil {
 					return spec{}, err

pkg/tasks/prometheus.go

@@ -306,6 +306,24 @@ func (t *PrometheusTask) Run(ctx context.Context) error {
 		}
 	}
 
+	telemetrySecret, err := t.factory.PrometheusK8sTelemetrySecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Prometheus telemetry secret failed")
+	}
+
+	if t.config.ClusterMonitoringConfiguration.TelemeterClientConfig.IsEnabled() && t.config.RemoteWrite {
+		klog.V(4).Info("updating Prometheus telemetry secret")
+		if err = t.client.CreateOrUpdateSecret(ctx, telemetrySecret); err != nil {
+			return errors.Wrap(err, "reconciling Prometheus telemetry secret failed")
+		}
+	} else {
+		klog.V(4).Info("deleting Prometheus telemetry secret")
+		if err = t.client.DeleteSecret(ctx, telemetrySecret); err != nil {
+			return errors.Wrap(err, "deleting Prometheus telemetry secret failed")
+		}
+		telemetrySecret = nil
+	}
+
 	{
 		// Create trusted CA bundle ConfigMap.
 		trustedCA, err := t.factory.PrometheusK8sTrustedCABundle()
@@ -334,7 +352,7 @@ func (t *PrometheusTask) Run(ctx context.Context) error {
 		}
 
 		klog.V(4).Info("initializing Prometheus object")
-		p, err := t.factory.PrometheusK8s(s, trustedCA)
+		p, err := t.factory.PrometheusK8s(s, trustedCA, telemetrySecret)
 		if err != nil {
 			return errors.Wrap(err, "initializing Prometheus object failed")
 		}