Add PodMonitor + ServiceMonitor TargetLabels

controllers/postgres_controller.go

@@ -56,12 +56,19 @@ import (
 const (
 	postgresExporterServiceName                    string = "postgres-exporter"
 	postgresExporterServicePortName                string = "metrics"
+	postgresExporterServicePortKeyName             string = "postgres-exporter-service-port"
+	postgresExporterServiceTargetPortKeyName       string = "postgres-exporter-service-target-port"
+	walGExporterServicePortName                    string = "backup-metrics"
+	walGExporterServicePortKeyName                 string = "wal-g-exporter-service-port"
+	walGExporterServiceTargetPortKeyName           string = "wal-g-exporter-service-target-port"
 	postgresExporterServiceTenantAnnotationName    string = pg.TenantLabelName
 	postgresExporterServiceProjectIDAnnotationName string = pg.ProjectIDLabelName
 	storageEncryptionKeyName                       string = "storage-encryption-key"
 	storageEncryptionKeyFinalizerName              string = "postgres.database.fits.cloud/secret-finalizer"
 	walGEncryptionSecretNamePostfix                string = "-walg-encryption"
 	walGEncryptionSecretKeyName                    string = "key"
+	podMonitorName                                 string = "patroni"
+	podMonitorPort                                 string = "8008"
 	initDBName                                     string = "postgres-initdb"
 	initDBSQLDummy                                 string = `SELECT 'NOOP';`
 	debugLogLevel                                  int    = 1
@@ -102,6 +109,7 @@ type PostgresReconciler struct {
 	EnableCustomTLSCert                 bool
 	TLSClusterIssuer                    string
 	TLSSubDomain                        string
+	EnableWalGExporter                  bool
 }
 
 type PatroniStandbyCluster struct {
@@ -300,6 +308,12 @@ func (r *PostgresReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		return ctrl.Result{}, fmt.Errorf("error while creating sidecars servicemonitor %v: %w", namespace, err)
 	}
 
+	// Add pod monitor
+	if err := r.createOrUpdatePatroniPodMonitor(ctx, namespace, instance); err != nil {
+		r.recorder.Eventf(instance, "Warning", "Error", "failed to create podmonitor: %v", err)
+		return ctrl.Result{}, fmt.Errorf("error while creating podmonitor %v: %w", namespace, err)
+	}
+
 	// Make sure the storage secret exist, if necessary
 	if err := r.ensureStorageEncryptionSecret(log, ctx, instance); err != nil {
 		r.recorder.Eventf(instance, "Warning", "Error", "failed to create storage secret: %v", err)
@@ -1473,16 +1487,28 @@ func (r *PostgresReconciler) createOrUpdateNetPol(ctx context.Context, instance
 
 // createOrUpdateExporterSidecarServices ensures the necessary services to access the sidecars exist
 func (r *PostgresReconciler) createOrUpdateExporterSidecarServices(log logr.Logger, ctx context.Context, namespace string, c *corev1.ConfigMap, in *pg.Postgres) error {
-	exporterServicePort, error := strconv.ParseInt(c.Data["postgres-exporter-service-port"], 10, 32)
+	pesPort, error := strconv.ParseInt(c.Data[postgresExporterServicePortKeyName], 10, 32)
+	if error != nil {
+		log.Error(error, "postgres-exporter-service-port could not be parsed to int32, falling back to default value")
+		pesPort = 9187
+	}
+
+	pesTargetPort, error := strconv.ParseInt(c.Data[postgresExporterServiceTargetPortKeyName], 10, 32)
+	if error != nil {
+		log.Error(error, "postgres-exporter-service-target-port could not be parsed to int32, falling back to default value")
+		pesTargetPort = pesPort
+	}
+
+	wgesPort, error := strconv.ParseInt(c.Data[walGExporterServicePortKeyName], 10, 32)
 	if error != nil {
-		// todo log error
-		exporterServicePort = 9187
+		log.Error(error, "wal-g-exporter-service-port could not be parsed to int32, falling back to default value")
+		pesPort = 9351
 	}
 
-	exporterServiceTargetPort, error := strconv.ParseInt(c.Data["postgres-exporter-service-target-port"], 10, 32)
+	wgesTargetPort, error := strconv.ParseInt(c.Data[walGExporterServiceTargetPortKeyName], 10, 32)
 	if error != nil {
-		// todo log error
-		exporterServiceTargetPort = exporterServicePort
+		log.Error(error, "wal-g-exporter-service-target-port could not be parsed to int32, falling back to default value")
+		pesTargetPort = pesPort
 	}
 
 	labels := map[string]string{
@@ -1506,11 +1532,20 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServices(log logr.Logg
 	pes.Spec.Ports = []corev1.ServicePort{
 		{
 			Name:       postgresExporterServicePortName,
-			Port:       int32(exporterServicePort), //nolint
+			Port:       int32(pesPort), //nolint
 			Protocol:   corev1.ProtocolTCP,
-			TargetPort: intstr.FromInt(int(exporterServiceTargetPort)),
+			TargetPort: intstr.FromInt(int(pesTargetPort)),
 		},
 	}
+	if r.EnableWalGExporter {
+		wgesp := corev1.ServicePort{
+			Name:       walGExporterServicePortName,
+			Port:       int32(wgesPort), //nolint
+			Protocol:   corev1.ProtocolTCP,
+			TargetPort: intstr.FromInt(int(wgesTargetPort)),
+		}
+		pes.Spec.Ports = append(pes.Spec.Ports, wgesp)
+	}
 	selector := map[string]string{
 		pg.ApplicationLabelName: pg.ApplicationLabelValue,
 	}
@@ -1596,6 +1631,10 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServiceMonitor(log log
 	pesm.Spec.Selector = metav1.LabelSelector{
 		MatchLabels: matchLabels,
 	}
+	pesm.Spec.TargetLabels = []string{
+		"postgres_partition_id=" + in.Spec.PartitionID,
+		"is_primary=" + strconv.FormatBool(in.IsReplicationPrimaryOrStandalone()),
+	}
 
 	// try to fetch any existing postgres-exporter service
 	ns := types.NamespacedName{
@@ -1623,6 +1662,69 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServiceMonitor(log log
 	return nil
 }
 
+// createOrUpdatePatroniPodMonitor ensures the servicemonitors for the sidecars exist
+func (r *PostgresReconciler) createOrUpdatePatroniPodMonitor(ctx context.Context, namespace string, in *pg.Postgres) error {
+	log := r.Log.WithValues("namespace", namespace)
+
+	labels := map[string]string{
+		"app":     "postgres-exporter",
+		"release": "prometheus",
+	}
+
+	annotations := map[string]string{
+		postgresExporterServiceTenantAnnotationName:    in.Spec.Tenant,
+		postgresExporterServiceProjectIDAnnotationName: in.Spec.ProjectID,
+	}
+
+	pm := &coreosv1.PodMonitor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        podMonitorName,
+			Namespace:   namespace,
+			Labels:      labels,
+			Annotations: annotations,
+		},
+	}
+
+	pm.Spec.PodMetricsEndpoints = []coreosv1.PodMetricsEndpoint{
+		{
+			Port: podMonitorPort,
+		},
+	}
+	pm.Spec.NamespaceSelector = coreosv1.NamespaceSelector{
+		MatchNames: []string{namespace},
+	}
+	matchLabels := map[string]string{
+		"application": "spilo",
+	}
+	pm.Spec.Selector = metav1.LabelSelector{
+		MatchLabels: matchLabels,
+	}
+
+	// try to fetch any existing podmonitor
+	ns := types.NamespacedName{
+		Namespace: namespace,
+		Name:      podMonitorName,
+	}
+	old := &coreosv1.PodMonitor{}
+	if err := r.SvcClient.Get(ctx, ns, old); err == nil {
+		// Copy the resource version
+		pm.ObjectMeta.ResourceVersion = old.ObjectMeta.ResourceVersion
+		if err := r.SvcClient.Update(ctx, pm); err != nil {
+			return fmt.Errorf("error while updating the podmonitor: %w", err)
+		}
+		log.Info("pod monitor updated")
+		return nil
+	}
+
+	// local podmonitor does not exist, creating it
+	if err := r.SvcClient.Create(ctx, pm); err != nil {
+		return fmt.Errorf("error while creating the podmonitor: %w", err)
+	}
+	log.Info("podmonitor created")
+
+	return nil
+}
+
 func (r *PostgresReconciler) deleteExporterSidecarService(log logr.Logger, ctx context.Context, namespace string) error {
 	s := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{

main.go

@@ -89,6 +89,7 @@ const (
 	tlsSubDomainFlg                        = "tls-sub-domain"
 	enablePatroniFailsafeModeFlg           = "enable-patroni-failsafe-mode"
 	enableFsGroupChangePolicyWebhookFlg    = "enable-fsgroup-change-policy-webhook"
+	enableWalGExporterFlg                  = "enable-wal-g-exporter"
 )
 
 var (
@@ -150,6 +151,7 @@ func main() {
 		enableSuperUserForDBO               bool
 		enablePatroniFailsafeMode           bool
 		enableFsGroupChangePolicyWebhook    bool
+		enableWalGExporter                  bool
 
 		portRangeStart                        int32
 		portRangeSize                         int32
@@ -315,6 +317,9 @@ func main() {
 	viper.SetDefault(enableFsGroupChangePolicyWebhookFlg, true)
 	enableFsGroupChangePolicyWebhook = viper.GetBool(enableFsGroupChangePolicyWebhookFlg)
 
+	viper.SetDefault(enableWalGEncryptionFlg, true)
+	enableWalGExporter = viper.GetBool(enableWalGExporterFlg)
+
 	ctrl.Log.Info("flag",
 		metricsAddrSvcMgrFlg, metricsAddrSvcMgr,
 		metricsAddrCtrlMgrFlg, metricsAddrCtrlMgr,
@@ -360,6 +365,7 @@ func main() {
 		tlsSubDomainFlg, tlsSubDomain,
 		enablePatroniFailsafeModeFlg, enablePatroniFailsafeMode,
 		enableFsGroupChangePolicyWebhookFlg, enableFsGroupChangePolicyWebhook,
+		enableWalGExporterFlg, enableWalGExporter,
 	)
 
 	svcClusterConf := ctrl.GetConfigOrDie()
@@ -478,6 +484,7 @@ func main() {
 		EnableCustomTLSCert:                 enableCustomTLSCert,
 		TLSClusterIssuer:                    tlsClusterIssuer,
 		TLSSubDomain:                        tlsSubDomain,
+		EnableWalGExporter:                  enableWalGExporter,
 	}).SetupWithManager(ctrlPlaneClusterMgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Postgres")
 		os.Exit(1)