deploy: add prometheus metrics for deployment conditions

pkg/deploy/api/types.go

@@ -448,6 +448,9 @@ const (
 	// FailedRcCreateReason is added in a deployment config when it cannot create a new replication
 	// controller.
 	FailedRcCreateReason DeploymentConditionReason = "ReplicationControllerCreateError"
+	// PendingDeployerPodReason is added when the replication controller waits for the
+	// deployer pod to run.
+	PendingDeployerPodReason DeploymentConditionReason = "PendingDeployerPod"
 	// NewReplicationControllerReason is added in a deployment config when it creates a new replication
 	// controller.
 	NewReplicationControllerReason DeploymentConditionReason = "NewReplicationControllerCreated"

pkg/deploy/controller/deploymentconfig/deploymentconfig_controller.go

@@ -395,7 +395,7 @@ func calculateStatus(config *deployapi.DeploymentConfig, rcs []*v1.ReplicationCo
 
 	updateConditions(config, &status, latestRC)
 	for _, cond := range additional {
-		deployutil.SetDeploymentCondition(&status, cond)
+		deployutil.SetDeploymentCondition(&status, cond, deploymentConditionCounter)
 	}
 
 	return status
@@ -405,27 +405,27 @@ func updateConditions(config *deployapi.DeploymentConfig, newStatus *deployapi.D
 	// Availability condition.
 	if newStatus.AvailableReplicas >= config.Spec.Replicas-deployutil.MaxUnavailable(config) && newStatus.AvailableReplicas > 0 {
 		minAvailability := deployutil.NewDeploymentCondition(deployapi.DeploymentAvailable, kapi.ConditionTrue, "", "Deployment config has minimum availability.")
-		deployutil.SetDeploymentCondition(newStatus, *minAvailability)
+		deployutil.SetDeploymentCondition(newStatus, *minAvailability, deploymentConditionCounter)
 	} else {
 		noMinAvailability := deployutil.NewDeploymentCondition(deployapi.DeploymentAvailable, kapi.ConditionFalse, "", "Deployment config does not have minimum availability.")
-		deployutil.SetDeploymentCondition(newStatus, *noMinAvailability)
+		deployutil.SetDeploymentCondition(newStatus, *noMinAvailability, deploymentConditionCounter)
 	}
 
 	// Condition about progress.
 	if latestRC != nil {
 		switch deployutil.DeploymentStatusFor(latestRC) {
 		case deployapi.DeploymentStatusPending:
 			msg := fmt.Sprintf("replication controller %q is waiting for pod %q to run", latestRC.Name, deployutil.DeployerPodNameForDeployment(latestRC.Name))
-			condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionUnknown, "", msg)
-			deployutil.SetDeploymentCondition(newStatus, *condition)
+			condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionUnknown, deployapi.PendingDeployerPodReason, msg)
+			deployutil.SetDeploymentCondition(newStatus, *condition, deploymentConditionCounter)
 		case deployapi.DeploymentStatusRunning:
 			if deployutil.IsProgressing(config, newStatus) {
-				deployutil.RemoveDeploymentCondition(newStatus, deployapi.DeploymentProgressing)
+				deployutil.RemoveDeploymentCondition(newStatus, deployapi.DeploymentProgressing, deploymentConditionCounter)
 				msg := fmt.Sprintf("replication controller %q is progressing", latestRC.Name)
 				condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionTrue, deployapi.ReplicationControllerUpdatedReason, msg)
 				// TODO: Right now, we use lastTransitionTime for storing the last time we had any progress instead
 				// of the last time the condition transitioned to a new status. We should probably change that.
-				deployutil.SetDeploymentCondition(newStatus, *condition)
+				deployutil.SetDeploymentCondition(newStatus, *condition, deploymentConditionCounter)
 			}
 		case deployapi.DeploymentStatusFailed:
 			var condition *deployapi.DeploymentCondition
@@ -436,11 +436,11 @@ func updateConditions(config *deployapi.DeploymentConfig, newStatus *deployapi.D
 				msg := fmt.Sprintf("replication controller %q has failed progressing", latestRC.Name)
 				condition = deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionFalse, deployapi.TimedOutReason, msg)
 			}
-			deployutil.SetDeploymentCondition(newStatus, *condition)
+			deployutil.SetDeploymentCondition(newStatus, *condition, deploymentConditionCounter)
 		case deployapi.DeploymentStatusComplete:
 			msg := fmt.Sprintf("replication controller %q successfully rolled out", latestRC.Name)
 			condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionTrue, deployapi.NewRcAvailableReason, msg)
-			deployutil.SetDeploymentCondition(newStatus, *condition)
+			deployutil.SetDeploymentCondition(newStatus, *condition, deploymentConditionCounter)
 		}
 	}
 }

pkg/deploy/controller/deploymentconfig/metrics.go

@@ -0,0 +1,22 @@
+package deploymentconfig
+
+import "github.com/prometheus/client_golang/prometheus"
+
+// DeploymentConfigControllerSubsystem is how this controller is represented in
+// prometheus metrics.
+const DeploymentConfigControllerSubsystem = "deploymentconfig_controller"
+
+var (
+	deploymentConditionCounter = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Subsystem: DeploymentConfigControllerSubsystem,
+			Name:      "set_condition",
+			Help:      "Gauge measuring changes to deployment conditions per condition, reason and the status",
+		},
+		[]string{"condition", "reason", "status"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(deploymentConditionCounter)
+}

pkg/deploy/util/util.go

@@ -21,6 +21,7 @@ import (
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
 	deployapiv1 "github.com/openshift/origin/pkg/deploy/api/v1"
 	"github.com/openshift/origin/pkg/util/namer"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
@@ -53,9 +54,17 @@ func GetDeploymentCondition(status deployapi.DeploymentConfigStatus, condType de
 	return nil
 }
 
+func ConditionToMetrics(condition deployapi.DeploymentCondition) []string {
+	return []string{
+		strings.ToLower(string(condition.Type)),
+		strings.ToLower(string(condition.Reason)),
+		strings.ToLower(string(condition.Status)),
+	}
+}
+
 // SetDeploymentCondition updates the deployment to include the provided condition. If the condition that
 // we are about to add already exists and has the same status and reason then we are not going to update.
-func SetDeploymentCondition(status *deployapi.DeploymentConfigStatus, condition deployapi.DeploymentCondition) {
+func SetDeploymentCondition(status *deployapi.DeploymentConfigStatus, condition deployapi.DeploymentCondition, metrics *prometheus.GaugeVec) {
 	currentCond := GetDeploymentCondition(*status, condition.Type)
 	if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason {
 		return
@@ -66,11 +75,26 @@ func SetDeploymentCondition(status *deployapi.DeploymentConfigStatus, condition
 	}
 	newConditions := filterOutCondition(status.Conditions, condition.Type)
 	status.Conditions = append(newConditions, condition)
+	// Update the metrics with new condition
+	metrics.WithLabelValues(ConditionToMetrics(condition)...).Inc()
 }
 
 // RemoveDeploymentCondition removes the deployment condition with the provided type.
-func RemoveDeploymentCondition(status *deployapi.DeploymentConfigStatus, condType deployapi.DeploymentConditionType) {
+func RemoveDeploymentCondition(status *deployapi.DeploymentConfigStatus, condType deployapi.DeploymentConditionType, metrics *prometheus.GaugeVec) {
+	oldConditions := status.Conditions
 	status.Conditions = filterOutCondition(status.Conditions, condType)
+	// Decrease counters for conditions we are going to remove
+	for _, oldCondition := range oldConditions {
+		found := false
+		for _, newCondition := range status.Conditions {
+			if found = oldCondition == newCondition; found {
+				break
+			}
+		}
+		if !found {
+			metrics.WithLabelValues(ConditionToMetrics(oldCondition)...).Dec()
+		}
+	}
 }
 
 // filterOutCondition returns a new slice of deployment conditions without conditions with the provided type.