Skip to content

Commit

Permalink
feat(metrics) record CSV sync events
Browse files Browse the repository at this point in the history
This commit introduces a change so OLM will expose metrics about CSV
sync events.
  • Loading branch information
awgreene committed Oct 2, 2019
1 parent a611449 commit 8c851f8
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 8 deletions.
2 changes: 2 additions & 0 deletions pkg/controller/operators/olm/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -930,6 +930,8 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error)
})
logger.Debug("syncing CSV")

metrics.EmitCSVMetric(clusterServiceVersion)

if a.csvNotification != nil {
a.csvNotification.OnAddOrUpdate(clusterServiceVersion)
}
Expand Down
18 changes: 18 additions & 0 deletions pkg/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,16 @@ import (

"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
v1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
olmv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"

)

const (
NAME_LABEL = "name"
INSTALLED_LABEL = "installed"
VERSION_LABEL = "version"
PHASE_LABEL = "phase"
REASON_LABEL = "reason"
)

// TODO(alecmerdler): Can we use this to emit Kubernetes events?
Expand Down Expand Up @@ -145,10 +150,19 @@ var (
},
[]string{NAME_LABEL, INSTALLED_LABEL},
)

csvSyncCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "csv_sync_total",
Help: "Monotonic count of CSV syncs",
},
[]string{NAME_LABEL, VERSION_LABEL, PHASE_LABEL, REASON_LABEL},
)
)

func RegisterOLM() {
prometheus.MustRegister(csvCount)
prometheus.MustRegister(csvSyncCounter)
prometheus.MustRegister(CSVUpgradeCount)
}

Expand All @@ -162,3 +176,7 @@ func RegisterCatalog() {
func CounterForSubscription(name, installedCSV string) prometheus.Counter {
return SubscriptionSyncCount.WithLabelValues(name, installedCSV)
}

func EmitCSVMetric(csv *olmv1alpha1.ClusterServiceVersion){
csvSyncCounter.WithLabelValues(csv.Name, csv.Spec.Version.String(), string(csv.Status.Phase), string(csv.Status.Reason)).Inc()
}
54 changes: 46 additions & 8 deletions test/e2e/metrics_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,34 +6,72 @@ import (
"testing"

log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/net"

"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
)

// TestMetrics tests the metrics endpoint of the OLM pod.
func TestMetricsEndpoint(t *testing.T) {
c := newKubeClient(t)
crc := newCRClient(t)

failingCSV := v1alpha1.ClusterServiceVersion{
TypeMeta: metav1.TypeMeta{
Kind: v1alpha1.ClusterServiceVersionKind,
APIVersion: v1alpha1.ClusterServiceVersionAPIVersion,
},
ObjectMeta: metav1.ObjectMeta{
Name: genName("failing-csv-test-"),
},
Spec: v1alpha1.ClusterServiceVersionSpec{
InstallStrategy: v1alpha1.NamedInstallStrategy{
StrategyName: install.InstallStrategyNameDeployment,
StrategySpecRaw: strategyRaw,
},
},
}

cleanupCSV, err := createCSV(t, c, crc, failingCSV, testNamespace, false, false)
require.NoError(t, err)
defer cleanupCSV()

_, err = fetchCSV(t, crc, failingCSV.Name, testNamespace, csvFailedChecker)
require.NoError(t, err)

rawOutput, err := getMetricsFromPod(t, c, getOLMPodName(t, c), operatorNamespace, "8081")
if err != nil {
t.Fatalf("Metrics test failed: %v\n", err)
}

// Verify metrics have been emitted for packageserver csv
require.Contains(t, rawOutput, "csv_sync_total")
require.Contains(t, rawOutput, "name=\""+failingCSV.Name+"\"")
require.Contains(t, rawOutput, "phase=\"Failed\"")
require.Contains(t, rawOutput, "reason=\"UnsupportedOperatorGroup\"")
require.Contains(t, rawOutput, "version=\"0.0.0\"")
log.Info(rawOutput)
}

func getOLMPodName(t *testing.T, client operatorclient.ClientInterface) string {
listOptions := metav1.ListOptions{LabelSelector: "app=olm-operator"}
podList, err := c.KubernetesInterface().CoreV1().Pods(operatorNamespace).List(listOptions)
podList, err := client.KubernetesInterface().CoreV1().Pods(operatorNamespace).List(listOptions)
if err != nil {
log.Infof("Error %v\n", err)
t.Fatalf("Listing pods failed: %v\n", err)
}
if len(podList.Items) > 1 {
t.Fatalf("Expected only 1 olm-operator pod, got %v", len(podList.Items))
if len(podList.Items) != 1 {
t.Fatalf("Expected 1 olm-operator pod, got %v", len(podList.Items))
}

podName := podList.Items[0].GetName()
log.Infof("Looking at pod %v in namespace %v", podName, operatorNamespace)
return podName

rawOutput, err := getMetricsFromPod(t, c, podName, operatorNamespace, "8081")
if err != nil {
t.Fatalf("Metrics test failed: %v\n", err)
}
log.Info(rawOutput)
}

func getMetricsFromPod(t *testing.T, client operatorclient.ClientInterface, podName string, namespace string, port string) (string, error) {
Expand Down

0 comments on commit 8c851f8

Please sign in to comment.