Skip to content

Commit

Permalink
Use env RUNBOOK_URL_TEMPLATE for the runbooks URL template (#307)
Browse files Browse the repository at this point in the history
Signed-off-by: assafad <aadmi@redhat.com>
  • Loading branch information
assafad committed May 9, 2023
1 parent f4b9755 commit 7565d9a
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 14 deletions.
5 changes: 4 additions & 1 deletion pkg/controller/hostpathprovisioner/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1097,13 +1097,16 @@ func verifyCreatePrometheusResources(cl client.Client) {
Expect(r.Annotations).To(BeNil())
}
}

runbookURLTemplate := getRunbookURLTemplate()

hppDownAlert := promv1.Rule{
Alert: "HPPOperatorDown",
Expr: intstr.FromString("kubevirt_hpp_operator_up_total == 0"),
For: "5m",
Annotations: map[string]string{
"summary": "Hostpath Provisioner operator is down",
"runbook_url": runbookURLBasePath + "HPPOperatorDown",
"runbook_url": fmt.Sprintf(runbookURLTemplate, "HPPOperatorDown"),
},
Labels: map[string]string{
"severity": "warning",
Expand Down
44 changes: 31 additions & 13 deletions pkg/controller/hostpathprovisioner/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ package hostpathprovisioner

import (
"context"
"errors"
"fmt"
"os"
"reflect"
"strings"

promv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
"github.com/go-logr/logr"
Expand All @@ -29,7 +31,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

"k8s.io/apimachinery/pkg/api/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
Expand All @@ -41,7 +43,8 @@ const (
rbacName = "hostpath-provisioner-monitoring"
monitorName = "service-monitor-hpp"
defaultMonitoringNs = "monitoring"
runbookURLBasePath = "https://kubevirt.io/monitoring/runbooks/"
defaultRunbookURLTemplate = "https://kubevirt.io/monitoring/runbooks/%s"
runbookURLTemplateEnv = "RUNBOOK_URL_TEMPLATE"
severityAlertLabelKey = "severity"
healthImpactAlertLabelKey = "operator_health_impact"
partOfAlertLabelKey = "kubernetes_operator_part_of"
Expand Down Expand Up @@ -76,7 +79,7 @@ func (r *ReconcileHostPathProvisioner) reconcilePrometheusResource(reqLogger log
setLastAppliedConfiguration(desired)
// Check if this PrometheusRule already exists
err := r.client.Get(context.TODO(), client.ObjectKeyFromObject(found), found)
if err != nil && errors.IsNotFound(err) {
if err != nil && k8serrors.IsNotFound(err) {
reqLogger.Info("Creating a new PrometheusResource", "Name", found.GetName())
err = r.client.Create(context.TODO(), desired)
if err != nil {
Expand Down Expand Up @@ -131,7 +134,7 @@ func (r *ReconcileHostPathProvisioner) deletePrometheusResources(namespace strin
Namespace: namespace,
},
}
if err := r.client.Delete(context.TODO(), rule); err != nil && !errors.IsNotFound(err) {
if err := r.client.Delete(context.TODO(), rule); err != nil && !k8serrors.IsNotFound(err) {
return err
}

Expand All @@ -141,7 +144,7 @@ func (r *ReconcileHostPathProvisioner) deletePrometheusResources(namespace strin
Namespace: namespace,
},
}
if err := r.client.Delete(context.TODO(), role); err != nil && !errors.IsNotFound(err) {
if err := r.client.Delete(context.TODO(), role); err != nil && !k8serrors.IsNotFound(err) {
return err
}

Expand All @@ -151,7 +154,7 @@ func (r *ReconcileHostPathProvisioner) deletePrometheusResources(namespace strin
Namespace: namespace,
},
}
if err := r.client.Delete(context.TODO(), roleBinding); err != nil && !errors.IsNotFound(err) {
if err := r.client.Delete(context.TODO(), roleBinding); err != nil && !k8serrors.IsNotFound(err) {
return err
}

Expand All @@ -161,7 +164,7 @@ func (r *ReconcileHostPathProvisioner) deletePrometheusResources(namespace strin
Namespace: namespace,
},
}
if err := r.client.Delete(context.TODO(), monitor); err != nil && !errors.IsNotFound(err) {
if err := r.client.Delete(context.TODO(), monitor); err != nil && !k8serrors.IsNotFound(err) {
return err
}

Expand All @@ -171,7 +174,7 @@ func (r *ReconcileHostPathProvisioner) deletePrometheusResources(namespace strin
Namespace: namespace,
},
}
if err := r.client.Delete(context.TODO(), service); err != nil && !errors.IsNotFound(err) {
if err := r.client.Delete(context.TODO(), service); err != nil && !k8serrors.IsNotFound(err) {
return err
}

Expand Down Expand Up @@ -208,15 +211,15 @@ func getRecordRules(namespace string) []promv1.Rule {
return recordRules
}

func getAlertRules() []promv1.Rule {
func getAlertRules(runbookURLTemplate string) []promv1.Rule {
return []promv1.Rule{
generateAlertRule(
"HPPOperatorDown",
"kubevirt_hpp_operator_up_total == 0",
"5m",
map[string]string{
"summary": "Hostpath Provisioner operator is down",
"runbook_url": runbookURLBasePath + "HPPOperatorDown",
"runbook_url": fmt.Sprintf(runbookURLTemplate, "HPPOperatorDown"),
},
map[string]string{
severityAlertLabelKey: "warning",
Expand All @@ -231,7 +234,7 @@ func getAlertRules() []promv1.Rule {
"5m",
map[string]string{
"summary": "Hostpath Provisioner is not available to use",
"runbook_url": runbookURLBasePath + "HPPNotReady",
"runbook_url": fmt.Sprintf(runbookURLTemplate, "HPPNotReady"),
},
map[string]string{
severityAlertLabelKey: "warning",
Expand All @@ -246,7 +249,7 @@ func getAlertRules() []promv1.Rule {
"1m",
map[string]string{
"summary": "HPP pool path sharing a filesystem with OS, fix to prevent HPP PVs from causing disk pressure and affecting node operation",
"runbook_url": runbookURLBasePath + "HPPSharingPoolPathWithOS",
"runbook_url": fmt.Sprintf(runbookURLTemplate, "HPPSharingPoolPathWithOS"),
},
map[string]string{
severityAlertLabelKey: "warning",
Expand All @@ -262,6 +265,8 @@ func createPrometheusRule(namespace string) *promv1.PrometheusRule {
labels := getRecommendedLabels()
labels[PrometheusLabelKey] = PrometheusLabelValue

runbookURLTemplate := getRunbookURLTemplate()

return &promv1.PrometheusRule{
TypeMeta: metav1.TypeMeta{
APIVersion: promv1.SchemeGroupVersion.String(),
Expand All @@ -276,7 +281,7 @@ func createPrometheusRule(namespace string) *promv1.PrometheusRule {
Groups: []promv1.RuleGroup{
{
Name: "hpp.rules",
Rules: append(getRecordRules(namespace), getAlertRules()...),
Rules: append(getRecordRules(namespace), getAlertRules(runbookURLTemplate)...),
},
},
},
Expand Down Expand Up @@ -438,3 +443,16 @@ func (r *ReconcileHostPathProvisioner) checkPrometheusUsed() (bool, error) {
}
return true, nil
}

func getRunbookURLTemplate() string {
runbookURLTemplate, exists := os.LookupEnv(runbookURLTemplateEnv)
if !exists {
runbookURLTemplate = defaultRunbookURLTemplate
}

if strings.Count(runbookURLTemplate, "%s") != 1 {
panic(errors.New("runbook URL template must have exactly 1 %s substring"))
}

return runbookURLTemplate
}
50 changes: 50 additions & 0 deletions pkg/controller/hostpathprovisioner/prometheus_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package hostpathprovisioner

import (
"fmt"
"os"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

var _ = Describe("Prometheus", func() {
BeforeEach(func() {
os.Unsetenv(runbookURLTemplateEnv)
})

AfterEach(func() {
os.Unsetenv(runbookURLTemplateEnv)
})

It("should use the default runbook URL template when no ENV Variable is set", func() {
promRule := createPrometheusRule("mynamespace")

for _, group := range promRule.Spec.Groups {
for _, rule := range group.Rules {
if rule.Alert != "" {
if rule.Annotations["runbook_url"] != "" {
Expect(rule.Annotations["runbook_url"]).To(Equal(fmt.Sprintf(defaultRunbookURLTemplate, rule.Alert)))
}
}
}
}
})

It("should use the desired runbook URL template when its ENV Variable is set", func() {
desiredRunbookURLTemplate := "desired/runbookURL/template/%s"
os.Setenv(runbookURLTemplateEnv, desiredRunbookURLTemplate)

promRule := createPrometheusRule("mynamespace")

for _, group := range promRule.Spec.Groups {
for _, rule := range group.Rules {
if rule.Alert != "" {
if rule.Annotations["runbook_url"] != "" {
Expect(rule.Annotations["runbook_url"]).To(Equal(fmt.Sprintf(desiredRunbookURLTemplate, rule.Alert)))
}
}
}
}
})
})

0 comments on commit 7565d9a

Please sign in to comment.