From 855e70e69c67cd338f83add9b0b18026e3395184 Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Wed, 15 Nov 2023 15:34:46 -0700 Subject: [PATCH] feat: add alertmanager sink (#107) * feat: alertmanager sink Signed-off-by: Tyler Gillson * feat: alertmanager sink Signed-off-by: Tyler Gillson * refactor: reduce VR update error verbosity Signed-off-by: Tyler Gillson * feat: alertmanager sink Signed-off-by: Tyler Gillson * docs: add screenshot to README.md Signed-off-by: Tyler Gillson * docs: clarify detail & failure annotations Signed-off-by: Tyler Gillson * feat: add basic auth support Signed-off-by: Tyler Gillson * feat: add TLS support Signed-off-by: Tyler Gillson * refactor: remove unused var from sink.Configure; add unit tests Signed-off-by: Tyler Gillson * test: increase coverage Signed-off-by: Tyler Gillson * test: increase coverage Signed-off-by: Tyler Gillson * refactor: misc. tidying Signed-off-by: Tyler Gillson * docs: update README Signed-off-by: Tyler Gillson * chore: log removal of path from alertmanager endpoint Signed-off-by: Tyler Gillson --------- Signed-off-by: Tyler Gillson --- .vscode/launch.json | 2 +- README.md | 80 ++++++++- api/v1alpha1/validatorconfig_types.go | 2 +- ...on.spectrocloud.labs_validatorconfigs.yaml | 1 + chart/validator/templates/sink-secret.yaml | 7 +- chart/validator/values.yaml | 11 +- ...on.spectrocloud.labs_validatorconfigs.yaml | 1 + .../controller/validationresult_controller.go | 4 +- internal/sinks/alertmanager.go | 161 ++++++++++++++++++ internal/sinks/alertmanager_test.go | 137 +++++++++++++++ internal/sinks/sink.go | 7 +- internal/sinks/sink_test.go | 39 +++++ internal/sinks/slack.go | 2 +- 13 files changed, 442 insertions(+), 12 deletions(-) create mode 100644 internal/sinks/alertmanager.go create mode 100644 internal/sinks/alertmanager_test.go create mode 100644 internal/sinks/sink_test.go diff --git a/.vscode/launch.json b/.vscode/launch.json index 99630a3d..6273c21b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,6 +31,6 @@ // "KUBECONFIG": "", "KUBEBUILDER_ASSETS": "/Users/tylergillson/spectrocloud/repos/oss/spectrocloud-labs/validation/validator/bin/k8s/1.27.1-darwin-arm64" } - }, + }, ] } \ No newline at end of file diff --git a/README.md b/README.md index 2b5ef5b9..e73de70c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Plugins: ## Installation ```bash -helm repo add validator https://spectrocloud-labs.github.io/validator/ +helm repo add validator https://spectrocloud-labs.github.io/validator helm repo update helm install validator validator/validator -n validator --create-namespace ``` @@ -31,6 +31,79 @@ helm install validator validator/validator -n validator --create-namespace ## Sinks Validator can be configured to emit updates to various event sinks whenever a `ValidationResult` is created or updated. See configuration details below for each supported sink. +### Alertmanager +Integrate with the Alertmanager API to emit alerts to all [supported Alertmanager receivers](https://prometheus.io/docs/alerting/latest/configuration/#receiver-integration-settings), including generic webhooks. The only required configuration is an Alertmanager endpoint. HTTP basic authentication and TLS are also supported. See [values.yaml](https://github.com/spectrocloud-labs/validator/blob/main/chart/validator/values.yaml) for configuration details. + +#### Sample Output +![Screen Shot 2023-11-15 at 10 42 20 AM](https://github.com/spectrocloud-labs/validator/assets/1795270/ce958b8e-96d7-4f5e-8efc-80e2fc2b0b4d) + +#### Setup +1. Install Alertmanager in your cluster (if it isn't installed already) +2. Configure Alertmanager alert content. Alerts can be formatted/customized via the following labels and annotations: + + Labels + - alertname + - plugin + - validation_result + - expected_results + + Annotations + - state + - validation_rule + - validation_type + - message + - status + - detail + - pipe-delimited array of detail messages, see sample config for parsing example + - failure (also pipe-delimited) + - last_validation_time + + Example Alertmanager ConfigMap used to produce the sample output above: + ```yaml + apiVersion: v1 + data: + alertmanager.yml: | + global: + slack_api_url: https://slack.com/api/chat.postMessage + receivers: + - name: default-receiver + slack_configs: + - channel: + text: |- + {{ range .Alerts.Firing -}} + *Validation Result: {{ .Labels.validation_result }}/{{ .Labels.expected_results }}* + + {{ range $k, $v := .Annotations }} + {{- if $v }}*{{ $k | title }}*: + {{- if match "\\|" $v }} + - {{ reReplaceAll "\\|" "\n- " $v -}} + {{- else }} + {{- printf " %s" $v -}} + {{- end }} + {{- end }} + {{ end }} + + {{ end }} + title: "{{ (index .Alerts 0).Labels.plugin }}: {{ (index .Alerts 0).Labels.alertname }}\n" + http_config: + authorization: + credentials: xoxb--- + send_resolved: false + route: + group_interval: 10s + group_wait: 10s + receiver: default-receiver + repeat_interval: 1h + templates: + - /etc/alertmanager/*.tmpl + kind: ConfigMap + metadata: + name: alertmanager + namespace: alertmanager + ``` + +2. Install validator and/or upgrade your validator Helm release, configuring `values.sink` accordingly. + ### Slack #### Sample Output @@ -38,7 +111,6 @@ Validator can be configured to emit updates to various event sinks whenever a `V Screen Shot 2023-11-10 at 4 18 22 PM #### Setup - 1. Go to https://api.slack.com/apps and click **Create New App**, then select **From scratch**. Pick an App Name and Slack Workspace, then click **Create App**. @@ -53,8 +125,8 @@ Validator can be configured to emit updates to various event sinks whenever a `V 4. Install validator and/or upgrade your validator Helm release, configuring `values.sink` accordingly. -## Getting Started -You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. +## Development +You’ll need a Kubernetes cluster to run against. You can use [kind](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). ### Running on the cluster diff --git a/api/v1alpha1/validatorconfig_types.go b/api/v1alpha1/validatorconfig_types.go index 66f94fc6..4c5dda76 100644 --- a/api/v1alpha1/validatorconfig_types.go +++ b/api/v1alpha1/validatorconfig_types.go @@ -28,7 +28,7 @@ type ValidatorConfigSpec struct { } type Sink struct { - // +kubebuilder:validation:Enum=slack + // +kubebuilder:validation:Enum=alertmanager;slack Type string `json:"type"` // Name of a K8s secret containing configuration details for the sink SecretName string `json:"secretName"` diff --git a/chart/validator/crds/validation.spectrocloud.labs_validatorconfigs.yaml b/chart/validator/crds/validation.spectrocloud.labs_validatorconfigs.yaml index 0d4acfad..9f939f23 100644 --- a/chart/validator/crds/validation.spectrocloud.labs_validatorconfigs.yaml +++ b/chart/validator/crds/validation.spectrocloud.labs_validatorconfigs.yaml @@ -65,6 +65,7 @@ spec: type: string type: enum: + - alertmanager - slack type: string required: diff --git a/chart/validator/templates/sink-secret.yaml b/chart/validator/templates/sink-secret.yaml index 550cf7d6..cc61233d 100644 --- a/chart/validator/templates/sink-secret.yaml +++ b/chart/validator/templates/sink-secret.yaml @@ -4,7 +4,12 @@ kind: Secret metadata: name: {{ required ".Values.sink.secretName is required!" .Values.sink.secretName }} stringData: - {{- if eq .Values.sink.type "slack" }} + {{- if eq .Values.sink.type "alertmanager" }} + endpoint: {{ required ".Values.sink.endpoint is required!" .Values.sink.endpoint }} + caCert: {{ .Values.sink.caCert }} + username: {{ .Values.sink.username }} + password: {{ .Values.sink.password }} + {{- else if eq .Values.sink.type "slack" }} apiToken: {{ required ".Values.sink.apiToken is required!" .Values.sink.apiToken }} channelId: {{ required ".Values.sink.channelId is required!" .Values.sink.channelId }} {{- end }} diff --git a/chart/validator/values.yaml b/chart/validator/values.yaml index c42a854f..4f4a981c 100644 --- a/chart/validator/values.yaml +++ b/chart/validator/values.yaml @@ -54,10 +54,19 @@ metricsService: # Optional sink configuration sink: {} + # type: alertmanager + # secretName: alertmanager-sink-secret + # endpoint: "http://alertmanager.alertmanager.svc.cluster.local:9093" + # caCert: "" # (TLS CA certificate, optional) + # username: "" # (HTTP basic auth, optional) + # password: "" # (HTTP basic auth, optional) + + # OR # type: slack - # secretName: "slack-secret" + # secretName: slack-sink-secret # apiToken: "" # channelId: "" + # By default, a secret will be created. Leave the above fields blank and specify 'createSecret: false' to use an existing secret. # WARNING: the existing secret must match the format used in sink-secret.yaml # createSecret: true diff --git a/config/crd/bases/validation.spectrocloud.labs_validatorconfigs.yaml b/config/crd/bases/validation.spectrocloud.labs_validatorconfigs.yaml index 0d4acfad..9f939f23 100644 --- a/config/crd/bases/validation.spectrocloud.labs_validatorconfigs.yaml +++ b/config/crd/bases/validation.spectrocloud.labs_validatorconfigs.yaml @@ -65,6 +65,7 @@ spec: type: string type: enum: + - alertmanager - slack type: string required: diff --git a/internal/controller/validationresult_controller.go b/internal/controller/validationresult_controller.go index ae4f3370..d9a13fd2 100644 --- a/internal/controller/validationresult_controller.go +++ b/internal/controller/validationresult_controller.go @@ -132,7 +132,7 @@ func (r *ValidationResultReconciler) Reconcile(ctx context.Context, req ctrl.Req sinkConfig = sinkSecret.Data } - if err := sink.Configure(*r.SinkClient, *vc, sinkConfig); err != nil { + if err := sink.Configure(*r.SinkClient, sinkConfig); err != nil { r.Log.Error(err, "failed to configure sink") return ctrl.Result{}, err } @@ -177,7 +177,7 @@ func (r *ValidationResultReconciler) updateStatus(ctx context.Context) error { vr.Status.SinkState = sinkState if err := r.Status().Update(context.Background(), vr); err != nil { - r.Log.V(0).Error(err, "failed to update ValidationResult status") + r.Log.V(1).Info("warning: failed to update ValidationResult status", "error", err.Error()) return err } diff --git a/internal/sinks/alertmanager.go b/internal/sinks/alertmanager.go new file mode 100644 index 00000000..b7caee9b --- /dev/null +++ b/internal/sinks/alertmanager.go @@ -0,0 +1,161 @@ +package sinks + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + + "github.com/spectrocloud-labs/validator/api/v1alpha1" +) + +type AlertmanagerSink struct { + client Client + log logr.Logger + + endpoint string + username string + password string +} + +type Alert struct { + Annotations map[string]string `json:"annotations"` + Labels map[string]string `json:"labels"` +} + +var ( + InvalidEndpoint = errors.New("invalid Alertmanager config: endpoint scheme and host are required") + EndpointRequired = errors.New("invalid Alertmanager config: endpoint required") +) + +func (s *AlertmanagerSink) Configure(c Client, config map[string][]byte) error { + // endpoint + endpoint, ok := config["endpoint"] + if !ok { + return EndpointRequired + } + u, err := url.Parse(string(endpoint)) + if err != nil { + return errors.Wrap(err, "invalid Alertmanager config: failed to parse endpoint") + } + if u.Scheme == "" || u.Host == "" { + return InvalidEndpoint + } + if u.Path != "" { + s.log.V(1).Info("stripping path from Alertmanager endpoint", "path", u.Path) + u.Path = "" + } + s.endpoint = fmt.Sprintf("%s/api/v2/alerts", u.String()) + + // basic auth + s.username = string(config["username"]) + s.password = string(config["password"]) + + // tls + var caCertPool *x509.CertPool + var insecureSkipVerify bool + + insecure, ok := config["insecureSkipVerify"] + if ok { + insecureSkipVerify, err = strconv.ParseBool(string(insecure)) + if err != nil { + return errors.Wrap(err, "invalid Alertmanager config: failed to parse insecureSkipVerify") + } + } + caCert, ok := config["caCert"] + if ok { + caCertPool, err = x509.SystemCertPool() + if err != nil { + return errors.Wrap(err, "invalid Alertmanager config: failed to get system cert pool") + } + caCertPool.AppendCertsFromPEM(caCert) + } + + c.hclient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecureSkipVerify, + MinVersion: tls.VersionTLS12, + RootCAs: caCertPool, + }, + } + s.client = c + + return nil +} + +func (s *AlertmanagerSink) Emit(r v1alpha1.ValidationResult) error { + alerts := make([]Alert, 0, len(r.Status.Conditions)) + + for i, c := range r.Status.Conditions { + alerts = append(alerts, Alert{ + Labels: map[string]string{ + "alertname": r.Name, + "plugin": r.Spec.Plugin, + "validation_result": strconv.Itoa(i + 1), + "expected_results": strconv.Itoa(r.Spec.ExpectedResults), + }, + Annotations: map[string]string{ + "state": string(r.Status.State), + "validation_rule": c.ValidationRule, + "validation_type": c.ValidationType, + "message": c.Message, + "status": string(c.Status), + "detail": strings.Join(c.Details, "|"), + "failure": strings.Join(c.Failures, "|"), + "last_validation_time": c.LastValidationTime.String(), + }, + }) + } + + body, err := json.Marshal(alerts) + if err != nil { + s.log.Error(err, "failed to marshal alerts", "alerts", alerts) + return err + } + s.log.V(1).Info("Alertmanager message", "payload", body) + + req, err := http.NewRequest(http.MethodPost, s.endpoint, bytes.NewReader(body)) + if err != nil { + s.log.Error(err, "failed to create HTTP POST request", "endpoint", s.endpoint) + return err + } + req.Header.Add("Content-Type", "application/json") + + if s.username != "" && s.password != "" { + req.Header.Add(basicAuthHeader(s.username, s.password)) + } + + resp, err := s.client.hclient.Do(req) + defer func() { + if resp != nil { + _ = resp.Body.Close() + } + }() + if err != nil { + s.log.Error(err, "failed to post alert", "endpoint", s.endpoint) + return err + } + if resp.StatusCode != 200 { + s.log.V(0).Info("failed to post alert", "endpoint", s.endpoint, "status", resp.Status, "code", resp.StatusCode) + return SinkEmissionFailed + } + + s.log.V(0).Info("Successfully posted alert to Alertmanager", "endpoint", s.endpoint, "status", resp.Status, "code", resp.StatusCode) + return nil +} + +func basicAuthHeader(username, password string) (string, string) { + auth := base64.StdEncoding.EncodeToString( + bytes.Join([][]byte{[]byte(username), []byte(password)}, []byte(":")), + ) + return "Authorization", fmt.Sprintf("Basic %s", auth) +} diff --git a/internal/sinks/alertmanager_test.go b/internal/sinks/alertmanager_test.go new file mode 100644 index 00000000..a37f666a --- /dev/null +++ b/internal/sinks/alertmanager_test.go @@ -0,0 +1,137 @@ +package sinks + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + + "github.com/spectrocloud-labs/validator/api/v1alpha1" +) + +var sinkClient = NewClient(1 * time.Second) + +func TestAlertmanagerConfigure(t *testing.T) { + cs := []struct { + name string + sink AlertmanagerSink + config map[string][]byte + expected error + }{ + { + name: "Pass", + sink: AlertmanagerSink{}, + config: map[string][]byte{ + "endpoint": []byte("http://fake.alertmanager.com:9093/api/v2/alerts"), + "caCert": []byte("_fake_ca_cert"), + }, + expected: nil, + }, + { + name: "Fail (no endpoint)", + sink: AlertmanagerSink{}, + config: map[string][]byte{}, + expected: EndpointRequired, + }, + { + name: "Fail (invalid endpoint)", + sink: AlertmanagerSink{}, + config: map[string][]byte{ + "endpoint": []byte("_not_an_endpoint_"), + }, + expected: InvalidEndpoint, + }, + { + name: "Fail (invalid insecureSkipVerify)", + sink: AlertmanagerSink{}, + config: map[string][]byte{ + "endpoint": []byte("https://fake.com"), + "insecureSkipVerify": []byte("_not_a_bool_"), + }, + expected: errors.New(`invalid Alertmanager config: failed to parse insecureSkipVerify: strconv.ParseBool: parsing "_not_a_bool_": invalid syntax`), + }, + } + for _, c := range cs { + t.Log(c.name) + err := c.sink.Configure(*sinkClient, c.config) + if err != nil && !reflect.DeepEqual(err.Error(), c.expected.Error()) { + t.Errorf("expected (%v), got (%v)", c.expected, err) + } + } +} + +func TestAlertManagerEmit(t *testing.T) { + cs := []struct { + name string + sink AlertmanagerSink + res v1alpha1.ValidationResult + server *httptest.Server + expected error + }{ + { + name: "Pass", + sink: AlertmanagerSink{}, + res: v1alpha1.ValidationResult{ + Status: v1alpha1.ValidationResultStatus{ + Conditions: []v1alpha1.ValidationCondition{ + { + Status: corev1.ConditionTrue, + }, + }, + }, + }, + server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "ok") + })), + expected: nil, + }, + { + name: "Fail", + sink: AlertmanagerSink{}, + res: v1alpha1.ValidationResult{}, + server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "invalid auth", http.StatusUnauthorized) + })), + expected: SinkEmissionFailed, + }, + } + for _, c := range cs { + t.Log(c.name) + defer c.server.Close() + _ = c.sink.Configure(*sinkClient, map[string][]byte{ + "endpoint": []byte(c.server.URL), + }) + err := c.sink.Emit(c.res) + if err != nil && !reflect.DeepEqual(err.Error(), c.expected.Error()) { + t.Errorf("expected (%v), got (%v)", c.expected, err) + } + } +} + +func TestBasicAuthHeader(t *testing.T) { + cs := []struct { + name string + username string + password string + expected string + }{ + { + name: "Pass", + username: "bob", + password: "frogs", + expected: "Basic Ym9iOmZyb2dz", + }, + } + for _, c := range cs { + t.Log(c.name) + _, v := basicAuthHeader(c.username, c.password) + if !reflect.DeepEqual(c.expected, v) { + t.Errorf("expected (%s), got (%s)", c.expected, v) + } + } +} diff --git a/internal/sinks/sink.go b/internal/sinks/sink.go index fa69d0fb..ed217c31 100644 --- a/internal/sinks/sink.go +++ b/internal/sinks/sink.go @@ -5,17 +5,22 @@ import ( "time" "github.com/go-logr/logr" + "github.com/pkg/errors" "github.com/spectrocloud-labs/validator/api/v1alpha1" ) +var SinkEmissionFailed = errors.New("sink emission failed") + type Sink interface { - Configure(c Client, vc v1alpha1.ValidatorConfig, config map[string][]byte) error + Configure(c Client, config map[string][]byte) error Emit(result v1alpha1.ValidationResult) error } func NewSink(sinkType string, log logr.Logger) Sink { switch sinkType { + case "alertmanager": + return &AlertmanagerSink{log: log} case "slack": return &SlackSink{log: log} default: diff --git a/internal/sinks/sink_test.go b/internal/sinks/sink_test.go new file mode 100644 index 00000000..b5563544 --- /dev/null +++ b/internal/sinks/sink_test.go @@ -0,0 +1,39 @@ +package sinks + +import ( + "reflect" + "testing" + + "github.com/go-logr/logr" +) + +func TestNewSink(t *testing.T) { + cs := []struct { + name string + sinkType string + expected Sink + }{ + { + name: "Pass (slack)", + sinkType: "slack", + expected: &SlackSink{}, + }, + { + name: "Pass (alertmanager)", + sinkType: "alertmanager", + expected: &AlertmanagerSink{}, + }, + { + name: "Pass (default)", + sinkType: "foo", + expected: &SlackSink{}, + }, + } + for _, c := range cs { + t.Log(c.name) + sink := NewSink(c.sinkType, logr.Logger{}) + if !reflect.DeepEqual(sink, c.expected) { + t.Errorf("expected (%+v), got (%+v)", c.expected, sink) + } + } +} diff --git a/internal/sinks/slack.go b/internal/sinks/slack.go index 6c8448b3..6e71e13b 100644 --- a/internal/sinks/slack.go +++ b/internal/sinks/slack.go @@ -19,7 +19,7 @@ type SlackSink struct { log logr.Logger } -func (s *SlackSink) Configure(c Client, vc v1alpha1.ValidatorConfig, config map[string][]byte) error { +func (s *SlackSink) Configure(c Client, config map[string][]byte) error { apiToken, ok := config["apiToken"] if !ok { return errors.New("invalid Slack configuration: apiToken required")