feat: use new Slack module to send slack notifications

Makefile

@@ -55,7 +55,6 @@ docker-push:
 tidy:
 	go mod tidy
 	git add go.mod go.sum
-	cd hack/generate-schemas && go mod tidy && 	git add go.mod go.sum
 
 .PHONY: compress
 compress: .bin/upx

go.mod

@@ -59,6 +59,7 @@ require (
 	github.com/samber/lo v1.47.0
 	github.com/samber/oops v1.13.1
 	github.com/samber/slog-echo v1.14.4
+	github.com/slack-go/slack v0.14.0
 	github.com/tg123/go-htpasswd v1.2.2
 	github.com/timberio/go-datemath v0.1.0
 	go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.44.0
@@ -155,6 +156,7 @@ require (
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect

go.sum

@@ -974,8 +974,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
-github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
-github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
@@ -1136,6 +1136,7 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q=
 github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
@@ -1516,6 +1517,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A=
 github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
+github.com/slack-go/slack v0.14.0 h1:6c0UTfbRnvRssZUsZ2qe0Iu07VAMPjRqOa6oX8ewF4k=
+github.com/slack-go/slack v0.14.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=

notification/events.go

@@ -240,7 +240,7 @@ func sendNotifications(ctx context.Context, events models.Events) models.Events
 			e.SetError(err.Error())
 			failedEvents = append(failedEvents, e)
 			notificationContext.WithError(err.Error())
-		} else if err := SendNotification(notificationContext, payload, celEnv); err != nil {
+		} else if err := PrepareAndSendEventNotification(notificationContext, payload, celEnv); err != nil {
 			e.SetError(err.Error())
 			failedEvents = append(failedEvents, e)
 			notificationContext.WithError(err.Error())

notification/send.go

@@ -1,12 +1,14 @@
 package notification
 
 import (
+	"embed"
 	"encoding/json"
 	"fmt"
 	"net/url"
 	"strings"
 	"time"
 
+	"github.com/flanksource/commons/collections"
 	"github.com/flanksource/commons/utils"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/models"
@@ -20,6 +22,9 @@ import (
 	"github.com/flanksource/incident-commander/utils/expression"
 )
 
+//go:embed templates/*
+var templates embed.FS
+
 // List of all possible variables for any expression related to notifications
 var allEnvVars = []string{"agent", "config", "check", "canary", "component", "incident", "team", "responder", "comment", "evidence", "hypothesis", "permalink"}
 
@@ -55,27 +60,13 @@ func (t *NotificationEventPayload) FromMap(m map[string]string) {
 	_ = json.Unmarshal(b, &t)
 }
 
-// SendNotification generates the notification from the given event and sends it.
-func SendNotification(ctx *Context, payload NotificationEventPayload, celEnv map[string]any) error {
-	templater := ctx.NewStructTemplater(celEnv, "", nil)
-
+// PrepareAndSendEventNotification generates the notification from the given event and sends it.
+func PrepareAndSendEventNotification(ctx *Context, payload NotificationEventPayload, celEnv map[string]any) error {
 	notification, err := GetNotification(ctx.Context, payload.NotificationID.String())
 	if err != nil {
 		return err
 	}
 
-	defaultTitle, defaultBody := defaultTitleAndBody(payload.EventName)
-
-	data := NotificationTemplate{
-		Title:      utils.Coalesce(notification.Title, defaultTitle),
-		Message:    utils.Coalesce(notification.Template, defaultBody),
-		Properties: notification.Properties,
-	}
-
-	if err := templater.Walk(&data); err != nil {
-		return fmt.Errorf("error templating notification: %w", err)
-	}
-
 	if payload.PersonID != nil {
 		ctx.WithPersonID(payload.PersonID).WithRecipientType(RecipientTypePerson)
 		var emailAddress string
@@ -84,7 +75,7 @@ func SendNotification(ctx *Context, payload NotificationEventPayload, celEnv map
 		}
 
 		smtpURL := fmt.Sprintf("%s?ToAddresses=%s", api.SystemSMTP, url.QueryEscape(emailAddress))
-		return Send(ctx, "", smtpURL, data.Title, data.Message, data.Properties)
+		return sendEventNotificationWithMetrics(ctx, celEnv, "", smtpURL, payload.EventName, notification, nil)
 	}
 
 	if payload.TeamID != "" {
@@ -99,11 +90,7 @@ func SendNotification(ctx *Context, payload NotificationEventPayload, celEnv map
 				continue
 			}
 
-			if err := templater.Walk(&cn); err != nil {
-				return fmt.Errorf("error templating notification: %w", err)
-			}
-
-			return Send(ctx, cn.Connection, cn.URL, data.Title, data.Message, data.Properties, cn.Properties)
+			return sendEventNotificationWithMetrics(ctx, celEnv, cn.Connection, cn.URL, payload.EventName, notification, cn.Properties)
 		}
 	}
 
@@ -114,17 +101,78 @@ func SendNotification(ctx *Context, payload NotificationEventPayload, celEnv map
 	// (SA4004: the surrounding loop is unconditionally terminated)
 	for _, cn := range notification.CustomNotifications {
 		ctx.WithRecipientType(RecipientTypeCustom)
+		return sendEventNotificationWithMetrics(ctx, celEnv, cn.Connection, cn.URL, payload.EventName, notification, cn.Properties)
+	}
 
-		if err := templater.Walk(&cn); err != nil {
-			return fmt.Errorf("error templating notification: %w", err)
-		}
+	return nil
+}
+
+// SendEventNotification is a wrapper around sendEventNotification() for better error handling & metrics collection purpose.
+func sendEventNotificationWithMetrics(ctx *Context, celEnv map[string]any, connectionName, shoutrrrURL, eventName string, notification *NotificationWithSpec, customProperties map[string]string) error {
+	start := time.Now()
 
-		return Send(ctx, cn.Connection, cn.URL, data.Title, data.Message, data.Properties, cn.Properties)
+	service, err := sendEventNotification(ctx, celEnv, connectionName, shoutrrrURL, eventName, notification, customProperties)
+	if err != nil {
+		notificationSendFailureCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
+		return err
 	}
 
+	notificationSentCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
+	notificationSendDuration.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Observe(time.Since(start).Seconds())
+
 	return nil
 }
 
+func sendEventNotification(ctx *Context, celEnv map[string]any, connectionName, shoutrrrURL, eventName string, notification *NotificationWithSpec, customProperties map[string]string) (string, error) {
+	defaultTitle, defaultBody := defaultTitleAndBody(eventName)
+	customProperties = collections.MergeMap(notification.Properties, customProperties)
+	data := NotificationTemplate{
+		Title:      utils.Coalesce(notification.Title, defaultTitle),
+		Message:    utils.Coalesce(notification.Template, defaultBody),
+		Properties: customProperties,
+	}
+
+	return SendNotification(ctx, connectionName, shoutrrrURL, celEnv, data)
+}
+
+func SendNotification(ctx *Context, connectionName, shoutrrrURL string, celEnv map[string]any, data NotificationTemplate) (string, error) {
+	if celEnv == nil {
+		celEnv = make(map[string]any)
+	}
+
+	var connection *models.Connection
+	var err error
+	if connectionName != "" {
+		connection, err = ctx.HydrateConnectionByURL(connectionName)
+		if err != nil {
+			return "", err
+		} else if connection == nil {
+			return "", fmt.Errorf("connection (%s) not found", connectionName)
+		}
+
+		shoutrrrURL = connection.URL
+		data.Properties = collections.MergeMap(connection.Properties, data.Properties)
+	}
+
+	if connection != nil && connection.Type == models.ConnectionTypeSlack {
+		// We know we are sending to slack.
+		// Send the notification with slack-api and don't go through Shoutrrr.
+		celEnv["channel"] = "slack"
+		templater := ctx.NewStructTemplater(celEnv, "", templateFuncs)
+		if err := templater.Walk(&data); err != nil {
+			return "", fmt.Errorf("error templating notification: %w", err)
+		}
+		return "slack", SlackSend(ctx, connection.Password, connection.Username, data)
+	}
+
+	service, err := shoutrrrSend(ctx, celEnv, shoutrrrURL, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to send message with Shoutrrr: %w", err)
+	}
+
+	return service, nil
+}
+
 // labelsTemplate is a helper func to generate the template for displaying labels
 func labelsTemplate(field string) string {
 	return fmt.Sprintf("{{if %s}}### Labels: \n{{range $k, $v := %s}}**{{$k}}**: {{$v}} \n{{end}}{{end}}", field, field)
@@ -133,24 +181,16 @@ func labelsTemplate(field string) string {
 // defaultTitleAndBody returns the default title and body for notification
 // based on the given event.
 func defaultTitleAndBody(event string) (title string, body string) {
+	content, _ := templates.ReadFile(fmt.Sprintf("templates/%s", event))
+
 	switch event {
 	case api.EventCheckPassed:
-		title = "Check {{.check.name}} has passed"
-		body = fmt.Sprintf(`Canary: {{.canary.name}}
-{{if .agent}}Agent: {{.agent.name}}{{end}}
-{{if .status.message}}Message: {{.status.message}} {{end}}
-%s
-
-[Reference]({{.permalink}})`, labelsTemplate(".check.labels"))
+		title = `{{ if ne channel "slack"}}Check {{.check.name}} has passed{{end}}`
+		body = string(content)
 
 	case api.EventCheckFailed:
-		title = "Check {{.check.name}} has failed"
-		body = fmt.Sprintf(`Canary: {{.canary.name}}
-{{if .agent}}Agent: {{.agent.name}}{{end}}
-Error: {{.status.error}}
-%s
-
-[Reference]({{.permalink}})`, labelsTemplate(".check.labels"))
+		title = `{{ if ne channel "slack"}}Check {{.check.name}} has failed{{end}}`
+		body = string(content)
 
 	case api.EventConfigHealthy, api.EventConfigUnhealthy, api.EventConfigWarning, api.EventConfigUnknown:
 		title = "{{.config.type}} {{.config.name}} is {{.config.health}}"

notification/shoutrrr.go

@@ -6,12 +6,10 @@ import (
 	"os"
 	"strconv"
 	"strings"
-	"time"
 
 	stripmd "github.com/adityathebe/go-strip-markdown/v2"
 	"github.com/containrrr/shoutrrr"
 	"github.com/containrrr/shoutrrr/pkg/types"
-	"github.com/flanksource/commons/collections"
 	"github.com/flanksource/commons/utils"
 	"github.com/flanksource/incident-commander/api"
 	"github.com/flanksource/incident-commander/mail"
@@ -42,33 +40,10 @@ func setSystemSMTPCredential(shoutrrrURL string) (string, error) {
 	return shoutrrrURL, nil
 }
 
-func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) error {
-	start := time.Now()
-
-	service, err := send(ctx, connectionName, shoutrrrURL, title, message, properties...)
-	if err != nil {
-		notificationSendFailureCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
-		return err
-	}
-
-	notificationSentCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
-	notificationSendDuration.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Observe(time.Since(start).Seconds())
-
-	return nil
-}
-
-// send sends a notification and returns the service it sent the notification to
-func send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) (string, error) {
-	if connectionName != "" {
-		connection, err := ctx.HydrateConnectionByURL(connectionName)
-		if err != nil {
-			return "", err
-		} else if connection == nil {
-			return "", fmt.Errorf("connection (%s) not found", connectionName)
-		}
-
-		shoutrrrURL = connection.URL
-		properties = append([]map[string]string{connection.Properties}, properties...)
+// shoutrrrSend sends a notification and returns the service it sent the notification to
+func shoutrrrSend(ctx *Context, celEnv map[string]any, shoutrrrURL string, data NotificationTemplate) (string, error) {
+	if celEnv == nil {
+		celEnv = make(map[string]any)
 	}
 
 	if strings.HasPrefix(shoutrrrURL, api.SystemSMTP) {
@@ -91,29 +66,30 @@ func send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 
 	switch service {
 	case "smtp":
-		message = icUtils.MarkdownToHTML(message)
-		properties = append(properties, map[string]string{"UseHTML": "true"}) // enforce HTML for smtp
+		data.Message = icUtils.MarkdownToHTML(data.Message)
+		data.Properties["UseHTML"] = "true" // enforce HTML for smtp
 
 	case "telegram":
-		properties = append(properties, map[string]string{"ParseMode": "MarkdownV2"})
+		data.Properties["ParseMode"] = "MarkdownV2"
 
 	default:
-		message = stripmd.StripOptions(message, stripmd.Options{KeepURL: true})
+		data.Message = stripmd.StripOptions(data.Message, stripmd.Options{KeepURL: true})
 	}
 
-	ctx.WithMessage(message)
-
-	var allProps map[string]string
-	for _, prop := range properties {
-		prop = GetPropsForService(service, prop)
-		allProps = collections.MergeMap(allProps, prop)
+	celEnv["channel"] = service
+	templater := ctx.NewStructTemplater(celEnv, "", templateFuncs)
+	if err := templater.Walk(&data); err != nil {
+		return "", fmt.Errorf("error templating notification: %w", err)
 	}
 
-	injectTitleIntoProperties(service, title, allProps)
+	ctx.WithMessage(data.Message)
+
+	data.Properties = GetPropsForService(service, data.Properties)
+	injectTitleIntoProperties(service, data.Title, data.Properties)
 
 	params := &types.Params{}
-	if properties != nil {
-		params = (*types.Params)(&allProps)
+	if data.Properties != nil {
+		params = (*types.Params)(&data.Properties)
 	}
 
 	// NOTE: Until shoutrrr fixes the "UseHTML" props, we'll use the mailer package
@@ -132,13 +108,13 @@ func send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 			port, _     = strconv.Atoi(parsedURL.Port())
 		)
 
-		m := mail.New(to, title, message, `text/html; charset="UTF-8"`).
+		m := mail.New(to, data.Title, data.Message, `text/html; charset="UTF-8"`).
 			SetFrom(fromName, from).
 			SetCredentials(parsedURL.Hostname(), port, parsedURL.User.Username(), password)
 		return service, m.Send()
 	}
 
-	sendErrors := sender.Send(message, params)
+	sendErrors := sender.Send(data.Message, params)
 	for _, err := range sendErrors {
 		if err != nil {
 			return "", fmt.Errorf("error publishing notification (service=%s): %w", service, err)