Add PagerDuty integration platform

cmd/botkube-agent/main.go

@@ -328,6 +328,14 @@ func run(ctx context.Context) (err error) {
 
 			sinkNotifiers = append(sinkNotifiers, wh)
 		}
+		if commGroupCfg.PagerDuty.Enabled {
+			pd, err := sink.NewPagerDuty(commGroupLogger.WithField(sinkLogFieldKey, "PagerDuty"), commGroupMeta.Index, commGroupCfg.PagerDuty, conf.Settings.ClusterName, analyticsReporter)
+			if err != nil {
+				return reportFatalError("while creating PagerDuty sink", err)
+			}
+
+			sinkNotifiers = append(sinkNotifiers, pd)
+		}
 	}
 	healthChecker.SetNotifiers(getHealthNotifiers(bots, sinkNotifiers))
 

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/DanielTitkov/go-adaptive-cards v0.2.2
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/Masterminds/semver/v3 v3.2.1
+	github.com/PagerDuty/go-pagerduty v1.8.0
 	github.com/alexflint/go-arg v1.4.3
 	github.com/avast/retry-go/v4 v4.3.3
 	github.com/aws/aws-sdk-go v1.44.122
@@ -15,7 +16,7 @@ require (
 	github.com/charmbracelet/log v0.2.2
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
-	github.com/fatih/color v1.15.0
+	github.com/fatih/color v1.16.0
 	github.com/go-playground/locales v0.14.1
 	github.com/go-playground/universal-translator v0.18.1
 	github.com/go-playground/validator/v10 v10.14.0
@@ -43,6 +44,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/common v0.44.0
 	github.com/r3labs/diff/v3 v3.0.1
 	github.com/sanity-io/litter v1.5.5
 	github.com/segmentio/analytics-go v3.1.0+incompatible
@@ -214,7 +216,6 @@ require (
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
@@ -223,7 +224,7 @@ require (
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect

go.sum

@@ -224,6 +224,8 @@ github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYx
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PagerDuty/go-pagerduty v1.8.0 h1:MTFqTffIcAervB83U7Bx6HERzLbyaSPL/+oxH3zyluI=
+github.com/PagerDuty/go-pagerduty v1.8.0/go.mod h1:nzIeAqyFSJAFkjWKvMzug0JtwDg+V+UoCWjFrfFH5mI=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
@@ -400,8 +402,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
@@ -413,8 +415,8 @@ github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6
 github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
-github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
-github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -1074,8 +1076,8 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
-github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

internal/source/scheduler.go

@@ -224,6 +224,12 @@ func (d *Scheduler) generateConfigs(ctx context.Context) error {
 				}
 			}
 		}
+
+		if commGroupCfg.PagerDuty.Enabled {
+			if err := d.generateSourceConfigs(ctx, false, commGroupCfg.PagerDuty.Bindings.Sources); err != nil {
+				return err
+			}
+		}
 	}
 
 	// Schedule all sources used by actions

pkg/config/config.go

@@ -102,6 +102,9 @@ const (
 
 	// WebhookCommPlatformIntegration defines an outgoing webhook integration.
 	WebhookCommPlatformIntegration CommPlatformIntegration = "webhook"
+
+	// PagerDutyCommPlatformIntegration defines an outgoing PagerDuty integration.
+	PagerDutyCommPlatformIntegration CommPlatformIntegration = "pagerDuty"
 )
 
 func (c CommPlatformIntegration) IsInteractive() bool {
@@ -482,6 +485,7 @@ type Communications struct {
 	CloudTeams    CloudTeams    `yaml:"cloudTeams,omitempty"`
 	Webhook       Webhook       `yaml:"webhook,omitempty"`
 	Elasticsearch Elasticsearch `yaml:"elasticsearch,omitempty"`
+	PagerDuty     PagerDuty     `yaml:"pagerDuty,omitempty"`
 }
 
 // SocketSlack configuration to authentication and send notifications
@@ -594,6 +598,18 @@ type Webhook struct {
 	Bindings SinkBindings `yaml:"bindings" validate:"required_if=Enabled true"`
 }
 
+// PagerDuty describes the PagerDuty sink.
+type PagerDuty struct {
+	// Enabled indicates if the PagerDuty sink is enabled.
+	Enabled bool `yaml:"enabled"`
+	// Bindings are the bindings for the PagerDuty sink.
+	Bindings SinkBindings `yaml:"bindings" validate:"required_if=Enabled true"`
+	// IntegrationKey is the PagerDuty integration key generated for Events v2 API.
+	IntegrationKey string `yaml:"integrationKey" validate:"required_if=Enabled true"`
+	// V2EventsAPIBasePath is the Events v2 API URL base path. Defaults to https://events.pagerduty.com.
+	V2EventsAPIBasePath string
+}
+
 // CfgWatcher describes configuration for watching the configuration.
 type CfgWatcher struct {
 	Enabled   bool                `yaml:"enabled"`

pkg/plugin/collector.go

@@ -77,6 +77,12 @@ func (c *Collector) GetAllEnabledAndUsedPlugins(cfg *config.Config) ([]string, [
 				}
 			}
 		}
+
+		if commGroupCfg.PagerDuty.Enabled {
+			for _, name := range commGroupCfg.PagerDuty.Bindings.Sources {
+				boundSources[name] = struct{}{}
+			}
+		}
 	}
 
 	// Collect all used executors/sources by actions

pkg/sink/pager_duty.go

@@ -0,0 +1,235 @@
+package sink
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/PagerDuty/go-pagerduty"
+	"github.com/mitchellh/mapstructure"
+	"github.com/sirupsen/logrus"
+
+	"github.com/kubeshop/botkube/internal/health"
+	"github.com/kubeshop/botkube/pkg/config"
+	"github.com/kubeshop/botkube/pkg/sliceutil"
+)
+
+// PagerDuty provides functionality to notify PagerDuty service about new events.
+type PagerDuty struct {
+	log      logrus.FieldLogger
+	reporter AnalyticsReporter
+
+	bindings config.SinkBindings
+
+	integrationKey string
+	clusterName    string
+	pagerDutyCli   *pagerduty.Client
+
+	status        health.PlatformStatusMsg
+	failureReason health.FailureReasonMsg
+	statusMux     sync.Mutex
+}
+
+// EventLink represents a link in a ChangeEvent and Alert event
+// https://developer.pagerduty.com/docs/events-api-v2/send-change-events/#the-links-property
+type EventLink struct {
+	Href string `json:"href"`
+	Text string `json:"text,omitempty"`
+}
+
+type incomingEvent struct {
+	Source    string
+	Data      any
+	Timestamp time.Time
+}
+
+// NewPagerDuty creates a new PagerDuty instance.
+func NewPagerDuty(log logrus.FieldLogger, commGroupIdx int, c config.PagerDuty, clusterName string, reporter AnalyticsReporter) (*PagerDuty, error) {
+	var opts []pagerduty.ClientOptions
+	if c.V2EventsAPIBasePath != "" {
+		opts = append(opts, pagerduty.WithV2EventsAPIEndpoint(c.V2EventsAPIBasePath))
+	}
+	notifier := &PagerDuty{
+		log:      log,
+		reporter: reporter,
+
+		bindings:       c.Bindings,
+		clusterName:    clusterName,
+		integrationKey: c.IntegrationKey,
+
+		status:        health.StatusUnknown,
+		failureReason: "",
+
+		// We only dispatch events using integration key, we don't need a token.
+		pagerDutyCli: pagerduty.NewClient("", opts...),
+	}
+
+	err := reporter.ReportSinkEnabled(notifier.IntegrationName(), commGroupIdx)
+	if err != nil {
+		log.WithError(err).Error("Failed to report analytics")
+	}
+
+	return notifier, nil
+}
+
+// SendEvent sends an event to a configured server.
+func (w *PagerDuty) SendEvent(ctx context.Context, rawData any, sources []string) error {
+	if !w.shouldNotify(sources) {
+		return nil
+	}
+
+	in := &incomingEvent{
+		Source:    strings.Join(sources, ","),
+		Data:      rawData,
+		Timestamp: time.Now(),
+	}
+
+	resp, err := w.postEvent(ctx, in)
+	if err != nil {
+		w.setFailureReason(health.FailureReasonConnectionError)
+		return fmt.Errorf("while sending message to PagerDuty: %w", err)
+	}
+
+	w.markHealthy()
+	w.log.WithField("response", resp).Debug("Message successfully sent")
+	return nil
+}
+
+// IntegrationName describes the notifier integration name.
+func (w *PagerDuty) IntegrationName() config.CommPlatformIntegration {
+	return config.PagerDutyCommPlatformIntegration
+}
+
+// Type describes the notifier type.
+func (w *PagerDuty) Type() config.IntegrationType {
+	return config.SinkIntegrationType
+}
+
+// GetStatus gets sink status.
+func (w *PagerDuty) GetStatus() health.PlatformStatus {
+	return health.PlatformStatus{
+		Status:   w.status,
+		Restarts: "0/0",
+		Reason:   w.failureReason,
+	}
+}
+
+func (w *PagerDuty) shouldNotify(sourceBindings []string) bool {
+	return sliceutil.Intersect(sourceBindings, w.bindings.Sources)
+}
+
+func (w *PagerDuty) resolveEventMeta(in *incomingEvent) eventMetadata {
+	out := eventMetadata{
+		Summary: fmt.Sprintf("Event from %s source", in.Source),
+		IsAlert: true,
+	}
+
+	var ev eventPayload
+	err := mapstructure.Decode(in.Data, &ev)
+	if err != nil {
+		// we failed, so let's treat it as an error
+		w.log.WithError(err).Error("Failed to decode event. Forwarding it to PagerDuty as an alert.")
+		return out
+	}
+
+	if ev.k8sEventPayload.Level != "" {
+		return enrichWithK8sEventMetadata(out, ev.k8sEventPayload)
+	}
+
+	if len(ev.argoPayload.Message.Sections) > 0 {
+		return enrichWithArgoCDEventMetadata(out, ev.argoPayload)
+	}
+
+	if len(ev.prometheusEventPayload.Annotations) > 0 {
+		return enrichWithPrometheusEventMetadata(out, ev.prometheusEventPayload)
+	}
+	return out
+}
+
+func (w *PagerDuty) postEvent(ctx context.Context, in *incomingEvent) (any, error) {
+	meta := w.resolveEventMeta(in)
+	if meta.IsAlert {
+		return w.triggerAlert(ctx, in, meta)
+	}
+	return w.triggerChange(ctx, in, meta)
+}
+
+func (w *PagerDuty) triggerAlert(ctx context.Context, in *incomingEvent, meta eventMetadata) (*pagerduty.V2EventResponse, error) {
+	return w.pagerDutyCli.ManageEventWithContext(ctx, &pagerduty.V2Event{
+		// required
+		RoutingKey: w.integrationKey,
+		Action:     "trigger",
+
+		// optional
+		Client:    "Botkube",
+		ClientURL: "https://app.botkube.io",
+
+		Payload: &pagerduty.V2Payload{
+			// required
+			Summary: meta.Summary,
+			// The unique location of the affected system, preferably a hostname or FQDN.
+			Source: fmt.Sprintf("%s/%s", w.clusterName, in.Source),
+			// The perceived severity of the status the event is describing with respect to the affected system. This can be critical, error, warning or info.
+			Severity: "error",
+
+			// optional
+			Timestamp: in.Timestamp.Format(time.RFC3339),
+			// Logical grouping of components of a service.
+			Group:     w.clusterName,
+			Component: meta.Component,
+			Details:   in,
+		},
+	})
+}
+
+func (w *PagerDuty) triggerChange(ctx context.Context, in *incomingEvent, meta eventMetadata) (*pagerduty.ChangeEventResponse, error) {
+	customDetails := map[string]any{
+		"group":   w.clusterName,
+		"details": in,
+	}
+
+	if meta.Component != "" {
+		customDetails["component"] = meta.Component
+	}
+
+	return w.pagerDutyCli.CreateChangeEventWithContext(ctx, pagerduty.ChangeEvent{
+		// required
+		RoutingKey: w.integrationKey,
+		Payload: pagerduty.ChangeEventPayload{
+			// required
+			Summary: meta.Summary,
+			// The unique location of the affected system, preferably a hostname or FQDN.
+			Source: fmt.Sprintf("%s/%s", w.clusterName, in.Source),
+
+			// optional
+			Timestamp:     in.Timestamp.Format(time.RFC3339),
+			CustomDetails: customDetails,
+		},
+	})
+}
+
+func (w *PagerDuty) setFailureReason(reason health.FailureReasonMsg) {
+	if reason == "" {
+		return
+	}
+
+	w.statusMux.Lock()
+	defer w.statusMux.Unlock()
+
+	w.status = health.StatusUnHealthy
+	w.failureReason = reason
+}
+
+func (w *PagerDuty) markHealthy() {
+	if w.status == health.StatusHealthy {
+		return
+	}
+
+	w.statusMux.Lock()
+	defer w.statusMux.Unlock()
+
+	w.status = health.StatusHealthy
+	w.failureReason = ""
+}