From 29a6c32fc41c463fba45bc98e9ee1c1e9f6c3bb6 Mon Sep 17 00:00:00 2001 From: KeisukeYamashita <19yamashita15@gmail.com> Date: Wed, 23 Dec 2020 23:31:37 +0900 Subject: [PATCH 1/2] Add pagerduty support Signed-off-by: KeisukeYamashita <19yamashita15@gmail.com> For https://github.com/falcosecurity/falcosidekick/issues/163 --- README.md | 10 ++++++++++ config.go | 6 ++++++ go.mod | 1 + go.sum | 4 ++++ handlers.go | 4 ++++ main.go | 11 +++++++++++ outputs/client.go | 2 ++ outputs/pagerduty.go | 47 ++++++++++++++++++++++++++++++++++++++++++++ stats.go | 1 + types/types.go | 10 ++++++++++ 10 files changed, 96 insertions(+) create mode 100644 outputs/pagerduty.go diff --git a/README.md b/README.md index 133958232..634397858 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,13 @@ kafka: topic: "" # Name of the topic, if not empty, Kafka output is enabled # partition: 0 # Partition number of the topic. # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + +pagerduty: + # apikey: # Pagerduty API Key, if not empty, Pagerduty is enabled + service: "" # Service to create an incident + assignee: "" # A list of comma separated users to assign. Cannot be provided if pagerduty.escalationpolicy is already specified. + escalationpolicy: "" # Escalation policy to assign. Cannot be provided if pagerduty.escalationpolicy is already specified + # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) ``` Usage : @@ -384,6 +391,9 @@ The *env vars* "match" field names in *yaml file with this structure (**take car * **KAFKA_TOPIC**: The name of the Kafka topic * **KAFKA_PARTITION**: The number of the Kafka partition * **KAFKA_MINIMUMPRIORITY**: minimum priority of event for using this output, order is `emergency|alert|critical|error|warning|notice|informational|debug or "" (default)` +* **PAGERDUTY_ASSIGNEE**: A list of comma separated users to assign. Cannot be provided if `PAGERDUTY_ESCALATION_POLICY` is already specified. If not empty, Pagerduty is *enabled* +* **PAGERDUTY_ESCALATION_POLICY**: Escalation policy to assign. Cannot be provided if `PAGERDUTY_ASSIGNEE` is already specified.If not empty, Pagerduty is *enabled* +* **PAGERDUTY_MINIMUMPRIORITY**: minimum priority of event for using this output, order is `emergency|alert|critical|error|warning|notice|informational|debug or "" (default)` #### Slack/Rocketchat/Mattermost/Googlechat Message Formatting diff --git a/config.go b/config.go index 6f27278b2..0c2ba7c20 100644 --- a/config.go +++ b/config.go @@ -127,6 +127,11 @@ func getConfig() *types.Configuration { v.SetDefault("Kafka.Topic", "") v.SetDefault("Kafka.Partition", 0) v.SetDefault("Kafka.MinimumPriority", "") + v.SetDefault("Pagerduty.APIKey", "") + v.SetDefault("Pagerduty.Service", "") + v.SetDefault("Pagerduty.Assignee", []string{}) + v.SetDefault("Pagerduty.EscalationPolicy", "") + v.SetDefault("Pagerduty.MinimumPriority", "") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.AutomaticEnv() @@ -192,6 +197,7 @@ func getConfig() *types.Configuration { c.GCP.PubSub.MinimumPriority = checkPriority(c.GCP.PubSub.MinimumPriority) c.Googlechat.MinimumPriority = checkPriority(c.Googlechat.MinimumPriority) c.Kafka.MinimumPriority = checkPriority(c.Kafka.MinimumPriority) + c.Pagerduty.MinimumPriority = checkPriority(c.Kafka.MinimumPriority) c.Slack.MessageFormatTemplate = getMessageFormatTemplate("Slack", c.Slack.MessageFormat) c.Rocketchat.MessageFormatTemplate = getMessageFormatTemplate("Rocketchat", c.Rocketchat.MessageFormat) diff --git a/go.mod b/go.mod index 4b5e31df7..38e9ac36b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go/pubsub v1.8.3 github.com/Azure/azure-event-hubs-go/v3 v3.3.3 github.com/DataDog/datadog-go v4.2.0+incompatible + github.com/PagerDuty/go-pagerduty v1.3.0 github.com/aws/aws-sdk-go v1.35.30 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 github.com/emersion/go-smtp v0.14.0 diff --git a/go.sum b/go.sum index ac6b2b252..49f111185 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/DataDog/datadog-go v4.2.0+incompatible h1:Q73jzyKHwyA04Gf4SSukRF+KR4w github.com/DataDog/datadog-go v4.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PagerDuty/go-pagerduty v1.3.0 h1:2vWajLxpmGeP8pmsyZ0MjFneHa8ASJrztJRSe3FNOzg= +github.com/PagerDuty/go-pagerduty v1.3.0/go.mod h1:W5hSIIPrzSgAkNBDiuymWN5g9yQVzimL7BUBL44f3RY= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -247,6 +249,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= diff --git a/handlers.go b/handlers.go index 0e7fd1962..dd6191f81 100644 --- a/handlers.go +++ b/handlers.go @@ -219,4 +219,8 @@ func forwardEvent(falcopayload types.FalcoPayload) { if config.Kafka.HostPort != "" && (priorityMap[strings.ToLower(falcopayload.Priority)] >= priorityMap[strings.ToLower(config.Kafka.MinimumPriority)] || falcopayload.Rule == TestRule) { go kafkaClient.KafkaProduce(falcopayload) } + + if (len(config.Pagerduty.Assignee) > 0 || config.Pagerduty.EscalationPolicy != "") && (priorityMap[strings.ToLower(falcopayload.Priority)] >= priorityMap[strings.ToLower(config.Pagerduty.MinimumPriority)] || falcopayload.Rule == TestRule) { + go kafkaClient.PagerdutyPost(falcopayload) + } } diff --git a/main.go b/main.go index 224e26865..08d210e95 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ var ( gcpClient *outputs.Client googleChatClient *outputs.Client kafkaClient *outputs.Client + pagerdutyClient *outputs.Client statsdClient, dogstatsdClient *statsd.Client config *types.Configuration @@ -315,6 +316,16 @@ func init() { } } + if config.Pagerduty.Service != "" { + var err error + pagerdutyClient, err = outputs.NewPagerdutyClient(config, stats, promStats, statsdClient, dogstatsdClient) + if err != nil { + config.Pagerduty.Service = "" + } else { + enabledOutputsText += "Pagerduty " + } + } + log.Printf("%v\n", enabledOutputsText) } diff --git a/outputs/client.go b/outputs/client.go index 79f061a4f..9ab4e531d 100644 --- a/outputs/client.go +++ b/outputs/client.go @@ -14,6 +14,7 @@ import ( "cloud.google.com/go/pubsub" "github.com/DataDog/datadog-go/statsd" + "github.com/PagerDuty/go-pagerduty" "github.com/aws/aws-sdk-go/aws/session" "github.com/segmentio/kafka-go" @@ -53,6 +54,7 @@ type Client struct { DogstatsdClient *statsd.Client GCPTopicClient *pubsub.Topic KafkaProducer *kafka.Conn + PagerdutyClient *pagerduty.Client } // NewClient returns a new output.Client for accessing the different API. diff --git a/outputs/pagerduty.go b/outputs/pagerduty.go new file mode 100644 index 000000000..a5f22af2d --- /dev/null +++ b/outputs/pagerduty.go @@ -0,0 +1,47 @@ +package outputs + +import ( + "fmt" + "log" + + "github.com/DataDog/datadog-go/statsd" + "github.com/PagerDuty/go-pagerduty" + "github.com/falcosecurity/falcosidekick/types" +) + +// NewPagerdutyClient returns a new output.Client for accessing the Pagerduty API. +func NewPagerdutyClient(config *types.Configuration, stats *types.Statistics, promStats *types.PromStatistics, statsdClient, dogstatsdClient *statsd.Client) (*Client, error) { + if len(config.Pagerduty.Assignee) > 0 && config.Pagerduty.EscalationPolicy != "" { + return nil, fmt.Errorf("assignee and escalation policy cannot be both configured") + } + + return &Client{ + OutputType: "GCP", + Config: config, + Stats: stats, + PromStats: promStats, + StatsdClient: statsdClient, + DogstatsdClient: dogstatsdClient, + PagerdutyClient: pagerduty.NewClient(config.Pagerduty.APIKey), + }, nil +} + +// PagerdutyPost posts incident to Pagerduty +func (c *Client) PagerdutyPost(falcopayload types.FalcoPayload) { + c.Stats.Pagerduty.Add(Total, 1) + + // TODO: Implement pagerduty post + err := c.Post(newDatadogPayload(falcopayload)) + if err != nil { + go c.CountMetric(Outputs, 1, []string{"output:pagerduty", "status:error"}) + c.Stats.Pagerduty.Add(Error, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "pagerduty", "status": Error}).Inc() + log.Printf("[ERROR] : Pagerduty - %v\n", err) + return + } + + go c.CountMetric(Outputs, 1, []string{"output:pagerduty", "status:ok"}) + c.Stats.Pagerduty.Add(OK, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "pagerduty", "status": OK}).Inc() + log.Printf("[INFO] : Pagerduty - Publish OK\n") +} diff --git a/stats.go b/stats.go index 438c95553..d11438740 100644 --- a/stats.go +++ b/stats.go @@ -47,6 +47,7 @@ func getInitStats() *types.Statistics { GCPPubSub: getOutputNewMap("gcppubsub"), GoogleChat: getOutputNewMap("googlechat"), Kafka: getOutputNewMap("kafka"), + Pagerduty: getOutputNewMap("pagerduty"), } stats.Falco.Add(outputs.Emergency, 0) stats.Falco.Add(outputs.Alert, 0) diff --git a/types/types.go b/types/types.go index 0cbe478ee..e62b84fd1 100644 --- a/types/types.go +++ b/types/types.go @@ -45,6 +45,7 @@ type Configuration struct { GCP gcpOutputConfig Googlechat GooglechatConfig Kafka kafkaConfig + Pagerduty pagerdutyConfig } // SlackOutputConfig represents parameters for Slack @@ -239,6 +240,14 @@ type kafkaConfig struct { MinimumPriority string } +type pagerdutyConfig struct { + APIKey string + Service string + Assignee []string + EscalationPolicy string + MinimumPriority string +} + // Statistics is a struct to store stastics type Statistics struct { Requests *expvar.Map @@ -270,6 +279,7 @@ type Statistics struct { GCPPubSub *expvar.Map GoogleChat *expvar.Map Kafka *expvar.Map + Pagerduty *expvar.Map } // PromStatistics is a struct to store prometheus metrics From c5154ad52873cd1d2e4ad98896b570cbbe3f3c9d Mon Sep 17 00:00:00 2001 From: KeisukeYamashita <19yamashita15@gmail.com> Date: Thu, 24 Dec 2020 00:52:48 +0900 Subject: [PATCH 2/2] Fix pagerduty handler condition Signed-off-by: KeisukeYamashita <19yamashita15@gmail.com> --- README.md | 7 +++++-- config.go | 2 +- handlers.go | 4 ++-- main.go | 3 ++- outputs/pagerduty.go | 43 +++++++++++++++++++++++++++++++++++-------- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 634397858..2bb243cab 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Currently available outputs are : * [**GCP PubSub**](https://cloud.google.com/pubsub) * [**Google Chat**](https://workspace.google.com/products/chat/) * [**Apache Kafka**](https://kafka.apache.org/) +* [**PagerDuty**](https://pagerduty.com/) ## Usage @@ -268,8 +269,8 @@ kafka: # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) pagerduty: - # apikey: # Pagerduty API Key, if not empty, Pagerduty is enabled - service: "" # Service to create an incident + # apikey: # Pagerduty API Key, if not empty, Pagerduty output is enabled + service: "" # Service to create an incident (mandatory) assignee: "" # A list of comma separated users to assign. Cannot be provided if pagerduty.escalationpolicy is already specified. escalationpolicy: "" # Escalation policy to assign. Cannot be provided if pagerduty.escalationpolicy is already specified # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) @@ -391,6 +392,8 @@ The *env vars* "match" field names in *yaml file with this structure (**take car * **KAFKA_TOPIC**: The name of the Kafka topic * **KAFKA_PARTITION**: The number of the Kafka partition * **KAFKA_MINIMUMPRIORITY**: minimum priority of event for using this output, order is `emergency|alert|critical|error|warning|notice|informational|debug or "" (default)` +* **PAGERDUTY_APIKEY**: Pagerduty API Key, if not empty, Pagerduty output is *enabled* +* **PAGERDUTY_SERVICE**: Service to create an incident (mandatory) * **PAGERDUTY_ASSIGNEE**: A list of comma separated users to assign. Cannot be provided if `PAGERDUTY_ESCALATION_POLICY` is already specified. If not empty, Pagerduty is *enabled* * **PAGERDUTY_ESCALATION_POLICY**: Escalation policy to assign. Cannot be provided if `PAGERDUTY_ASSIGNEE` is already specified.If not empty, Pagerduty is *enabled* * **PAGERDUTY_MINIMUMPRIORITY**: minimum priority of event for using this output, order is `emergency|alert|critical|error|warning|notice|informational|debug or "" (default)` diff --git a/config.go b/config.go index 0c2ba7c20..00ca28725 100644 --- a/config.go +++ b/config.go @@ -197,7 +197,7 @@ func getConfig() *types.Configuration { c.GCP.PubSub.MinimumPriority = checkPriority(c.GCP.PubSub.MinimumPriority) c.Googlechat.MinimumPriority = checkPriority(c.Googlechat.MinimumPriority) c.Kafka.MinimumPriority = checkPriority(c.Kafka.MinimumPriority) - c.Pagerduty.MinimumPriority = checkPriority(c.Kafka.MinimumPriority) + c.Pagerduty.MinimumPriority = checkPriority(c.Pagerduty.MinimumPriority) c.Slack.MessageFormatTemplate = getMessageFormatTemplate("Slack", c.Slack.MessageFormat) c.Rocketchat.MessageFormatTemplate = getMessageFormatTemplate("Rocketchat", c.Rocketchat.MessageFormat) diff --git a/handlers.go b/handlers.go index dd6191f81..ace74e525 100644 --- a/handlers.go +++ b/handlers.go @@ -220,7 +220,7 @@ func forwardEvent(falcopayload types.FalcoPayload) { go kafkaClient.KafkaProduce(falcopayload) } - if (len(config.Pagerduty.Assignee) > 0 || config.Pagerduty.EscalationPolicy != "") && (priorityMap[strings.ToLower(falcopayload.Priority)] >= priorityMap[strings.ToLower(config.Pagerduty.MinimumPriority)] || falcopayload.Rule == TestRule) { - go kafkaClient.PagerdutyPost(falcopayload) + if config.Pagerduty.APIKey != "" && config.Pagerduty.Service != "" && (priorityMap[strings.ToLower(falcopayload.Priority)] >= priorityMap[strings.ToLower(config.Pagerduty.MinimumPriority)] || falcopayload.Rule == TestRule) { + go pagerdutyClient.PagerdutyCreateIncident(falcopayload) } } diff --git a/main.go b/main.go index 08d210e95..5793ff889 100644 --- a/main.go +++ b/main.go @@ -316,10 +316,11 @@ func init() { } } - if config.Pagerduty.Service != "" { + if config.Pagerduty.APIKey != "" && config.Pagerduty.Service != "" { var err error pagerdutyClient, err = outputs.NewPagerdutyClient(config, stats, promStats, statsdClient, dogstatsdClient) if err != nil { + config.Pagerduty.APIKey = "" config.Pagerduty.Service = "" } else { enabledOutputsText += "Pagerduty " diff --git a/outputs/pagerduty.go b/outputs/pagerduty.go index a5f22af2d..806b2c0d4 100644 --- a/outputs/pagerduty.go +++ b/outputs/pagerduty.go @@ -16,7 +16,7 @@ func NewPagerdutyClient(config *types.Configuration, stats *types.Statistics, pr } return &Client{ - OutputType: "GCP", + OutputType: "PagerDuty", Config: config, Stats: stats, PromStats: promStats, @@ -26,22 +26,49 @@ func NewPagerdutyClient(config *types.Configuration, stats *types.Statistics, pr }, nil } -// PagerdutyPost posts incident to Pagerduty -func (c *Client) PagerdutyPost(falcopayload types.FalcoPayload) { +// PagerdutyCreateIncident posts incident to Pagerduty +func (c *Client) PagerdutyCreateIncident(falcopayload types.FalcoPayload) { c.Stats.Pagerduty.Add(Total, 1) - // TODO: Implement pagerduty post - err := c.Post(newDatadogPayload(falcopayload)) - if err != nil { + opts := &pagerduty.CreateIncidentOptions{ + Type: "incident", + Title: falcopayload.Output, + Service: &pagerduty.APIReference{ + ID: c.Config.Pagerduty.Service, + Type: "service_reference", + }, + } + + if len(c.Config.Pagerduty.Assignee) > 0 { + assignments := make([]pagerduty.Assignee, len(c.Config.Pagerduty.Assignee)) + for i, a := range c.Config.Pagerduty.Assignee { + assignments[i] = pagerduty.Assignee{ + Assignee: pagerduty.APIObject{ + ID: a, + Type: "user_reference", + }, + } + } + opts.Assignments = assignments + } + + if policy := c.Config.Pagerduty.EscalationPolicy; policy != "" { + opts.EscalationPolicy = &pagerduty.APIReference{ + ID: policy, + Type: "escalation_policy_reference", + } + } + + if _, err := c.PagerdutyClient.CreateIncident("falcosidekick", opts); err != nil { go c.CountMetric(Outputs, 1, []string{"output:pagerduty", "status:error"}) c.Stats.Pagerduty.Add(Error, 1) c.PromStats.Outputs.With(map[string]string{"destination": "pagerduty", "status": Error}).Inc() - log.Printf("[ERROR] : Pagerduty - %v\n", err) + log.Printf("[ERROR] : PagerDuty - %v\n", err) return } go c.CountMetric(Outputs, 1, []string{"output:pagerduty", "status:ok"}) c.Stats.Pagerduty.Add(OK, 1) c.PromStats.Outputs.With(map[string]string{"destination": "pagerduty", "status": OK}).Inc() - log.Printf("[INFO] : Pagerduty - Publish OK\n") + log.Printf("[INFO] : Pagerduty - Create Incident OK\n") }