From 98ccff147af27006c2de3eaa40288530ff5aad79 Mon Sep 17 00:00:00 2001 From: louism517 Date: Wed, 3 Jul 2019 10:25:51 +0100 Subject: [PATCH] Threshold alerts (#49) * first stab * add support for threshold alerts * move alert creation inside config, add severity checks * break from loop --- README.md | 5 - go.mod | 2 +- go.sum | 6 + .../spaceapegames/go-wavefront/alert.go | 34 +++- .../spaceapegames/go-wavefront/event.go | 2 +- .../spaceapegames/go-wavefront/target.go | 2 +- .../spaceapegames/go-wavefront/version | 2 +- vendor/modules.txt | 2 +- wavefront/resource_alert.go | 104 +++++++++- wavefront/resource_alert_test.go | 191 ++++++++++++++++++ 10 files changed, 326 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0f42f01..8202688 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,6 @@ Once you have the plugin you should remove the `_os_arch` from the end of the fi Valid provider filenames are `terraform-provider-NAME_X.X.X` or `terraform-provider-NAME_vX.X.X` -## Known Issues - -To ensure that applies of large batches of Alerts are successful you can use the `-parallelism` flag to prevent parallel resource creations -`terraform apply -parallelism=1` - ## Building and Testing ### Build the plugin. diff --git a/go.mod b/go.mod index 50a19b3..c48bb04 100644 --- a/go.mod +++ b/go.mod @@ -3,5 +3,5 @@ module github.com/spaceapegames/terraform-provider-wavefront require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/terraform v0.12.0 - github.com/spaceapegames/go-wavefront v0.0.0-20190424105721-72e8eb185145 + github.com/spaceapegames/go-wavefront v1.6.2 ) diff --git a/go.sum b/go.sum index 5f44b52..8efcdbe 100644 --- a/go.sum +++ b/go.sum @@ -325,6 +325,12 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaceapegames/go-wavefront v0.0.0-20190424105721-72e8eb185145 h1:iN5Rc2gJLJFZ3Io4n5qxwe6uVKyHRfTGWc0yTauAH60= github.com/spaceapegames/go-wavefront v0.0.0-20190424105721-72e8eb185145/go.mod h1:Q9wbY/SM99cOM4NvoJ2SN1bzcJUJOEzta5C7dhnT5IM= +github.com/spaceapegames/go-wavefront v0.0.0-20190627103934-9f0b48af3f8f h1:EG1F9wt5cMZ9az8TDMlMRD08HP8zdjn8f3fmsVBBhf4= +github.com/spaceapegames/go-wavefront v0.0.0-20190627103934-9f0b48af3f8f/go.mod h1:Q9wbY/SM99cOM4NvoJ2SN1bzcJUJOEzta5C7dhnT5IM= +github.com/spaceapegames/go-wavefront v1.6.1 h1:5ZezLy0NiiOFDhrgJA01c5/vZZbFcj2IL2ymdRXJzT8= +github.com/spaceapegames/go-wavefront v1.6.1/go.mod h1:Q9wbY/SM99cOM4NvoJ2SN1bzcJUJOEzta5C7dhnT5IM= +github.com/spaceapegames/go-wavefront v1.6.2 h1:5AHElze/Y6oi4DdpkYfksS4F0ItsBpV7WwuB3r159cY= +github.com/spaceapegames/go-wavefront v1.6.2/go.mod h1:Q9wbY/SM99cOM4NvoJ2SN1bzcJUJOEzta5C7dhnT5IM= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= diff --git a/vendor/github.com/spaceapegames/go-wavefront/alert.go b/vendor/github.com/spaceapegames/go-wavefront/alert.go index 72604ad..26be4ca 100644 --- a/vendor/github.com/spaceapegames/go-wavefront/alert.go +++ b/vendor/github.com/spaceapegames/go-wavefront/alert.go @@ -6,6 +6,11 @@ import ( "io/ioutil" ) +const ( + AlertTypeThreshold = "THRESHOLD" + AlertTypeClassic = "CLASSIC" +) + // Alert represents a single Wavefront Alert type Alert struct { // Name is the name given to an Alert @@ -14,15 +19,26 @@ type Alert struct { // ID is the Wavefront-assigned ID of an existing Alert ID *string `json:"id,omitempty"` + // AlertType should be either CLASSIC or THRESHOLD + AlertType string `json:"alertType,omitempty"` + // AdditionalInfo is any extra information about the Alert AdditionalInfo string `json:"additionalInformation"` // Target is a comma-separated list of targets for the Alert - Target string `json:"target"` + Target string `json:"target,omitempty"` + + // For THRESHOLD alerts. Targets is a map[string]string. This maps severity to lists of targets. + // Valid keys are: severe, smoke, warn or info + Targets map[string]string `json:"targets"` // Condition is the condition under which the Alert will fire Condition string `json:"condition"` + // For THRESHOLD alerts. Conditions is a map[string]string. This maps severity to respective conditions. + // Valid keys are: severe, smoke, warn or info + Conditions map[string]string `json:"conditions"` + // DisplayExpression is the ts query to generate a graph of this Alert, in the UI DisplayExpression string `json:"displayExpression,omitempty"` @@ -33,19 +49,31 @@ type Alert struct { // ResolveAfterMinutes is the number of minutes the Condition must be un-met // before the Alert is considered resolved ResolveAfterMinutes int `json:"resolveAfterMinutes,omitempty"` - + // Minutes to wait before re-sending notification of firing alert. NotificationResendFrequencyMinutes int `json:"notificationResendFrequencyMinutes"` // Severity is the severity of the Alert, and can be one of SEVERE, // SMOKE, WARN or INFO - Severity string `json:"severity"` + Severity string `json:"severity,omitempty"` + + // For THRESHOLD alerts. SeverityList is a list of strings. Different severities applicable to this alert. + // Valid elements are: SEVERE, SMOKE, WARN or INFO + SeverityList []string `json:"severityList"` // Status is the current status of the Alert Status []string `json:"status"` // Tags are the tags applied to the Alert Tags []string + + FailingHostLabelPairs []SourceLabelPair `json:"failingHostLabelPairs,omitempty"` + InMaintenanceHostLabelPairs []SourceLabelPair `json:"inMaintenanceHostLabelPairs,omitempty"` +} + +type SourceLabelPair struct { + Host string `json:"host"` + Firing int `json:"firing"` } // Alerts is used to perform alert-related operations against the Wavefront API diff --git a/vendor/github.com/spaceapegames/go-wavefront/event.go b/vendor/github.com/spaceapegames/go-wavefront/event.go index 3c2d707..2690981 100644 --- a/vendor/github.com/spaceapegames/go-wavefront/event.go +++ b/vendor/github.com/spaceapegames/go-wavefront/event.go @@ -114,7 +114,7 @@ func (e Events) Find(filter []*SearchCondition, timeRange *TimeRange) ([]*Event, return results, nil } -// FindByID returns the Error with the Wavefront-assigned ID. +// FindByID returns the Event with the Wavefront-assigned ID. // If not found an error is returned func (e Events) FindByID(id string) (*Event, error) { res, err := e.Find([]*SearchCondition{ diff --git a/vendor/github.com/spaceapegames/go-wavefront/target.go b/vendor/github.com/spaceapegames/go-wavefront/target.go index 1176538..08262e9 100644 --- a/vendor/github.com/spaceapegames/go-wavefront/target.go +++ b/vendor/github.com/spaceapegames/go-wavefront/target.go @@ -33,7 +33,7 @@ type Target struct { // EmailSubject is the subject of the email which will be sent for this Target // (EMAIL targets only) EmailSubject string `json:"emailSubject"` - + // IsHTMLContent is a boolean value for wavefront to add HTML Boilerplate // while using HTML Templates as email. // (EMAIL targets only) diff --git a/vendor/github.com/spaceapegames/go-wavefront/version b/vendor/github.com/spaceapegames/go-wavefront/version index 26ca594..9c6d629 100644 --- a/vendor/github.com/spaceapegames/go-wavefront/version +++ b/vendor/github.com/spaceapegames/go-wavefront/version @@ -1 +1 @@ -1.5.1 +1.6.1 diff --git a/vendor/modules.txt b/vendor/modules.txt index 6e8859d..3c2eb55 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -202,7 +202,7 @@ github.com/posener/complete github.com/posener/complete/cmd/install github.com/posener/complete/cmd github.com/posener/complete/match -# github.com/spaceapegames/go-wavefront v0.0.0-20190424105721-72e8eb185145 +# github.com/spaceapegames/go-wavefront v1.6.2 github.com/spaceapegames/go-wavefront # github.com/spf13/afero v1.2.1 github.com/spf13/afero diff --git a/wavefront/resource_alert.go b/wavefront/resource_alert.go index cef7950..108f19c 100644 --- a/wavefront/resource_alert.go +++ b/wavefront/resource_alert.go @@ -23,15 +23,28 @@ func resourceAlert() *schema.Resource { Type: schema.TypeString, Required: true, }, + "alert_type": { + Type: schema.TypeString, + Optional: true, + Default: wavefront.AlertTypeClassic, + }, "target": { Type: schema.TypeString, - Required: true, + Optional: true, }, "condition": { Type: schema.TypeString, - Required: true, + Optional: true, StateFunc: trimSpaces, }, + "threshold_conditions": { + Type: schema.TypeMap, + Optional: true, + }, + "threshold_targets": { + Type: schema.TypeMap, + Optional: true, + }, "additional_information": { Type: schema.TypeString, Optional: true, @@ -56,7 +69,7 @@ func resourceAlert() *schema.Resource { }, "severity": { Type: schema.TypeString, - Required: true, + Optional: true, }, "tags": { Type: schema.TypeSet, @@ -71,6 +84,14 @@ func trimSpaces(d interface{}) string { return strings.TrimSpace(d.(string)) } +func trimSpacesMap(m map[string]interface{}) map[string]string { + trimmed := map[string]string{} + for key, v := range m { + trimmed[key] = trimSpaces(v) + } + return trimmed +} + func resourceAlertCreate(d *schema.ResourceData, m interface{}) error { alerts := m.(*wavefrontClient).client.Alerts() @@ -81,19 +102,21 @@ func resourceAlertCreate(d *schema.ResourceData, m interface{}) error { a := &wavefront.Alert{ Name: d.Get("name").(string), - Target: d.Get("target").(string), - Condition: trimSpaces(d.Get("condition").(string)), AdditionalInfo: trimSpaces(d.Get("additional_information").(string)), DisplayExpression: trimSpaces(d.Get("display_expression").(string)), Minutes: d.Get("minutes").(int), ResolveAfterMinutes: d.Get("resolve_after_minutes").(int), NotificationResendFrequencyMinutes: d.Get("notification_resend_frequency_minutes").(int), - Severity: d.Get("severity").(string), - Tags: tags, + Tags: tags, + } + + err := validateAlertConditions(a, d) + if err != nil { + return err } // Create the alert on Wavefront - err := alerts.Create(a) + err = alerts.Create(a) if err != nil { return fmt.Errorf("error creating Alert %s. %s", d.Get("name"), err) } @@ -129,6 +152,9 @@ func resourceAlertRead(d *schema.ResourceData, m interface{}) error { d.Set("notification_resend_frequency_minutes", tmpAlert.NotificationResendFrequencyMinutes) d.Set("severity", tmpAlert.Severity) d.Set("tags", tmpAlert.Tags) + d.Set("alert_type", tmpAlert.AlertType) + d.Set("threshold_conditions", tmpAlert.Conditions) + d.Set("threshold_targets", tmpAlert.Targets) return nil } @@ -151,16 +177,18 @@ func resourceAlertUpdate(d *schema.ResourceData, m interface{}) error { a := tmpAlert a.Name = d.Get("name").(string) - a.Target = d.Get("target").(string) - a.Condition = trimSpaces(d.Get("condition").(string)) a.AdditionalInfo = trimSpaces(d.Get("additional_information").(string)) a.DisplayExpression = trimSpaces(d.Get("display_expression").(string)) a.Minutes = d.Get("minutes").(int) a.ResolveAfterMinutes = d.Get("resolve_after_minutes").(int) a.NotificationResendFrequencyMinutes = d.Get("notification_resend_frequency_minutes").(int) - a.Severity = d.Get("severity").(string) a.Tags = tags + err = validateAlertConditions(&a, d) + if err != nil { + return err + } + // Update the alert on Wavefront err = alerts.Update(&a) if err != nil { @@ -188,3 +216,57 @@ func resourceAlertDelete(d *schema.ResourceData, m interface{}) error { d.SetId("") return nil } + +func validateAlertConditions(a *wavefront.Alert, d *schema.ResourceData) error { + if d.Get("alert_type") == wavefront.AlertTypeThreshold { + a.AlertType = wavefront.AlertTypeThreshold + if conditions, ok := d.GetOk("threshold_conditions"); ok { + a.Conditions = trimSpacesMap(conditions.(map[string]interface{})) + err := validateThresholdLevels(a.Conditions) + if err != nil { + return err + } + } else { + return fmt.Errorf("threshold_conditions must be supplied for threshold alerts") + } + + if targets, ok := d.GetOk("threshold_targets"); ok { + a.Targets = trimSpacesMap(targets.(map[string]interface{})) + return validateThresholdLevels(a.Targets) + } + + } else if d.Get("alert_type") == wavefront.AlertTypeClassic { + a.AlertType = wavefront.AlertTypeClassic + + if d.Get("condition") == "" { + return fmt.Errorf("condition must be supplied for classic alerts") + } + a.Condition = trimSpaces(d.Get("condition").(string)) + + if d.Get("severity") == "" { + return fmt.Errorf("severity must be supplied for classic alerts") + } + a.Severity = d.Get("severity").(string) + a.Target = d.Get("target").(string) + } else { + return fmt.Errorf("alert_type must be CLASSIC or THRESHOLD") + } + + return nil +} + +func validateThresholdLevels(m map[string]string) error { + for key := range m { + ok := false + for _, level := range []string{"severe", "warn", "info", "smoke"} { + if key == level { + ok = true + break + } + } + if !ok { + return fmt.Errorf("invalid severity: %s", key) + } + } + return nil +} diff --git a/wavefront/resource_alert_test.go b/wavefront/resource_alert_test.go index 04176e3..76dff20 100644 --- a/wavefront/resource_alert_test.go +++ b/wavefront/resource_alert_test.go @@ -2,6 +2,7 @@ package wavefront_plugin import ( "fmt" + "github.com/hashicorp/terraform/helper/schema" "testing" "github.com/hashicorp/terraform/helper/resource" @@ -168,6 +169,139 @@ func TestAccWavefrontAlert_Multiple(t *testing.T) { }) } +func TestAccWavefrontAlert_Threshold(t *testing.T) { + var record wavefront.Alert + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckWavefrontAlertDestroy, + Steps: []resource.TestStep{ + { + Config: testAccCheckWavefrontAlert_threshold(), + Check: resource.ComposeTestCheckFunc( + testAccCheckWavefrontAlertExists("wavefront_alert.test_threshold_alert", &record), + testAccCheckWavefrontThresholdAlertAttributes(&record), + + //Check against state that the attributes are as we expect + resource.TestCheckResourceAttr( + "wavefront_alert.test_threshold_alert", "threshold_conditions.%", "3"), + resource.TestCheckResourceAttr( + "wavefront_alert.test_threshold_alert", "threshold_targets.%", "1"), + ), + }, + }, + }) +} + +func TestResourceAlert_validateAlertConditions(t *testing.T) { + + cases := []struct { + name string + conf *schema.ResourceData + errorMessage string + }{ + { + "invalid alert type", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "WRONG") + return d + }(), + "alert_type must be CLASSIC or THRESHOLD", + }, + { + "classic alert missing condition", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "CLASSIC") + d.Set("severity", "severe") + return d + }(), + "condition must be supplied for classic alerts", + }, + { + "classic alert missing severity", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "CLASSIC") + d.Set("condition", "ts()") + return d + }(), + "severity must be supplied for classic alerts", + }, + { + "classic alert", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "CLASSIC") + d.Set("condition", "ts()") + d.Set("severity", "severe") + return d + }(), + "", + }, + { + "threshold alert missing conditions", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "THRESHOLD") + return d + }(), + "threshold_conditions must be supplied for threshold alerts", + }, + { + "threshold alert", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "THRESHOLD") + d.Set("threshold_conditions", map[string]interface{}{"severe": "ts()"}) + return d + }(), + "", + }, + { + "threshold alert invalid condition", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "THRESHOLD") + d.Set("threshold_conditions", map[string]interface{}{"banana": "ts()"}) + return d + }(), + "invalid severity: banana", + }, + { + "threshold alert invalid target", + func() *schema.ResourceData { + d := resourceAlert().TestResourceData() + d.Set("alert_type", "THRESHOLD") + d.Set("threshold_conditions", map[string]interface{}{"severe": "ts()"}) + d.Set("threshold_targets", map[string]interface{}{"banana": "ts()"}) + + return d + }(), + "invalid severity: banana", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := validateAlertConditions(&wavefront.Alert{}, c.conf) + + m := "" + if err == nil { + m = "" + } else { + m = err.Error() + } + + if m != c.errorMessage { + t.Errorf("expected error '%s', got '%s'", c.errorMessage, err.Error()) + } + }) + } +} + func testAccCheckWavefrontAlertDestroy(s *terraform.State) error { alerts := testAccProvider.Meta().(*wavefrontClient).client.Alerts() @@ -199,6 +333,21 @@ func testAccCheckWavefrontAlertAttributes(alert *wavefront.Alert) resource.TestC } } +func testAccCheckWavefrontThresholdAlertAttributes(alert *wavefront.Alert) resource.TestCheckFunc { + return func(s *terraform.State) error { + + if val, ok := alert.Conditions["severe"]; ok { + if val != "100-ts(\"cpu.usage_idle\", environment=preprod and cpu=cpu-total ) > 80" { + return fmt.Errorf("bad value: %s", alert.Conditions["severe"]) + } + } else { + return fmt.Errorf("target not set") + } + + return nil + } +} + func testAccCheckWavefrontAlertAttributesUpdated(alert *wavefront.Alert) resource.TestCheckFunc { return func(s *terraform.State) error { @@ -395,3 +544,45 @@ resource "wavefront_alert" "test_alert3" { } `) } + +func testAccCheckWavefrontAlert_threshold() string { + return fmt.Sprintf(` +resource "wavefront_alert_target" "test_target" { + name = "Terraform Test Target" + description = "Test target" + method = "EMAIL" + recipient = "test@example.com" + email_subject = "This is a test" + is_html_content = true + template = "{}" + triggers = [ + "ALERT_OPENED", + "ALERT_RESOLVED" + ] +} + + +resource "wavefront_alert" "test_threshold_alert" { + name = "Terraform Test Alert" + alert_type = "THRESHOLD" + additional_information = "This is a Terraform Test Alert" + display_expression = "100-ts(\"cpu.usage_idle\", environment=preprod and cpu=cpu-total )" + minutes = 5 + resolve_after_minutes = 5 + + threshold_conditions = { + "severe" = "100-ts(\"cpu.usage_idle\", environment=preprod and cpu=cpu-total ) > 80" + "warn" = "100-ts(\"cpu.usage_idle\", environment=preprod and cpu=cpu-total ) > 60" + "info" = "100-ts(\"cpu.usage_idle\", environment=preprod and cpu=cpu-total ) > 50" + } + + threshold_targets = { + "severe" = "target:${wavefront_alert_target.test_target.id}" + } + + tags = [ + "terraform" + ] +} +`) +}