Stn/correctly mark api silences

api/v2/api.go

@@ -53,6 +53,8 @@ type API struct {
 	peer           *cluster.Peer
 	silences       *silence.Silences
 	alerts         provider.Alerts
+	muter          types.Muter
+	marker         types.Marker
 	getAlertStatus getAlertStatusFn
 	uptime         time.Time
 
@@ -72,9 +74,17 @@ type API struct {
 type getAlertStatusFn func(prometheus_model.Fingerprint) types.AlertStatus
 
 // NewAPI returns a new Alertmanager API v2
-func NewAPI(alerts provider.Alerts, sf getAlertStatusFn, silences *silence.Silences, peer *cluster.Peer, l log.Logger) (*API, error) {
+func NewAPI(
+	alerts provider.Alerts,
+	marker types.Marker,
+	sf getAlertStatusFn,
+	silences *silence.Silences,
+	peer *cluster.Peer,
+	l log.Logger,
+) (*API, error) {
 	api := API{
 		alerts:         alerts,
+		marker:         marker,
 		getAlertStatus: sf,
 		peer:           peer,
 		silences:       silences,
@@ -115,13 +125,14 @@ func NewAPI(alerts provider.Alerts, sf getAlertStatusFn, silences *silence.Silen
 }
 
 // Update sets the configuration string to a new value.
-func (api *API) Update(cfg *config.Config, resolveTimeout time.Duration) error {
+func (api *API) Update(cfg *config.Config, resolveTimeout time.Duration, muter types.Muter) error {
 	api.mtx.Lock()
 	defer api.mtx.Unlock()
 
 	api.resolveTimeout = resolveTimeout
 	api.alertmanagerConfig = cfg
 	api.route = dispatch.NewRoute(cfg.Route, nil)
+	api.muter = muter
 	return nil
 }
 
@@ -238,6 +249,28 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re
 			continue
 		}
 
+		// TODO: Add a timestamp now()+X, after which the Mutes() will
+		// be refreshed?
+		// Will check if the alert is inhibited, and update its status.
+		api.muter.Mutes(a.Labels)
+		// Will check if the alert is silenced, and update its status.
+		sils, err := api.silences.Query(
+			silence.QState(types.SilenceStateActive),
+			silence.QMatches(a.Labels),
+		)
+		if err != nil {
+			level.Error(api.logger).Log("msg", "Querying silences failed", "err", err)
+		}
+
+		if len(sils) > 0 {
+			ids := make([]string, len(sils))
+			for i, s := range sils {
+				ids[i] = s.Id
+			}
+			api.marker.SetSilenced(a.Labels.Fingerprint(), ids...)
+		}
+
+		// Get alert's current status after seeing if it is suppressed.
 		status := api.getAlertStatus(a.Fingerprint())
 
 		if !*params.Active && status.State == types.AlertStateActive {

cmd/alertmanager/main.go

@@ -302,6 +302,7 @@ func run() int {
 
 	apiV2, err := apiv2.NewAPI(
 		alerts,
+		marker,
 		marker.Status,
 		silences,
 		peer,
@@ -350,16 +351,6 @@ func run() int {
 
 		hash = md5HashAsMetricValue(plainCfg)
 
-		err = apiV1.Update(conf, time.Duration(conf.Global.ResolveTimeout))
-		if err != nil {
-			return err
-		}
-
-		err = apiV2.Update(conf, time.Duration(conf.Global.ResolveTimeout))
-		if err != nil {
-			return err
-		}
-
 		tmpl, err = template.FromGlobs(conf.Templates...)
 		if err != nil {
 			return err
@@ -381,6 +372,17 @@ func run() int {
 			peer,
 			logger,
 		)
+
+		err = apiV1.Update(conf, time.Duration(conf.Global.ResolveTimeout))
+		if err != nil {
+			return err
+		}
+
+		err = apiV2.Update(conf, time.Duration(conf.Global.ResolveTimeout), inhibitor)
+		if err != nil {
+			return err
+		}
+
 		disp = dispatch.NewDispatcher(alerts, dispatch.NewRoute(conf.Route, nil), pipeline, marker, timeoutFunc, logger)
 
 		go disp.Run()

test/with_api_v2/acceptance.go

@@ -289,6 +289,11 @@ func (amc *AlertmanagerCluster) Start() error {
 	return nil
 }
 
+// Members returns the underlying slice of cluster members.
+func (amc *AlertmanagerCluster) Members() []*Alertmanager {
+	return amc.ams
+}
+
 // Start the alertmanager and wait until it is ready to receive.
 func (am *Alertmanager) Start(additionalArg []string) error {
 	am.t.Helper()
@@ -555,6 +560,10 @@ func (am *Alertmanager) GenericAPIV2Call(at float64, f func()) {
 	am.t.Do(at, f)
 }
 
+func (am *Alertmanager) Client() *apiclient.Alertmanager {
+	return am.clientV2
+}
+
 func (am *Alertmanager) getURL(path string) string {
 	return fmt.Sprintf("http://%s%s%s", am.apiAddr, am.opts.RoutePrefix, path)
 }

test/with_api_v2/acceptance/api_test.go

@@ -0,0 +1,150 @@
+// Copyright 2018 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/go-openapi/strfmt"
+	a "github.com/prometheus/alertmanager/test/with_api_v2"
+	"github.com/prometheus/alertmanager/test/with_api_v2/api_v2_client/client/alert"
+	"github.com/prometheus/alertmanager/test/with_api_v2/api_v2_client/client/silence"
+	"github.com/prometheus/alertmanager/test/with_api_v2/api_v2_client/models"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAlertGetReturnsCurrentStatus checks that querying the API returns the
+// current status of each alert, i.e. if it is silenced or inhibited.
+func TestAlertGetReturnsCurrentAlertStatus(t *testing.T) {
+	t.Parallel()
+
+	conf := `
+route:
+  receiver: "default"
+  group_by: []
+  group_wait:      1s
+  group_interval:  10m
+  repeat_interval: 1h
+
+inhibit_rules:
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: ['alertname']
+
+receivers:
+- name: "default"
+  webhook_configs:
+  - url: 'http://%s'
+`
+
+	at := a.NewAcceptanceTest(t, &a.AcceptanceOpts{
+		Tolerance: 1 * time.Second,
+	})
+	co := at.Collector("webhook")
+	wh := a.NewWebhook(co)
+
+	amc := at.AlertmanagerCluster(fmt.Sprintf(conf, wh.Address()), 1)
+	require.NoError(t, amc.Start())
+	defer amc.Terminate()
+
+	am := amc.Members()[0]
+
+	labelName := "alertname"
+	labelValue := "test1"
+
+	now := time.Now()
+	startsAt := strfmt.DateTime(now)
+	endsAt := strfmt.DateTime(now.Add(5 * time.Minute))
+
+	labels := models.LabelSet(map[string]string{labelName: labelValue, "severity": "warning"})
+	fp := model.LabelSet{model.LabelName(labelName): model.LabelValue(labelValue), "severity": "warning"}.Fingerprint()
+	pa := &models.PostableAlert{
+		StartsAt: startsAt,
+		EndsAt:   endsAt,
+		Alert:    models.Alert{Labels: labels},
+	}
+	alertParams := alert.NewPostAlertsParams()
+	alertParams.Alerts = models.PostableAlerts{pa}
+	_, err := am.Client().Alert.PostAlerts(alertParams)
+	require.NoError(t, err)
+
+	resp, err := am.Client().Alert.GetAlerts(nil)
+	require.NoError(t, err)
+	// No silence has been created or inhibiting alert sent, alert should
+	// be active.
+	for _, alert := range resp.Payload {
+		require.Equal(t, models.AlertStatusStateActive, *alert.Status.State)
+	}
+
+	// Wait for group_wait, so that we are in the group_interval period,
+	// when the pipeline won't update the alert's status.
+	time.Sleep(2 * time.Second)
+
+	// Create silence and verify that the alert is immediately marked
+	// silenced via the API.
+	silenceParams := silence.NewPostSilencesParams()
+
+	cm := "a"
+	isRegex := false
+	ps := &models.PostableSilence{
+		Silence: models.Silence{
+			StartsAt:  &startsAt,
+			EndsAt:    &endsAt,
+			Comment:   &cm,
+			CreatedBy: &cm,
+			Matchers: models.Matchers{
+				&models.Matcher{Name: &labelName, Value: &labelValue, IsRegex: &isRegex},
+			},
+		},
+	}
+	silenceParams.Silence = ps
+	silenceResp, err := am.Client().Silence.PostSilences(silenceParams)
+	require.NoError(t, err)
+	silenceID := silenceResp.Payload.SilenceID
+
+	resp, err = am.Client().Alert.GetAlerts(nil)
+	require.NoError(t, err)
+	for _, alert := range resp.Payload {
+		require.Equal(t, models.AlertStatusStateSuppressed, *alert.Status.State)
+		require.Equal(t, fp.String(), *alert.Fingerprint)
+		require.Equal(t, 1, len(alert.Status.SilencedBy))
+		require.Equal(t, silenceID, alert.Status.SilencedBy[0])
+	}
+
+	// Create inhibiting alert and verify that original alert is
+	// immediately marked as inhibited.
+	labels["severity"] = "critical"
+	_, err = am.Client().Alert.PostAlerts(alertParams)
+	require.NoError(t, err)
+
+	inhibitingFP := model.LabelSet{model.LabelName(labelName): model.LabelValue(labelValue), "severity": "critical"}.Fingerprint()
+
+	resp, err = am.Client().Alert.GetAlerts(nil)
+	require.NoError(t, err)
+	for _, alert := range resp.Payload {
+		require.Equal(t, 1, len(alert.Status.SilencedBy))
+		require.Equal(t, silenceID, alert.Status.SilencedBy[0])
+		if fp.String() == *alert.Fingerprint {
+			require.Equal(t, models.AlertStatusStateSuppressed, *alert.Status.State)
+			require.Equal(t, fp.String(), *alert.Fingerprint)
+			require.Equal(t, 1, len(alert.Status.InhibitedBy))
+			require.Equal(t, inhibitingFP.String(), alert.Status.InhibitedBy[0])
+		}
+	}
+}