reset alert converge reduce cache when restored alert trigger (#1281)

apache · Jan 16, 2024 · a717913 · a717913
1 parent 03255fb
commit a717913
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 9 deletions.
diff --git a/alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java b/alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java
@@ -224,7 +224,6 @@ private void handleRecoveredAlert(long currentTimeMilli, long monitorId, String
         if (notResolvedAlert != null) {
             // Sending an alarm Restore
             Map<String, String> tags = notResolvedAlert.getTags();
-            tags.put(CommonConstants.TAG_ALARM_TYPE, "recover");
             String content = this.bundle.getString("alerter.alarm.recover") + " : " + expr;
             Alert resumeAlert = Alert.builder()
                                         .tags(tags)
@@ -262,6 +261,7 @@ private void afterThresholdRuleMatch(long currentTimeMilli, long monitorId, Stri
             Map<String, String> tags = new HashMap<>(6);
             tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
             tags.put(CommonConstants.TAG_MONITOR_APP, app);
+            tags.put(CommonConstants.TAG_POLICY_ID, String.valueOf(define.getId()));
             if (!CollectionUtils.isEmpty(define.getTags())) {
                 for (TagItem tagItem : define.getTags()) {
                     fieldValueMap.put(tagItem.getName(), tagItem.getValue());
@@ -320,6 +320,7 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr
             Map<String, String> tags = new HashMap<>(6);
             tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
             tags.put(CommonConstants.TAG_MONITOR_APP, app);
+            tags.put(CommonConstants.TAG_POLICY_ID, String.valueOf(avaAlertDefine.getId()));
             tags.put("metrics", CommonConstants.AVAILABILITY);
             tags.put("code", metricsData.getCode().name());
             Map<String, Object> valueMap = tags.entrySet().stream()
@@ -334,7 +335,7 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr
             if (preAlert == null) {
                 Alert.AlertBuilder alertBuilder = Alert.builder()
                                                           .tags(tags)
-                                                          .priority(CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY)
+                                                          .priority(avaAlertDefine.getPriority())
                                                           .status(ALERT_STATUS_CODE_PENDING)
                                                           .target(CommonConstants.AVAILABILITY)
                                                           .content(AlertTemplateUtil.render(avaAlertDefine.getTemplate(), valueMap))
@@ -377,16 +378,10 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr
             Alert notResolvedAlert = notRecoveredAlertMap.remove(notResolvedAlertKey);
             if (notResolvedAlert != null) {
                 // Sending an alarm Restore
-                Map<String, String> tags = new HashMap<>(6);
-                tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
-                tags.put(CommonConstants.TAG_MONITOR_APP, app);
-                tags.put(CommonConstants.TAG_ALARM_TYPE, "recover");
+                Map<String, String> tags = notResolvedAlert.getTags();
                 if (!avaAlertDefine.isRecoverNotice()) {
                     tags.put(CommonConstants.IGNORE, CommonConstants.IGNORE);
                 }
-                if (notResolvedAlert.getTags() != null) {
-                    tags.putAll(notResolvedAlert.getTags());
-                }
                 String content = this.bundle.getString("alerter.availability.recover");
                 Alert resumeAlert = Alert.builder()
                                             .tags(tags)

diff --git a/alerter/src/main/java/org/dromara/hertzbeat/alert/reduce/AlarmConvergeReduce.java b/alerter/src/main/java/org/dromara/hertzbeat/alert/reduce/AlarmConvergeReduce.java
@@ -43,6 +43,22 @@ public boolean filterConverge(Alert currentAlert) {
         if (currentAlert.getTags() != null && currentAlert.getTags().containsKey(CommonConstants.IGNORE)) {
             return true;
         }
+        if (currentAlert.getStatus() == CommonConstants.ALERT_STATUS_CODE_RESTORED) {
+            // restored alert
+            int alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_CRITICAL)
+                    + Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
+                    + Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
+            converageAlertMap.remove(alertHash);
+            alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY)
+                    + Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
+                    + Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
+            converageAlertMap.remove(alertHash);
+            alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_WARNING)
+                    + Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
+                    + Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
+            converageAlertMap.remove(alertHash);
+            return true;
+        }
         ICacheService<String, Object> convergeCache = CacheFactory.getAlertConvergeCache();
         List<AlertConverge> alertConvergeList = (List<AlertConverge>) convergeCache.get(CommonConstants.CACHE_ALERT_CONVERGE);
         if (alertConvergeList == null) {

diff --git a/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java b/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java
@@ -247,6 +247,11 @@ public interface CommonConstants {
      */
     String TAG_MONITOR_NAME = "monitorName";
 
+    /**
+     * 内有标签: policyId 告警阈值规则ID
+     */
+    String TAG_POLICY_ID = "policyId";
+
     /**
      * 内有标签: app 监控类型
      */
@@ -304,6 +309,7 @@ public interface CommonConstants {
 
     /**
      * ignore label
+     * 处理未配置恢复告警，但需要使用恢复告警变更监控状态的情况
      */
     String IGNORE = "ignore";