Skip to content

Commit

Permalink
[feature] support alert silence config (#873)
Browse files Browse the repository at this point in the history
* [webapp] update alert config ui i18

* support alert silence feature

* [webapp] fix silence bug and i18n

* [webapp] add silence matchAll

* support alert silence feature

* support alert silence feature

* optimize caffeine cache (#875)

* 优化caffeine cache

* 移动cache到common模块

* 修改clear实现

* 修改clear实现

* perfect comment

* add cache test

---------

Co-authored-by: hujiaofen <hujiaofen@2dfire.com>

* support alert silence feature

* [webapp] update alert config ui i18

* support alert silence feature

* [webapp] fix silence bug and i18n

* [webapp] add silence matchAll

* support alert silence feature

* support alert silence feature

* support alert silence feature

* use alert silence cache

* add hzb_history sql script

* update history column length

---------

Co-authored-by: zcx <48920254+Ceilzcx@users.noreply.github.com>
Co-authored-by: hujiaofen <hujiaofen@2dfire.com>
  • Loading branch information
3 people authored Apr 19, 2023
1 parent d45bf8e commit 4f1e441
Show file tree
Hide file tree
Showing 33 changed files with 1,509 additions and 47 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,15 @@ public class CalculateAlarm {
private final CommonDataQueue dataQueue;
private final AlertDefineService alertDefineService;
private final AlerterProperties alerterProperties;
private final SilenceAlarm silenceAlarm;
private final ResourceBundle bundle;

public CalculateAlarm (AlerterWorkerPool workerPool, CommonDataQueue dataQueue,
public CalculateAlarm (AlerterWorkerPool workerPool, CommonDataQueue dataQueue, SilenceAlarm silenceAlarm,
AlertDefineService alertDefineService, AlertMonitorDao monitorDao,
AlerterProperties alerterProperties) {
this.workerPool = workerPool;
this.dataQueue = dataQueue;
this.silenceAlarm = silenceAlarm;
this.alertDefineService = alertDefineService;
this.alerterProperties = alerterProperties;
this.bundle = ResourceBundleUtil.getBundle("alerter");
Expand Down Expand Up @@ -179,7 +181,7 @@ private void calculate(CollectRep.MetricsData metricsData) {
int defineTimes = define.getTimes() == null ? 1 : define.getTimes();
if (times >= defineTimes) {
triggeredAlertMap.remove(monitorAlertKey);
dataQueue.addAlertData(triggeredAlert);
silenceAlarm.filterSilenceAndSendData(triggeredAlert);
}
} else {
fieldValueMap.put("app", app);
Expand All @@ -203,7 +205,7 @@ private void calculate(CollectRep.MetricsData metricsData) {
.build();
int defineTimes = define.getTimes() == null ? 1 : define.getTimes();
if (1 >= defineTimes) {
dataQueue.addAlertData(alert);
silenceAlarm.filterSilenceAndSendData(alert);
} else {
triggeredAlertMap.put(monitorAlertKey, alert);
}
Expand Down Expand Up @@ -269,7 +271,7 @@ private void handlerAvailableMetrics(long monitorId, String app, String metrics,
.lastTriggerTime(currentTimeMilli)
.times(1)
.build();
dataQueue.addAlertData(resumeAlert);
silenceAlarm.filterSilenceAndSendData(resumeAlert);
}
}
}
Expand Down Expand Up @@ -301,7 +303,7 @@ private void handlerMonitorAvailableAlert(long monitorId, String app, CollectRep
.nextEvalInterval(alerterProperties.getAlertEvalIntervalBase())
.times(1);
if (avaAlertDefine.getTimes() == null || avaAlertDefine.getTimes() <= 1) {
dataQueue.addAlertData(alertBuilder.build().clone());
silenceAlarm.filterSilenceAndSendData(alertBuilder.build().clone());
} else {
alertBuilder.status(CommonConstants.ALERT_STATUS_CODE_NOT_REACH);
}
Expand All @@ -318,7 +320,7 @@ private void handlerMonitorAvailableAlert(long monitorId, String app, CollectRep
int defineTimes = avaAlertDefine.getTimes() == null ? 1 : avaAlertDefine.getTimes();
if (times >= defineTimes) {
preAlert.setStatus(CommonConstants.ALERT_STATUS_CODE_PENDING);
dataQueue.addAlertData(preAlert);
silenceAlarm.filterSilenceAndSendData(preAlert);
} else {
preAlert.setStatus(CommonConstants.ALERT_STATUS_CODE_NOT_REACH);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package org.dromara.hertzbeat.alert.calculate;

import lombok.RequiredArgsConstructor;
import org.dromara.hertzbeat.alert.dao.AlertSilenceDao;
import org.dromara.hertzbeat.common.cache.CacheFactory;
import org.dromara.hertzbeat.common.cache.ICacheService;
import org.dromara.hertzbeat.common.entity.alerter.Alert;
import org.dromara.hertzbeat.common.entity.alerter.AlertSilence;
import org.dromara.hertzbeat.common.entity.manager.TagItem;
import org.dromara.hertzbeat.common.queue.CommonDataQueue;
import org.dromara.hertzbeat.common.util.CommonConstants;
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.List;
import java.util.Map;

/**
* silence alarm
* @author tom
* @date 2023/4/12 20:00
*/
@Service
@RequiredArgsConstructor
public class SilenceAlarm {

private final AlertSilenceDao alertSilenceDao;

private final CommonDataQueue dataQueue;

@SuppressWarnings("unchecked")
public void filterSilenceAndSendData(Alert alert) {
ICacheService<String, Object> silenceCache = CacheFactory.getAlertSilenceCache();
List<AlertSilence> alertSilenceList = (List<AlertSilence>) silenceCache.get(CommonConstants.CACHE_ALERT_SILENCE);
if (alertSilenceList == null) {
alertSilenceList = alertSilenceDao.findAll();
silenceCache.put(CommonConstants.CACHE_ALERT_SILENCE, alertSilenceList);
}
for (AlertSilence alertSilence : alertSilenceList) {
// if match the silence rule, return
boolean match = alertSilence.isMatchAll();
if (!match) {
List<TagItem> tags = alertSilence.getTags();
if (alert.getTags() != null && !alert.getTags().isEmpty()) {
Map<String, String> alertTagMap = alert.getTags();
match = tags.stream().anyMatch(item -> {
if (alertTagMap.containsKey(item.getName())) {
String tagValue = alertTagMap.get(item.getName());
if (tagValue == null && item.getValue() == null) {
return true;
} else {
return tagValue != null && tagValue.equals(item.getValue());
}
} else {
return false;
}
});
}
if (match && alertSilence.getPriorities() != null && !alertSilence.getPriorities().isEmpty()) {
match = alertSilence.getPriorities().stream().anyMatch(item -> item != null && item == alert.getPriority());
}
}
if (match) {
LocalDateTime nowDate = LocalDateTime.now();
if (alertSilence.getType() == 0) {
// once time
if (alertSilence.getPeriodStart() != null && alertSilence.getPeriodEnd() != null) {
if (nowDate.isAfter(alertSilence.getPeriodStart().toLocalDateTime())
&& nowDate.isBefore(alertSilence.getPeriodEnd().toLocalDateTime())) {
int times = alertSilence.getTimes() == null ? 0 : alertSilence.getTimes();
alertSilence.setTimes(times + 1);
alertSilenceDao.save(alertSilence);
return;
}
}
} else if (alertSilence.getType() == 1) {
// cyc time
int currentDayOfWeek = nowDate.toLocalDate().getDayOfWeek().getValue();
if (alertSilence.getDays() != null && !alertSilence.getDays().isEmpty()) {
boolean dayMatch = alertSilence.getDays().stream().anyMatch(item -> item == currentDayOfWeek);
if (dayMatch && alertSilence.getPeriodStart() != null && alertSilence.getPeriodEnd() != null ) {
LocalTime nowTime = nowDate.toLocalTime();

if (nowTime.isAfter(alertSilence.getPeriodStart().toLocalTime())
&& nowTime.isBefore(alertSilence.getPeriodEnd().toLocalTime())) {
int times = alertSilence.getTimes() == null ? 0 : alertSilence.getTimes();
alertSilence.setTimes(times + 1);
alertSilenceDao.save(alertSilence);
return;
}
}
}
}
}
}
dataQueue.addAlertData(alert);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dromara.hertzbeat.alert.controller;

import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.dromara.hertzbeat.alert.service.AlertSilenceService;
import org.dromara.hertzbeat.common.entity.alerter.AlertSilence;
import org.dromara.hertzbeat.common.entity.dto.Message;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import javax.validation.Valid;

import static org.dromara.hertzbeat.common.util.CommonConstants.MONITOR_NOT_EXIST_CODE;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;

/**
* Alarm Silence management API
* 告警静默管理API
* @author tom
* @date 2023/04/12 22:32
*/
@Tag(name = "Alert Silence API | 告警静默管理API")
@RestController
@RequestMapping(path = "/api/alert/silence", produces = {APPLICATION_JSON_VALUE})
public class AlertSilenceController {

@Autowired
private AlertSilenceService alertSilenceService;

@PostMapping
@Operation(summary = "New Alarm Silence | 新增告警静默", description = "Added an alarm Silence | 新增一个告警静默")
public ResponseEntity<Message<Void>> addNewAlertSilence(@Valid @RequestBody AlertSilence alertSilence) {
alertSilenceService.validate(alertSilence, false);
alertSilenceService.addAlertSilence(alertSilence);
return ResponseEntity.ok(new Message<>("Add success"));
}

@PutMapping
@Operation(summary = "Modifying an Alarm Silence | 修改告警静默", description = "Modify an existing alarm Silence | 修改一个已存在告警静默")
public ResponseEntity<Message<Void>> modifyAlertSilence(@Valid @RequestBody AlertSilence alertSilence) {
alertSilenceService.validate(alertSilence, true);
alertSilenceService.modifyAlertSilence(alertSilence);
return ResponseEntity.ok(new Message<>("Modify success"));
}

@GetMapping(path = "/{id}")
@Operation(summary = "Querying Alarm Silence | 查询告警静默",
description = "You can obtain alarm Silence information based on the alarm Silence ID | 根据告警静默ID获取告警静默信息")
public ResponseEntity<Message<AlertSilence>> getAlertSilence(
@Parameter(description = "Alarm Silence ID | 告警静默ID", example = "6565463543") @PathVariable("id") long id) {
AlertSilence alertSilence = alertSilenceService.getAlertSilence(id);
Message.MessageBuilder<AlertSilence> messageBuilder = Message.builder();
if (alertSilence == null) {
messageBuilder.code(MONITOR_NOT_EXIST_CODE).msg("AlertSilence not exist.");
} else {
messageBuilder.data(alertSilence);
}
return ResponseEntity.ok(messageBuilder.build());
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dromara.hertzbeat.alert.controller;

import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.dromara.hertzbeat.alert.service.AlertSilenceService;
import org.dromara.hertzbeat.common.entity.alerter.AlertSilence;
import org.dromara.hertzbeat.common.entity.dto.Message;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.Predicate;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;

/**
* Silence the batch API for alarms
* 静默告警批量API
* @author tom
* @date 2023/04/12 22:03
*/
@Tag(name = "Alert Silence Batch API | 告警静默管理API")
@RestController
@RequestMapping(path = "/api/alert/silences", produces = {APPLICATION_JSON_VALUE})
public class AlertSilencesController {

@Autowired
private AlertSilenceService alertSilenceService;

@GetMapping
@Operation(summary = "Query the alarm silence list | 查询告警静默列表",
description = "You can obtain the list of alarm silence by querying filter items | 根据查询过滤项获取告警静默信息列表")
public ResponseEntity<Message<Page<AlertSilence>>> getAlertSilences(
@Parameter(description = "Alarm Silence ID | 告警静默ID", example = "6565463543") @RequestParam(required = false) List<Long> ids,
@Parameter(description = "Sort field, default id | 排序字段,默认id", example = "id") @RequestParam(defaultValue = "id") String sort,
@Parameter(description = "Sort mode: asc: ascending, desc: descending | 排序方式,asc:升序,desc:降序", example = "desc") @RequestParam(defaultValue = "desc") String order,
@Parameter(description = "List current page | 列表当前分页", example = "0") @RequestParam(defaultValue = "0") int pageIndex,
@Parameter(description = "Number of list pages | 列表分页数量", example = "8") @RequestParam(defaultValue = "8") int pageSize) {

Specification<AlertSilence> specification = (root, query, criteriaBuilder) -> {
List<Predicate> andList = new ArrayList<>();
if (ids != null && !ids.isEmpty()) {
CriteriaBuilder.In<Long> inPredicate= criteriaBuilder.in(root.get("id"));
for (long id : ids) {
inPredicate.value(id);
}
andList.add(inPredicate);
}
Predicate[] predicates = new Predicate[andList.size()];
return criteriaBuilder.and(andList.toArray(predicates));
};
Sort sortExp = Sort.by(new Sort.Order(Sort.Direction.fromString(order), sort));
PageRequest pageRequest = PageRequest.of(pageIndex, pageSize, sortExp);
Page<AlertSilence> alertSilencePage = alertSilenceService.getAlertSilences(specification,pageRequest);
Message<Page<AlertSilence>> message = new Message<>(alertSilencePage);
return ResponseEntity.ok(message);
}

@DeleteMapping
@Operation(summary = "Delete alarm silence in batches | 批量删除告警静默",
description = "Delete alarm silence in batches based on the alarm silence ID list | 根据告警静默ID列表批量删除告警静默")
public ResponseEntity<Message<Void>> deleteAlertDefines(
@Parameter(description = "Alarm Silence IDs | 告警静默IDs", example = "6565463543") @RequestParam(required = false) List<Long> ids
) {
if (ids != null && !ids.isEmpty()) {
alertSilenceService.deleteAlertSilences(new HashSet<>(ids));
}
Message<Void> message = new Message<>();
return ResponseEntity.ok(message);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dromara.hertzbeat.alert.dao;

import org.dromara.hertzbeat.common.entity.alerter.AlertSilence;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;

import java.util.Set;

/**
* AlertSilence 数据库操作
* @author tom
* @date 2023/04/12 22:03
*/
public interface AlertSilenceDao extends JpaRepository<AlertSilence, Long>, JpaSpecificationExecutor<AlertSilence> {

/**
* Delete alarm silence based on the ID list
* @param silenceIds alert silence id list
*/
@Modifying
void deleteAlertSilencesByIdIn(Set<Long> silenceIds);
}
Loading

0 comments on commit 4f1e441

Please sign in to comment.