Skip to content

Commit

Permalink
auto update alert status (#1104)
Browse files Browse the repository at this point in the history
  • Loading branch information
@l646505418
l646505418 authored Jul 17, 2023
1 parent 027c74e commit 3b7f667
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 9 deletions.
66 changes: 57 additions & 9 deletions alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
import com.googlecode.aviator.exception.ExpressionRuntimeException;
import com.googlecode.aviator.exception.ExpressionSyntaxErrorException;
import org.dromara.hertzbeat.alert.AlerterWorkerPool;
import org.dromara.hertzbeat.alert.dao.AlertDao;
import org.dromara.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.dromara.hertzbeat.alert.service.AlertService;
import org.dromara.hertzbeat.common.queue.CommonDataQueue;
import org.dromara.hertzbeat.alert.dao.AlertMonitorDao;
import org.dromara.hertzbeat.common.entity.alerter.Alert;
Expand All @@ -37,16 +39,21 @@
import org.dromara.hertzbeat.common.util.ResourceBundleUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.domain.Specification;

import javax.persistence.criteria.Predicate;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

import static org.dromara.hertzbeat.common.constants.CommonConstants.ALERT_STATUS_CODE_PENDING;
import static org.dromara.hertzbeat.common.constants.CommonConstants.ALERT_STATUS_CODE_SOLVED;

/**
* Calculate alarms based on the alarm definition rules and collected data
* 根据告警定义规则和采集数据匹配计算告警
* @author tom
*
* @author tom
*/
@Configuration
@Slf4j
Expand All @@ -65,14 +72,16 @@ public class CalculateAlarm {
private final AlertDefineService alertDefineService;
private final AlarmCommonReduce alarmCommonReduce;
private final ResourceBundle bundle;
private final AlertService alertService;

public CalculateAlarm (AlerterWorkerPool workerPool, CommonDataQueue dataQueue,
AlertDefineService alertDefineService, AlertMonitorDao monitorDao,
AlarmCommonReduce alarmCommonReduce) {
public CalculateAlarm(AlerterWorkerPool workerPool, CommonDataQueue dataQueue,
AlertDefineService alertDefineService, AlertMonitorDao monitorDao,
AlarmCommonReduce alarmCommonReduce, AlertService alertService) {
this.workerPool = workerPool;
this.dataQueue = dataQueue;
this.alarmCommonReduce = alarmCommonReduce;
this.alertDefineService = alertDefineService;
this.alertService = alertService;
this.bundle = ResourceBundleUtil.getBundle("alerter");
this.triggeredAlertMap = new ConcurrentHashMap<>(128);
this.unAvailableMonitors = Collections.synchronizedSet(new HashSet<>(16));
Expand Down Expand Up @@ -154,7 +163,8 @@ private void calculate(CollectRep.MetricsData metricsData) {
try {
Expression expression = AviatorEvaluator.compile(expr, true);
match = (Boolean) expression.execute(fieldValueMap);
} catch (CompileExpressionErrorException | ExpressionSyntaxErrorException compileException) {
} catch (CompileExpressionErrorException |
ExpressionSyntaxErrorException compileException) {
log.error("Alert Define Rule: {} Compile Error: {}.", expr, compileException.getMessage());
} catch (ExpressionRuntimeException expressionRuntimeException) {
log.error("Alert Define Rule: {} Run Error: {}.", expr, expressionRuntimeException.getMessage());
Expand Down Expand Up @@ -188,7 +198,7 @@ private void calculate(CollectRep.MetricsData metricsData) {
.tags(tags)
.alertDefineId(define.getId())
.priority(define.getPriority())
.status(CommonConstants.ALERT_STATUS_CODE_PENDING)
.status(ALERT_STATUS_CODE_PENDING)
.target(app + "." + metrics + "." + define.getField())
.triggerTimes(1)
.firstAlarmTime(currentTimeMilli)
Expand Down Expand Up @@ -263,6 +273,10 @@ private void handlerAvailableMetrics(long monitorId, String app, String metrics,
.triggerTimes(1)
.build();
alarmCommonReduce.reduceAndSendAlarm(resumeAlert);
Runnable updateStatusJob = () -> {
updateAvailabilityAlertStatus(monitorId, resumeAlert);
};
workerPool.executeJob(updateStatusJob);
}
}
}
Expand All @@ -285,7 +299,7 @@ private void handlerMonitorAvailableAlert(long monitorId, String app, CollectRep
Alert.AlertBuilder alertBuilder = Alert.builder()
.tags(tags)
.priority(CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY)
.status(CommonConstants.ALERT_STATUS_CODE_PENDING)
.status(ALERT_STATUS_CODE_PENDING)
.target(CommonConstants.AVAILABILITY)
.content(AlertTemplateUtil.render(avaAlertDefine.getTemplate(), valueMap))
.firstAlarmTime(currentTimeMill)
Expand All @@ -300,7 +314,7 @@ private void handlerMonitorAvailableAlert(long monitorId, String app, CollectRep
triggeredAlertMap.put(String.valueOf(monitorId), alertBuilder.build());
} else {
int times = preAlert.getTriggerTimes() + 1;
if (preAlert.getStatus() == CommonConstants.ALERT_STATUS_CODE_PENDING) {
if (preAlert.getStatus() == ALERT_STATUS_CODE_PENDING) {
times = 1;
preAlert.setContent(AlertTemplateUtil.render(avaAlertDefine.getTemplate(), valueMap));
preAlert.setTags(tags);
Expand All @@ -310,12 +324,46 @@ private void handlerMonitorAvailableAlert(long monitorId, String app, CollectRep
preAlert.setLastAlarmTime(currentTimeMill);
int defineTimes = avaAlertDefine.getTimes() == null ? 1 : avaAlertDefine.getTimes();
if (times >= defineTimes) {
preAlert.setStatus(CommonConstants.ALERT_STATUS_CODE_PENDING);
preAlert.setStatus(ALERT_STATUS_CODE_PENDING);
alarmCommonReduce.reduceAndSendAlarm(preAlert.clone());
unAvailableMonitors.add(monitorId);
} else {
preAlert.setStatus(CommonConstants.ALERT_STATUS_CODE_NOT_REACH);
}
}
}


private void updateAvailabilityAlertStatus(long monitorId, Alert restoreAlert) {
List<Alert> availabilityAlerts = queryAvailabilityAlerts(monitorId, restoreAlert);
availabilityAlerts.stream().parallel().forEach(alert -> {
log.info("updating alert id:{}",alert.getId());
alertService.editAlertStatus(ALERT_STATUS_CODE_SOLVED, List.of(alert.getId()));
});
}

private List<Alert> queryAvailabilityAlerts(long monitorId, Alert restoreAlert) {
//create query condition
Specification<Alert> specification = (root, query, criteriaBuilder) -> {
List<Predicate> andList = new ArrayList<>();

Predicate predicateTags = criteriaBuilder.like(root.get("tags").as(String.class), "%" + monitorId + "%");
andList.add(predicateTags);

Predicate predicatePriority = criteriaBuilder.equal(root.get("priority"), CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY);
andList.add(predicatePriority);

Predicate predicateStatus = criteriaBuilder.equal(root.get("status"), ALERT_STATUS_CODE_PENDING);
andList.add(predicateStatus);

Predicate predicateAlertTime = criteriaBuilder.lessThanOrEqualTo(root.get("lastAlarmTime"), restoreAlert.getLastAlarmTime());
andList.add(predicateAlertTime);

Predicate[] predicates = new Predicate[andList.size()];
return criteriaBuilder.and(andList.toArray(predicates));
};

//query results
return alertService.getAlerts(specification);
}
}
10 changes: 10 additions & 0 deletions alerter/src/main/java/org/dromara/hertzbeat/alert/service/AlertService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,4 +91,14 @@ public interface AlertService {
*/
void addNewAlertReport(AlertReport alertReport);

/**
* Dynamic conditional query
* 动态条件查询
*
* @param specification Query conditions 查询条件
* @return search result 查询结果
*/
List<Alert> getAlerts(Specification<Alert> specification);


}
6 changes: 6 additions & 0 deletions alerter/src/main/java/org/dromara/hertzbeat/alert/service/impl/AlertServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,12 @@ public void addNewAlertReport(AlertReport alertReport) {
alarmCommonReduce.reduceAndSendAlarm(buildAlertData(alertReport));
}

@Override
public List<Alert> getAlerts(Specification<Alert> specification) {

return alertDao.findAll(specification);
}

/**
* The external alarm information is converted to Alert 对外告警信息 转换为Alert
* @param alertReport 对外告警信息
Expand Down

0 comments on commit 3b7f667

Please sign in to comment.