Skip to content

Commit

Permalink
support alert threshold rule config system value row count (#1180)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomsun28
tomsun28 committed Jan 16, 2024
1 parent 77f24bd commit d593ba4
Show file tree
Hide file tree
Showing 10 changed files with 195 additions and 106 deletions.
187 changes: 109 additions & 78 deletions alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.springframework.context.event.EventListener;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import javax.persistence.criteria.Predicate;
import java.util.*;
Expand All @@ -60,6 +61,8 @@
@Slf4j
public class CalculateAlarm {

private static final String SYSTEM_VALUE_ROW_COUNT = "system_value_row_count";

/**
* The alarm in the process is triggered
* 触发中告警信息
Expand Down Expand Up @@ -134,101 +137,129 @@ private void calculate(CollectRep.MetricsData metricsData) {
}
List<CollectRep.Field> fields = metricsData.getFieldsList();
Map<String, Object> fieldValueMap = new HashMap<>(16);
for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) {
if (!valueRow.getColumnsList().isEmpty()) {
fieldValueMap.clear();
String instance = valueRow.getInstance();
if (!"".equals(instance)) {
fieldValueMap.put("instance", instance);
int valueRowCount = metricsData.getValuesCount();
for (Map.Entry<String, List<AlertDefine>> entry : defineMap.entrySet()) {
List<AlertDefine> defines = entry.getValue();
for (AlertDefine define : defines) {
String expr = define.getExpr();
if (!StringUtils.hasText(expr)) {
continue;
}
for (int index = 0; index < valueRow.getColumnsList().size(); index++) {
String valueStr = valueRow.getColumns(index);
CollectRep.Field field = fields.get(index);
if (field.getType() == CommonConstants.TYPE_NUMBER) {
Double doubleValue = CommonUtil.parseStrDouble(valueStr);
if (doubleValue != null) {
fieldValueMap.put(field.getName(), doubleValue);
}
} else {
if (!"".equals(valueStr)) {
fieldValueMap.put(field.getName(), valueStr);
if (expr.contains(SYSTEM_VALUE_ROW_COUNT)) {
fieldValueMap.put(SYSTEM_VALUE_ROW_COUNT, valueRowCount);
try {
boolean match = execAlertExpression(fieldValueMap, expr);
if (match) {
// If the threshold rule matches, the number of times the threshold has been triggered is determined and an alarm is triggered
// 阈值规则匹配，判断已触发阈值次数，触发告警
afterThresholdRuleMatch(currentTimeMilli, monitorId, app, metrics, fieldValueMap, define);
// 若此阈值已被触发，则其它数据行的触发忽略
continue;
}
} catch (Exception e) {
log.warn(e.getMessage(), e);
}
}
for (Map.Entry<String, List<AlertDefine>> entry : defineMap.entrySet()) {
List<AlertDefine> defines = entry.getValue();
for (AlertDefine define : defines) {
String expr = define.getExpr();
try {
Boolean match = false;
try {
Expression expression = AviatorEvaluator.compile(expr, true);
match = (Boolean) expression.execute(fieldValueMap);
} catch (CompileExpressionErrorException |
ExpressionSyntaxErrorException compileException) {
log.error("Alert Define Rule: {} Compile Error: {}.", expr, compileException.getMessage());
} catch (ExpressionRuntimeException expressionRuntimeException) {
log.error("Alert Define Rule: {} Run Error: {}.", expr, expressionRuntimeException.getMessage());
} catch (Exception e) {
log.error("Alert Define Rule: {} Run Error: {}.", e, e.getMessage());
}

if (match != null && match) {
// If the threshold rule matches, the number of times the threshold has been triggered is determined and an alarm is triggered
// 阈值规则匹配，判断已触发阈值次数，触发告警
String monitorAlertKey = String.valueOf(monitorId) + define.getId();
Alert triggeredAlert = triggeredAlertMap.get(monitorAlertKey);
if (triggeredAlert != null) {
int times = triggeredAlert.getTriggerTimes() + 1;
triggeredAlert.setTriggerTimes(times);
triggeredAlert.setFirstAlarmTime(currentTimeMilli);
triggeredAlert.setLastAlarmTime(currentTimeMilli);
int defineTimes = define.getTimes() == null ? 1 : define.getTimes();
if (times >= defineTimes) {
triggeredAlertMap.remove(monitorAlertKey);
alarmCommonReduce.reduceAndSendAlarm(triggeredAlert.clone());
for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) {
if (!valueRow.getColumnsList().isEmpty()) {
fieldValueMap.clear();
fieldValueMap.put(SYSTEM_VALUE_ROW_COUNT, valueRowCount);
String instance = valueRow.getInstance();
if (!"".equals(instance)) {
fieldValueMap.put("instance", instance);
}
for (int index = 0; index < valueRow.getColumnsList().size(); index++) {
String valueStr = valueRow.getColumns(index);
if (!CommonConstants.NULL_VALUE.equals(valueStr)) {
CollectRep.Field field = fields.get(index);
if (field.getType() == CommonConstants.TYPE_NUMBER) {
Double doubleValue = CommonUtil.parseStrDouble(valueStr);
if (doubleValue != null) {
fieldValueMap.put(field.getName(), doubleValue);
}
} else {
fieldValueMap.put("app", app);
fieldValueMap.put("metrics", metrics);
fieldValueMap.put("metric", define.getField());
Map<String, String> tags = new HashMap<>(6);
tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
tags.put(CommonConstants.TAG_MONITOR_APP, app);
Alert alert = Alert.builder()
.tags(tags)
.alertDefineId(define.getId())
.priority(define.getPriority())
.status(ALERT_STATUS_CODE_PENDING)
.target(app + "." + metrics + "." + define.getField())
.triggerTimes(1)
.firstAlarmTime(currentTimeMilli)
.lastAlarmTime(currentTimeMilli)
// Keyword matching and substitution in the template
// 模板中关键字匹配替换
.content(AlertTemplateUtil.render(define.getTemplate(), fieldValueMap))
.build();
int defineTimes = define.getTimes() == null ? 1 : define.getTimes();
if (1 >= defineTimes) {
alarmCommonReduce.reduceAndSendAlarm(alert);
} else {
triggeredAlertMap.put(monitorAlertKey, alert);
if (!"".equals(valueStr)) {
fieldValueMap.put(field.getName(), valueStr);
}
}
// Threshold rules below this priority are ignored
// 此优先级以下的阈值规则则忽略
}
}
}
try {
boolean match = execAlertExpression(fieldValueMap, expr);
if (match) {
// If the threshold rule matches, the number of times the threshold has been triggered is determined and an alarm is triggered
// 阈值规则匹配，判断已触发阈值次数，触发告警
afterThresholdRuleMatch(currentTimeMilli, monitorId, app, metrics, fieldValueMap, define);
// 若此阈值已被触发，则其它数据行的触发忽略
break;
}
} catch (Exception e) {
log.warn(e.getMessage(), e);
}
}
}

}
}
}


private void afterThresholdRuleMatch(long currentTimeMilli, long monitorId, String app, String metrics, Map<String, Object> fieldValueMap, AlertDefine define) {
String monitorAlertKey = String.valueOf(monitorId) + define.getId();
Alert triggeredAlert = triggeredAlertMap.get(monitorAlertKey);
if (triggeredAlert != null) {
int times = triggeredAlert.getTriggerTimes() + 1;
triggeredAlert.setTriggerTimes(times);
triggeredAlert.setFirstAlarmTime(currentTimeMilli);
triggeredAlert.setLastAlarmTime(currentTimeMilli);
int defineTimes = define.getTimes() == null ? 1 : define.getTimes();
if (times >= defineTimes) {
triggeredAlertMap.remove(monitorAlertKey);
alarmCommonReduce.reduceAndSendAlarm(triggeredAlert.clone());
}
} else {
fieldValueMap.put("app", app);
fieldValueMap.put("metrics", metrics);
fieldValueMap.put("metric", define.getField());
Map<String, String> tags = new HashMap<>(6);
tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
tags.put(CommonConstants.TAG_MONITOR_APP, app);
Alert alert = Alert.builder()
.tags(tags)
.alertDefineId(define.getId())
.priority(define.getPriority())
.status(ALERT_STATUS_CODE_PENDING)
.target(app + "." + metrics + "." + define.getField())
.triggerTimes(1)
.firstAlarmTime(currentTimeMilli)
.lastAlarmTime(currentTimeMilli)
// Keyword matching and substitution in the template
// 模板中关键字匹配替换
.content(AlertTemplateUtil.render(define.getTemplate(), fieldValueMap))
.build();
int defineTimes = define.getTimes() == null ? 1 : define.getTimes();
if (1 >= defineTimes) {
alarmCommonReduce.reduceAndSendAlarm(alert);
} else {
triggeredAlertMap.put(monitorAlertKey, alert);
}
}
}

private boolean execAlertExpression(Map<String, Object> fieldValueMap, String expr) {
Boolean match = false;
try {
Expression expression = AviatorEvaluator.compile(expr, true);
match = (Boolean) expression.execute(fieldValueMap);
} catch (CompileExpressionErrorException |
ExpressionSyntaxErrorException compileException) {
log.error("Alert Define Rule: {} Compile Error: {}.", expr, compileException.getMessage());
} catch (ExpressionRuntimeException expressionRuntimeException) {
log.error("Alert Define Rule: {} Run Error: {}.", expr, expressionRuntimeException.getMessage());
} catch (Exception e) {
log.error("Alert Define Rule: {} Run Error: {}.", e, e.getMessage());
}
return match;
}

private void handlerAvailableMetrics(long monitorId, String app, String metrics, CollectRep.MetricsData metricsData) {
if (metricsData.getCode() != CollectRep.Code.SUCCESS) {
// Collection and abnormal
Expand Down
14 changes: 8 additions & 6 deletions common/src/main/java/org/dromara/hertzbeat/common/config/AviatorConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.springframework.context.annotation.Configuration;

import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;

/**
Expand Down Expand Up @@ -72,7 +73,7 @@ public String getName() {
});

AviatorEvaluator.getInstance().addFunction(new StrContainsFunction());
AviatorEvaluator.getInstance().addFunction(new StrExistsFunction());
AviatorEvaluator.getInstance().addFunction(new ObjectExistsFunction());
AviatorEvaluator.getInstance().addFunction(new StrMatchesFunction());
}

Expand Down Expand Up @@ -125,20 +126,21 @@ public String getName() {
}

/**
* 自定义aviator判断环境中是否存在字符串
* 自定义aviator判断环境中此对象是否存在值
*/
private static class StrExistsFunction extends AbstractFunction {
private static class ObjectExistsFunction extends AbstractFunction {
@Override
public AviatorObject call(Map<String, Object> env, AviatorObject arg) {
if (arg == null) {
return AviatorBoolean.FALSE;
}
Object keyTmp = arg.getValue(env);
if (keyTmp == null) {
if (Objects.isNull(keyTmp)) {
return AviatorBoolean.FALSE;
} else {
String key = String.valueOf(keyTmp);
return AviatorBoolean.valueOf(StringUtils.isNotEmpty(key));
}
String key = String.valueOf(keyTmp);
return AviatorBoolean.valueOf(env.containsKey(key));
}
@Override
public String getName() {
Expand Down
16 changes: 12 additions & 4 deletions common/src/test/java/org/dromara/hertzbeat/common/config/AviatorConfigurationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,19 @@ void testCustomStringFunctions() {
// test StrExistsFunction
String expr6 = "exists('DNE_Key1')";
Boolean res6 = (Boolean) AviatorEvaluator.compile(expr6).execute(env);
Assertions.assertFalse(res6);
Assertions.assertTrue(res6);

String expr7 = "exists('k6')";
String expr7 = "exists(k6)";
Boolean res7 = (Boolean) AviatorEvaluator.compile(expr7).execute(env);
Assertions.assertTrue(res7);
Assertions.assertFalse(res7);

String expr21 = "exists('k5')";
Boolean res21 = (Boolean) AviatorEvaluator.compile(expr21).execute(env);
Assertions.assertTrue(res21);

String expr22 = "exists(k5)";
Boolean res22 = (Boolean) AviatorEvaluator.compile(expr22).execute(env);
Assertions.assertTrue(res22);

// test StrMatchesFunction
String regex1 = "'^[a-zA-Z0-9]+$'"; // only alphanumeric
Expand Down Expand Up @@ -88,4 +96,4 @@ void testCustomStringFunctions() {
Boolean res13 = (Boolean) AviatorEvaluator.compile(expr10).execute(env);
Assertions.assertFalse(res13);
}
}
}
6 changes: 4 additions & 2 deletions ...house/src/main/java/org/dromara/hertzbeat/warehouse/controller/MetricsDataController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,9 @@ public ResponseEntity<Message<MetricsData>> getMetricsData(
dataBuilder.fields(fields);
List<ValueRow> valueRows = storageData.getValuesList().stream().map(redisValueRow ->
ValueRow.builder().instance(redisValueRow.getInstance())
.values(redisValueRow.getColumnsList().stream().map(Value::new).collect(Collectors.toList()))
.values(redisValueRow.getColumnsList().stream()
.map(origin -> CommonConstants.NULL_VALUE.equals(origin) ? new Value()
: new Value(origin)).collect(Collectors.toList()))
.build()).collect(Collectors.toList());
dataBuilder.valueRows(valueRows);
return ResponseEntity.ok().body(new Message<>(dataBuilder.build()));
Expand Down Expand Up @@ -165,7 +167,7 @@ public ResponseEntity<Message<MetricsHistoryData>> getMetricHistoryData(
if (history == null) {
history = "6h";
}
Map<String, List<Value>> instanceValuesMap = null;
Map<String, List<Value>> instanceValuesMap;
if (interval == null || !interval) {
instanceValuesMap = historyDataStorage.getHistoryMetricData(monitorId, app, metrics, metric, instance, history);
} else {
Expand Down
Loading

0 comments on commit d593ba4

Please sign in to comment.