From 628cb4ce07c705e208e287eaf552f17f79c7b8dd Mon Sep 17 00:00:00 2001 From: Smliexx Date: Thu, 30 Nov 2023 14:02:05 +0800 Subject: [PATCH] add time type to support query_time of mysql and mariadb (#1364) Signed-off-by: Clownsw --- .../alert/calculate/CalculateAlarm.java | 120 ++++++++++-------- .../collector/dispatch/MetricsCollect.java | 29 +++-- .../common/constants/CommonConstants.java | 29 +++-- .../hertzbeat/common/util/CommonUtil.java | 70 ++++++++-- .../src/main/resources/define/app-mariadb.yml | 4 +- .../src/main/resources/define/app-mysql.yml | 4 +- .../alert-setting.component.html | 44 ++++++- .../alert-setting/alert-setting.component.ts | 2 +- web-app/src/assets/i18n/en-US.json | 1 + web-app/src/assets/i18n/zh-CN.json | 1 + web-app/src/assets/i18n/zh-TW.json | 1 + 11 files changed, 204 insertions(+), 101 deletions(-) diff --git a/alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java b/alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java index 2d54bec64ed..17b0a179484 100644 --- a/alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java +++ b/alerter/src/main/java/org/dromara/hertzbeat/alert/calculate/CalculateAlarm.java @@ -22,29 +22,29 @@ import com.googlecode.aviator.exception.CompileExpressionErrorException; import com.googlecode.aviator.exception.ExpressionRuntimeException; import com.googlecode.aviator.exception.ExpressionSyntaxErrorException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.dromara.hertzbeat.alert.AlerterWorkerPool; +import org.dromara.hertzbeat.alert.dao.AlertMonitorDao; import org.dromara.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.dromara.hertzbeat.alert.service.AlertDefineService; import org.dromara.hertzbeat.alert.service.AlertService; -import org.dromara.hertzbeat.common.entity.manager.TagItem; -import org.dromara.hertzbeat.common.queue.CommonDataQueue; -import org.dromara.hertzbeat.alert.dao.AlertMonitorDao; +import org.dromara.hertzbeat.alert.util.AlertTemplateUtil; +import org.dromara.hertzbeat.common.constants.CommonConstants; import org.dromara.hertzbeat.common.entity.alerter.Alert; import org.dromara.hertzbeat.common.entity.alerter.AlertDefine; -import org.dromara.hertzbeat.alert.service.AlertDefineService; -import org.dromara.hertzbeat.alert.util.AlertTemplateUtil; import org.dromara.hertzbeat.common.entity.manager.Monitor; +import org.dromara.hertzbeat.common.entity.manager.TagItem; import org.dromara.hertzbeat.common.entity.message.CollectRep; -import org.dromara.hertzbeat.common.constants.CommonConstants; +import org.dromara.hertzbeat.common.queue.CommonDataQueue; import org.dromara.hertzbeat.common.support.event.MonitorDeletedEvent; import org.dromara.hertzbeat.common.support.event.SystemConfigChangeEvent; import org.dromara.hertzbeat.common.util.CommonUtil; import org.dromara.hertzbeat.common.util.ResourceBundleUtil; -import lombok.extern.slf4j.Slf4j; import org.springframework.context.event.EventListener; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; import javax.persistence.criteria.Predicate; import java.util.*; @@ -86,7 +86,7 @@ public class CalculateAlarm { public CalculateAlarm(AlerterWorkerPool workerPool, CommonDataQueue dataQueue, AlertDefineService alertDefineService, AlertMonitorDao monitorDao, - AlarmCommonReduce alarmCommonReduce, AlertService alertService) { + AlarmCommonReduce alarmCommonReduce, AlertService alertService) { this.workerPool = workerPool; this.dataQueue = dataQueue; this.alarmCommonReduce = alarmCommonReduce; @@ -120,7 +120,7 @@ private void startCalculate() { calculate(metricsData); } } catch (InterruptedException ignored) { - + } catch (Exception e) { log.error("calculate alarm error: {}.", e.getMessage(), e); } @@ -154,10 +154,12 @@ private void calculate(CollectRep.MetricsData metricsData) { for (Map.Entry> entry : defineMap.entrySet()) { List defines = entry.getValue(); for (AlertDefine define : defines) { - String expr = define.getExpr(); - if (!StringUtils.hasText(expr)) { + final String expr = define.getExpr(); + + if (StringUtils.isBlank(expr)) { continue; } + if (expr.contains(SYSTEM_VALUE_ROW_COUNT)) { fieldValueMap.put(SYSTEM_VALUE_ROW_COUNT, valueRowCount); try { @@ -189,18 +191,26 @@ private void calculate(CollectRep.MetricsData metricsData) { if (CommonConstants.NULL_VALUE.equals(valueStr)) { continue; } - CollectRep.Field field = fields.get(index); - if (field.getType() == CommonConstants.TYPE_NUMBER) { - Double doubleValue = CommonUtil.parseStrDouble(valueStr); - if (doubleValue != null) { + final CollectRep.Field field = fields.get(index); + final int fieldType = field.getType(); + + if (fieldType == CommonConstants.TYPE_NUMBER) { + final Double doubleValue; + if ((doubleValue = CommonUtil.parseStrDouble(valueStr)) != null) { fieldValueMap.put(field.getName(), doubleValue); } + } else if (fieldType == CommonConstants.TYPE_TIME) { + final Integer integerValue; + if ((integerValue = CommonUtil.parseStrInteger(valueStr)) != null) { + fieldValueMap.put(field.getName(), integerValue); + } } else { - if (!"".equals(valueStr)) { + if (StringUtils.isNotEmpty(valueStr)) { fieldValueMap.put(field.getName(), valueStr); } } + if (field.getLabel()) { instanceBuilder.append(valueStr).append("-"); } @@ -232,15 +242,15 @@ private void handleRecoveredAlert(long currentTimeMilli, long monitorId, String Map tags = notResolvedAlert.getTags(); String content = this.bundle.getString("alerter.alarm.recover") + " : " + expr; Alert resumeAlert = Alert.builder() - .tags(tags) - .target(define.getApp() + "." + define.getMetric() + "." + define.getField()) - .content(content) - .priority(CommonConstants.ALERT_PRIORITY_CODE_WARNING) - .status(CommonConstants.ALERT_STATUS_CODE_RESTORED) - .firstAlarmTime(currentTimeMilli) - .lastAlarmTime(currentTimeMilli) - .triggerTimes(1) - .build(); + .tags(tags) + .target(define.getApp() + "." + define.getMetric() + "." + define.getField()) + .content(content) + .priority(CommonConstants.ALERT_PRIORITY_CODE_WARNING) + .status(CommonConstants.ALERT_STATUS_CODE_RESTORED) + .firstAlarmTime(currentTimeMilli) + .lastAlarmTime(currentTimeMilli) + .triggerTimes(1) + .build(); alarmCommonReduce.reduceAndSendAlarm(resumeAlert); } } @@ -275,17 +285,17 @@ private void afterThresholdRuleMatch(long currentTimeMilli, long monitorId, Stri } } Alert alert = Alert.builder() - .tags(tags) - .priority(define.getPriority()) - .status(ALERT_STATUS_CODE_PENDING) - .target(app + "." + metrics + "." + define.getField()) - .triggerTimes(1) - .firstAlarmTime(currentTimeMilli) - .lastAlarmTime(currentTimeMilli) - // Keyword matching and substitution in the template - // 模板中关键字匹配替换 - .content(AlertTemplateUtil.render(define.getTemplate(), fieldValueMap)) - .build(); + .tags(tags) + .priority(define.getPriority()) + .status(ALERT_STATUS_CODE_PENDING) + .target(app + "." + metrics + "." + define.getField()) + .triggerTimes(1) + .firstAlarmTime(currentTimeMilli) + .lastAlarmTime(currentTimeMilli) + // Keyword matching and substitution in the template + // 模板中关键字匹配替换 + .content(AlertTemplateUtil.render(define.getTemplate(), fieldValueMap)) + .build(); int defineTimes = define.getTimes() == null ? 1 : define.getTimes(); if (1 >= defineTimes) { String notResolvedAlertKey = String.valueOf(monitorId) + define.getId() + fieldValueMap.get("instance"); @@ -329,7 +339,7 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr tags.put("metrics", CommonConstants.AVAILABILITY); tags.put("code", metricsData.getCode().name()); Map valueMap = tags.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); if (!CollectionUtils.isEmpty(avaAlertDefine.getTags())) { for (TagItem tagItem : avaAlertDefine.getTags()) { @@ -339,14 +349,14 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr } if (preAlert == null) { Alert.AlertBuilder alertBuilder = Alert.builder() - .tags(tags) - .priority(avaAlertDefine.getPriority()) - .status(ALERT_STATUS_CODE_PENDING) - .target(CommonConstants.AVAILABILITY) - .content(AlertTemplateUtil.render(avaAlertDefine.getTemplate(), valueMap)) - .firstAlarmTime(currentTimeMill) - .lastAlarmTime(currentTimeMill) - .triggerTimes(1); + .tags(tags) + .priority(avaAlertDefine.getPriority()) + .status(ALERT_STATUS_CODE_PENDING) + .target(CommonConstants.AVAILABILITY) + .content(AlertTemplateUtil.render(avaAlertDefine.getTemplate(), valueMap)) + .firstAlarmTime(currentTimeMill) + .lastAlarmTime(currentTimeMill) + .triggerTimes(1); if (avaAlertDefine.getTimes() == null || avaAlertDefine.getTimes() <= 1) { String notResolvedAlertKey = monitorId + CommonConstants.AVAILABILITY; notRecoveredAlertMap.put(notResolvedAlertKey, alertBuilder.build()); @@ -389,15 +399,15 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr } String content = this.bundle.getString("alerter.availability.recover"); Alert resumeAlert = Alert.builder() - .tags(tags) - .target(CommonConstants.AVAILABILITY) - .content(content) - .priority(CommonConstants.ALERT_PRIORITY_CODE_WARNING) - .status(CommonConstants.ALERT_STATUS_CODE_RESTORED) - .firstAlarmTime(currentTimeMill) - .lastAlarmTime(currentTimeMill) - .triggerTimes(1) - .build(); + .tags(tags) + .target(CommonConstants.AVAILABILITY) + .content(content) + .priority(CommonConstants.ALERT_PRIORITY_CODE_WARNING) + .status(CommonConstants.ALERT_STATUS_CODE_RESTORED) + .firstAlarmTime(currentTimeMill) + .lastAlarmTime(currentTimeMill) + .triggerTimes(1) + .build(); alarmCommonReduce.reduceAndSendAlarm(resumeAlert); Runnable updateStatusJob = () -> { // todo update pre all type alarm status diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/MetricsCollect.java b/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/MetricsCollect.java index 3b4e6b71358..ae62e98123c 100644 --- a/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/MetricsCollect.java +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/MetricsCollect.java @@ -32,11 +32,8 @@ import org.dromara.hertzbeat.common.entity.job.Job; import org.dromara.hertzbeat.common.entity.job.Metrics; import org.dromara.hertzbeat.common.entity.message.CollectRep; -import org.dromara.hertzbeat.common.constants.CommonConstants; import org.dromara.hertzbeat.common.util.CommonUtil; import org.dromara.hertzbeat.common.util.Pair; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; import java.util.*; import java.util.stream.Collectors; @@ -282,7 +279,7 @@ private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder col value = String.valueOf(objValue); } } catch (Exception e) { - log.info("[calculates execute warning] {}.", e.getMessage()); + log.info("[calculates execute warning] {}.", e.getMessage()); } } else { // does not exist then map the alias value @@ -293,13 +290,23 @@ private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder col } else { value = aliasFieldValueMap.get(realField); } - if (CommonConstants.TYPE_NUMBER == field.getType() && value != null) { - CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil - .extractDoubleAndUnitFromStr(value); - value = String.valueOf(doubleAndUnit.getValue()); - aliasFieldUnit = doubleAndUnit.getUnit(); + + if (value != null) { + final byte fieldType = field.getType(); + + if (fieldType == CommonConstants.TYPE_NUMBER) { + CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil + .extractDoubleAndUnitFromStr(value); + final Double tempValue = doubleAndUnit.getValue(); + value = tempValue == null ? null : String.valueOf(tempValue); + aliasFieldUnit = doubleAndUnit.getUnit(); + } else if (fieldType == CommonConstants.TYPE_TIME) { + final int tempValue; + value = (tempValue = CommonUtil.parseTimeStrToSecond(value)) == -1 ? null : String.valueOf(tempValue); + } } } + // 单位处理 Pair unitPair = fieldUnitMap.get(realField); if (aliasFieldUnit != null) { @@ -335,7 +342,6 @@ private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder col /** - * * @param cal * @param fieldAliasMap * @return @@ -343,7 +349,7 @@ private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder col private Object[] transformCal(String cal, Map fieldAliasMap) { int splitIndex = cal.indexOf("="); String field = cal.substring(0, splitIndex).trim(); - String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#","#"); + String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#", "#"); Expression expression; try { expression = AviatorEvaluator.compile(expressionStr, true); @@ -357,6 +363,7 @@ private Object[] transformCal(String cal, Map fieldAliasMap) { /** * transform unit + * * @param unit * @return */ diff --git a/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java b/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java index 5117994641a..6a3f4ab4dcc 100644 --- a/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java +++ b/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java @@ -20,6 +20,7 @@ /** * Public Constant * + * */ public interface CommonConstants { @@ -150,6 +151,12 @@ public interface CommonConstants { */ byte TYPE_SECRET = 2; + /** + * Field parameter type: time + * 字段参数类型: 时间 + */ + byte TYPE_TIME = 3; + /** * Collection indicator value: null placeholder for empty value * 采集指标值：null空值占位符 @@ -157,13 +164,11 @@ public interface CommonConstants { String NULL_VALUE = " "; /** - * * */ String PROM_TIME = "timestamp"; /** - * * */ String PROM_VALUE = "value"; @@ -190,12 +195,12 @@ public interface CommonConstants { * 参数类型 密码 */ byte PARAM_TYPE_PASSWORD = 2; - + /** * Parameter Type Map values */ byte PARAM_TYPE_MAP = 3; - + /** * Parameter Type arrays values */ @@ -244,7 +249,7 @@ public interface CommonConstants { * 内有标签: app 监控类型 */ String TAG_MONITOR_APP = "app"; - + /** * 内有标签: alarm type */ @@ -264,37 +269,37 @@ public interface CommonConstants { * cache key notice_rule */ String CACHE_NOTICE_RULE = "notice_rule"; - + /** * cache key alert silence */ String CACHE_ALERT_SILENCE = "alert_silence"; - + /** * cache key alert converge */ String CACHE_ALERT_CONVERGE = "alert_converge"; - + /** * collector status online 0 */ byte COLLECTOR_STATUS_ONLINE = 0; - + /** * collector status offline 1 */ byte COLLECTOR_STATUS_OFFLINE = 1; - + /** * default main collector name */ String MAIN_COLLECTOR_NODE = "main-default-collector"; - + /** * locale spilt */ String LOCALE_SEPARATOR = "_"; - + /** * ignore label * 处理未配置恢复告警，但需要使用恢复告警变更任务状态的情况 diff --git a/common/src/main/java/org/dromara/hertzbeat/common/util/CommonUtil.java b/common/src/main/java/org/dromara/hertzbeat/common/util/CommonUtil.java index 7a7cb275927..d97609e27fd 100644 --- a/common/src/main/java/org/dromara/hertzbeat/common/util/CommonUtil.java +++ b/common/src/main/java/org/dromara/hertzbeat/common/util/CommonUtil.java @@ -18,9 +18,11 @@ package org.dromara.hertzbeat.common.util; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.LocalTime; import java.util.Map; import java.util.Optional; import java.util.regex.Matcher; @@ -40,15 +42,36 @@ public class CommonUtil { private static final int PHONE_LENGTH = 11; + /** + * 将字符串str转换为int数字类型 + * + * @param str string + * @return double 数字 + */ + public static Integer parseStrInteger(final String str) { + if (StringUtils.isBlank(str)) { + return null; + } + + try { + return Integer.parseInt(str); + } catch (Exception e) { + log.debug(e.getMessage(), e); + return null; + } + } + /** * 将字符串str转换为double数字类型 + * * @param str string * @return double 数字 */ - public static Double parseStrDouble(String str) { - if (str == null || "".equals(str)) { + public static Double parseStrDouble(final String str) { + if (StringUtils.isBlank(str)) { return null; } + try { return Double.parseDouble(str); } catch (Exception e) { @@ -57,17 +80,38 @@ public static Double parseStrDouble(String str) { } } + /** + * 将时间字符串str转换为秒 + * + * @param str string + * @return double 数字 + */ + public static int parseTimeStrToSecond(final String str) { + if (StringUtils.isEmpty(str)) { + return -1; + } + + try { + return LocalTime.parse(str).toSecondOfDay(); + } catch (Exception e) { + log.debug(e.getMessage(), e); + return -1; + } + } + /** * 将字符串str,此字符串可能带单位,转换为double数字类型 * 将数值小数点限制到4位 - * @param str string + * + * @param str string * @param unit 字符串单位 * @return string格式的 double 数字 小数点最大到4位 */ public static String parseDoubleStr(String str, String unit) { - if (str == null || "".equals(str)) { + if (StringUtils.isBlank(str)) { return null; } + try { if (unit != null && str.endsWith(unit)) { str = str.substring(0, str.length() - unit.length()); @@ -82,26 +126,30 @@ public static String parseDoubleStr(String str, String unit) { /** * 邮箱格式校验 + * * @param email 邮箱 * @return 是否校验成功 */ - public static boolean validateEmail(String email) { - if (email == null || "".equals(email)) { + public static boolean validateEmail(final String email) { + if (StringUtils.isBlank(email)) { return false; } + Matcher m = EMAIL_PATTERN.matcher(email); return m.find(); } /** * 手机号格式校验 + * * @param phoneNum 手机号 * @return 是否校验成功 */ - public static boolean validatePhoneNum(String phoneNum) { - if (phoneNum == null || "".equals(phoneNum) || phoneNum.length() != PHONE_LENGTH) { + public static boolean validatePhoneNum(final String phoneNum) { + if (StringUtils.isBlank(phoneNum) || phoneNum.length() != PHONE_LENGTH) { return false; } + Matcher m = PHONE_PATTERN.matcher(phoneNum); return m.find(); } @@ -136,14 +184,14 @@ public static String removeBlankLine(String value) { } return value.replaceAll("(?m)^\\s*$(\\n|\\r\\n)", ""); } - + public static String getLangMappingValueFromI18nMap(String lang, Map i18nMap) { if (i18nMap == null || i18nMap.isEmpty()) { return null; } return Optional.ofNullable(i18nMap.get(lang)) - .orElse(i18nMap.values().stream() - .findFirst().orElse(null)); + .orElse(i18nMap.values().stream() + .findFirst().orElse(null)); } } diff --git a/manager/src/main/resources/define/app-mariadb.yml b/manager/src/main/resources/define/app-mariadb.yml index c2b8803a140..acfd4611b05 100644 --- a/manager/src/main/resources/define/app-mariadb.yml +++ b/manager/src/main/resources/define/app-mariadb.yml @@ -639,13 +639,13 @@ metrics: type: 1 label: true - field: sql_text - type: 0 + type: 1 - field: start_time type: 1 - field: db type: 1 - field: query_time - type: 1 + type: 3 aliasFields: - sql_text - start_time diff --git a/manager/src/main/resources/define/app-mysql.yml b/manager/src/main/resources/define/app-mysql.yml index 38ae3d17933..4c6f9a39c97 100644 --- a/manager/src/main/resources/define/app-mysql.yml +++ b/manager/src/main/resources/define/app-mysql.yml @@ -902,7 +902,7 @@ metrics: zh-CN: ID en-US: ID - field: sql_text - type: 0 + type: 1 i18n: zh-CN: SQL 文本 en-US: SQL Text @@ -917,7 +917,7 @@ metrics: zh-CN: 数据库 en-US: Database - field: query_time - type: 1 + type: 3 i18n: zh-CN: 查询时间 en-US: Query Time diff --git a/web-app/src/app/routes/alert/alert-setting/alert-setting.component.html b/web-app/src/app/routes/alert/alert-setting/alert-setting.component.html index 436d5786e45..a30fa9be1df 100644 --- a/web-app/src/app/routes/alert/alert-setting/alert-setting.component.html +++ b/web-app/src/app/routes/alert/alert-setting/alert-setting.component.html @@ -241,7 +241,13 @@ > {{ item.label ? item.label : item.value }} - {{ item.type === 0 ? ('alert.setting.number' | i18n) : ('alert.setting.string' | i18n) }} + {{ + item.type === 0 + ? ('alert.setting.number' | i18n) + : item.type === 3 + ? ('alert.setting.time' | i18n) + : ('alert.setting.string' | i18n) + }} {{ item.unit }} @@ -259,12 +265,36 @@ [nzDropdownStyle]="{ 'text-align': 'center', 'font-weight': 'bolder', 'font-size': 'larger' }" [nzPlaceHolder]="'alert.setting.rule.operator' | i18n" > - - - - - - + + + + + +