Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

update label structure store in victoria metrics, make it prometheus like #1426

Merged
merged 6 commits into from
Dec 17, 2023
Merged

update label structure store in victoria metrics, make it prometheus like #1426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b2dc520
update label structure store in victoria metrics, make it prometheus …
tomsun28 Dec 16, 2023
fb30c44
Merge branch 'master' into promethues_prefix
tomsun28 Dec 16, 2023
d253f7e
fix unit test
tomsun28 Dec 16, 2023
d1a0f1e
Merge remote-tracking branch 'origin/promethues_prefix' into promethu…
tomsun28 Dec 16, 2023
ea0f588
update doc
tomsun28 Dec 16, 2023
a9b8583
Merge branch 'master' into promethues_prefix
tomsun28 Dec 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,4 +319,14 @@ public interface CommonConstants {
* collector auth failed message
*/
String COLLECTOR_AUTH_FAILED = "Auth Failed";

/**
* for prometheus task name prefix
*/
String PROMETHEUS_APP_PREFIX = "_prometheus_";

/**
* prometheus
*/
String PROMETHEUS = "prometheus";
}
2 changes: 1 addition & 1 deletion home/i18n/en/code.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@
"message": "HertzBeat's stand-alone and cluster version code are all open source, unlimited and anyone who are interested in it are very welcome to contribute. No matter how small the contribution is, whether it is a document code or a typo, we respect everyone and learn and grow together. {br} We hope that the code we participate in can be deployed to thousands of machines, leaving our code footprints in this world.{br}{github} {gitee}"
},
"slogan": {
"message": "Open Source Real-time Monitoring Tool"
"message": "Open Source Real-time Monitoring System"
},
"Who uses HertzBeat?": {
"message": "Who uses HertzBeat?"
Expand Down
3 changes: 3 additions & 0 deletions manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/CollectorAndJobScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ public void collectorGoOnline(String identity, CollectorInfo collectorInfo) {
try {
// 构造采集任务Job实体
Job appDefine = appService.getAppDefine(monitor.getApp());
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setMonitorId(monitor.getId());
appDefine.setInterval(monitor.getIntervals());
appDefine.setCyclic(true);
Expand Down
3 changes: 3 additions & 0 deletions manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/SchedulerInit.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ public void run(String... args) throws Exception {
try {
// 构造采集任务Job实体
Job appDefine = appService.getAppDefine(monitor.getApp());
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setId(monitor.getJobId());
appDefine.setMonitorId(monitor.getId());
appDefine.setInterval(monitor.getIntervals());
Expand Down
9 changes: 9 additions & 0 deletions manager/src/main/java/org/dromara/hertzbeat/manager/service/AppService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import java.util.List;
import java.util.Map;
import java.util.Optional;

/**
* Monitoring Type Management Interface
Expand Down Expand Up @@ -62,6 +63,14 @@ public interface AppService {
*/
Job getAppDefine(String app) throws IllegalArgumentException;

/**
* Get monitor structure definition based on monitor type name
*
* @param app Monitoring type name
* @return Monitoring Structure Definition Optional
*/
Optional<Job> getAppDefineOption(String app);

/**
* 获取app定义的指标
*
Expand Down
9 changes: 9 additions & 0 deletions manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,15 @@ public Job getAppDefine(String app) throws IllegalArgumentException {
return appDefine.clone();
}

@Override
public Optional<Job> getAppDefineOption(String app) {
if (StringUtils.hasText(app)) {
Job appDefine = appDefines.get(app.toLowerCase());
return Optional.ofNullable(appDefine);
}
return Optional.empty();
}

@Override
public List<String> getAppDefineMetricNames(String app) {
List<String> metricNames = new ArrayList<>(16);
Expand Down
27 changes: 23 additions & 4 deletions manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/MonitorServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ public void detectMonitor(Monitor monitor, List<Param> params, String collector)
monitorId = MONITOR_ID_TMP;
}
Job appDefine = appService.getAppDefine(monitor.getApp());
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setMonitorId(monitorId);
appDefine.setCyclic(false);
appDefine.setTimestamp(System.currentTimeMillis());
Expand Down Expand Up @@ -172,8 +175,11 @@ public void addMonitor(Monitor monitor, List<Param> params, String collector) th
}
tags.add(Tag.builder().name(CommonConstants.TAG_MONITOR_ID).value(String.valueOf(monitorId)).type((byte) 0).build());
tags.add(Tag.builder().name(CommonConstants.TAG_MONITOR_NAME).value(String.valueOf(monitor.getName())).type((byte) 0).build());
// Construct the collection task Job entity 构造采集任务Job实体
// Construct the collection task Job entity
Job appDefine = appService.getAppDefine(monitor.getApp());
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setMonitorId(monitorId);
appDefine.setInterval(monitor.getIntervals());
appDefine.setCyclic(true);
Expand Down Expand Up @@ -317,17 +323,21 @@ public void validate(MonitorDto monitorDto, Boolean isModify) throws IllegalArgu
param.setValue(value);
})
.collect(Collectors.toMap(Param::getField, param -> param));
// Check name uniqueness 校验名称唯一性
// Check name uniqueness and can not equal app type
if (isModify != null) {
Optional<Job> defineOptional = appService.getAppDefineOption(monitor.getName());
if (defineOptional.isPresent()) {
throw new IllegalArgumentException("Monitoring name cannot be the existed monitoring type name!");
}
Optional<Monitor> monitorOptional = monitorDao.findMonitorByNameEquals(monitor.getName());
if (monitorOptional.isPresent()) {
Monitor existMonitor = monitorOptional.get();
if (isModify) {
if (!existMonitor.getId().equals(monitor.getId())) {
throw new IllegalArgumentException("Monitoring name cannot be repeated!");
throw new IllegalArgumentException("Monitoring name already exists!");
}
} else {
throw new IllegalArgumentException("Monitoring name cannot be repeated!");
throw new IllegalArgumentException("Monitoring name already exists!");
}
}
}
Expand Down Expand Up @@ -509,6 +519,9 @@ public void modifyMonitor(Monitor monitor, List<Param> params, String collector)
// Construct the collection task Job entity
// 构造采集任务Job实体
Job appDefine = appService.getAppDefine(monitor.getApp());
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setId(preMonitor.getJobId());
appDefine.setMonitorId(monitorId);
appDefine.setInterval(monitor.getIntervals());
Expand Down Expand Up @@ -659,6 +672,9 @@ public void enableManageMonitors(HashSet<Long> ids) {
// Construct the collection task Job entity
// 构造采集任务Job实体
Job appDefine = appService.getAppDefine(monitor.getApp());
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setMonitorId(monitor.getId());
appDefine.setInterval(monitor.getIntervals());
appDefine.setCyclic(true);
Expand Down Expand Up @@ -765,6 +781,9 @@ public void updateAppCollectJob(Job job) {
log.error("update monitor job error when template modify, define | id | jobId is null. continue");
continue;
}
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setId(monitor.getJobId());
appDefine.setMonitorId(monitor.getId());
appDefine.setInterval(monitor.getIntervals());
Expand Down
2 changes: 1 addition & 1 deletion manager/src/test/java/org/dromara/hertzbeat/manager/service/MonitorServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ void validateMonitorName() {
try {
monitorService.validate(dto, isModify);
} catch (IllegalArgumentException e) {
assertEquals("Monitoring name cannot be repeated!", e.getMessage());
assertEquals("Monitoring name already exists!", e.getMessage());
}
}

Expand Down
64 changes: 39 additions & 25 deletions ...rc/main/java/org/dromara/hertzbeat/warehouse/store/HistoryVictoriaMetricsDataStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,13 @@ public class HistoryVictoriaMetricsDataStorage extends AbstractHistoryDataStorag
private static final String STATUS_PATH = "/api/v1/status/tsdb";
private static final String STATUS = "status";
private static final String STATUS_SUCCESS = "success";
private static final String NAME_KEY = "__name__";
private static final String LABEL_KEY_NAME = "__name__";
private static final String LABEL_KEY_JOB = "job";
private static final String LABEL_KEY_INSTANCE = "instance";
private static final String SPILT = "_";
private static final String BASIC = "Basic";
private static final String MONITOR_TYPE_KEY = "__app__";
private static final String MONITOR_METRICS_KEY = "__metrics__";
private static final String MONITOR_METRIC_KEY = "__metric__";
private static final String MONITOR_ID_KEY = "__monitorId__";

private final WarehouseProperties.StoreProperties.VictoriaMetricsProperties victoriaMetricsProp;

Expand Down Expand Up @@ -125,9 +125,16 @@ public void saveData(CollectRep.MetricsData metricsData) {
return;
}
Map<String, String> defaultLabels = new HashMap<>(8);
defaultLabels.put(MONITOR_ID_KEY, String.valueOf(metricsData.getId()));
defaultLabels.put(MONITOR_TYPE_KEY, metricsData.getApp());
defaultLabels.put(MONITOR_METRICS_KEY, metricsData.getMetrics());
boolean isPrometheusAuto = false;
if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
isPrometheusAuto = true;
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()
.substring(CommonConstants.PROMETHEUS_APP_PREFIX.length()));
} else {
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp());
}
defaultLabels.put(LABEL_KEY_INSTANCE, String.valueOf(metricsData.getId()));

List<CollectRep.Field> fields = metricsData.getFieldsList();
Long[] timestamp = new Long[]{metricsData.getTime()};
Expand All @@ -154,7 +161,9 @@ public void saveData(CollectRep.MetricsData metricsData) {
if (entry.getKey() != null && entry.getValue() != null) {
try {
labels.putAll(defaultLabels);
labels.put(NAME_KEY, metricsData.getApp() + SPILT + metricsData.getMetrics() + SPILT + entry.getKey());
String labelName = isPrometheusAuto ? metricsData.getMetrics()
: metricsData.getMetrics() + SPILT + entry.getKey();
labels.put(LABEL_KEY_NAME, labelName);
labels.put(MONITOR_METRIC_KEY, entry.getKey());
VictoriaMetricsContent content = VictoriaMetricsContent.builder()
.metric(labels)
Expand Down Expand Up @@ -191,8 +200,13 @@ public void destroy() {}

@Override
public Map<String, List<Value>> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) {
String timeSeriesSelector = NAME_KEY + "=\"" + app + SPILT + metrics + SPILT + metric + "\"" +
"," + MONITOR_ID_KEY + "=\"" + monitorId + "\"";
String labelName = metrics + SPILT + metric;
if (CommonConstants.PROMETHEUS.equals(app)) {
labelName = metrics;
}
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\"" +
"," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"" +
"," + MONITOR_METRIC_KEY + "=\"" + metric + "\"";
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpHeaders headers = new HttpHeaders();
Expand All @@ -219,9 +233,9 @@ public Map<String, List<Value>> getHistoryMetricData(Long monitorId, String app,
).collect(Collectors.toList());
for (VictoriaMetricsContent content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(NAME_KEY);
labels.remove(MONITOR_ID_KEY);
labels.remove(MONITOR_TYPE_KEY);
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
Expand Down Expand Up @@ -275,8 +289,8 @@ public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, Str
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6));
startTime = dateTime.toEpochSecond();
}
String timeSeriesSelector = NAME_KEY + "=\"" + app + SPILT + metrics + SPILT + metric + "\"" +
"," + MONITOR_ID_KEY + "=\"" + monitorId + "\"";
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + app + SPILT + metrics + SPILT + metric + "\"" +
"," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"";
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpHeaders headers = new HttpHeaders();
Expand Down Expand Up @@ -304,9 +318,9 @@ public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, Str
List<VictoriaMetricsQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (VictoriaMetricsQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(NAME_KEY);
labels.remove(MONITOR_ID_KEY);
labels.remove(MONITOR_TYPE_KEY);
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
Expand Down Expand Up @@ -339,9 +353,9 @@ public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, Str
List<VictoriaMetricsQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (VictoriaMetricsQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(NAME_KEY);
labels.remove(MONITOR_ID_KEY);
labels.remove(MONITOR_TYPE_KEY);
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
Expand Down Expand Up @@ -373,9 +387,9 @@ public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, Str
List<VictoriaMetricsQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (VictoriaMetricsQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(NAME_KEY);
labels.remove(MONITOR_ID_KEY);
labels.remove(MONITOR_TYPE_KEY);
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
Expand Down Expand Up @@ -407,9 +421,9 @@ public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, Str
List<VictoriaMetricsQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (VictoriaMetricsQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(NAME_KEY);
labels.remove(MONITOR_ID_KEY);
labels.remove(MONITOR_TYPE_KEY);
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
Expand Down
Loading