Skip to content

Commit

Permalink
[monitor] feature update numeric metrics all contain decimal points (#…
Browse files Browse the repository at this point in the history 
…217)

* [warehouse] support custom tdengine tableStrColumnDefineMaxLength

* [monitor] feature update numeric metrics all contain decimal points
  • Loading branch information
@tomsun28
tomsun28 committed Jan 16, 2024
1 parent 35dee28 commit f11f24d
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 32 deletions.
2 changes: 1 addition & 1 deletion alerter/src/main/java/com/usthe/alert/calculate/CalculateAlarm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ private void calculate(CollectRep.MetricsData metricsData) {
String valueStr = valueRow.getColumns(index);
CollectRep.Field field = fields.get(index);
if (field.getType() == CommonConstants.TYPE_NUMBER) {
Double doubleValue = CommonUtil.parseDoubleStr(valueStr);
Double doubleValue = CommonUtil.parseStrDouble(valueStr);
if (doubleValue != null) {
fieldValueMap.put(field.getName(), doubleValue);
}
Expand Down
2 changes: 1 addition & 1 deletion collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder col
// 存在计算表达式 则计算值
if (CommonConstants.TYPE_NUMBER == field.getType()) {
for (String variable : expression.getVariableNames()) {
Double doubleValue = CommonUtil.parseDoubleStr(aliasFieldValueMap.get(variable));
Double doubleValue = CommonUtil.parseStrDouble(aliasFieldValueMap.get(variable));
if (doubleValue != null) {
fieldValueMap.put(variable, doubleValue);
}
Expand Down
5 changes: 2 additions & 3 deletions common/src/main/java/com/usthe/common/util/CommonUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class CommonUtil {
* @param str string
* @return double 数字
*/
public static Double parseDoubleStr(String str) {
public static Double parseStrDouble(String str) {
if (str == null || "".equals(str)) {
return null;
}
Expand Down Expand Up @@ -54,8 +54,7 @@ public static String parseDoubleStr(String str, String unit) {
str = str.substring(0, str.length() - unit.length());
}
BigDecimal bigDecimal = new BigDecimal(str);
double value = bigDecimal.setScale(4, RoundingMode.HALF_UP).doubleValue();
return String.valueOf(value);
return bigDecimal.setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
} catch (Exception e) {
log.debug(e.getMessage(), e);
return null;
Expand Down
22 changes: 17 additions & 5 deletions warehouse/src/main/java/com/usthe/warehouse/WarehouseProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,25 +218,29 @@ public void setOrg(String org) {

public static class TdEngineProperties {
/**
* TdEngine数据存储是否启动
* Whether the TdEngine data store is enabled
*/
private boolean enabled = true;
/**
* TdEngine的连接服务器url
* TdEngine connect url
*/
private String url = "jdbc:TAOS-RS://localhost:6041/demo";
/**
* 驱动类路径
* tdengine driver, default restful driver
*/
private String driverClassName = "com.taosdata.jdbc.rs.RestfulDriver";
/**
* 数据库用户名
* tdengine username
*/
private String username;
/**
* 数据库密码
* tdengine password
*/
private String password;
/**
* auto create table's string column define max length : NCHAR(200)
*/
private int tableStrColumnDefineMaxLength = 200;

public boolean isEnabled() {
return enabled;
Expand Down Expand Up @@ -277,6 +281,14 @@ public String getPassword() {
public void setPassword(String password) {
this.password = password;
}

public int getTableStrColumnDefineMaxLength() {
return tableStrColumnDefineMaxLength;
}

public void setTableStrColumnDefineMaxLength(int tableStrColumnDefineMaxLength) {
this.tableStrColumnDefineMaxLength = tableStrColumnDefineMaxLength;
}
}

public static class RedisProperties {
Expand Down
23 changes: 12 additions & 11 deletions warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

/**
* 指标数据查询接口
*
* @author tom
* @date 2021/12/5 15:52
*/
Expand Down Expand Up @@ -68,30 +69,30 @@ public ResponseEntity<Message<Void>> getWarehouseStorageServerStatus(
}

@GetMapping("/api/monitor/{monitorId}/metrics/{metrics}")
@ApiOperation(value = "查询监控指标组的指标数据", notes = "查询监控指标组的指标数据")
@ApiOperation(value = "Query Real Time Metrics Data", notes = "查询监控指标组的指标数据")
public ResponseEntity<Message<MetricsData>> getMetricsData(
@ApiParam(value = "监控ID", example = "343254354")
@ApiParam(value = "Monitor Id", example = "343254354")
@PathVariable Long monitorId,
@ApiParam(value = "监控指标组", example = "cpu")
@ApiParam(value = "Metrics Name", example = "cpu")
@PathVariable String metrics) {
CollectRep.MetricsData redisData = memoryDataStorage.getCurrentMetricsData(monitorId, metrics);
if (redisData == null) {
CollectRep.MetricsData storageData = memoryDataStorage.getCurrentMetricsData(monitorId, metrics);
if (storageData == null) {
return ResponseEntity.ok().body(new Message<>("query metrics data is empty"));
}
{
MetricsData.MetricsDataBuilder dataBuilder = MetricsData.builder();
dataBuilder.id(redisData.getId()).app(redisData.getApp()).metric(redisData.getMetrics())
.time(redisData.getTime());
List<Field> fields = redisData.getFieldsList().stream().map(redisField ->
dataBuilder.id(storageData.getId()).app(storageData.getApp()).metric(storageData.getMetrics())
.time(storageData.getTime());
List<Field> fields = storageData.getFieldsList().stream().map(redisField ->
Field.builder().name(redisField.getName())
.type(Integer.valueOf(redisField.getType()).byteValue())
.build())
.collect(Collectors.toList());
dataBuilder.fields(fields);
List<ValueRow> valueRows = redisData.getValuesList().stream().map(redisValueRow ->
List<ValueRow> valueRows = storageData.getValuesList().stream().map(redisValueRow ->
ValueRow.builder().instance(redisValueRow.getInstance())
.values(redisValueRow.getColumnsList().stream().map(Value::new).collect(Collectors.toList()))
.build()).collect(Collectors.toList());
.build()).collect(Collectors.toList());
dataBuilder.valueRows(valueRows);
return ResponseEntity.ok().body(new Message<>(dataBuilder.build()));
}
Expand All @@ -110,7 +111,7 @@ public ResponseEntity<Message<MetricsHistoryData>> getMetricHistoryData(
@RequestParam(required = false) String history,
@ApiParam(value = "是否计算聚合数据,需查询时间段大于1周以上,默认不开启,聚合降样时间窗口默认为4小时", example = "false")
@RequestParam(required = false) Boolean interval
) {
) {
String[] names = metricFull.split("\\.");
if (names.length != METRIC_FULL_LENGTH) {
throw new IllegalArgumentException("metrics full name: " + metricFull + " is illegal.");
Expand Down
30 changes: 19 additions & 11 deletions warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;

import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
Expand All @@ -23,7 +24,7 @@
import java.util.regex.Pattern;

/**
* influxdb存储采集数据
* tdengine 存储采集数据
* @author tom
* @date 2021/11/24 18:23
*/
Expand All @@ -49,12 +50,12 @@ public class TdEngineDataStorage implements DisposableBean {

private static final String TABLE_NOT_EXIST
= "Table does not exist";
private static final Integer SQL_STRING_VALUE_MAX_LENGTH = 200;

private HikariDataSource hikariDataSource;
private WarehouseWorkerPool workerPool;
private MetricsDataExporter dataExporter;
private boolean serverAvailable = false;
private boolean serverAvailable;
private int tableStrColumnDefineMaxLength;

public TdEngineDataStorage(WarehouseWorkerPool workerPool, WarehouseProperties properties,
MetricsDataExporter dataExporter) {
Expand All @@ -64,6 +65,7 @@ public TdEngineDataStorage(WarehouseWorkerPool workerPool, WarehouseProperties p
log.error("init error, please config Warehouse TdEngine props in application.yml");
throw new IllegalArgumentException("please config Warehouse TdEngine props");
}
tableStrColumnDefineMaxLength = properties.getStore().getTdEngine().getTableStrColumnDefineMaxLength();
serverAvailable = initTdEngineDatasource(properties.getStore().getTdEngine());
startStorageData(serverAvailable);
}
Expand Down Expand Up @@ -181,14 +183,15 @@ public void saveData(CollectRep.MetricsData metricsData) {
// 超级表未创建 创建对应超级表
StringBuilder fieldSqlBuilder = new StringBuilder("(");
fieldSqlBuilder.append("ts TIMESTAMP, ");
fieldSqlBuilder.append("instance NCHAR(200), ");
fieldSqlBuilder.append("instance NCHAR(").append(tableStrColumnDefineMaxLength).append("), ");
for (int index = 0; index < fields.size(); index++) {
CollectRep.Field field = fields.get(index);
String fieldName = field.getName();
if (field.getType() == CommonConstants.TYPE_NUMBER) {
fieldSqlBuilder.append(fieldName).append(" ").append("DOUBLE");
} else {
fieldSqlBuilder.append(fieldName).append(" ").append("NCHAR(200)");
fieldSqlBuilder.append(fieldName).append(" ").append("NCHAR(")
.append(tableStrColumnDefineMaxLength).append(")");
}
if (index != fields.size() - 1) {
fieldSqlBuilder.append(", ");
Expand Down Expand Up @@ -220,14 +223,14 @@ public void saveData(CollectRep.MetricsData metricsData) {
private String formatStringValue(String value){
String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
// bugfix Argument list too long
if (formatValue != null && formatValue.length() > SQL_STRING_VALUE_MAX_LENGTH) {
formatValue = formatValue.substring(0, SQL_STRING_VALUE_MAX_LENGTH);
if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) {
formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength);
}
return formatValue;
}

@Override
public void destroy() throws Exception {
public void destroy() {
if (hikariDataSource != null) {
hikariDataSource.close();
}
Expand Down Expand Up @@ -268,8 +271,9 @@ public Map<String, List<Value>> getHistoryMetricData(Long monitorId, String app,
instanceValue = "NULL";
}
double value = resultSet.getDouble(3);
String strValue = new BigDecimal(value).stripTrailingZeros().toPlainString();
List<Value> valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
valueList.add(new Value(String.valueOf(value), ts.getTime()));
valueList.add(new Value(strValue, ts.getTime()));
}
resultSet.close();
return instanceValuesMap;
Expand Down Expand Up @@ -348,12 +352,16 @@ public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, Str
while (resultSet.next()) {
Timestamp ts = resultSet.getTimestamp(1);
double origin = resultSet.getDouble(2);
String originStr = new BigDecimal(origin).stripTrailingZeros().toPlainString();
double avg = resultSet.getDouble(3);
String avgStr = new BigDecimal(avg).stripTrailingZeros().toPlainString();
double min = resultSet.getDouble(4);
String minStr = new BigDecimal(min).stripTrailingZeros().toPlainString();
double max = resultSet.getDouble(5);
String maxStr = new BigDecimal(max).stripTrailingZeros().toPlainString();
Value value = Value.builder()
.origin(String.valueOf(origin)).mean(String.valueOf(avg))
.min(String.valueOf(min)).max(String.valueOf(max))
.origin(originStr).mean(avgStr)
.min(minStr).max(maxStr)
.time(ts.getTime())
.build();
values.add(value);
Expand Down

0 comments on commit f11f24d

Please sign in to comment.