From eae123249ec3f4c0c20a9c22bb7a7fc34259f424 Mon Sep 17 00:00:00 2001 From: "zhouyoulin12@163.com" Date: Tue, 17 Jan 2023 16:37:46 +0800 Subject: [PATCH 1/5] add test unit case by zhouyoulin --- .../collector/util/JsonPathParserTest.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java b/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java index 193182d815a..155054102bf 100644 --- a/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java +++ b/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java @@ -1,21 +1,125 @@ package com.usthe.collector.util; +import com.jayway.jsonpath.TypeRef; import com.usthe.collector.collect.common.cache.CommonCache; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; + import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; /** * Test case for {@link JsonPathParser} + * @author zhouyl + * modified by zhouyl at 2023-01-17 */ class JsonPathParserTest { + private static final String JSON_ARRAY = "[{'name': 'tom', 'speed': '433'},{'name': 'lili', 'speed': '543'}]"; + public final static String JSON_OBJECT = + "{ \"store\": {\n" + + " \"book\": [ \n" + + " { \"category\": \"reference\",\n" + + " \"author\": \"Nigel Rees\",\n" + + " \"title\": \"Sayings of the Century\",\n" + + " \"price\": 8.95\n" + + " },\n" + + " { \"category\": \"fiction\",\n" + + " \"author\": \"Evelyn Waugh\",\n" + + " \"title\": \"Sword of Honour\",\n" + + " \"price\": 12.99\n" + + " },\n" + + " { \"category\": \"fiction\",\n" + + " \"author\": \"Herman Melville\",\n" + + " \"title\": \"Moby Dick\",\n" + + " \"isbn\": \"0-553-21311-3\",\n" + + " \"price\": 8.99\n" + + " },\n" + + " { \"category\": \"fiction\",\n" + + " \"author\": \"J. R. R. Tolkien\",\n" + + " \"title\": \"The Lord of the Rings\",\n" + + " \"isbn\": \"0-395-19395-8\",\n" + + " \"price\": 22.99\n" + + " }\n" + + " ],\n" + + " \"bicycle\": {\n" + + " \"color\": \"red\",\n" + + " \"price\": 19.95\n," + + " \"gears\": [23, 50]\n," + + " \"extra\": {\"x\": 0}\n," + + " \"escape\" : \"Esc\\b\\f\\n\\r\\t\\u002A\",\n" + + " \"nullValue\": null\n" + + " }\n" + + " }\n" + + "}"; @BeforeEach void setUp() { } @Test void parseContentWithJsonPath() { + // process array + List tom = JsonPathParser.parseContentWithJsonPath(JSON_ARRAY,"$[0].name"); + assertNotNull(tom); + assertEquals("tom",tom.get(0)); + // get json array map + List entry = JsonPathParser.parseContentWithJsonPath(JSON_ARRAY,"$[1]"); + assertNotNull(entry); + entry.forEach(e -> { + assertInstanceOf(Map.class, e); + assertEquals("543",((Map)e).get("speed")); + }); + // process object + List author = JsonPathParser.parseContentWithJsonPath(JSON_OBJECT,"$.store.book[0].author"); + assertNotNull(author); + assertEquals("Nigel Rees",author.get(0)); + // get json object map + List book = JsonPathParser.parseContentWithJsonPath(JSON_OBJECT,"$.store.book[1]"); + assertNotNull(book); + book.forEach(e -> { + assertInstanceOf(Map.class, e); + assertEquals("Sword of Honour",((Map)e).get("title")); + }); + } + + /** + * @author zhouyl + * modified by zhouyl at 2023-01-17 + * @throws java.lang.UnsupportedOperationException: Json-smart provider does not support TypeRef! Use a Jackson or Gson based provider + * need provid an provider to support TypeRef,like this: + * final Configuration configuration = Configuration.builder()// + * .jsonProvider(new JacksonJsonProvider(Json.mapper()))// + * .mappingProvider(new JacksonMappingProvider(Json.mapper()))// + * .build(); + */ +// @Test + void parseContentWithJsonPath2() { + TypeRef> typeStringRef = new TypeRef>() {}; + // process array + List tom = JsonPathParser.parseContentWithJsonPath(JSON_ARRAY,"$[0].name",typeStringRef); + assertNotNull(tom); + assertEquals("tom",tom.get(0)); + TypeRef> typeMapRef = new TypeRef>() {}; + // get json array map + List entry = JsonPathParser.parseContentWithJsonPath(JSON_ARRAY,"$[1]",typeMapRef); + assertNotNull(entry); + entry.forEach(e -> { + assertEquals("543",e.get("speed")); + }); + TypeRef> typeStrRef = new TypeRef>() {}; + // process object + List author = JsonPathParser.parseContentWithJsonPath(JSON_OBJECT,"$.store.book[0].author",typeStrRef); + assertNotNull(author); + assertEquals("Nigel Rees",author.get(0)); + TypeRef> typeObjMapRef = new TypeRef>() {}; + // get json object map + List book = JsonPathParser.parseContentWithJsonPath(JSON_OBJECT,"$.store.book[1]",typeObjMapRef); + assertNotNull(book); + book.forEach(e -> { + assertEquals("Sword of Honour",e.get("title")); + }); } } \ No newline at end of file From 4e13ef1f5020535a746d0f8ecfb5689e66590e2d Mon Sep 17 00:00:00 2001 From: "zhouyoulin12@163.com" Date: Tue, 17 Jan 2023 17:45:24 +0800 Subject: [PATCH 2/5] add test unit case --- .../java/com/usthe/collector/util/JsonPathParserTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java b/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java index 155054102bf..8976a41a55a 100644 --- a/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java +++ b/collector/src/test/java/com/usthe/collector/util/JsonPathParserTest.java @@ -1,7 +1,6 @@ package com.usthe.collector.util; import com.jayway.jsonpath.TypeRef; -import com.usthe.collector.collect.common.cache.CommonCache; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -13,8 +12,6 @@ /** * Test case for {@link JsonPathParser} - * @author zhouyl - * modified by zhouyl at 2023-01-17 */ class JsonPathParserTest { @@ -86,8 +83,6 @@ void parseContentWithJsonPath() { } /** - * @author zhouyl - * modified by zhouyl at 2023-01-17 * @throws java.lang.UnsupportedOperationException: Json-smart provider does not support TypeRef! Use a Jackson or Gson based provider * need provid an provider to support TypeRef,like this: * final Configuration configuration = Configuration.builder()// From 332081fdc3ed32656d8fda56f9fe72f3d66bbf7c Mon Sep 17 00:00:00 2001 From: "zhouyoulin12@163.com" Date: Tue, 14 Mar 2023 17:08:24 +0800 Subject: [PATCH 3/5] fix bug: 714, see link(https://github.com/dromara/hertzbeat/issues/714) --- warehouse/pom.xml | 2 +- .../store/HistoryTdEngineDataStorage.java | 24 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 97d8b9948e2..6cfb6d5ab21 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -28,7 +28,7 @@ 0.13.3 3.4.0 3.0.5 - 2.0.42 + 3.0.0 4.0.0 diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java index dc4f7a82aad..4380fc70ab9 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java @@ -24,6 +24,7 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -66,6 +67,12 @@ public class HistoryTdEngineDataStorage extends AbstractHistoryDataStorage { private static final String TABLE_NOT_EXIST = "Table does not exist"; + private static final String SINGLE_QUOTATION_MARKS + = "'"; + + private static final String CONVERT_SINGLE_QUOTATION_MARKS + = "\\\\'"; + private HikariDataSource hikariDataSource; private final int tableStrColumnDefineMaxLength; @@ -127,10 +134,11 @@ public void saveData(CollectRep.MetricsData metricsData) { for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { StringBuilder sqlRowBuffer = new StringBuilder("("); sqlRowBuffer.append(metricsData.getTime() + i++).append(", "); - sqlRowBuffer.append("'").append(valueRow.getInstance()).append("', "); + String instance = convertStr(valueRow.getInstance()); + sqlRowBuffer.append("'").append(instance).append("', "); for (int index = 0; index < fields.size(); index++) { CollectRep.Field field = fields.get(index); - String value = valueRow.getColumns(index); + String value = convertStr(valueRow.getColumns(index)); if (field.getType() == CommonConstants.TYPE_NUMBER) { // number data if (CommonConstants.NULL_VALUE.equals(value)) { @@ -210,6 +218,18 @@ public void saveData(CollectRep.MetricsData metricsData) { } } + /** + * 单引号转义 + * @param source + * @return + */ + private String convertStr(String source){ + if (StringUtils.isNotEmpty(source) && source.contains(SINGLE_QUOTATION_MARKS)){ + return source.replaceAll(SINGLE_QUOTATION_MARKS,CONVERT_SINGLE_QUOTATION_MARKS); + } + return source; + } + private String formatStringValue(String value){ String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0"); // bugfix Argument list too long From b3c07425a43f9035b49886af9fcbbe2e2807b8be Mon Sep 17 00:00:00 2001 From: "zhouyoulin12@163.com" Date: Tue, 14 Mar 2023 17:16:25 +0800 Subject: [PATCH 4/5] fix bug: 714, see link(https://github.com/dromara/hertzbeat/issues/714) --- .../store/HistoryTdEngineDataStorage.java | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java index 4380fc70ab9..00d0d78c376 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryTdEngineDataStorage.java @@ -24,7 +24,6 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -67,12 +66,6 @@ public class HistoryTdEngineDataStorage extends AbstractHistoryDataStorage { private static final String TABLE_NOT_EXIST = "Table does not exist"; - private static final String SINGLE_QUOTATION_MARKS - = "'"; - - private static final String CONVERT_SINGLE_QUOTATION_MARKS - = "\\\\'"; - private HikariDataSource hikariDataSource; private final int tableStrColumnDefineMaxLength; @@ -134,11 +127,11 @@ public void saveData(CollectRep.MetricsData metricsData) { for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { StringBuilder sqlRowBuffer = new StringBuilder("("); sqlRowBuffer.append(metricsData.getTime() + i++).append(", "); - String instance = convertStr(valueRow.getInstance()); + String instance = formatStringValue(valueRow.getInstance()); sqlRowBuffer.append("'").append(instance).append("', "); for (int index = 0; index < fields.size(); index++) { CollectRep.Field field = fields.get(index); - String value = convertStr(valueRow.getColumns(index)); + String value = valueRow.getColumns(index); if (field.getType() == CommonConstants.TYPE_NUMBER) { // number data if (CommonConstants.NULL_VALUE.equals(value)) { @@ -218,18 +211,6 @@ public void saveData(CollectRep.MetricsData metricsData) { } } - /** - * 单引号转义 - * @param source - * @return - */ - private String convertStr(String source){ - if (StringUtils.isNotEmpty(source) && source.contains(SINGLE_QUOTATION_MARKS)){ - return source.replaceAll(SINGLE_QUOTATION_MARKS,CONVERT_SINGLE_QUOTATION_MARKS); - } - return source; - } - private String formatStringValue(String value){ String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0"); // bugfix Argument list too long From 7056e48256468e6f5a0e0bc3e7f02e3e49ccb642 Mon Sep 17 00:00:00 2001 From: "zhouyoulin12@163.com" Date: Mon, 20 Mar 2023 18:40:51 +0800 Subject: [PATCH 5/5] fix bug: 742, see link(https://github.com/dromara/hertzbeat/issues/742) --- .../main/resources/define/app-springboot3.yml | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 manager/src/main/resources/define/app-springboot3.yml diff --git a/manager/src/main/resources/define/app-springboot3.yml b/manager/src/main/resources/define/app-springboot3.yml new file mode 100644 index 00000000000..6ac16d2bd1c --- /dev/null +++ b/manager/src/main/resources/define/app-springboot3.yml @@ -0,0 +1,183 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +category: service +app: springboot3 +name: + zh-CN: SpringBoot3.0 + en-US: SpringBoot3.0 +params: + # field-字段名称标识符 + - field: host + # name-参数字段显示名称 + name: + zh-CN: 主机Host + en-US: Host + # type-字段类型,样式(大部分映射input标签type属性) + type: host + # 是否是必输项 true-必填 false-可选 + required: true + - field: port + name: + zh-CN: 端口 + en-US: Port + type: number + # 当type为number时,用range表示范围 + range: '[0,65535]' + required: true + defaultValue: 8080 + - field: ssl + name: + zh-CN: 启动SSL + en-US: SSL + # 当type为boolean时,前端用switch展示开关 + type: boolean + required: false + - field: base_path + name: + zh-CN: Base Path + en-US: Base Path + type: text + defaultValue: /actuator + required: true + hide: true +metrics: + - name: thread_state + visible: false + priority: 1 + fields: + - field: state + type: 1 + protocol: http + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^base_path^_^/metrics/jvm.threads.states + # 请求方式 GET POST PUT DELETE PATCH + method: GET + # 是否启用ssl/tls,即是http还是https,默认false + ssl: ^_^ssl^_^ + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控 + parseType: jsonPath + parseScript: '$.availableTags[?(@.tag == "state")].values[*]' + + - name: threads + priority: 2 + fields: + - field: state + type: 1 + - field: number + type: 0 + aliasFields: + - $.measurements[?(@.statistic == "VALUE")].value + calculates: + - state='^o^state^o^' + - number=#`$.measurements[?(@.statistic == "VALUE")].value` + protocol: http + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^base_path^_^/metrics/jvm.threads.states?tag=state:^o^state^o^ + # 请求方式 GET POST PUT DELETE PATCH + method: GET + # 是否启用ssl/tls,即是http还是https,默认false + ssl: ^_^ssl^_^ + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控 + parseType: jsonPath + parseScript: '$' + + - name: space_name + visible: false + priority: 3 + fields: + - field: id + type: 1 + protocol: http + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^base_path^_^/metrics/jvm.memory.used + # 请求方式 GET POST PUT DELETE PATCH + method: GET + # 是否启用ssl/tls,即是http还是https,默认false + ssl: ^_^ssl^_^ + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控 + parseType: jsonPath + parseScript: '$.availableTags[?(@.tag == "id")].values[*]' + + - name: memory_used + priority: 4 + fields: + - field: space + type: 1 + - field: mem_used + type: 0 + unit: MB + aliasFields: + - $.measurements[?(@.statistic == "VALUE")].value + calculates: + - space="^o^id^o^" + - mem_used=#`$.measurements[?(@.statistic == "VALUE")].value` + units: + - mem_used=B->MB + protocol: http + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^base_path^_^/metrics/jvm.memory.used?tag=id:^o^id^o^ + # 请求方式 GET POST PUT DELETE PATCH + method: GET + # 是否启用ssl/tls,即是http还是https,默认false + ssl: ^_^ssl^_^ + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控 + parseType: jsonPath + parseScript: '$' + + - name: health + # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集 + # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度 + priority: 4 + # 指标组中的具体监控指标 + fields: + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: status + type: 1 + protocol: http + # 当protocol为http协议时具体的采集配置 + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^base_path^_^/health + # 请求方式 GET POST PUT DELETE PATCH + method: GET + # 是否启用ssl/tls,即是http还是https,默认false + ssl: ^_^ssl^_^ + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控 + parseType: default