Skip to content

Commit

Permalink
refactor verify http collect response status code (#1014)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomsun28
tomsun28 authored Jun 7, 2023
1 parent 40d198c commit 03320e2
Show file tree
Hide file tree
Showing 17 changed files with 273 additions and 67 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@ Thanks these wonderful people, welcome to join us:

## 💬 Join discussion

HertzBeat is a project under the [Dromara Open Source Community](https://dromara.org/).
HertzBeat is a top project under the [Dromara Open Source Community](https://dromara.org/).

##### Channel

Expand Down
2 changes: 1 addition & 1 deletion README_CN.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ Thanks these wonderful people, welcome to join us:

## 💬 社区交流

HertzBeat 赫兹跳动是 [Dromara开源社区](https://dromara.org/) 下项目。
HertzBeat 赫兹跳动是 [Dromara开源社区](https://dromara.org/) 下顶级项目。

##### 微信交流群

Expand Down
94 changes: 53 additions & 41 deletions collector/src/main/java/org/dromara/hertzbeat/collector/collect/http/HttpCollectImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,10 @@
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.dromara.hertzbeat.common.constants.SignConstants.RIGHT_DASH;

Expand All @@ -94,6 +92,10 @@
@Slf4j
public class HttpCollectImpl extends AbstractCollect {

private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());

public HttpCollectImpl() {
}

Expand All @@ -117,45 +119,44 @@ public void collect(CollectRep.MetricsData.Builder builder,
int statusCode = response.getStatusLine().getStatusCode();
boolean isSuccessInvoke = checkSuccessInvoke(metrics, statusCode);
log.debug("http response status: {}", statusCode);
if ((statusCode < HttpStatus.SC_OK || statusCode >= HttpStatus.SC_BAD_REQUEST) && !isSuccessInvoke) {
// 1XX 4XX 5XX 状态码且不在successCodes中的状态码 失败
if (!isSuccessInvoke) {
// 状态码不在successCodes中的状态码为失败
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return;
} else {
// 2xx 3xx 状态码 或在successCodes中的状态码成功
// todo 这里直接将InputStream转为了String, 对于prometheus exporter大数据来说, 会生成大对象, 可能会严重影响JVM内存空间
// todo 方法一、使用InputStream进行解析, 代码改动大; 方法二、手动触发gc, 可以参考dubbo for long i
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
// 根据不同的解析方式解析
if (resp == null || "".equals(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
Long responseTime = System.currentTimeMillis() - startTime;
String parseType = metrics.getHttp().getParseType();
try {
if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) {
parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) {
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_PROM_QL.equalsIgnoreCase(parseType)) {
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
} else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
parseResponseByPrometheusExporter(resp, metrics.getAliasFields(), builder);
} else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) {
parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
} else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)) {
parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) {
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
} else {
parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
}
} catch (Exception e) {
log.info("parse error: {}.", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
}
// 在successCodes中的状态码成功
// todo 这里直接将InputStream转为了String, 对于prometheus exporter大数据来说, 会生成大对象, 可能会严重影响JVM内存空间
// todo 方法一、使用InputStream进行解析, 代码改动大; 方法二、手动触发gc, 可以参考dubbo for long i
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
// 根据不同的解析方式解析
if (resp == null || "".equals(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
Long responseTime = System.currentTimeMillis() - startTime;
String parseType = metrics.getHttp().getParseType();
try {
if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) {
parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) {
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_PROM_QL.equalsIgnoreCase(parseType)) {
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
} else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
parseResponseByPrometheusExporter(resp, metrics.getAliasFields(), builder);
} else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) {
parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
} else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)) {
parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) {
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
} else {
parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
}
} catch (Exception e) {
log.info("parse error: {}.", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
}
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
Expand Down Expand Up @@ -210,7 +211,7 @@ private void validateParams(Metrics metrics) throws Exception {
}

if (CollectionUtils.isEmpty(httpProtocol.getSuccessCodes())) {
httpProtocol.setSuccessCodes(List.of(200));
httpProtocol.setSuccessCodes(List.of("200"));
}
}

Expand Down Expand Up @@ -589,6 +590,17 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
}

private boolean checkSuccessInvoke(Metrics metrics, int statusCode) {
return metrics.getHttp().getSuccessCodes().stream().parallel().filter(code -> code == statusCode).findAny().isPresent();
List<String> successCodes = metrics.getHttp().getSuccessCodes();
Set<Integer> successCodeSet = successCodes != null ? successCodes.stream().map(code -> {
try {
return Integer.valueOf(code);
} catch (Exception ignored) {
return null;
}
}).filter(Objects::nonNull).collect(Collectors.toSet()) : defaultSuccessStatusCodes;
if (successCodeSet.isEmpty()) {
successCodeSet = defaultSuccessStatusCodes;
}
return successCodeSet.contains(statusCode);
}
}
24 changes: 19 additions & 5 deletions collector/src/main/java/org/dromara/hertzbeat/collector/util/CollectUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ public static JsonElement replaceSmilingPlaceholder(JsonElement jsonElement, Map
if (key != null && key.startsWith(SMILING_PLACEHOLDER) && key.endsWith(SMILING_PLACEHOLDER)) {
key = key.replaceAll(SMILING_PLACEHOLDER_REX, "");
Configmap param = configmap.get(key);
if (param != null && param.getType() == (byte) 3) {
if (param != null && param.getType() == CommonConstants.PARAM_TYPE_MAP) {
String jsonValue = (String) param.getValue();
TypeReference<Map<String, String>> typeReference = new TypeReference<>() {};
Map<String, String> map = JsonUtil.fromJson(jsonValue, typeReference);
Expand Down Expand Up @@ -337,17 +337,17 @@ public static JsonElement replaceSmilingPlaceholder(JsonElement jsonElement, Map
}
} else if (jsonElement.isJsonArray()) {
JsonArray jsonArray = jsonElement.getAsJsonArray();
Iterator<JsonElement> iterator = jsonArray.iterator();
int index = 0;
while (iterator.hasNext()) {
JsonElement element = iterator.next();
while (index < jsonArray.size()) {
JsonElement element = jsonArray.get(index);
if (element.isJsonPrimitive()) {
// Check if there are special characters Replace
// 判断是否含有特殊字符 替换
String value = element.getAsString();
Matcher smilingMatcher = SMILING_PLACEHOLDER_REGEX_PATTERN.matcher(value);
if (smilingMatcher.find()) {
smilingMatcher.reset();
String[] arrayValues = null;
while (smilingMatcher.find()) {
String group = smilingMatcher.group();
String replaceField = group.replaceAll(SMILING_PLACEHOLDER_REX, "");
Expand All @@ -360,12 +360,26 @@ public static JsonElement replaceSmilingPlaceholder(JsonElement jsonElement, Map
} else {
value = value.replace(group, "");
}
} else if (param.getType() == CommonConstants.PARAM_TYPE_ARRAY) {
arrayValues = String.valueOf(param.getValue()).split(",");
} else {
value = value.replace(group, (String) param.getValue());
}
} else {
value = null;
break;
}
}
jsonArray.set(index, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));
if (arrayValues != null) {
jsonArray.remove(index);
index--;
for (String arrayValue : arrayValues) {
jsonArray.add(arrayValue);
index++;
}
} else {
jsonArray.set(index, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));
}
}
} else {
jsonArray.set(index, replaceSmilingPlaceholder(element, configmap));
Expand Down
10 changes: 10 additions & 0 deletions common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,16 @@ public interface CommonConstants {
* 参数类型 密码
*/
byte PARAM_TYPE_PASSWORD = 2;

/**
* Parameter Type Map values
*/
byte PARAM_TYPE_MAP = 3;

/**
* Parameter Type arrays values
*/
byte PARAM_TYPE_ARRAY = 4;

/**
* Authentication type Account password
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/java/org/dromara/hertzbeat/common/entity/job/Configmap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public class Configmap {

/**
* Parameter type 0: number 1: string 2: encrypted string 3: json string mapped by map
* 参数类型 0:数字 1:字符串 2:加密串 3:map映射的json串
* 参数类型 0:数字 1:字符串 2:加密串 3:map映射的json串 4:arrays string
* number,string,secret
* 数字,非加密字符串,加密字符串
*/
Expand Down
12 changes: 6 additions & 6 deletions common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/HttpProtocol.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public class HttpProtocol {
/**
* 响应数据解析方式
* default - 自有的数据解析规则
* json_path 自定义jsonPath脚本 https://www.jsonpath.cn/
* json_path 自定义jsonPath脚本 <a href="https://www.jsonpath.cn/">...</a>
* xml_path 自定义xmlPath脚本
* prometheus Prometheus数据规则
*/
Expand All @@ -96,7 +96,7 @@ public class HttpProtocol {
* http success status code. default 200
* successCode means what http response status code we consider it collect success.
*/
private List<Integer> successCodes ;
private List<String> successCodes;

/**
* 认证信息
Expand All @@ -114,19 +114,19 @@ public static class Authorization {
*/
private String bearerTokenToken;
/**
* Basic Auth's username
* Basic Auth 's username
*/
private String basicAuthUsername;
/**
* Basic Auth's password
* Basic Auth 's password
*/
private String basicAuthPassword;
/**
* Digest Auth's username
* Digest Auth 's username
*/
private String digestAuthUsername;
/**
* Digest Auth's password
* Digest Auth 's password
*/
private String digestAuthPassword;
}
Expand Down
3 changes: 1 addition & 2 deletions common/src/main/java/org/dromara/hertzbeat/common/entity/manager/Param.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,8 @@ public class Param {
* Parameter type 0: number 1: string 2: encrypted string 3: json string mapped by map
* 参数类型 0:数字 1:字符串 2:加密串 3:map映射的json串
*/
@Schema(title = "参数类型 0:数字 1:字符串 2:加密串 3:map映射的json串", accessMode = READ_WRITE)
@Schema(title = "参数类型 0:数字 1:字符串 2:加密串 3:map映射的json串 4:arrays string", accessMode = READ_WRITE)
@Min(0)
@Max(3)
private byte type;

/**
Expand Down
Loading

0 comments on commit 03320e2

Please sign in to comment.