Skip to content

Commit

Permalink
[hertzbeat] add Prometheus exporter metrics parser and IoTDB monitor (#…
Browse files Browse the repository at this point in the history 
…505)

  prometheus exporter metrics parser

  prometheus exporter metrics parser

  prometheus exporter metrics parser

  prometheus exporter metrics parser

  optimize ExporterParser.java

  optimize ExporterParser.java

  optimize ExporterParser.java

  handle INF parser

  添加iotdb的exporter方式监听

  [manager] iotdb monitor add cluster_node_status metric

  [manager] fix actuator mail health check error when startup

  [manager] use prometheus instead of prometheus_exporter

  [manager] add iotdb monitoring metrics

  [collector] add metric type - info, update parser

  [collector] update

  ExporterParser change to singleton mode

  add ExporterParser currentMetricFamily

  格式 优 代码, 使用map管理ExporterParser防止无限制创建对象, 使用锁机制反正多线程读写冲突

  修改注解格式

  格式 注解

  修改注解格式

  解决魔法数问题
  • Loading branch information
@Ceilzcx
Ceilzcx authored Dec 20, 2022
1 parent 1d19857 commit e1c9ad4
Show file tree
Hide file tree
Showing 13 changed files with 1,192 additions and 17 deletions.
2 changes: 1 addition & 1 deletion collector/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
<groupId>com.usthe.tancloud</groupId>
<artifactId>common</artifactId>
<version>1.0</version>
<scope>provided</scope>
<!-- <scope>provided</scope>-->
</dependency>
<!-- validation -->
<dependency>
Expand Down
70 changes: 57 additions & 13 deletions collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import com.google.gson.JsonParser;
import com.usthe.collector.collect.AbstractCollect;
import com.usthe.collector.collect.common.http.CommonHttpClient;
import com.usthe.collector.collect.http.promethus.exporter.ExporterParser;
import com.usthe.collector.collect.http.promethus.exporter.MetricFamily;
import com.usthe.collector.dispatch.DispatchConstants;
import com.usthe.collector.util.CollectUtil;
import com.usthe.collector.util.CollectorConstants;
Expand Down Expand Up @@ -76,6 +78,8 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;


/**
Expand All @@ -86,7 +90,8 @@
@Slf4j
public class HttpCollectImpl extends AbstractCollect {

public HttpCollectImpl() {}
public HttpCollectImpl() {
}

@Override
public void collect(CollectRep.MetricsData.Builder builder,
Expand Down Expand Up @@ -114,12 +119,14 @@ public void collect(CollectRep.MetricsData.Builder builder,
return;
} else {
// 2xx 3xx 状态码 成功
// todo 这里直接将InputStream转为了String, 对于prometheus exporter大数据来说, 会生成大对象, 可能会严重影响JVM内存空间
// todo 方法一、使用InputStream进行解析, 代码改动大; 方法二、手动触发gc, 可以参考dubbo for long i
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
// 根据不同的解析方式解析
if (resp == null || "".equals(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
Long responseTime = System.currentTimeMillis() - startTime;
Long responseTime = System.currentTimeMillis() - startTime;
String parseType = metrics.getHttp().getParseType();
try {
if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) {
Expand All @@ -128,9 +135,11 @@ public void collect(CollectRep.MetricsData.Builder builder,
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_PROM_QL.equalsIgnoreCase(parseType)) {
parseResponseByPromQL(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
} else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
parseResponseByPrometheusExporter(resp, metrics.getAliasFields(), builder);
} else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) {
parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
} else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){
} else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)) {
parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
} else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) {
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
Expand Down Expand Up @@ -316,7 +325,7 @@ private void parseResponseByJsonPath(String resp, List<String> aliasFields, Http
continue;
}
if (objectValue instanceof Map) {
Map<String, Object> stringMap = (Map<String, Object>)objectValue;
Map<String, Object> stringMap = (Map<String, Object>) objectValue;
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
Object value = stringMap.get(alias);
Expand Down Expand Up @@ -361,7 +370,44 @@ private void parseResponseByJsonPath(String resp, List<String> aliasFields, Http
private void parseResponseByPromQL(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder) {
AbstractPrometheusParse prometheusParser = PrometheusParseCreater.getPrometheusParse();
prometheusParser.handle(resp,aliasFields,http,builder);
prometheusParser.handle(resp, aliasFields, http, builder);
}

private static final Map<Long, ExporterParser> EXPORTER_PARSER_TABLE = new ConcurrentHashMap<>();

private void parseResponseByPrometheusExporter(String resp, List<String> aliasFields,
CollectRep.MetricsData.Builder builder) {
if (!EXPORTER_PARSER_TABLE.containsKey(builder.getId())) {
EXPORTER_PARSER_TABLE.put(builder.getId(), new ExporterParser());
}
ExporterParser parser = EXPORTER_PARSER_TABLE.get(builder.getId());
Map<String, MetricFamily> metricFamilyMap = parser.textToMetric(resp);
String metrics = builder.getMetrics();
if (metricFamilyMap.containsKey(metrics)) {
MetricFamily metricFamily = metricFamilyMap.get(metrics);
for (MetricFamily.Metric metric : metricFamily.getMetricList()) {
Map<String, String> labelMap = metric.getLabelPair()
.stream()
.collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String aliasField : aliasFields) {
if ("value".equals(aliasField)) {
if (metric.getCounter() != null) {
valueRowBuilder.addColumns(metric.getCounter().getValue() + "");
} else if (metric.getGauge() != null) {
valueRowBuilder.addColumns(metric.getGauge().getValue() + "");
} else if (metric.getUntyped() != null) {
valueRowBuilder.addColumns(metric.getUntyped().getValue() + "");
} else if (metric.getInfo() != null) {
valueRowBuilder.addColumns(metric.getInfo().getValue() + "");
}
} else {
valueRowBuilder.addColumns(labelMap.get(aliasField));
}
}
builder.addValues(valueRowBuilder.build());
}
}
}

private void parseResponseByDefault(String resp, List<String> aliasFields, HttpProtocol http,
Expand Down Expand Up @@ -436,6 +482,7 @@ public HttpContext createHttpContext(HttpProtocol httpProtocol) {

/**
* 根据http配置参数构造请求头
*
* @param httpProtocol http参数配置
* @return 请求体
*/
Expand All @@ -444,9 +491,9 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
// method
String httpMethod = httpProtocol.getMethod().toUpperCase();
if (HttpMethod.GET.matches(httpMethod)) {
requestBuilder = RequestBuilder.get();
requestBuilder = RequestBuilder.get();
} else if (HttpMethod.POST.matches(httpMethod)) {
requestBuilder = RequestBuilder.post();
requestBuilder = RequestBuilder.post();
} else if (HttpMethod.PUT.matches(httpMethod)) {
requestBuilder = RequestBuilder.put();
} else if (HttpMethod.DELETE.matches(httpMethod)) {
Expand All @@ -471,7 +518,7 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
// The default request header can be overridden if customized
// keep-alive
requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive");
requestBuilder.addHeader(HttpHeaders.USER_AGENT,"Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");
requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");
// headers The custom request header is overwritten here
Map<String, String> headers = httpProtocol.getHeaders();
if (headers != null && !headers.isEmpty()) {
Expand All @@ -488,9 +535,6 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, "application/json");
} else if (DispatchConstants.PARSE_XML_PATH.equals(httpProtocol.getParseType())) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/xml,application/xml");
} else if (DispatchConstants.PARSE_PROMETHEUS.equals(httpProtocol.getParseType())) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, DispatchConstants.PARSE_PROMETHEUS_ACCEPT);
requestBuilder.addHeader(HttpHeaders.ACCEPT_ENCODING, "gzip");
} else {
requestBuilder.addHeader(HttpHeaders.ACCEPT, "*/*");
}
Expand All @@ -507,13 +551,13 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
}
}
}

// 请求内容，会覆盖post协议的params
if(StringUtils.hasLength(httpProtocol.getPayload())){
if (StringUtils.hasLength(httpProtocol.getPayload())) {
requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(), StandardCharsets.UTF_8));
}

Expand Down
12 changes: 12 additions & 0 deletions collector/src/main/java/com/usthe/collector/collect/http/promethus/ParseException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package com.usthe.collector.collect.http.promethus;

/**
* @author ceilzcx
* @since 7/11/2022
*/
public class ParseException extends RuntimeException {

public ParseException(String msg) {
super(msg);
}
}
Loading

0 comments on commit e1c9ad4

Please sign in to comment.