Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[feature] support collect data from header #2266

Merged
merged 4 commits into from
Jul 13, 2024
Merged

[feature] support collect data from header #2266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2f19805
[feature] support collect data from header
Calvin979 Jul 10, 2024
0d4c50d
[feature] support collect data from header
Calvin979 Jul 10, 2024
adbabbb
[refactor] fix backend ci failed
Calvin979 Jul 11, 2024
e9477a5
Merge branch 'master' into feature/collect_header_data
zqr10159 Jul 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 37 additions & 22 deletions collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.Header;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
Expand Down Expand Up @@ -149,7 +150,7 @@ public void collect(CollectRep.MetricsData.Builder builder,
case DispatchConstants.PARSE_XML_PATH ->
parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
case DispatchConstants.PARSE_WEBSITE ->
parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime, response);
case DispatchConstants.PARSE_SITE_MAP ->
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
default ->
Expand Down Expand Up @@ -197,22 +198,46 @@ public String supportProtocol() {
return DispatchConstants.PROTOCOL_HTTP;
}

private void parseResponseByWebsite(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
private void parseResponseByWebsite(String resp, Metrics metrics, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime,
CloseableHttpResponse response) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
for (String alias : aliasFields) {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
for (String alias : metrics.getAliasFields()) {
if ("summary".equalsIgnoreCase(metrics.getName())) {
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
} else if ("header".equalsIgnoreCase(metrics.getName())) {
addColumnFromHeader(valueRowBuilder, alias, response);
}
}
builder.addValues(valueRowBuilder.build());
}

private void addColumnFromHeader(CollectRep.ValueRow.Builder valueRowBuilder, String alias, CloseableHttpResponse response) {
if (!StringUtils.hasText(alias)) {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
return;
}

final Header firstHeader = response.getFirstHeader(alias);
if (Objects.isNull(firstHeader)) {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
return;
}

valueRowBuilder.addColumns(firstHeader.getValue());
}

private void addColumnForSummary(Long responseTime, CollectRep.ValueRow.Builder valueRowBuilder, int keywordNum, String alias) {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
}
}

private void parseResponseBySiteMap(String resp, List<String> aliasFields,
CollectRep.MetricsData.Builder builder) {
List<String> siteUrls = new LinkedList<>();
Expand Down Expand Up @@ -326,12 +351,8 @@ private void parseResponseByJsonPath(String resp, List<String> aliasFields, Http
} else {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
}
} else if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
}
}
}
Expand Down Expand Up @@ -422,13 +443,7 @@ private void getValueFromJson(List<String> aliasFields, CollectRep.MetricsData.B
String value = valueElement.getAsString();
valueRowBuilder.addColumns(value);
} else {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
}
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
}
}
builder.addValues(valueRowBuilder.build());
Expand Down
45 changes: 45 additions & 0 deletions manager/src/main/resources/define/app-api.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,3 +309,48 @@ metrics:
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: website
keyword: ^_^keyword^_^
- name: header
i18n:
zh-CN: 请求头
en-US: Header
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 1
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: Content-Type
type: 1
i18n:
zh-CN: Content Type
en-US: Content Type
- field: Content-Length
type: 1
i18n:
zh-CN: 响应内容长度
en-US: Content Length
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# the config content when protocol is http
http:
# http host: ipv4 ipv6 domain
host: ^_^host^_^
# http port
port: ^_^port^_^
# http url
url: ^_^uri^_^
timeout: ^_^timeout^_^
# http method: GET POST PUT DELETE PATCH
method: GET
# if enabled https
ssl: ^_^ssl^_^
authorization:
# http auth type: Basic Auth, Digest Auth, Bearer Token
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: website
keyword: ^_^keyword^_^
45 changes: 45 additions & 0 deletions manager/src/main/resources/define/app-website.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,48 @@ metrics:
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: website
keyword: ^_^keyword^_^
- name: header
i18n:
zh-CN: 请求头
en-US: Header
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 1
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: Content-Type
type: 1
i18n:
zh-CN: Content Type
en-US: Content Type
- field: Content-Length
type: 1
i18n:
zh-CN: 响应内容长度
en-US: Content Length
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# the config content when protocol is http
http:
# http host: ipv4 ipv6 domain
host: ^_^host^_^
# http port
port: ^_^port^_^
# http url
url: ^_^uri^_^
timeout: ^_^timeout^_^
# http method: GET POST PUT DELETE PATCH
method: GET
# if enabled https
ssl: ^_^ssl^_^
authorization:
# http auth type: Basic Auth, Digest Auth, Bearer Token
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: website
keyword: ^_^keyword^_^
Loading