From f4186d73769085d8516cf7b1b2d1729b4b74423b Mon Sep 17 00:00:00 2001 From: a-little-fool <2030509072@qq.com> Date: Thu, 14 Dec 2023 11:38:58 +0800 Subject: [PATCH 1/3] feature: support monitoring nginx metrics and add a help doc --- .../collect/nginx/NginxCollectImpl.java | 343 ++++++++++++++++++ .../collector/dispatch/DispatchConstants.java | 4 + ...ertzbeat.collector.collect.AbstractCollect | 1 + .../collect/nginx/NginxCollectImplTest.java | 129 +++++++ .../hertzbeat/common/entity/job/Metrics.java | 4 + .../entity/job/protocol/NginxProtocol.java | 41 +++ home/docs/help/nginx.md | 112 ++++++ .../src/main/resources/define/app-nginx.yml | 271 ++++++++++++++ 8 files changed, 905 insertions(+) create mode 100644 collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java create mode 100644 collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java create mode 100644 common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/NginxProtocol.java create mode 100644 home/docs/help/nginx.md create mode 100644 manager/src/main/resources/define/app-nginx.yml diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java new file mode 100644 index 00000000000..eee4b17807d --- /dev/null +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java @@ -0,0 +1,343 @@ +package org.dromara.hertzbeat.collector.collect.nginx; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; +import org.dromara.hertzbeat.collector.collect.AbstractCollect; +import org.dromara.hertzbeat.collector.collect.common.http.CommonHttpClient; +import org.dromara.hertzbeat.collector.dispatch.DispatchConstants; +import org.dromara.hertzbeat.collector.util.CollectUtil; +import org.dromara.hertzbeat.common.constants.CollectorConstants; +import org.dromara.hertzbeat.common.constants.CommonConstants; +import org.dromara.hertzbeat.common.entity.job.Metrics; +import org.dromara.hertzbeat.common.entity.job.protocol.NginxProtocol; +import org.dromara.hertzbeat.common.entity.message.CollectRep; +import org.dromara.hertzbeat.common.util.CommonUtil; +import org.dromara.hertzbeat.common.util.IpDomainUtil; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.dromara.hertzbeat.common.constants.SignConstants.RIGHT_DASH; + +/** + * nginx collect + * @author zhoushusheng + */ +@Slf4j +public class NginxCollectImpl extends AbstractCollect { + + private final static int SUCCESS_CODE = 200; + private final static String NGINX_STATUS_NAME = "nginx_status"; + private final static String REQ_STATUS_NAME = "req_status"; + private final static String AVAILABLE = "available"; + private final static String CONNECTIONS = "connections"; + private final static String ACTIVE = "active"; + private final static String GET = "get"; + private final static String FIELD_SPLIT = "_"; + private final static String REGEX_KEYS = "server\\s+(\\w+)\\s+(\\w+)\\s+(\\w+)"; + private final static String REGEX_VALUES = "(\\d+) (\\d+) (\\d+)"; + private final static String REGEX_SERVER = "(\\w+): (\\d+)"; + private final static String REGEX_SPLIT = "\\r?\\n"; + private final static String REGEX_LINE_SPLIT = "\\s+"; + + + public NginxCollectImpl() { + + } + + @Override + public void collect(CollectRep.MetricsData.Builder builder, long monitorId, String app, Metrics metrics) { + long startTime = System.currentTimeMillis(); + + // 校验参数 + try { + validateParams(metrics); + } catch (Exception e) { + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(e.getMessage()); + return; + } + + HttpContext httpContext = createHttpContext(metrics.getNginx()); + HttpUriRequest request = createHttpRequest(metrics.getNginx()); + try { + // 发起http请求，获取响应数据 + CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != SUCCESS_CODE) { + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg("StatusCode " + statusCode); + return; + } + String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + + Long responseTime = System.currentTimeMillis() - startTime; + // 根据metrics name选择调用不同解析方法 + if (metrics.getName().equals(NGINX_STATUS_NAME) || metrics.getName().equals(AVAILABLE)) { + parseNginxStatusResponse(builder, resp, metrics, responseTime); + } else if (metrics.getName().equals(REQ_STATUS_NAME)) { + parseReqStatusResponse(builder, resp, metrics, responseTime); + } + } catch (IOException e1) { + String errorMsg = CommonUtil.getMessageFromThrowable(e1); + log.info(errorMsg); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(errorMsg); + } catch (Exception e2) { + String errorMsg = CommonUtil.getMessageFromThrowable(e2); + log.info(errorMsg); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(errorMsg); + } + + } + + @Override + public String supportProtocol() { + return DispatchConstants.PROTOCOL_NGINX; + } + + private void validateParams(Metrics metrics) throws Exception { + NginxProtocol nginxProtocol = metrics.getNginx(); + if (metrics == null || nginxProtocol == null || nginxProtocol.isInValid()) { + throw new Exception("Nginx collect must has nginx params"); + } + if (nginxProtocol.getUrl() == null + || "".equals(nginxProtocol.getUrl()) + || !nginxProtocol.getUrl().startsWith(RIGHT_DASH)) { + nginxProtocol.setUrl(nginxProtocol.getUrl() == null ? RIGHT_DASH : RIGHT_DASH + nginxProtocol.getUrl().trim()); + } + } + + private HttpContext createHttpContext(NginxProtocol nginxProtocol) { + HttpHost host = new HttpHost(nginxProtocol.getHost(), Integer.parseInt(nginxProtocol.getPort())); + HttpClientContext httpClientContext = new HttpClientContext(); + httpClientContext.setTargetHost(host); + return httpClientContext; + } + + private HttpUriRequest createHttpRequest(NginxProtocol nginxProtocol) { + RequestBuilder requestBuilder = RequestBuilder.get(); + // uri + String uri = CollectUtil.replaceUriSpecialChar(nginxProtocol.getUrl()); + if (IpDomainUtil.isHasSchema(nginxProtocol.getHost())) { + requestBuilder.setUri(nginxProtocol.getHost() + ":" + nginxProtocol.getPort() + uri); + } else { + String ipAddressType = IpDomainUtil.checkIpAddressType(nginxProtocol.getHost()); + String baseUri = CollectorConstants.IPV6.equals(ipAddressType) + ? String.format("[%s]:%s", nginxProtocol.getHost(), nginxProtocol.getPort() + uri) + : String.format("%s:%s", nginxProtocol.getHost(), nginxProtocol.getPort() + uri); + + requestBuilder.setUri(CollectorConstants.HTTP_HEADER + baseUri); + } + + requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive"); + requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36"); + + requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/plain"); + + Integer timeout = Integer.parseInt(nginxProtocol.getTimeout()); + if (timeout > 0) { + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(timeout) + .setSocketTimeout(timeout) + .setRedirectsEnabled(true) + .build(); + requestBuilder.setConfig(requestConfig); + } + return requestBuilder.build(); + } + + /** + * 解析nginx自带ngx_http_stub_status_module模块暴露信息 + * @param builder + * @param resp + * @param metrics + * @param responseTime + */ + private void parseNginxStatusResponse(CollectRep.MetricsData.Builder builder, String resp, Metrics metrics, + Long responseTime) { + /** example + * Active connections: 2 + * server accepts handled requests + * 4 4 2 + * Reading: 0 Writing: 1 Waiting: 1 + */ + List aliasFields = metrics.getAliasFields(); + Map metricMap = regexNginxStatusMatch(resp); + // 返回数据 + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String alias : aliasFields) { + Object value = metricMap.get(alias); + if (value != null) { + valueRowBuilder.addColumns(String.valueOf(value)); + } else { + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(responseTime.toString()); + } else { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + } + } + } + builder.addValues(valueRowBuilder.build()); + } + + /** + * 解析ngx_http_reqstat_module模块暴露信息 + * @param builder + * @param resp + * @param metrics + * @param responseTime + */ + private void parseReqStatusResponse(CollectRep.MetricsData.Builder builder, String resp, Metrics metrics, + Long responseTime) { + /** example + * zone_name key max_active max_bw traffic requests active bandwidth + * imgstore_appid 43 27 6M 63G 374063 0 0 + * imgstore_appid 53 329 87M 2058G 7870529 50 25M + * server_addr 10.128.1.17 2 8968 24M 1849 0 0 + * server_addr 127.0.0.1 1 6M 5G 912 1 0 + * server_addr 180.96.x.1 3358 934M 27550G 141277391 891 356M + * server_addr 180.96.x.2 78 45M 220G 400704 0 0 + * server_addr 180.96.x.3 242 58M 646G 2990547 42 7M + * server_name d.123.sogou.com 478 115M 2850G 30218726 115 39M + * server_name dl.pinyin.sogou.com 913 312M 8930G 35345453 225 97M + * server_name download.ie.sogou.com 964 275M 7462G 7979817 297 135M + */ + List reqSatusResponses = regexReqStatusMatch(resp); + List aliasFields = metrics.getAliasFields(); + + for (ReqSatusResponse reqSatusResponse : reqSatusResponses) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String alias : aliasFields) { + if (CollectorConstants.RESPONSE_TIME.equals(alias)) { + valueRowBuilder.addColumns(String.valueOf(responseTime)); + } else { + try { + String methodName = reqSatusResponse.getFieldMethodName(alias); + Object value = reflect(reqSatusResponse, methodName); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumns(String.valueOf(value)); + } catch (NoSuchMethodException e1) { + String errorMsg = CommonUtil.getMessageFromThrowable(e1); + log.info(errorMsg); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(errorMsg); + } catch (InvocationTargetException e2) { + String errorMsg = CommonUtil.getMessageFromThrowable(e2); + log.info(errorMsg); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(errorMsg); + } catch (IllegalAccessException e3) { + String errorMsg = CommonUtil.getMessageFromThrowable(e3); + log.info(errorMsg); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(errorMsg); + } + } + } + builder.addValues(valueRowBuilder.build()); + } + } + + private Object reflect(ReqSatusResponse reqSatusResponse, String methodName) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Class clazz = reqSatusResponse.getClass(); + Method method = clazz.getMethod(methodName); + return method.invoke(reqSatusResponse); + } + + private Map regexNginxStatusMatch(String resp) { + Map metricsMap = new HashMap<>(); + // 正则提取监控信息 + Pattern pattern = Pattern.compile(REGEX_SERVER); + Matcher matcher = pattern.matcher(resp); + while (matcher.find()) { + String key = StringUtils.lowerCase(matcher.group(1)); + String value = matcher.group(2); + metricsMap.put(key.equals(CONNECTIONS) ? ACTIVE : key, value); + } + Pattern pattern1 = Pattern.compile(REGEX_KEYS); + Matcher matcher1 = pattern1.matcher(resp); + Pattern pattern2 = Pattern.compile(REGEX_VALUES); + Matcher matcher2 = pattern2.matcher(resp); + if (matcher1.find() && matcher2.find()) { + for (int i = 0; i < 3; i++) { + metricsMap.put(matcher1.group(i + 1), matcher2.group(i + 1)); + } + } + return metricsMap; + } + + private List regexReqStatusMatch(String resp) { + List reqSatusResponses = new ArrayList<>(); + + String[] lines = resp.split(REGEX_SPLIT); + for (int i = 1; i < lines.length; i++) { + String[] values = lines[i].split(REGEX_LINE_SPLIT); + ReqSatusResponse reqSatusResponse = ReqSatusResponse.builder() + .zoneName(values[0]) + .key(values[1]) + .maxActive(values[2]) + .maxBw(values[3]) + .traffic(values[4]) + .requests(values[5]) + .active(values[6]) + .bandwidth(values[7]) + .build(); + reqSatusResponses.add(reqSatusResponse); + } + return reqSatusResponses; + } + + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + static class ReqSatusResponse { + private String zoneName; // zone_name + + private String maxActive; // max_active + + private String key; // key + + private String maxBw; // max_bw + + private String traffic; // traffic + + private String requests; // requests + + private String active; // active + + private String bandwidth; // bandwidth + + public String getFieldMethodName(String name) { + String[] words = name.split(FIELD_SPLIT); + StringBuilder result = new StringBuilder(); + for (String word : words) { + result.append(Character.toUpperCase(word.charAt(0))).append(word.substring(1)); + } + return GET + result; + } + } +} diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/DispatchConstants.java b/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/DispatchConstants.java index 7fc2490c4c5..cbef92430ce 100644 --- a/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/DispatchConstants.java +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/DispatchConstants.java @@ -35,6 +35,10 @@ public interface DispatchConstants { * protocol telnet */ String PROTOCOL_TELNET = "telnet"; + /** + * protocol nginx + */ + String PROTOCOL_NGINX = "nginx"; /** * protocol smtp */ diff --git a/collector/src/main/resources/META-INF/services/org.dromara.hertzbeat.collector.collect.AbstractCollect b/collector/src/main/resources/META-INF/services/org.dromara.hertzbeat.collector.collect.AbstractCollect index aea2549476a..5167f7cdf54 100644 --- a/collector/src/main/resources/META-INF/services/org.dromara.hertzbeat.collector.collect.AbstractCollect +++ b/collector/src/main/resources/META-INF/services/org.dromara.hertzbeat.collector.collect.AbstractCollect @@ -16,3 +16,4 @@ org.dromara.hertzbeat.collector.collect.mq.RocketmqSingleCollectImpl org.dromara.hertzbeat.collector.collect.udp.UdpCollectImpl org.dromara.hertzbeat.collector.collect.push.PushCollectImpl org.dromara.hertzbeat.collector.collect.dns.DnsCollectImpl +org.dromara.hertzbeat.collector.collect.nginx.NginxCollectImpl diff --git a/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java b/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java new file mode 100644 index 00000000000..53e002e77b8 --- /dev/null +++ b/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java @@ -0,0 +1,129 @@ +package org.dromara.hertzbeat.collector.collect.nginx; + +import org.dromara.hertzbeat.common.entity.job.Metrics; +import org.dromara.hertzbeat.common.entity.job.protocol.NginxProtocol; +import org.dromara.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Test case for {@link NginxCollectImpl} + * @author zhoushusheng + */ +public class NginxCollectImplTest { + private NginxCollectImpl nginxCollect; + private NginxProtocol nginxProtocol; + + @BeforeEach + public void setup() { + nginxCollect = new NginxCollectImpl(); + nginxProtocol = NginxProtocol.builder() + .host("127.0.0.1") + .port("80") + .timeout("6000") + .url("/nginx-status") + .build(); + } + + @Test + public void testNginxCollect() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + long monitorId = 999; + String app = "testNginx"; + + Metrics metrics = new Metrics(); + metrics.setNginx(nginxProtocol); + nginxCollect.collect(builder, monitorId, app, metrics); + } + + @Test + public void testNginxStatusMatch() { + String status = "Active connections: 2\n" + + "server accepts handled requests\n" + + "4 4 2\n" + + "Reading: 0 Writing: 1 Waiting: 1"; + + // 使用正则表达式匹配并提取所需的键和对应的值 + Pattern keyValuePattern = Pattern.compile("(\\w+): (\\d+)"); + Matcher keyValueMatcher = keyValuePattern.matcher(status); + + while (keyValueMatcher.find()) { + String key = keyValueMatcher.group(1); + String value = keyValueMatcher.group(2); + + System.out.println(key + ": " + value); + } + + // 使用正则表达式匹配并提取"accepts"、"handled"和"requests"的键和对应的值 + Pattern valuesPattern = Pattern.compile("server\\s+(\\w+)\\s+(\\w+)\\s+(\\w+)"); + Matcher valuesMatcher = valuesPattern.matcher(status); + + if (valuesMatcher.find()) { + String accepts = valuesMatcher.group(1); + String handled = valuesMatcher.group(2); + String requests = valuesMatcher.group(3); + + System.out.println("accepts: " + accepts); + System.out.println("handled: " + handled); + System.out.println("requests: " + requests); + } + + Pattern valuePattern = Pattern.compile("(\\d+) (\\d+) (\\d+)"); + Matcher valueMatcher = valuePattern.matcher(status); + if (valueMatcher.find()) { + int accepts = Integer.parseInt(valueMatcher.group(1)); + int handled = Integer.parseInt(valueMatcher.group(2)); + int requests = Integer.parseInt(valueMatcher.group(3)); + System.out.println("accepts: " + accepts); + System.out.println("handled: " + handled); + System.out.println("requests: " + requests); + } + } + + @Test + public void testReqStatusMatch() { + String urlContent = "zone_name\tkey\tmax_active\tmax_bw\ttraffic\trequests\tactive\tbandwidth\n" + + "server_addr\t172.17.0.3\t2\t 440\t68K\t23\t1\t 0\n" + + "server_name\tlocalhost\t2\t 440\t68K\t23\t1\t 0\n" + + "server_url\tlocalhost/\t1\t 0\t 0\t4\t0\t 0\n" + + "server_url\tlocalhost/index.html\t1\t 104\t27K\t4\t0\t 0\n" + + "server_url\tlocalhost/nginx-status\t1\t 32\t 9896\t5\t0\t 0\n" + + "server_url\tlocalhost/req-status\t1\t 0\t31K\t10\t1\t 0"; + + String[] lines = urlContent.split("\\r?\\n"); + List zoneNames = new ArrayList<>(); + List keys = new ArrayList<>(); + List maxActives = new ArrayList<>(); + List maxBws = new ArrayList<>(); + List traffics = new ArrayList<>(); + List requests = new ArrayList<>(); + List actives = new ArrayList<>(); + List bandwidths = new ArrayList<>(); + + for (int i = 1; i < lines.length; i++) { + String[] values = lines[i].split("\\s+"); + zoneNames.add(values[0]); + keys.add(values[1]); + maxActives.add(values[2]); + maxBws.add(values[3]); + traffics.add(values[4]); + requests.add(values[5]); + actives.add(values[6]); + bandwidths.add(values[7]); + } + + System.out.println("zone_name: " + zoneNames); + System.out.println("key: " + keys); + System.out.println("max_active: " + maxActives); + System.out.println("max_bw: " + maxBws); + System.out.println("traffic: " + traffics); + System.out.println("requests: " + requests); + System.out.println("active: " + actives); + System.out.println("bandwidth: " + bandwidths); + } +} diff --git a/common/src/main/java/org/dromara/hertzbeat/common/entity/job/Metrics.java b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/Metrics.java index a3b7fae82ca..3f42ab1e8ee 100644 --- a/common/src/main/java/org/dromara/hertzbeat/common/entity/job/Metrics.java +++ b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/Metrics.java @@ -186,6 +186,10 @@ public class Metrics { * Monitoring configuration information using the public DNS protocol */ private DnsProtocol dns; + /** + * Monitoring configuration information using the public DNS protocol + */ + private NginxProtocol nginx; /** * collector use - Temporarily store subTask metrics response data diff --git a/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/NginxProtocol.java b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/NginxProtocol.java new file mode 100644 index 00000000000..22af654a566 --- /dev/null +++ b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/NginxProtocol.java @@ -0,0 +1,41 @@ +package org.dromara.hertzbeat.common.entity.job.protocol; + +import org.apache.commons.lang3.StringUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class NginxProtocol { + /** + * nginx主机ip或域名 + */ + private String host; + + /** + * nginx主机端口 + */ + private String port; + + /** + * 超时时间 + */ + private String timeout; + + /** + * 监控模块页面url + */ + private String url; + + /** + * 校验相关参数 + * @return + */ + public boolean isInValid() { + return StringUtils.isAnyBlank(host, port, timeout); + } +} diff --git a/home/docs/help/nginx.md b/home/docs/help/nginx.md new file mode 100644 index 00000000000..76456eb650d --- /dev/null +++ b/home/docs/help/nginx.md @@ -0,0 +1,112 @@ +--- +id: nginx +title: Monitoring Nginx +sidebar_label: Nginx Monitor +keywords: [open source monitoring tool, open source java spark monitoring tool, monitoring nginx metrics] +--- + +> Collect and monitor the general performance Metrics of Nginx. + +**Protocol Use：Nginx** + +### Nginx App Enable `ngx_http_stub_status_module` And `ngx_http_reqstat_module` Configure + +If you want to monitor information in 'Nginx' with this monitoring type, you need to modify your nginx configure file for enable the module monitor. + +**1、Add `ngx_http_stub_status_module` Configure:** + +```shell +# modify nginx.conf +server { + listen 80; # port + server_name localhost; + location /nginx-status { + stub_status on; + access_log on; + #allow 127.0.0.1; #only allow requests from localhost + #deny all; #deny all other hosts + } +} +``` + + + +**2、Add `ngx_http_reqstat_module` Configure:** + +```shell +# install `ngx_http_reqstat_module` +wget https://github.com/zls0424/ngx_req_status/archive/master.zip -O ngx_req_status.zip + +unzip ngx_req_status.zip + +patch -p1 < ../ngx_req_status-master/write_filter.patch + +./configure --prefix=/usr/local/nginx-1.4.2 --add-module=../ngx_req_status-master + +make -j2 + +make install +``` + +```shell +# modify nginx.conf +http { + req_status_zone server_name $server_name 256k; + req_status_zone server_addr $server_addr 256k; + req_status_zone server_url $server_name$uri 256k; + req_status server_name server_addr server_url; + server { + server_name xxx; // server_name + location /req-status { + req_status_show on; + } + } +} +``` + +**⚠️`ngx_http_reqstat_module` need to download it ourselves, If we don't need to monitor this module, we can only collect information about the `ngx_http_stub_status_module` module** + +**There is help_doc: https://blog.csdn.net/weixin_55985097/article/details/116722309** + + +### Configuration parameter + +| Parameter name | Parameter help description | +|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Monitoring Host | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://) | +| Monitoring name | Identify the name of this monitoring. The name needs to be unique | +| Port | Port provided by JMX | +| Timeout | Allow collection response time | +| Collection interval | Interval time of monitor periodic data collection, unit: second, and the minimum interval that can be set is 30 seconds | +| Whether to detect | Whether to detect and check the availability of monitoring before adding monitoring. Adding and modifying operations will continue only after the detection is successful | +| Description remarks | For more information about identifying and describing this monitoring, users can note information here | + +### Collection Metrics + +#### Metrics Set：nginx_status + +| Metric name | Metric unit | Metric help description | +|-------------|-------------|------------------------------------------| +| accepts | | Accepted connections | +| handled | | Successfully processed connections | +| active | | Currently active connections | +| dropped | | Discarded connections | +| requests | | Client requests | +| reading | | Connections performing read operations | +| writing | | Connections performing write operations | +| waiting | | Waiting connections | + +#### Metrics Set：req_status + +| Metric name | Metric unit | Metric help description | +|-------------|-------------|---------------------------------| +| zone_name | | Group category | +| key | | Group name | +| max_active | | Maximum concurrent connections | +| max_bw | kb | Maximum bandwidth | +| traffic | kb | Total traffic | +| requests | | Total requests | +| active | | Current concurrent connections | +| bandwidth | kb | Current bandwidth | + + diff --git a/manager/src/main/resources/define/app-nginx.yml b/manager/src/main/resources/define/app-nginx.yml new file mode 100644 index 00000000000..2a32e4fb9df --- /dev/null +++ b/manager/src/main/resources/define/app-nginx.yml @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 监控类型所属类别：service-应用服务 program-应用程序 db-数据库 custom-自定义 os-操作系统 bigdata-大数据 mid-中间件 webserver-web服务器 cache-缓存 cn-云原生 network-网络监控等等 +category: service +# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws... +app: nginx +# The app api i18n name +# app api国际化名称 +name: + zh-CN: Nginx + en-US: Nginx +# 监控类型的帮助描述信息 +help: + zh-CN: HertzBeat 使用 Nginx 通过配置 ngx_http_stub_status_module 对 nginx 的通用性能指标(nginx_status、req_status等)进行采集监控。
您可以点击“新建 nginx”并进行配置，或者选择“更多操作”，导入已有配置。 + en-US: HertzBeat uses Nginx to configure ngx_http_stub_status_module for collecting general metrics of Nginx (nginx_status, req_status.).
You can click "New Nginx" and configure it, or select "More Operations" to import the existing configuration. + zh-TW: HertzBeat 使用 Nginx 通過配置 ngx_http_stub_status_module 對 Nginx 數據庫的通用性能指標(nginx信息、请求信息等)進行采集監控。
您可以點擊“新建 Nginx”並進行配置，或者選擇“更多操作”，導入已有配置。 +helpLink: + zh-CN: https://hertzbeat.com/zh-cn/docs/help/nginx + en-US: https://hertzbeat.com/docs/help/nginx +# app api所需输入参数定义(根据定义渲染页面UI) +# Input params define for app api(render web ui by the definition) +params: + # field-param field key + # field-字段名称标识符 + - field: host + # name-param field display i18n name + # name-参数字段显示名称 + name: + zh-CN: 目标Host + en-US: Target Host + # type-param field type(most mapping the html input type) + # type-字段类型,样式(大部分映射input标签type属性) + type: host + # required-true or false + # 是否是必输项 true-必填 false-可选 + required: true + # field-param field key + # field-变量字段标识符 + - field: port + # name-param field display i18n name + # name-参数字段显示名称 + name: + zh-CN: 端口 + en-US: Port + # type-param field type(most mapping the html input type) + # type-字段类型,样式(大部分映射input标签type属性) + type: number + # when type is number, range is required + # 当type为number时,用range表示范围 + range: '[0,65535]' + # required-true or false + # required-是否是必输项 true-必填 false-可选 + required: true + # default value + # 默认值 + defaultValue: 80 + # field-param field key + # field-变量字段标识符 + - field: timeout + # name-param field display i18n name + # name-参数字段显示名称 + name: + zh-CN: 连接超时时间(ms) + en-US: Connect Timeout(ms) + # type-param field type(most mapping the html input type) + # type-字段类型,样式(大部分映射input标签type属性) + type: number + # when type is number, range is required + # 当type为number时,用range表示范围 + range: '[0,100000]' + # required-true or false + # 是否是必输项 true-必填 false-可选 + required: true + # default value 6000 + # 默认值 6000 + defaultValue: 6000 + +# collect metrics config list +# 采集指标配置列表 +metrics: + # metrics - available + # 监控指标 - available + - name: available + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + # priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue + # 指标采集调度优先级(0->127)->(优先级高->低) 优先级低的指标会等优先级高的指标采集完成后才会被调度, 相同优先级的指标会并行调度采集 + # 优先级为0的指标为可用性指标,即它会被首先调度,采集成功才会继续调度其它指标,采集失败则中断调度 + priority: 0 + # collect metrics content + # 具体监控指标列表 + fields: + # field-metric name, type-metric type(0-number,1-string), instance-is instance primary key, unit-metric unit + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: responseTime + type: 0 + unit: ms + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + # 采集协议,目前支持sql,ssh,http,telnet,wmi,snmp,sdk + protocol: nginx + # Specific collection configuration when protocol is http protocol + # 当protocol为http协议时具体的采集配置 + nginx: + # http host: ipv4 ipv6 domain + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # http port + # 端口 + port: ^_^port^_^ + # timeout + # 超时时间 + timeout: ^_^timeout^_^ + + - name: nginx_status + i18n: + zh-CN: nginx 状态信息 + en-US: Nginx Status + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + # priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue + # 指标采集调度优先级(0->127)->(优先级高->低) 优先级低的指标会等优先级高的指标采集完成后才会被调度, 相同优先级的指标会并行调度采集 + # 优先级为0的指标为可用性指标,即它会被首先调度,采集成功才会继续调度其它指标,采集失败则中断调度 + priority: 1 + fields: + - field: accepts + type: 0 + i18n: + zh-CN: 接受连接数 + en-US: Accepts + - field: handled + type: 0 + i18n: + zh-CN: 成功处理连接数 + en-US: Handled + - field: active + type: 0 + i18n: + zh-CN: 当前活跃连接数 + en-US: Active + - field: dropped + type: 0 + i18n: + zh-CN: 已丢弃连接数 + en-US: Dropped + - field: requests + type: 0 + i18n: + zh-CN: 客户端请求数 + en-US: Requests + - field: reading + type: 0 + i18n: + zh-CN: 正在执行读操作的连接数 + en-US: Reading + - field: writing + type: 0 + i18n: + zh-CN: 正在执行写操作的连接数 + en-US: Writing + - field: waiting + type: 0 + i18n: + zh-CN: 正在等待的连接数 + en-US: Waiting + # 指标别名列表，用于在查询结果中识别指标 + aliasFields: + - accepts + - handled + - active + - dropped + - requests + - reading + - writing + - waiting + # A list of calculation scripts for metric values. + # 计算指标值的脚本列表 + calculates: + - dropped=accepts - handled + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk, nginx + protocol: nginx + # the config content when protocol is http + nginx: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # http port + port: ^_^port^_^ + # timeout + # 超时时间 + timeout: ^_^timeout^_^ + # http url + # url请求接口路径 + url: /nginx-status + + - name: req_status + i18n: + zh-CN: 请求详细信息 + en-US: Req Status + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + # priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue + # 指标采集调度优先级(0->127)->(优先级高->低) 优先级低的指标会等优先级高的指标采集完成后才会被调度, 相同优先级的指标会并行调度采集 + # 优先级为0的指标为可用性指标,即它会被首先调度,采集成功才会继续调度其它指标,采集失败则中断调度 + priority: 2 + fields: + - field: zone_name + type: 1 + i18n: + zh-CN: 分组类别 + en-US: Zone Name + - field: key + type: 1 + i18n: + zh-CN: 分组名称 + en-US: Key + - field: max_active + type: 0 + i18n: + zh-CN: 最大并发连接数 + en-US: Max Active + - field: max_bw + type: 0 + unit: KB + i18n: + zh-CN: 最大带宽 + en-US: Max BW + - field: traffic + type: 0 + unit: KB + i18n: + zh-CN: 总流量 + en-US: Traffic + - field: requests + type: 0 + i18n: + zh-CN: 总请求数 + en-US: Requests + - field: active + type: 0 + i18n: + zh-CN: 当前并发连接数 + en-US: Active + - field: bandwidth + type: 0 + unit: KB + i18n: + zh-CN: 当前带宽 + en-US: Bandwidth + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: nginx + # the config content when protocol is http + nginx: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # http port + port: ^_^port^_^ + # timeout + # 超时时间 + timeout: ^_^timeout^_^ + # http url + # url请求接口路径 + url: /req-status From 0af139db3524808e12cc972a32b9d73b8597590e Mon Sep 17 00:00:00 2001 From: a-little-fool <2030509072@qq.com> Date: Thu, 14 Dec 2023 16:23:47 +0800 Subject: [PATCH 2/3] feature: fix build warnings --- .../collector/collect/nginx/NginxCollectImpl.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java index eee4b17807d..050358f78d4 100644 --- a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java @@ -94,9 +94,9 @@ public void collect(CollectRep.MetricsData.Builder builder, long monitorId, Stri Long responseTime = System.currentTimeMillis() - startTime; // 根据metrics name选择调用不同解析方法 - if (metrics.getName().equals(NGINX_STATUS_NAME) || metrics.getName().equals(AVAILABLE)) { + if (NGINX_STATUS_NAME.equals(metrics.getName()) || AVAILABLE.equals(metrics.getName())) { parseNginxStatusResponse(builder, resp, metrics, responseTime); - } else if (metrics.getName().equals(REQ_STATUS_NAME)) { + } else if (REQ_STATUS_NAME.equals(metrics.getName())) { parseReqStatusResponse(builder, resp, metrics, responseTime); } } catch (IOException e1) { @@ -185,7 +185,7 @@ private void parseNginxStatusResponse(CollectRep.MetricsData.Builder builder, St * Reading: 0 Writing: 1 Waiting: 1 */ List aliasFields = metrics.getAliasFields(); - Map metricMap = regexNginxStatusMatch(resp); + Map metricMap = regexNginxStatusMatch(resp, metrics.getAliasFields().size()); // 返回数据 CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); for (String alias : aliasFields) { @@ -267,22 +267,22 @@ private Object reflect(ReqSatusResponse reqSatusResponse, String methodName) thr return method.invoke(reqSatusResponse); } - private Map regexNginxStatusMatch(String resp) { - Map metricsMap = new HashMap<>(); + private Map regexNginxStatusMatch(String resp, Integer aliasFieldsSize) { + Map metricsMap = new HashMap<>(aliasFieldsSize); // 正则提取监控信息 Pattern pattern = Pattern.compile(REGEX_SERVER); Matcher matcher = pattern.matcher(resp); while (matcher.find()) { String key = StringUtils.lowerCase(matcher.group(1)); String value = matcher.group(2); - metricsMap.put(key.equals(CONNECTIONS) ? ACTIVE : key, value); + metricsMap.put(CONNECTIONS.equals(key) ? ACTIVE : key, value); } Pattern pattern1 = Pattern.compile(REGEX_KEYS); Matcher matcher1 = pattern1.matcher(resp); Pattern pattern2 = Pattern.compile(REGEX_VALUES); Matcher matcher2 = pattern2.matcher(resp); if (matcher1.find() && matcher2.find()) { - for (int i = 0; i < 3; i++) { + for (int i = 0; i < matcher1.groupCount(); i++) { metricsMap.put(matcher1.group(i + 1), matcher2.group(i + 1)); } } From 30455be1e52be1e6aec5ac3474124aec5f3c52dc Mon Sep 17 00:00:00 2001 From: a-little-fool <2030509072@qq.com> Date: Thu, 14 Dec 2023 16:44:24 +0800 Subject: [PATCH 3/3] feature: modify author name --- .../hertzbeat/collector/collect/nginx/NginxCollectImpl.java | 2 +- .../hertzbeat/collector/collect/nginx/NginxCollectImplTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java index 050358f78d4..7da77d4a805 100644 --- a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImpl.java @@ -42,7 +42,7 @@ /** * nginx collect - * @author zhoushusheng + * @author a-little-fool */ @Slf4j public class NginxCollectImpl extends AbstractCollect { diff --git a/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java b/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java index 53e002e77b8..8d487ab5f9d 100644 --- a/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java +++ b/collector/src/test/java/org/dromara/hertzbeat/collector/collect/nginx/NginxCollectImplTest.java @@ -13,7 +13,7 @@ /** * Test case for {@link NginxCollectImpl} - * @author zhoushusheng + * @author a-little-fool */ public class NginxCollectImplTest { private NginxCollectImpl nginxCollect;