From 25706294419c8e0847ff8f109f50adf2ba87edbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9C=E9=A3=8E?= Date: Fri, 29 Mar 2024 14:56:36 +0800 Subject: [PATCH 1/2] mongodb monitoring support custom connection timeout param --- .../mongodb/MongodbSingleCollectImpl.java | 97 ++++++++++++------- .../entity/job/protocol/MongodbProtocol.java | 5 + .../src/main/resources/define/app-mongodb.yml | 27 ++++++ 3 files changed, 92 insertions(+), 37 deletions(-) diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java index 8b8600a9a08..9def839e3d7 100644 --- a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java @@ -17,37 +17,39 @@ package org.dromara.hertzbeat.collector.collect.mongodb; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Optional; - +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; import com.mongodb.MongoServerUnavailableException; import com.mongodb.MongoTimeoutException; import com.mongodb.client.ClientSession; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.dromara.hertzbeat.collector.collect.AbstractCollect; import org.dromara.hertzbeat.collector.collect.common.cache.CacheIdentifier; import org.dromara.hertzbeat.collector.collect.common.cache.ConnectionCommonCache; import org.dromara.hertzbeat.collector.collect.common.cache.MongodbConnect; -import org.dromara.hertzbeat.common.util.CommonUtil; import org.dromara.hertzbeat.collector.dispatch.DispatchConstants; +import org.dromara.hertzbeat.common.constants.CommonConstants; import org.dromara.hertzbeat.common.entity.job.Metrics; import org.dromara.hertzbeat.common.entity.job.protocol.MongodbProtocol; import org.dromara.hertzbeat.common.entity.message.CollectRep; -import org.dromara.hertzbeat.common.constants.CommonConstants; -import org.bson.Document; +import org.dromara.hertzbeat.common.util.CommonUtil; import org.springframework.util.Assert; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; -import org.dromara.hertzbeat.collector.collect.AbstractCollect; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Optional; -import lombok.extern.slf4j.Slf4j; +import static java.util.concurrent.TimeUnit.MILLISECONDS; /** * Mongodb single collect - * - * + *

+ *

* see also https://www.mongodb.com/languages/java, * https://www.mongodb.com/docs/manual/reference/command/serverStatus/#metrics */ @@ -55,11 +57,12 @@ public class MongodbSingleCollectImpl extends AbstractCollect { /** - * 支持的 mongodb diagnostic 命令，排除internal/deprecated相关的命令 - * 可参考 ..., + * mongodb diagnostic commands supported, excluding internal/deprecated related commands + * can refer to ..., * ... - * 注意：一些命令需要相应的权限才能执行，否则执行虽然不会报错，但是返回的结果是空的， - * 详见 ... + * Note: Some commands require the corresponding permissions to execute, + * otherwise the execution will not report an error, but the return result is empty. + * seeDetails ... */ private static final String[] SUPPORTED_MONGODB_DIAGNOSTIC_COMMANDS = { "buildInfo", @@ -89,20 +92,24 @@ public void collect(CollectRep.MetricsData.Builder builder, long monitorId, Stri builder.setMsg(e.getMessage()); return; } - // command 命名规则约定为上述 mongodb diagnostic 支持的命令，同时也通过 . 支持子文档 - // 如果 command 不包括 . 则直接执行命令，使用其返回的文档，否则需要先执行 metricsParts[0] 命令，再获取相关的子文档 - // 例如 serverStatus.metrics.operation 支持 serverStatus 命令的 metrics 子文档下面的 operation 子文档 + // The command naming convention is the command supported by the above mongodb diagnostic. Support subdocument + // If the command does not include., execute the command directly and use the document it returns; + // otherwise, you need to execute the metricsParts[0] command first and then obtain the related subdocument + // For example serverStatus. Metrics. Operation support serverStatus command the metrics of the following document operation documents String[] metricsParts = metrics.getMongodb().getCommand().split("\\."); String command = metricsParts[0]; - // 检查 . 分割的第一部分是否是 mongodb diagnostic 支持的命令 + // Check whether the first part of the. Split is a command supported by the mongodb diagnostic if (Arrays.stream(SUPPORTED_MONGODB_DIAGNOSTIC_COMMANDS).noneMatch(command::equals)) { builder.setCode(CollectRep.Code.FAIL); builder.setMsg("unsupported mongodb diagnostic command: " + command); return; } ClientSession clientSession = null; + MongoClient mongoClient; + CacheIdentifier identifier= null; try { - MongoClient mongoClient = getClient(metrics); + identifier = getIdentifier(metrics.getMongodb()); + mongoClient = getClient(metrics,identifier); MongoDatabase mongoDatabase = mongoClient.getDatabase(metrics.getMongodb().getDatabase()); clientSession = mongoClient.startSession(); CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); @@ -116,11 +123,12 @@ public void collect(CollectRep.MetricsData.Builder builder, long monitorId, Stri } } if (document == null) { - throw new RuntimeException("the document get from command " + metrics.getMongodb().getCommand() + " is null."); + throw new RuntimeException("the document get from command " + metrics.getMongodb().getCommand() + " is null."); } fillBuilder(metrics, valueRowBuilder, document); builder.addValues(valueRowBuilder.build()); } catch (MongoServerUnavailableException | MongoTimeoutException unavailableException) { + ConnectionCommonCache.getInstance().removeCache(identifier); builder.setCode(CollectRep.Code.UN_CONNECTABLE); String message = CommonUtil.getMessageFromThrowable(unavailableException); builder.setMsg(message); @@ -144,7 +152,7 @@ public String supportProtocol() { } /** - * 使用 metrics 中配置的收集指标以及执行mongodb命令返回的文档，填充 valueRowBuilder + * Populate the valueRowBuilder with the collection metrics configured in Metrics and the documentation returned by executing the mongodb command */ private void fillBuilder(Metrics metrics, CollectRep.ValueRow.Builder valueRowBuilder, Document document) { metrics.getAliasFields().forEach(it -> { @@ -162,7 +170,7 @@ private void fillBuilder(Metrics metrics, CollectRep.ValueRow.Builder valueRowBu } /** - * 检查 metrics 中的 mongodb 连接信息是否完整 + * Check that the mongodb connection information in metrics is complete */ private void preCheck(Metrics metrics) { if (metrics == null || metrics.getMongodb() == null) { @@ -176,17 +184,22 @@ private void preCheck(Metrics metrics) { Assert.hasText(mongodbProtocol.getPassword(), "Mongodb Protocol password is required."); } + public static CacheIdentifier getIdentifier(MongodbProtocol mongodbProtocol){ + // try to reuse connection + return CacheIdentifier.builder() + .ip(mongodbProtocol.getHost()).port(mongodbProtocol.getPort()) + .username(mongodbProtocol.getUsername()).password(mongodbProtocol.getPassword()).build(); + } + /** - * 通过metrics中的mongodb连接信息获取 - * mongodb client本身不存在网络调用，和网络链接。对每次采集，我们需要新建session并使用后关闭它 + * Obtained from mongodb connection information in metrics + * The mongodb client itself does not have network calls and network links. + * For each collection, we need to create a new session and close it after use * mongodb client is thread pool, we need to create the session for each collect */ - private MongoClient getClient(Metrics metrics) { + private MongoClient getClient(Metrics metrics, CacheIdentifier identifier) { MongodbProtocol mongodbProtocol = metrics.getMongodb(); - // try to reuse connection - CacheIdentifier identifier = CacheIdentifier.builder() - .ip(mongodbProtocol.getHost()).port(mongodbProtocol.getPort()) - .username(mongodbProtocol.getUsername()).password(mongodbProtocol.getPassword()).build(); + Optional cacheOption = ConnectionCommonCache.getInstance().getCache(identifier, true); MongoClient mongoClient = null; if (cacheOption.isPresent()) { @@ -196,12 +209,22 @@ private MongoClient getClient(Metrics metrics) { if (mongoClient != null) { return mongoClient; } - // 复用失败则新建连接 connect to mongodb - // 密码可能包含特殊字符，需要使用类似js的encodeURIComponent进行编码，这里使用java的URLEncoder + // If the multiplexing fails, create a new connection to connect to mongodb + // Passwords may contain special characters and need to be encoded using JS-like encodeURIComponent, which uses java URLEncoder String url = String.format("mongodb://%s:%s@%s:%s/%s?authSource=%s", mongodbProtocol.getUsername(), URLEncoder.encode(mongodbProtocol.getPassword(), StandardCharsets.UTF_8), mongodbProtocol.getHost(), mongodbProtocol.getPort(), mongodbProtocol.getDatabase(), mongodbProtocol.getAuthenticationDatabase()); - mongoClient = MongoClients.create(url); + // Use the Mongo Client Settings builder to configure timeouts and other configurations + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(url)) + .applyToClusterSettings(builder -> + // Set the timeout period for server selection + builder.serverSelectionTimeout(Long.parseLong(mongodbProtocol.getTimeout()),MILLISECONDS)) + .build(); + + // CREATE THE MONGO CLIENT USING THE CONFIGURATION + mongoClient = MongoClients.create(settings); + MongodbConnect mongodbConnect = new MongodbConnect(mongoClient); ConnectionCommonCache.getInstance().addCache(identifier, mongodbConnect, 3600 * 1000L); return mongoClient; diff --git a/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/MongodbProtocol.java b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/MongodbProtocol.java index 20a272a7dd8..a5c4b5a87f5 100644 --- a/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/MongodbProtocol.java +++ b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/MongodbProtocol.java @@ -67,4 +67,9 @@ public class MongodbProtocol { */ private String command; + /** + * TIME OUT PERIOD + */ + private String timeout; + } diff --git a/manager/src/main/resources/define/app-mongodb.yml b/manager/src/main/resources/define/app-mongodb.yml index e8ff19da100..6d60f3266dd 100644 --- a/manager/src/main/resources/define/app-mongodb.yml +++ b/manager/src/main/resources/define/app-mongodb.yml @@ -102,6 +102,24 @@ params: limit: 20 required: true defaultValue: 'admin' + - field: timeout + # name-param field display i18n name + # name-参数字段显示名称 + name: + zh-CN: 连接超时时间(ms) + en-US: Connect Timeout(ms) + # type-param field type(most mapping the html input type) + # type-字段类型,样式(大部分映射input标签type属性) + type: number + # when type is number, range is required + # 当type为number时,用range表示范围 + range: '[0,100000]' + # required-true or false + # 是否是必输项 true-必填 false-可选 + required: true + # default value 6000 + # 默认值 6000 + defaultValue: 6000 # can get status information and other config params of MongoDB by running Diagnostic Commands # 可以通过执行 Diagnostic Commands 获取 mongodb 的状态信息以及一些性能、配置参数 # https://www.mongodb.com/docs/manual/reference/command/nav-diagnostic/ @@ -182,6 +200,9 @@ metrics: authenticationDatabase: ^_^authenticationDatabase^_^ #The buildInfo command is an administrative command which returns a build summary for the current mongod. command: buildInfo + # timeout unit：ms + # 超时时间 单位：ms + timeout: ^_^timeout^_^ # https://www.mongodb.com/docs/manual/reference/command/serverStatus/#metrics - name: server_document @@ -219,6 +240,7 @@ metrics: database: ^_^database^_^ authenticationDatabase: ^_^authenticationDatabase^_^ command: serverStatus.metrics.document + timeout: ^_^timeout^_^ - name: server_operation priority: 1 @@ -245,6 +267,7 @@ metrics: database: ^_^database^_^ authenticationDatabase: ^_^authenticationDatabase^_^ command: serverStatus.metrics.operation + timeout: ^_^timeout^_^ - name: server_ttl priority: 1 @@ -271,6 +294,7 @@ metrics: database: ^_^database^_^ authenticationDatabase: ^_^authenticationDatabase^_^ command: serverStatus.metrics.ttl + timeout: ^_^timeout^_^ # https://www.mongodb.com/docs/manual/reference/command/hostInfo/ - name: systemInfo @@ -330,6 +354,7 @@ metrics: database: ^_^database^_^ authenticationDatabase: ^_^authenticationDatabase^_^ command: hostInfo.system + timeout: ^_^timeout^_^ - name: os priority: 1 @@ -361,6 +386,7 @@ metrics: database: ^_^database^_^ authenticationDatabase: ^_^authenticationDatabase^_^ command: hostInfo.os + timeout: ^_^timeout^_^ - name: extra priority: 1 @@ -417,3 +443,4 @@ metrics: database: ^_^database^_^ authenticationDatabase: ^_^authenticationDatabase^_^ command: hostInfo.extra.extra + timeout: ^_^timeout^_^ \ No newline at end of file From 2f43f0ecca23a35a2643df0b2e8a6832d4430f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9C=E9=A3=8E?= Date: Fri, 29 Mar 2024 15:16:56 +0800 Subject: [PATCH 2/2] fix the Error of build . --- .../collector/collect/mongodb/MongodbSingleCollectImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java index 9def839e3d7..f650264c4f7 100644 --- a/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java +++ b/collector/src/main/java/org/dromara/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java @@ -109,7 +109,7 @@ public void collect(CollectRep.MetricsData.Builder builder, long monitorId, Stri CacheIdentifier identifier= null; try { identifier = getIdentifier(metrics.getMongodb()); - mongoClient = getClient(metrics,identifier); + mongoClient = getClient(metrics, identifier); MongoDatabase mongoDatabase = mongoClient.getDatabase(metrics.getMongodb().getDatabase()); clientSession = mongoClient.startSession(); CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); @@ -219,7 +219,7 @@ private MongoClient getClient(Metrics metrics, CacheIdentifier identifier) { .applyConnectionString(new ConnectionString(url)) .applyToClusterSettings(builder -> // Set the timeout period for server selection - builder.serverSelectionTimeout(Long.parseLong(mongodbProtocol.getTimeout()),MILLISECONDS)) + builder.serverSelectionTimeout(Long.parseLong(mongodbProtocol.getTimeout()), MILLISECONDS)) .build(); // CREATE THE MONGO CLIENT USING THE CONFIGURATION