From 5641c431fd6a888c9b2e384f458c299f606e24d2 Mon Sep 17 00:00:00 2001 From: Albumen Kevin Date: Tue, 23 May 2023 14:39:43 +0800 Subject: [PATCH 01/18] URLParamMap support toString (#12384) --- .../java/org/apache/dubbo/common/url/component/URLParam.java | 3 ++- .../test/java/org/apache/dubbo/common/url/URLParamTest.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/url/component/URLParam.java b/dubbo-common/src/main/java/org/apache/dubbo/common/url/component/URLParam.java index cf6ba675102..1500787fdaf 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/url/component/URLParam.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/url/component/URLParam.java @@ -22,6 +22,7 @@ import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.common.utils.StringUtils; +import java.util.AbstractMap; import java.util.Arrays; import java.util.BitSet; import java.util.Collection; @@ -247,7 +248,7 @@ public static Map> initMethodParameters(Map { + public static class URLParamMap extends AbstractMap { private URLParam urlParam; public URLParamMap(URLParam urlParam) { diff --git a/dubbo-common/src/test/java/org/apache/dubbo/common/url/URLParamTest.java b/dubbo-common/src/test/java/org/apache/dubbo/common/url/URLParamTest.java index e8818411525..3c1665cd799 100644 --- a/dubbo-common/src/test/java/org/apache/dubbo/common/url/URLParamTest.java +++ b/dubbo-common/src/test/java/org/apache/dubbo/common/url/URLParamTest.java @@ -220,6 +220,7 @@ void testURLParamMap() { Assertions.assertFalse(urlParam1.getParameters().containsKey("aaa")); Assertions.assertFalse(urlParam1.getParameters().containsKey("version")); Assertions.assertFalse(urlParam1.getParameters().containsKey(new Object())); + Assertions.assertEquals(new HashMap<>(urlParam1.getParameters()).toString(), urlParam1.getParameters().toString()); URLParam urlParam2 = URLParam.parse("aaa=aaa&version=1.0"); URLParam.URLParamMap urlParam2Map = (URLParam.URLParamMap) urlParam2.getParameters(); @@ -284,6 +285,10 @@ void testURLParamMap() { set.add(urlParam4.getParameters()); Assertions.assertEquals(2,set.size()); + + URLParam urlParam5 = URLParam.parse("version=1.0"); + Assertions.assertEquals(new HashMap<>(urlParam5.getParameters()).toString(), urlParam5.getParameters().toString()); + } @Test From e5d2a5798e5e6bf99bb5f42fc82b1741965ffa35 Mon Sep 17 00:00:00 2001 From: Albumen Kevin Date: Tue, 23 May 2023 14:52:49 +0800 Subject: [PATCH 02/18] Support config close timeout (#12386) * Support config close timeout * Fix spell * fix ut --- .../org/apache/dubbo/remoting/Constants.java | 2 ++ .../support/header/CloseTimerTask.java | 14 +++++------ .../support/header/HeaderExchangeServer.java | 17 ++++++------- .../apache/dubbo/remoting/utils/UrlUtils.java | 21 ++++++++++++++++ .../dubbo/remoting/utils/UrlUtilsTest.java | 24 +++++++++++++++++++ 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/Constants.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/Constants.java index 9920ab126af..bd2ad05220b 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/Constants.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/Constants.java @@ -144,8 +144,10 @@ public interface Constants { String TELNET_KEY = "telnet"; String HEARTBEAT_KEY = "heartbeat"; String HEARTBEAT_CONFIG_KEY = "dubbo.protocol.default-heartbeat"; + String CLOSE_TIMEOUT_CONFIG_KEY = "dubbo.protocol.default-close-timeout"; int DEFAULT_HEARTBEAT = 60 * 1000; String HEARTBEAT_TIMEOUT_KEY = "heartbeat.timeout"; + String CLOSE_TIMEOUT_KEY = "close.timeout"; String CONNECTIONS_KEY = "connections"; int DEFAULT_BACKLOG = 1024; diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/CloseTimerTask.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/CloseTimerTask.java index 7c2eea22896..aa4df8d0537 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/CloseTimerTask.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/CloseTimerTask.java @@ -32,11 +32,11 @@ public class CloseTimerTask extends AbstractTimerTask { private static final ErrorTypeAwareLogger logger = LoggerFactory.getErrorTypeAwareLogger(CloseTimerTask.class); - private final int idleTimeout; + private final int closeTimeout; - public CloseTimerTask(ChannelProvider channelProvider, HashedWheelTimer hashedWheelTimer, Long heartbeatTimeoutTick, int idleTimeout) { - super(channelProvider, hashedWheelTimer, heartbeatTimeoutTick); - this.idleTimeout = idleTimeout; + public CloseTimerTask(ChannelProvider channelProvider, HashedWheelTimer hashedWheelTimer, Long tick, int closeTimeout) { + super(channelProvider, hashedWheelTimer, tick); + this.closeTimeout = closeTimeout; } @Override @@ -46,9 +46,9 @@ protected void doTask(Channel channel) { Long lastWrite = lastWrite(channel); Long now = now(); // check ping & pong at server - if ((lastRead != null && now - lastRead > idleTimeout) - || (lastWrite != null && now - lastWrite > idleTimeout)) { - logger.warn(PROTOCOL_FAILED_RESPONSE, "", "", "Close channel " + channel + ", because idleCheck timeout: " + idleTimeout + "ms"); + if ((lastRead != null && now - lastRead > closeTimeout) + || (lastWrite != null && now - lastWrite > closeTimeout)) { + logger.warn(PROTOCOL_FAILED_RESPONSE, "", "", "Close channel " + channel + ", because idleCheck timeout: " + closeTimeout + "ms"); channel.close(); } } catch (Throwable t) { diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java index f27aae80e8a..3dca5a9746e 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java @@ -49,8 +49,7 @@ import static org.apache.dubbo.remoting.Constants.HEARTBEAT_CHECK_TICK; import static org.apache.dubbo.remoting.Constants.LEAST_HEARTBEAT_DURATION; import static org.apache.dubbo.remoting.Constants.TICKS_PER_WHEEL; -import static org.apache.dubbo.remoting.utils.UrlUtils.getHeartbeat; -import static org.apache.dubbo.remoting.utils.UrlUtils.getIdleTimeout; +import static org.apache.dubbo.remoting.utils.UrlUtils.getCloseTimeout; /** * ExchangeServerImpl @@ -211,11 +210,9 @@ public ChannelHandler getChannelHandler() { public void reset(URL url) { server.reset(url); try { - int currHeartbeat = getHeartbeat(getUrl()); - int currIdleTimeout = getIdleTimeout(getUrl()); - int heartbeat = getHeartbeat(url); - int idleTimeout = getIdleTimeout(url); - if (currHeartbeat != heartbeat || currIdleTimeout != idleTimeout) { + int currCloseTimeout = getCloseTimeout(getUrl()); + int closeTimeout = getCloseTimeout(url); + if (closeTimeout != currCloseTimeout) { cancelCloseTask(); startIdleCheckTask(url); } @@ -262,9 +259,9 @@ private long calculateLeastDuration(int time) { private void startIdleCheckTask(URL url) { if (!server.canHandleIdle()) { AbstractTimerTask.ChannelProvider cp = () -> unmodifiableCollection(HeaderExchangeServer.this.getChannels()); - int idleTimeout = getIdleTimeout(url); - long idleTimeoutTick = calculateLeastDuration(idleTimeout); - this.closeTimer = new CloseTimerTask(cp, IDLE_CHECK_TIMER.get(), idleTimeoutTick, idleTimeout); + int closeTimeout = getCloseTimeout(url); + long closeTimeoutTick = calculateLeastDuration(closeTimeout); + this.closeTimer = new CloseTimerTask(cp, IDLE_CHECK_TIMER.get(), closeTimeoutTick, closeTimeout); } } } diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/utils/UrlUtils.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/utils/UrlUtils.java index fe620b613fa..e3f510c7cc8 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/utils/UrlUtils.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/utils/UrlUtils.java @@ -36,6 +36,27 @@ public class UrlUtils { private static final String ALLOWED_SERIALIZATION_KEY = "allowedSerialization"; + public static int getCloseTimeout(URL url) { + String configuredCloseTimeout = System.getProperty(Constants.CLOSE_TIMEOUT_CONFIG_KEY); + int defaultCloseTimeout = -1; + if (StringUtils.isNotEmpty(configuredCloseTimeout)) { + try { + defaultCloseTimeout = Integer.parseInt(configuredCloseTimeout); + } catch (NumberFormatException e) { + // use default heartbeat + } + } + if (defaultCloseTimeout < 0) { + defaultCloseTimeout = getIdleTimeout(url); + } + int closeTimeout = url.getParameter(Constants.CLOSE_TIMEOUT_KEY, defaultCloseTimeout); + int heartbeat = getHeartbeat(url); + if (closeTimeout < heartbeat * 2) { + throw new IllegalStateException("closeTimeout < heartbeatInterval * 2"); + } + return closeTimeout; + } + public static int getIdleTimeout(URL url) { int heartBeat = getHeartbeat(url); // idleTimeout should be at least more than twice heartBeat because possible retries of client. diff --git a/dubbo-remoting/dubbo-remoting-api/src/test/java/org/apache/dubbo/remoting/utils/UrlUtilsTest.java b/dubbo-remoting/dubbo-remoting-api/src/test/java/org/apache/dubbo/remoting/utils/UrlUtilsTest.java index 2ba6f2b5066..6a957aee5e5 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/test/java/org/apache/dubbo/remoting/utils/UrlUtilsTest.java +++ b/dubbo-remoting/dubbo-remoting-api/src/test/java/org/apache/dubbo/remoting/utils/UrlUtilsTest.java @@ -46,4 +46,28 @@ void testConfiguredHeartbeat() { Assertions.assertEquals(200, UrlUtils.getHeartbeat(url)); System.clearProperty(Constants.HEARTBEAT_CONFIG_KEY); } + + @Test + void testGetCloseTimeout() { + URL url1 = URL.valueOf("dubbo://127.0.0.1:12345?heartbeat=10000"); + URL url2 = URL.valueOf("dubbo://127.0.0.1:12345?heartbeat=10000&heartbeat.timeout=50000"); + URL url3 = URL.valueOf("dubbo://127.0.0.1:12345?heartbeat=10000&heartbeat.timeout=10000"); + URL url4 = URL.valueOf("dubbo://127.0.0.1:12345?close.timeout=30000&heartbeat=10000&heartbeat.timeout=10000"); + URL url5 = URL.valueOf("dubbo://127.0.0.1:12345?close.timeout=40000&heartbeat=10000&heartbeat.timeout=50000"); + URL url6 = URL.valueOf("dubbo://127.0.0.1:12345?close.timeout=10000&heartbeat=10000&heartbeat.timeout=10000"); + Assertions.assertEquals(30000, UrlUtils.getCloseTimeout(url1)); + Assertions.assertEquals(50000, UrlUtils.getCloseTimeout(url2)); + Assertions.assertThrows(RuntimeException.class, () -> UrlUtils.getCloseTimeout(url3)); + Assertions.assertThrows(RuntimeException.class, () -> UrlUtils.getCloseTimeout(url4)); + Assertions.assertEquals(40000, UrlUtils.getCloseTimeout(url5)); + Assertions.assertThrows(RuntimeException.class, () -> UrlUtils.getCloseTimeout(url6)); + } + + @Test + void testConfiguredClose() { + System.setProperty(Constants.CLOSE_TIMEOUT_CONFIG_KEY, "180000"); + URL url = URL.valueOf("dubbo://127.0.0.1:12345"); + Assertions.assertEquals(180000, UrlUtils.getCloseTimeout(url)); + System.clearProperty(Constants.HEARTBEAT_CONFIG_KEY); + } } From 8c2c672f8193985ee0f5c67f63a0be6e99fe8018 Mon Sep 17 00:00:00 2001 From: Albumen Kevin Date: Tue, 23 May 2023 15:31:39 +0800 Subject: [PATCH 03/18] Fix netty4 server close timeout (#12387) --- .../apache/dubbo/remoting/transport/netty4/NettyServer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyServer.java b/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyServer.java index cbb547a5262..45b3ac90cf2 100644 --- a/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyServer.java +++ b/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyServer.java @@ -136,14 +136,13 @@ protected void initServerBootstrap(NettyServerHandler nettyServerHandler) { .childHandler(new ChannelInitializer() { @Override protected void initChannel(SocketChannel ch) throws Exception { - // FIXME: should we use getTimeout()? - int idleTimeout = UrlUtils.getIdleTimeout(getUrl()); + int closeTimeout = UrlUtils.getCloseTimeout(getUrl()); NettyCodecAdapter adapter = new NettyCodecAdapter(getCodec(), getUrl(), NettyServer.this); ch.pipeline().addLast("negotiation", new SslServerTlsHandler(getUrl())); ch.pipeline() .addLast("decoder", adapter.getDecoder()) .addLast("encoder", adapter.getEncoder()) - .addLast("server-idle-handler", new IdleStateHandler(0, 0, idleTimeout, MILLISECONDS)) + .addLast("server-idle-handler", new IdleStateHandler(0, 0, closeTimeout, MILLISECONDS)) .addLast("handler", nettyServerHandler); } }); From 2277c5826bee1161e617ef6d490aee66ee5c05d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:10:57 +0800 Subject: [PATCH 04/18] Bump fastjson2 from 2.0.31 to 2.0.32 (#12366) Bumps [fastjson2](https://github.com/alibaba/fastjson2) from 2.0.31 to 2.0.32. - [Release notes](https://github.com/alibaba/fastjson2/releases) - [Commits](https://github.com/alibaba/fastjson2/compare/2.0.31...2.0.32) --- updated-dependencies: - dependency-name: com.alibaba.fastjson2:fastjson2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-dependencies-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-dependencies-bom/pom.xml b/dubbo-dependencies-bom/pom.xml index bbb29805062..ba3f59819aa 100644 --- a/dubbo-dependencies-bom/pom.xml +++ b/dubbo-dependencies-bom/pom.xml @@ -102,7 +102,7 @@ 4.5.14 4.4.16 1.2.83 - 2.0.31 + 2.0.32 3.4.14 4.3.0 2.12.0 From 4820d357b072a2a71d0da64c8f101eae6479fcf3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:11:05 +0800 Subject: [PATCH 05/18] Bump jackson_version from 2.15.0 to 2.15.1 (#12367) Bumps `jackson_version` from 2.15.0 to 2.15.1. Updates `jackson-core` from 2.15.0 to 2.15.1 - [Release notes](https://github.com/FasterXML/jackson-core/releases) - [Changelog](https://github.com/FasterXML/jackson-core/blob/jackson-core-2.15.1/release.properties) - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.15.0...jackson-core-2.15.1) Updates `jackson-databind` from 2.15.0 to 2.15.1 - [Commits](https://github.com/FasterXML/jackson/commits) Updates `jackson-datatype-jsr310` from 2.15.0 to 2.15.1 Updates `jackson-annotations` from 2.15.0 to 2.15.1 - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.fasterxml.jackson.datatype:jackson-datatype-jsr310 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.fasterxml.jackson.core:jackson-annotations dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-dependencies-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-dependencies-bom/pom.xml b/dubbo-dependencies-bom/pom.xml index ba3f59819aa..69ea576a76c 100644 --- a/dubbo-dependencies-bom/pom.xml +++ b/dubbo-dependencies-bom/pom.xml @@ -184,7 +184,7 @@ 2.0.6 5.4.3 2.10.1 - 2.15.0 + 2.15.1 1.6 6.1.26 2.0 From 7561ab45fe6b8f848fb0b842dc5e570121016041 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:11:16 +0800 Subject: [PATCH 06/18] Bump spring-boot-starter-test from 2.7.11 to 2.7.12 (#12369) Bumps [spring-boot-starter-test](https://github.com/spring-projects/spring-boot) from 2.7.11 to 2.7.12. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.7.11...v2.7.12) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-test dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-config/dubbo-config-spring/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-config/dubbo-config-spring/pom.xml b/dubbo-config/dubbo-config-spring/pom.xml index 25252f67b3c..8e9957c4cdd 100644 --- a/dubbo-config/dubbo-config-spring/pom.xml +++ b/dubbo-config/dubbo-config-spring/pom.xml @@ -27,7 +27,7 @@ The spring config module of dubbo project false - 2.7.11 + 2.7.12 From 3ade9b087684c7807035c0f845bef48b60ca854b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:11:48 +0800 Subject: [PATCH 07/18] Bump spring-boot-maven-plugin from 2.7.11 to 2.7.12 (#12371) Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.7.11 to 2.7.12. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.7.11...v2.7.12) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-demo/dubbo-demo-annotation/pom.xml | 2 +- dubbo-demo/dubbo-demo-api/pom.xml | 2 +- dubbo-demo/dubbo-demo-spring-boot/pom.xml | 2 +- dubbo-demo/dubbo-demo-xml/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dubbo-demo/dubbo-demo-annotation/pom.xml b/dubbo-demo/dubbo-demo-annotation/pom.xml index fcdf966e93a..9376e585db6 100644 --- a/dubbo-demo/dubbo-demo-annotation/pom.xml +++ b/dubbo-demo/dubbo-demo-annotation/pom.xml @@ -30,7 +30,7 @@ true - 2.7.11 + 2.7.12 diff --git a/dubbo-demo/dubbo-demo-api/pom.xml b/dubbo-demo/dubbo-demo-api/pom.xml index 05db3e38db6..03753368f94 100644 --- a/dubbo-demo/dubbo-demo-api/pom.xml +++ b/dubbo-demo/dubbo-demo-api/pom.xml @@ -36,7 +36,7 @@ true - 2.7.11 + 2.7.12 dubbo-demo-api diff --git a/dubbo-demo/dubbo-demo-spring-boot/pom.xml b/dubbo-demo/dubbo-demo-spring-boot/pom.xml index e95aa0f50d4..8375c013c44 100644 --- a/dubbo-demo/dubbo-demo-spring-boot/pom.xml +++ b/dubbo-demo/dubbo-demo-spring-boot/pom.xml @@ -37,7 +37,7 @@ 8 true 2.7.11 - 2.7.11 + 2.7.12 1.11.0 diff --git a/dubbo-demo/dubbo-demo-xml/pom.xml b/dubbo-demo/dubbo-demo-xml/pom.xml index 0514421d5b5..67a9d940011 100644 --- a/dubbo-demo/dubbo-demo-xml/pom.xml +++ b/dubbo-demo/dubbo-demo-xml/pom.xml @@ -32,7 +32,7 @@ true - 2.7.11 + 2.7.12 From d228ae0b792a21cee094ba256b59fb86b6ebe3b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:11:56 +0800 Subject: [PATCH 08/18] Bump swagger_version from 1.6.10 to 1.6.11 (#12370) Bumps `swagger_version` from 1.6.10 to 1.6.11. Updates `swagger-annotations` from 1.6.10 to 1.6.11 Updates `swagger-jaxrs` from 1.6.10 to 1.6.11 --- updated-dependencies: - dependency-name: io.swagger:swagger-annotations dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.swagger:swagger-jaxrs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-dependencies-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-dependencies-bom/pom.xml b/dubbo-dependencies-bom/pom.xml index 69ea576a76c..29c88a4556b 100644 --- a/dubbo-dependencies-bom/pom.xml +++ b/dubbo-dependencies-bom/pom.xml @@ -177,7 +177,7 @@ 1.18.1 0.7.5 3.2.13 - 1.6.10 + 1.6.11 1.1.9.1 1.70 From 8f8a4a510574712524d6ffbf39311c290612c953 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:18:05 +0800 Subject: [PATCH 09/18] Bump spring-boot.version from 2.7.11 to 2.7.12 (#12368) Bumps `spring-boot.version` from 2.7.11 to 2.7.12. Updates `spring-boot-starter` from 2.7.11 to 2.7.12 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.7.11...v2.7.12) Updates `spring-boot-autoconfigure` from 2.7.11 to 2.7.12 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.7.11...v2.7.12) Updates `spring-boot-starter-logging` from 2.7.11 to 2.7.12 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.7.11...v2.7.12) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.springframework.boot:spring-boot-autoconfigure dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.springframework.boot:spring-boot-starter-logging dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../dubbo-demo-spring-boot-consumer/pom.xml | 2 +- .../dubbo-demo-spring-boot-provider/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-consumer/pom.xml b/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-consumer/pom.xml index 7771ce1be16..191af45a65e 100644 --- a/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-consumer/pom.xml +++ b/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-consumer/pom.xml @@ -31,7 +31,7 @@ 8 8 1.7.33 - 2.7.11 + 2.7.12 true diff --git a/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-provider/pom.xml b/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-provider/pom.xml index ec8c54ddd89..1a2a3ad0244 100644 --- a/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-provider/pom.xml +++ b/dubbo-demo/dubbo-demo-spring-boot/dubbo-demo-spring-boot-provider/pom.xml @@ -31,7 +31,7 @@ 8 8 1.7.33 - 2.7.11 + 2.7.12 true From 66f18b2f5e174b2a05f4c25fdca1f2df30682800 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:18:14 +0800 Subject: [PATCH 10/18] Bump kryo from 4.0.2 to 4.0.3 (#12373) Bumps [kryo](https://github.com/EsotericSoftware/kryo) from 4.0.2 to 4.0.3. - [Release notes](https://github.com/EsotericSoftware/kryo/releases) - [Commits](https://github.com/EsotericSoftware/kryo/compare/kryo-parent-4.0.2...kryo-parent-4.0.3) --- updated-dependencies: - dependency-name: com.esotericsoftware:kryo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-dependencies-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-dependencies-bom/pom.xml b/dubbo-dependencies-bom/pom.xml index 29c88a4556b..fb13029a77b 100644 --- a/dubbo-dependencies-bom/pom.xml +++ b/dubbo-dependencies-bom/pom.xml @@ -124,7 +124,7 @@ 7.0.5.Final 3.0.1-b12 1.1.1 - 4.0.2 + 4.0.3 0.45 2.57 1.11.1 From 88e5a29c9059ce1533770237777c62e7b2e7d763 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:18:23 +0800 Subject: [PATCH 11/18] Bump protobuf-java_version from 3.23.0 to 3.23.1 (#12374) Bumps `protobuf-java_version` from 3.23.0 to 3.23.1. Updates `protobuf-java` from 3.23.0 to 3.23.1 - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.23.0...v3.23.1) Updates `protobuf-java-util` from 3.23.0 to 3.23.1 --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.google.protobuf:protobuf-java-util dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-dependencies-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-dependencies-bom/pom.xml b/dubbo-dependencies-bom/pom.xml index fb13029a77b..b6339c1179c 100644 --- a/dubbo-dependencies-bom/pom.xml +++ b/dubbo-dependencies-bom/pom.xml @@ -114,7 +114,7 @@ 3.5.5 0.18.1 4.0.66 - 3.23.0 + 3.23.1 1.3.2 3.1.0 9.4.51.v20230217 From 0a27be1a3c1d22ea70b1ee1652e915bc16efaaea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 20:37:59 +0800 Subject: [PATCH 12/18] Bump spring-boot-dependencies from 2.7.11 to 2.7.12 (#12372) Bumps [spring-boot-dependencies](https://github.com/spring-projects/spring-boot) from 2.7.11 to 2.7.12. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.7.11...v2.7.12) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-dependencies dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dubbo-demo/dubbo-demo-spring-boot/pom.xml | 2 +- dubbo-spring-boot/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dubbo-demo/dubbo-demo-spring-boot/pom.xml b/dubbo-demo/dubbo-demo-spring-boot/pom.xml index 8375c013c44..230e3fc196a 100644 --- a/dubbo-demo/dubbo-demo-spring-boot/pom.xml +++ b/dubbo-demo/dubbo-demo-spring-boot/pom.xml @@ -36,7 +36,7 @@ 8 8 true - 2.7.11 + 2.7.12 2.7.12 1.11.0 diff --git a/dubbo-spring-boot/pom.xml b/dubbo-spring-boot/pom.xml index 443343f9b4c..49cebf9b278 100644 --- a/dubbo-spring-boot/pom.xml +++ b/dubbo-spring-boot/pom.xml @@ -40,7 +40,7 @@ - 2.7.11 + 2.7.12 ${revision} 2.20.0 From 6580b0185cb39bae755b46e3f5a4fd779166f504 Mon Sep 17 00:00:00 2001 From: huazhongming Date: Tue, 23 May 2023 21:02:21 +0800 Subject: [PATCH 13/18] Fix PojoUtils NPE problem (#12380) * Fix PojoUtils NPE problem Signed-off-by: crazyhzm * Fix code smell Signed-off-by: crazyhzm --------- Signed-off-by: crazyhzm --- .../apache/dubbo/common/utils/PojoUtils.java | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/PojoUtils.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/PojoUtils.java index 60a78061f47..30b454ebe8d 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/PojoUtils.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/PojoUtils.java @@ -359,11 +359,11 @@ private static Object realize1(Object pojo, Class type, Type genericType, fin Map mapGeneric = new HashMap<>(8); mapGeneric.putAll(mapParent); TypeVariable>[] typeParameters = type.getTypeParameters(); - if(genericType instanceof ParameterizedType && typeParameters.length > 0) { - ParameterizedType parameterizedType = (ParameterizedType)genericType; + if (genericType instanceof ParameterizedType && typeParameters.length > 0) { + ParameterizedType parameterizedType = (ParameterizedType) genericType; Type[] actualTypeArguments = parameterizedType.getActualTypeArguments(); for (int i = 0; i < typeParameters.length; i++) { - if(!(actualTypeArguments[i] instanceof TypeVariable)) { + if (!(actualTypeArguments[i] instanceof TypeVariable)) { mapGeneric.put(typeParameters[i].getTypeName(), actualTypeArguments[i]); } } @@ -534,26 +534,23 @@ private static Object realize1(Object pojo, Class type, Type genericType, fin Object value = entry.getValue(); if (value != null) { Method method = getSetterMethod(dest.getClass(), name, value.getClass()); - Field field = getField(dest.getClass(), name); + Field field = getAndCacheField(dest.getClass(), name); if (method != null) { if (!method.isAccessible()) { method.setAccessible(true); } Type containType = mapGeneric.get(field.getGenericType().getTypeName()); - if(containType != null) { + if (containType != null) { //is generic - if(containType instanceof ParameterizedType) { - value = realize1(value, (Class) ((ParameterizedType)containType).getRawType(), containType, mapGeneric, history); - } - else if (containType instanceof Class){ + if (containType instanceof ParameterizedType) { + value = realize1(value, (Class) ((ParameterizedType) containType).getRawType(), containType, mapGeneric, history); + } else if (containType instanceof Class) { value = realize1(value, (Class) containType, containType, mapGeneric, history); - } - else { + } else { Type ptype = method.getGenericParameterTypes()[0]; value = realize1(value, method.getParameterTypes()[0], ptype, mapGeneric, history); } - } - else { + } else { Type ptype = method.getGenericParameterTypes()[0]; value = realize1(value, method.getParameterTypes()[0], ptype, mapGeneric, history); } @@ -626,7 +623,7 @@ private static Object newThrowableInstance(Class cls, String message) { try { Constructor messagedConstructor = cls.getDeclaredConstructor(String.class); return messagedConstructor.newInstance(message); - } catch (Throwable t) { + } catch (Exception t) { return newInstance(cls); } } @@ -634,7 +631,7 @@ private static Object newThrowableInstance(Class cls, String message) { private static Object newInstance(Class cls) { try { return cls.getDeclaredConstructor().newInstance(); - } catch (Throwable t) { + } catch (Exception t) { Constructor[] constructors = cls.getDeclaredConstructors(); /* From Javadoc java.lang.Class#getDeclaredConstructors @@ -653,7 +650,7 @@ declared by the class represented by this Class object. constructor.setAccessible(true); Object[] parameters = Arrays.stream(constructor.getParameterTypes()).map(PojoUtils::getDefaultValue).toArray(); return constructor.newInstance(parameters); - } catch (Throwable e) { + } catch (Exception e) { lastError = e; } } @@ -704,12 +701,24 @@ private static Method getSetterMethod(Class cls, String property, Class va return method; } - private static Field getField(Class cls, String fieldName) { - Field result = null; + private static Field getAndCacheField(Class cls, String fieldName) { + Field result; if (CLASS_FIELD_CACHE.containsKey(cls) && CLASS_FIELD_CACHE.get(cls).containsKey(fieldName)) { return CLASS_FIELD_CACHE.get(cls).get(fieldName); } - for(Class acls = cls; acls != null; acls = acls.getSuperclass()) { + + result = getField(cls, fieldName); + + if (result != null) { + ConcurrentMap fields = CLASS_FIELD_CACHE.computeIfAbsent(cls, k -> new ConcurrentHashMap<>()); + fields.putIfAbsent(fieldName, result); + } + return result; + } + + private static Field getField(Class cls, String fieldName) { + Field result = null; + for (Class acls = cls; acls != null; acls = acls.getSuperclass()) { try { result = acls.getDeclaredField(fieldName); if (!Modifier.isPublic(result.getModifiers())) { @@ -718,7 +727,7 @@ private static Field getField(Class cls, String fieldName) { } catch (NoSuchFieldException e) { } } - if(result == null) { + if (result == null && cls != null) { for (Field field : cls.getFields()) { if (fieldName.equals(field.getName()) && ReflectUtils.isPublicInstanceField(field)) { result = field; @@ -726,11 +735,6 @@ private static Field getField(Class cls, String fieldName) { } } } - - if (result != null) { - ConcurrentMap fields = CLASS_FIELD_CACHE.computeIfAbsent(cls, k -> new ConcurrentHashMap<>()); - fields.putIfAbsent(fieldName, result); - } return result; } From d2391d2abde386dbf6387d02667ed8f2cb23939c Mon Sep 17 00:00:00 2001 From: wxbty <38374721+wxbty@users.noreply.github.com> Date: Wed, 24 May 2023 13:48:43 +0800 Subject: [PATCH 14/18] Metrics default val set 0 (#12379) * add comment * use dubbo nullable * fill default 0 * add licence * fix ci * fix ci * fix ci * add comment * fix ci * fix ci * modify warn --------- Co-authored-by: x-shadow-man <1494445739@qq.com> Co-authored-by: songxiaosheng --- .../metrics/data/MethodStatComposite.java | 16 ++++- .../metrics/data/ServiceStatComposite.java | 3 + .../dubbo/metrics/model/MetricsSupport.java | 27 ++++++++- .../metrics/model/key/CategoryOverall.java | 3 +- .../dubbo/metrics/model/key/MetricsCat.java | 13 +++- .../dubbo/metrics/model/key/MetricsKey.java | 6 -- .../metrics/model/key/MetricsKeyWrapper.java | 15 +++++ .../dubbo/metrics/MetricsSupportTest.java | 59 +++++++++++++++++++ .../dubbo/metrics/DefaultConstants.java | 6 +- .../metrics/event/DefaultSubDispatcher.java | 12 +--- .../collector/DefaultCollectorTest.java | 7 ++- .../metrics/filter/MetricsFilterTest.java | 22 ++++--- .../metadata/event/MetadataSubDispatcher.java | 7 +++ .../MetadataMetricsCollectorTest.java | 4 +- .../registry/event/RegistrySubDispatcher.java | 7 +++ .../RegistryMetricsCollectorTest.java | 22 +++---- 16 files changed, 177 insertions(+), 52 deletions(-) create mode 100644 dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/MetricsSupportTest.java diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/MethodStatComposite.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/MethodStatComposite.java index 7b029e8f951..a4c6bba709a 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/MethodStatComposite.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/MethodStatComposite.java @@ -17,9 +17,13 @@ package org.apache.dubbo.metrics.data; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; import org.apache.dubbo.common.utils.CollectionUtils; +import org.apache.dubbo.metrics.exception.MetricsNeverHappenException; import org.apache.dubbo.metrics.model.MethodMetric; import org.apache.dubbo.metrics.model.MetricsCategory; +import org.apache.dubbo.metrics.model.MetricsSupport; import org.apache.dubbo.metrics.model.key.MetricsKeyWrapper; import org.apache.dubbo.metrics.model.sample.CounterMetricSample; import org.apache.dubbo.metrics.model.sample.GaugeMetricSample; @@ -41,9 +45,12 @@ */ public class MethodStatComposite extends AbstractMetricsExport { + private static final Logger logger = LoggerFactory.getLogger(MethodStatComposite.class); + public MethodStatComposite(ApplicationModel applicationModel) { super(applicationModel); } + private final Map> methodNumStats = new ConcurrentHashMap<>(); public void initWrapper(List metricsKeyWrappers) { @@ -58,6 +65,7 @@ public void incrementMethodKey(MetricsKeyWrapper wrapper, Invocation invocation, return; } methodNumStats.get(wrapper).computeIfAbsent(new MethodMetric(getApplicationModel(), invocation), k -> new AtomicLong(0L)).getAndAdd(size); + MetricsSupport.fillZero(methodNumStats); } public List export(MetricsCategory category) { @@ -65,11 +73,13 @@ public List export(MetricsCategory category) { for (MetricsKeyWrapper wrapper : methodNumStats.keySet()) { Map stringAtomicLongMap = methodNumStats.get(wrapper); for (MethodMetric methodMetric : stringAtomicLongMap.keySet()) { - if (methodMetric.getSampleType() == MetricSample.Type.GAUGE) { + if (wrapper.getSampleType() == MetricSample.Type.COUNTER) { list.add(new CounterMetricSample<>(wrapper, - methodMetric.getTags(), category, stringAtomicLongMap.get(methodMetric))); - } else { + methodMetric.getTags(), category, stringAtomicLongMap.get(methodMetric))); + } else if (wrapper.getSampleType() == MetricSample.Type.GAUGE) { list.add(new GaugeMetricSample<>(wrapper, methodMetric.getTags(), category, stringAtomicLongMap, value -> value.get(methodMetric).get())); + } else { + throw new MetricsNeverHappenException("Unsupported metricSample type: " + wrapper.getSampleType()); } } diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/ServiceStatComposite.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/ServiceStatComposite.java index ef635666362..797270300c5 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/ServiceStatComposite.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/data/ServiceStatComposite.java @@ -19,6 +19,7 @@ import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.metrics.model.MetricsCategory; +import org.apache.dubbo.metrics.model.MetricsSupport; import org.apache.dubbo.metrics.model.ServiceKeyMetric; import org.apache.dubbo.metrics.model.key.MetricsKeyWrapper; import org.apache.dubbo.metrics.model.sample.GaugeMetricSample; @@ -57,6 +58,7 @@ public void incrementServiceKey(MetricsKeyWrapper wrapper, String serviceKey, in return; } serviceWrapperNumStats.get(wrapper).computeIfAbsent(new ServiceKeyMetric(getApplicationModel(), serviceKey), k -> new AtomicLong(0L)).getAndAdd(size); + MetricsSupport.fillZero(serviceWrapperNumStats); } public void setServiceKey(MetricsKeyWrapper wrapper, String serviceKey, int num) { @@ -64,6 +66,7 @@ public void setServiceKey(MetricsKeyWrapper wrapper, String serviceKey, int num) return; } serviceWrapperNumStats.get(wrapper).computeIfAbsent(new ServiceKeyMetric(getApplicationModel(), serviceKey), k -> new AtomicLong(0L)).set(num); + MetricsSupport.fillZero(serviceWrapperNumStats); } public List export(MetricsCategory category) { diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsSupport.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsSupport.java index dde1c958b46..a8f26efd131 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsSupport.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsSupport.java @@ -18,6 +18,7 @@ package org.apache.dubbo.metrics.model; import org.apache.dubbo.common.Version; +import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.metrics.collector.MethodMetricsCollector; import org.apache.dubbo.metrics.collector.ServiceMetricsCollector; import org.apache.dubbo.metrics.event.MetricsEvent; @@ -35,6 +36,9 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import static org.apache.dubbo.common.constants.CommonConstants.GROUP_CHAR_SEPARATOR; import static org.apache.dubbo.common.constants.CommonConstants.PATH_SEPARATOR; @@ -158,9 +162,9 @@ public static String getInterfaceName(Invocation invocation) { public static String getMethodName(Invocation invocation) { String methodName = invocation.getMethodName(); if (invocation instanceof RpcInvocation - && isGenericCall(((RpcInvocation) invocation).getParameterTypesDesc(), methodName) - && invocation.getArguments() != null - && invocation.getArguments().length == 3) { + && isGenericCall(((RpcInvocation) invocation).getParameterTypesDesc(), methodName) + && invocation.getArguments() != null + && invocation.getArguments().length == 3) { methodName = ((String) invocation.getArguments()[0]).trim(); } return methodName; @@ -224,4 +228,21 @@ public static void incrAndAddRt(MetricsKey metricsKey, MetricsPlaceValue placeTy collector.increment(event.getAttachmentValue(INVOCATION), new MetricsKeyWrapper(metricsKey, placeType), SELF_INCREMENT_SIZE); collector.addRt(event.getAttachmentValue(INVOCATION), placeType.getType(), event.getTimePair().calc()); } + + /** + * Generate a complete indicator item for an interface/method + */ + public static void fillZero(Map> data) { + if (CollectionUtils.isEmptyMap(data)) { + return; + } + Set allKeyMetrics = data.values().stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet()); + data.forEach((keyWrapper, mapVal) -> + { + for (T key : allKeyMetrics) { + mapVal.computeIfAbsent(key, k -> new AtomicLong(0)); + } + }); + + } } diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/CategoryOverall.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/CategoryOverall.java index c13973d17ad..6afec481060 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/CategoryOverall.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/CategoryOverall.java @@ -17,7 +17,8 @@ package org.apache.dubbo.metrics.model.key; -import io.micrometer.common.lang.Nullable; + +import org.apache.dubbo.common.lang.Nullable; /** * The overall event set, including the event processing functions in three stages diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsCat.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsCat.java index 88b19cd2f8c..cf1c8134199 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsCat.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsCat.java @@ -23,18 +23,27 @@ import java.util.function.BiFunction; import java.util.function.Function; +/** + * The behavior wrapper class of MetricsKey, + * which saves the complete content of the key {@link MetricsPlaceValue}, + * the corresponding collector {@link CombMetricsCollector}, + * and the event listener (generate function) at the key level {@link AbstractMetricsKeyListener} + */ public class MetricsCat { private MetricsPlaceValue placeType; private final Function eventFunc; + /** + * @param metricsKey The key corresponding to the listening event, not necessarily the export key(export key may be dynamic) + * @param biFunc Binary function, corresponding to MetricsKey with less content, corresponding to post event + */ public MetricsCat(MetricsKey metricsKey, BiFunction biFunc) { this.eventFunc = collector -> biFunc.apply(metricsKey, collector); } /** - * @param metricsKey The key that the current category listens to，not necessarily the export key(export key may be dynamic) - * @param tpFunc Build the func that outputs the MetricsListener by listen metricsKey + * @param tpFunc Ternary function, corresponding to finish and error events, because an additional record rt is required, and the type type of metricsKey is required */ public MetricsCat(MetricsKey metricsKey, TpFunction tpFunc) { this.eventFunc = collector -> tpFunc.apply(metricsKey, placeType, collector); diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java index 9470f6b31b7..09cb9881187 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java @@ -134,12 +134,6 @@ public final String getNameByType(String type) { return String.format(name, type); } - - public final MetricsKey formatName(String type) { - this.name = String.format(name, type); - return this; - } - public final String getDescription() { return this.description; } diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKeyWrapper.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKeyWrapper.java index 95d7a31d69e..21bb7648e90 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKeyWrapper.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKeyWrapper.java @@ -19,6 +19,7 @@ import io.micrometer.common.lang.Nullable; import org.apache.dubbo.metrics.model.MetricsSupport; +import org.apache.dubbo.metrics.model.sample.MetricSample; import org.apache.dubbo.rpc.model.ApplicationModel; import java.util.Map; @@ -39,6 +40,11 @@ public class MetricsKeyWrapper { */ private final MetricsPlaceValue placeType; + /** + * Exported sample type + */ + private MetricSample.Type sampleType = MetricSample.Type.COUNTER; + /** * When the MetricsPlaceType is null, it is equivalent to a single MetricsKey. * Use the decorator mode to share a container with MetricsKey @@ -48,6 +54,15 @@ public MetricsKeyWrapper(MetricsKey metricsKey, @Nullable MetricsPlaceValue plac this.placeType = placeType; } + public MetricsKeyWrapper setSampleType(MetricSample.Type sampleType) { + this.sampleType = sampleType; + return this; + } + + public MetricSample.Type getSampleType() { + return sampleType; + } + public MetricsPlaceValue getPlaceType() { return placeType; } diff --git a/dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/MetricsSupportTest.java b/dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/MetricsSupportTest.java new file mode 100644 index 00000000000..7bc12d6a497 --- /dev/null +++ b/dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/MetricsSupportTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.metrics; + +import org.apache.dubbo.common.constants.CommonConstants; +import org.apache.dubbo.config.ApplicationConfig; +import org.apache.dubbo.metrics.model.MetricsSupport; +import org.apache.dubbo.metrics.model.ServiceKeyMetric; +import org.apache.dubbo.metrics.model.key.MetricsKeyWrapper; +import org.apache.dubbo.metrics.model.key.MetricsLevel; +import org.apache.dubbo.metrics.model.key.MetricsPlaceValue; +import org.apache.dubbo.rpc.model.ApplicationModel; +import org.apache.dubbo.rpc.model.FrameworkModel; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.dubbo.metrics.model.key.MetricsKey.METRIC_REQUESTS; + +public class MetricsSupportTest { + + @Test + void testFillZero() { + ApplicationModel applicationModel = FrameworkModel.defaultModel().newApplication(); + ApplicationConfig config = new ApplicationConfig(); + config.setName("MockMetrics"); + applicationModel.getApplicationConfigManager().setApplication(config); + + Map> data = new HashMap<>(); + MetricsKeyWrapper key1 = new MetricsKeyWrapper(METRIC_REQUESTS, MetricsPlaceValue.of(CommonConstants.PROVIDER, MetricsLevel.METHOD)); + MetricsKeyWrapper key2 = new MetricsKeyWrapper(METRIC_REQUESTS, MetricsPlaceValue.of(CommonConstants.CONSUMER, MetricsLevel.METHOD)); + ServiceKeyMetric sm1 = new ServiceKeyMetric(applicationModel, "a.b.c"); + ServiceKeyMetric sm2 = new ServiceKeyMetric(applicationModel, "a.b.d"); + data.computeIfAbsent(key1, k -> new HashMap<>()).put(sm1, new AtomicLong(1)); + data.computeIfAbsent(key1, k -> new HashMap<>()).put(sm2, new AtomicLong(1)); + data.put(key2, new HashMap<>()); + Assertions.assertEquals(2, data.values().stream().mapToLong(map -> map.values().size()).sum()); + MetricsSupport.fillZero(data); + Assertions.assertEquals(4, data.values().stream().mapToLong(map -> map.values().size()).sum()); + } +} diff --git a/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/DefaultConstants.java b/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/DefaultConstants.java index c4e6f9485cc..4bb9245e580 100644 --- a/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/DefaultConstants.java +++ b/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/DefaultConstants.java @@ -21,6 +21,7 @@ import org.apache.dubbo.metrics.model.key.MetricsKeyWrapper; import org.apache.dubbo.metrics.model.key.MetricsLevel; import org.apache.dubbo.metrics.model.key.MetricsPlaceValue; +import org.apache.dubbo.metrics.model.sample.MetricSample; import java.util.Arrays; import java.util.List; @@ -46,8 +47,9 @@ public interface DefaultConstants { List METHOD_LEVEL_KEYS = Arrays.asList( new MetricsKeyWrapper(METRIC_REQUESTS, MetricsPlaceValue.of(CommonConstants.PROVIDER, MetricsLevel.METHOD)), new MetricsKeyWrapper(METRIC_REQUESTS, MetricsPlaceValue.of(CommonConstants.CONSUMER, MetricsLevel.METHOD)), - new MetricsKeyWrapper(METRIC_REQUESTS_PROCESSING, MetricsPlaceValue.of(CommonConstants.PROVIDER, MetricsLevel.METHOD)), - new MetricsKeyWrapper(METRIC_REQUESTS_PROCESSING, MetricsPlaceValue.of(CommonConstants.CONSUMER, MetricsLevel.METHOD)), + // METRIC_REQUESTS_PROCESSING use GAUGE + new MetricsKeyWrapper(METRIC_REQUESTS_PROCESSING, MetricsPlaceValue.of(CommonConstants.PROVIDER, MetricsLevel.METHOD)).setSampleType(MetricSample.Type.GAUGE), + new MetricsKeyWrapper(METRIC_REQUESTS_PROCESSING, MetricsPlaceValue.of(CommonConstants.CONSUMER, MetricsLevel.METHOD)).setSampleType(MetricSample.Type.GAUGE), new MetricsKeyWrapper(METRIC_REQUESTS_SUCCEED, MetricsPlaceValue.of(CommonConstants.PROVIDER, MetricsLevel.METHOD)), new MetricsKeyWrapper(METRIC_REQUESTS_SUCCEED, MetricsPlaceValue.of(CommonConstants.CONSUMER, MetricsLevel.METHOD)), new MetricsKeyWrapper(METRIC_REQUEST_BUSINESS_FAILED, MetricsPlaceValue.of(CommonConstants.PROVIDER, MetricsLevel.METHOD)), diff --git a/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/event/DefaultSubDispatcher.java b/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/event/DefaultSubDispatcher.java index 99c78e6c1d1..5bcbc204697 100644 --- a/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/event/DefaultSubDispatcher.java +++ b/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/event/DefaultSubDispatcher.java @@ -29,12 +29,8 @@ import org.apache.dubbo.metrics.model.key.MetricsKey; import org.apache.dubbo.metrics.model.key.MetricsLevel; import org.apache.dubbo.metrics.model.key.MetricsPlaceValue; -import org.apache.dubbo.metrics.model.sample.MetricSample; -import org.apache.dubbo.rpc.Invocation; import static org.apache.dubbo.metrics.DefaultConstants.METRIC_THROWABLE; -import static org.apache.dubbo.metrics.MetricsConstants.INVOCATION; -import static org.apache.dubbo.metrics.MetricsConstants.INVOCATION_METRICS_COUNTER; import static org.apache.dubbo.metrics.model.key.MetricsKey.METRIC_REQUESTS_SERVICE_UNAVAILABLE_FAILED; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -71,17 +67,13 @@ private CategoryOverall initMethodRequest() { { MetricsPlaceValue dynamicPlaceType = MetricsPlaceValue.of(event.getAttachmentValue(MetricsConstants.INVOCATION_SIDE), MetricsLevel.METHOD); MetricsSupport.increment(key, dynamicPlaceType, (MethodMetricsCollector) collector, event); - - // METRIC_REQUESTS_PROCESSING use GAUGE - Invocation invocation = event.getAttachmentValue(INVOCATION); - invocation.put(INVOCATION_METRICS_COUNTER, MetricSample.Type.GAUGE); MetricsSupport.increment(MetricsKey.METRIC_REQUESTS_PROCESSING, dynamicPlaceType, (MethodMetricsCollector) collector, event); })), new MetricsCat(MetricsKey.METRIC_REQUESTS_SUCCEED, (key, placeType, collector) -> AbstractMetricsKeyListener.onFinish(key, event -> { MetricsPlaceValue dynamicPlaceType = MetricsPlaceValue.of(event.getAttachmentValue(MetricsConstants.INVOCATION_SIDE), MetricsLevel.METHOD); - MetricsSupport.dec(MetricsKey.METRIC_REQUESTS_PROCESSING, dynamicPlaceType, (MethodMetricsCollector) collector, event); + MetricsSupport.dec(MetricsKey.METRIC_REQUESTS_PROCESSING, dynamicPlaceType, collector, event); Object throwableObj = event.getAttachmentValue(METRIC_THROWABLE); MetricsKey targetKey; @@ -99,7 +91,7 @@ private CategoryOverall initMethodRequest() { // Dynamic metricsKey && dynamicPlaceType MetricsPlaceValue dynamicPlaceType = MetricsPlaceValue.of(event.getAttachmentValue(MetricsConstants.INVOCATION_SIDE), MetricsLevel.METHOD); MetricsSupport.increment(MetricsKey.METRIC_REQUESTS_TOTAL_FAILED, dynamicPlaceType, (MethodMetricsCollector) collector, event); - MetricsSupport.dec(MetricsKey.METRIC_REQUESTS_PROCESSING, dynamicPlaceType, (MethodMetricsCollector) collector, event); + MetricsSupport.dec(MetricsKey.METRIC_REQUESTS_PROCESSING, dynamicPlaceType, collector, event); MetricsSupport.incrAndAddRt(targetKey, dynamicPlaceType, (MethodMetricsCollector) collector, event); } ))); diff --git a/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/DefaultCollectorTest.java b/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/DefaultCollectorTest.java index 9beb258115b..ab17adeae32 100644 --- a/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/DefaultCollectorTest.java +++ b/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/DefaultCollectorTest.java @@ -20,6 +20,7 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.constants.CommonConstants; import org.apache.dubbo.config.ApplicationConfig; +import org.apache.dubbo.metrics.DefaultConstants; import org.apache.dubbo.metrics.TestMetricsInvoker; import org.apache.dubbo.metrics.event.MetricsDispatcher; import org.apache.dubbo.metrics.event.RequestBeforeEvent; @@ -148,8 +149,8 @@ void testRequestEventNoRt() { // push finish rt +1 List metricSamples = collector.collect(); - //num(total+success+processing) + rt(5) = 8 - Assertions.assertEquals(8, metricSamples.size()); + //all METHOD_LEVEL_KEYS + rt(5) = 27 + Assertions.assertEquals(DefaultConstants.METHOD_LEVEL_KEYS.size() + 5, metricSamples.size()); List metricsNames = metricSamples.stream().map(MetricSample::getName).collect(Collectors.toList()); // No error will contain total+success+processing String REQUESTS = new MetricsKeyWrapper(METRIC_REQUESTS, MetricsPlaceValue.of(side, MetricsLevel.SERVICE)).targetKey(); @@ -191,7 +192,7 @@ void testRequestEventNoRt() { metricSamples = collector.collect(); // num(total+success+error+total_error+processing) + rt(5) = 5 - Assertions.assertEquals(10, metricSamples.size()); + Assertions.assertEquals(DefaultConstants.METHOD_LEVEL_KEYS.size() + 5, metricSamples.size()); String TIMEOUT = new MetricsKeyWrapper(METRIC_REQUESTS_TIMEOUT, MetricsPlaceValue.of(side, MetricsLevel.SERVICE)).targetKey(); String TOTAL_FAILED = new MetricsKeyWrapper(METRIC_REQUESTS_TOTAL_FAILED, MetricsPlaceValue.of(side, MetricsLevel.SERVICE)).targetKey(); diff --git a/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/filter/MetricsFilterTest.java b/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/filter/MetricsFilterTest.java index 887b7855552..488986afeb7 100644 --- a/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/filter/MetricsFilterTest.java +++ b/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/filter/MetricsFilterTest.java @@ -25,6 +25,9 @@ import org.apache.dubbo.metrics.event.MetricsEventBus; import org.apache.dubbo.metrics.event.RequestEvent; import org.apache.dubbo.metrics.model.key.MetricsKey; +import org.apache.dubbo.metrics.model.key.MetricsKeyWrapper; +import org.apache.dubbo.metrics.model.key.MetricsLevel; +import org.apache.dubbo.metrics.model.key.MetricsPlaceValue; import org.apache.dubbo.metrics.model.sample.CounterMetricSample; import org.apache.dubbo.metrics.model.sample.MetricSample; import org.apache.dubbo.rpc.AppResponse; @@ -125,7 +128,7 @@ void testUnknownFailedRequests() { Map metricsMap = getMetricsMap(); Assertions.assertTrue(metricsMap.containsKey(MetricsKey.METRIC_REQUESTS_FAILED.getNameByType(side))); - Assertions.assertFalse(metricsMap.containsKey(MetricsKey.METRIC_REQUESTS_SUCCEED.getNameByType(side))); + Assertions.assertTrue(metricsMap.containsKey(MetricsKey.METRIC_REQUESTS_SUCCEED.getNameByType(side))); MetricSample sample = metricsMap.get(MetricsKey.METRIC_REQUESTS_FAILED.getNameByType(side)); Map tags = sample.getTags(); @@ -152,7 +155,7 @@ void testBusinessFailedRequests() { Map metricsMap = getMetricsMap(); Assertions.assertTrue(metricsMap.containsKey(MetricsKey.METRIC_REQUEST_BUSINESS_FAILED.getNameByType(side))); - Assertions.assertFalse(metricsMap.containsKey(MetricsKey.METRIC_REQUESTS_SUCCEED.getNameByType(side))); + Assertions.assertTrue(metricsMap.containsKey(MetricsKey.METRIC_REQUESTS_SUCCEED.getNameByType(side))); MetricSample sample = metricsMap.get(MetricsKey.METRIC_REQUEST_BUSINESS_FAILED.getNameByType(side)); @@ -229,7 +232,7 @@ void testSucceedRequests() { filter.onResponse(result, invoker, invocation); Map metricsMap = getMetricsMap(); - Assertions.assertFalse(metricsMap.containsKey(MetricsKey.METRIC_REQUEST_BUSINESS_FAILED.getNameByType(side))); + Assertions.assertTrue(metricsMap.containsKey(MetricsKey.METRIC_REQUEST_BUSINESS_FAILED.getNameByType(side))); Assertions.assertTrue(metricsMap.containsKey(MetricsKey.METRIC_REQUESTS_SUCCEED.getNameByType(side))); MetricSample sample = metricsMap.get(MetricsKey.METRIC_REQUESTS_SUCCEED.getNameByType(side)); @@ -266,12 +269,13 @@ void testMissingGroup() { @Test public void testErrors() { - testFilterError(RpcException.SERIALIZATION_EXCEPTION, MetricsKey.METRIC_REQUESTS_CODEC_FAILED.formatName(side)); - testFilterError(RpcException.NETWORK_EXCEPTION, MetricsKey.METRIC_REQUESTS_NETWORK_FAILED.formatName(side)); + testFilterError(RpcException.SERIALIZATION_EXCEPTION, MetricsKey.METRIC_REQUESTS_CODEC_FAILED); + testFilterError(RpcException.NETWORK_EXCEPTION, MetricsKey.METRIC_REQUESTS_NETWORK_FAILED); } private void testFilterError(int errorCode, MetricsKey metricsKey) { + String targetKey = new MetricsKeyWrapper(metricsKey, MetricsPlaceValue.of(side, MetricsLevel.METHOD)).targetKey(); setup(); collector.setCollectEnabled(true); given(invoker.invoke(invocation)).willThrow(new RpcException(errorCode)); @@ -288,14 +292,14 @@ private void testFilterError(int errorCode, MetricsKey metricsKey) { } } Map metricsMap = getMetricsMap(); - Assertions.assertTrue(metricsMap.containsKey(metricsKey.getName())); + Assertions.assertTrue(metricsMap.containsKey(targetKey)); - MetricSample sample = metricsMap.get(metricsKey.getName()); + MetricSample sample = metricsMap.get(targetKey); - Assertions.assertSame(((CounterMetricSample) sample).getValue().longValue(), count); + Assertions.assertSame(((CounterMetricSample) sample).getValue().longValue(), count); - Assertions.assertTrue(metricsMap.containsKey(metricsKey.getName())); + Assertions.assertTrue(metricsMap.containsKey(targetKey)); Map tags = sample.getTags(); Assertions.assertEquals(tags.get(TAG_INTERFACE_KEY), INTERFACE_NAME); diff --git a/dubbo-metrics/dubbo-metrics-metadata/src/main/java/org/apache/dubbo/metrics/metadata/event/MetadataSubDispatcher.java b/dubbo-metrics/dubbo-metrics-metadata/src/main/java/org/apache/dubbo/metrics/metadata/event/MetadataSubDispatcher.java index f78f1965e1d..ebbe20f8a8b 100644 --- a/dubbo-metrics/dubbo-metrics-metadata/src/main/java/org/apache/dubbo/metrics/metadata/event/MetadataSubDispatcher.java +++ b/dubbo-metrics/dubbo-metrics-metadata/src/main/java/org/apache/dubbo/metrics/metadata/event/MetadataSubDispatcher.java @@ -50,6 +50,9 @@ public MetadataSubDispatcher(MetadataMetricsCollector collector) { } + /** + * A closer aggregation of MetricsCat, a summary collection of certain types of events + */ interface CategorySet { CategoryOverall APPLICATION_PUSH = new CategoryOverall(OP_TYPE_PUSH, MCat.APPLICATION_PUSH_POST, MCat.APPLICATION_PUSH_FINISH, MCat.APPLICATION_PUSH_ERROR); CategoryOverall APPLICATION_SUBSCRIBE = new CategoryOverall(OP_TYPE_SUBSCRIBE, MCat.APPLICATION_SUBSCRIBE_POST, MCat.APPLICATION_SUBSCRIBE_FINISH, MCat.APPLICATION_SUBSCRIBE_ERROR); @@ -58,6 +61,10 @@ interface CategorySet { List ALL = Arrays.asList(APPLICATION_PUSH, APPLICATION_SUBSCRIBE, SERVICE_SUBSCRIBE); } + /** + * {@link MetricsCat} MetricsCat collection, for better classification processing + * Except for a few custom functions, most of them can build standard event listening functions through the static methods of MetricsApplicationListener + */ interface MCat { // MetricsPushListener MetricsCat APPLICATION_PUSH_POST = new MetricsCat(MetricsKey.METADATA_PUSH_METRIC_NUM, MetricsApplicationListener::onPostEventBuild); diff --git a/dubbo-metrics/dubbo-metrics-metadata/src/test/java/org/apache/dubbo/metrics/metadata/MetadataMetricsCollectorTest.java b/dubbo-metrics/dubbo-metrics-metadata/src/test/java/org/apache/dubbo/metrics/metadata/MetadataMetricsCollectorTest.java index 3f3d54bb230..23d93254a91 100644 --- a/dubbo-metrics/dubbo-metrics-metadata/src/test/java/org/apache/dubbo/metrics/metadata/MetadataMetricsCollectorTest.java +++ b/dubbo-metrics/dubbo-metrics-metadata/src/test/java/org/apache/dubbo/metrics/metadata/MetadataMetricsCollectorTest.java @@ -211,7 +211,7 @@ void testStoreProviderMetadataMetrics() { List metricSamples = collector.collect(); // App(6) + service success(1) - Assertions.assertEquals(MetadataMetricsConstants.APP_LEVEL_KEYS.size() + 1, metricSamples.size()); + Assertions.assertEquals(MetadataMetricsConstants.APP_LEVEL_KEYS.size() + MetadataMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); Assertions.assertTrue(metricSamples.stream().allMatch(metricSample -> metricSample instanceof GaugeMetricSample)); Assertions.assertTrue(metricSamples.stream().anyMatch(metricSample -> ((GaugeMetricSample) metricSample).applyAsDouble() == 1)); return null; @@ -221,7 +221,7 @@ void testStoreProviderMetadataMetrics() { // push finish rt +1 List metricSamples = collector.collect(); // App(6) + service total/success(2) + rt(5) = 7 - Assertions.assertEquals(MetadataMetricsConstants.APP_LEVEL_KEYS.size() + 2 + 5, metricSamples.size()); + Assertions.assertEquals(MetadataMetricsConstants.APP_LEVEL_KEYS.size() + MetadataMetricsConstants.SERVICE_LEVEL_KEYS.size() + 5, metricSamples.size()); long c1 = metadataEvent.getTimePair().calc(); metadataEvent = MetadataEvent.toServiceSubscribeEvent(applicationModel, serviceKey); diff --git a/dubbo-metrics/dubbo-metrics-registry/src/main/java/org/apache/dubbo/metrics/registry/event/RegistrySubDispatcher.java b/dubbo-metrics/dubbo-metrics-registry/src/main/java/org/apache/dubbo/metrics/registry/event/RegistrySubDispatcher.java index 8481a199844..f0d8c74f411 100644 --- a/dubbo-metrics/dubbo-metrics-registry/src/main/java/org/apache/dubbo/metrics/registry/event/RegistrySubDispatcher.java +++ b/dubbo-metrics/dubbo-metrics-registry/src/main/java/org/apache/dubbo/metrics/registry/event/RegistrySubDispatcher.java @@ -58,6 +58,9 @@ public RegistrySubDispatcher(RegistryMetricsCollector collector) { }); } + /** + * A closer aggregation of MetricsCat, a summary collection of certain types of events + */ interface CategorySet { CategoryOverall APPLICATION_REGISTER = new CategoryOverall(OP_TYPE_REGISTER, MCat.APPLICATION_REGISTER_POST, MCat.APPLICATION_REGISTER_FINISH, MCat.APPLICATION_REGISTER_ERROR); CategoryOverall APPLICATION_SUBSCRIBE = new CategoryOverall(OP_TYPE_SUBSCRIBE, MCat.APPLICATION_SUBSCRIBE_POST, MCat.APPLICATION_SUBSCRIBE_FINISH, MCat.APPLICATION_SUBSCRIBE_ERROR); @@ -70,6 +73,10 @@ interface CategorySet { } + /** + * {@link MetricsCat} MetricsCat collection, for better classification processing + * Except for a few custom functions, most of them can build standard event listening functions through the static methods of MetricsApplicationListener + */ interface MCat { // MetricsRegisterListener MetricsCat APPLICATION_REGISTER_POST = new MetricsCat(MetricsKey.REGISTER_METRIC_REQUESTS, MetricsApplicationListener::onPostEventBuild); diff --git a/dubbo-metrics/dubbo-metrics-registry/src/test/java/org/apache/dubbo/metrics/registry/metrics/collector/RegistryMetricsCollectorTest.java b/dubbo-metrics/dubbo-metrics-registry/src/test/java/org/apache/dubbo/metrics/registry/metrics/collector/RegistryMetricsCollectorTest.java index 3dc6dad7d30..9b9de75e60e 100644 --- a/dubbo-metrics/dubbo-metrics-registry/src/test/java/org/apache/dubbo/metrics/registry/metrics/collector/RegistryMetricsCollectorTest.java +++ b/dubbo-metrics/dubbo-metrics-registry/src/test/java/org/apache/dubbo/metrics/registry/metrics/collector/RegistryMetricsCollectorTest.java @@ -140,9 +140,9 @@ void testServicePushMetrics() { List metricSamples = collector.collect(); // push success +1 - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 1, metricSamples.size()); + Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); // Service num only 1 and contains tag of interface - Assertions.assertEquals(1, metricSamples.stream().filter(metricSample -> serviceName.equals(metricSample.getTags().get("interface"))).count()); + Assertions.assertEquals(RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.stream().filter(metricSample -> serviceName.equals(metricSample.getTags().get("interface"))).count()); return null; } ); @@ -150,7 +150,7 @@ void testServicePushMetrics() { // push finish rt +1 List metricSamples = collector.collect(); // App(7) + rt(5) + service(total/success) = 14 - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + 2, metricSamples.size()); + Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); long c1 = registryEvent.getTimePair().calc(); registryEvent = RegistryEvent.toRsEvent(applicationModel, serviceName, 2); @@ -171,7 +171,7 @@ void testServicePushMetrics() { metricSamples = collector.collect(); // App(7) + rt(5) + service(total/success/failed) = 15 - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + 3, metricSamples.size()); + Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); // calc rt for (MetricSample sample : metricSamples) { @@ -201,10 +201,10 @@ void testServiceSubscribeMetrics() { List metricSamples = collector.collect(); Assertions.assertTrue(metricSamples.stream().allMatch(metricSample -> metricSample instanceof GaugeMetricSample)); Assertions.assertTrue(metricSamples.stream().anyMatch(metricSample -> ((GaugeMetricSample) metricSample).applyAsDouble() == 1)); - // App(7) + (service success +1) - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 1, metricSamples.size()); + // App(default=7) + (service success +1) + Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); // Service num only 1 and contains tag of interface - Assertions.assertEquals(1, metricSamples.stream().filter(metricSample -> serviceName.equals(metricSample.getTags().get("interface"))).count()); + Assertions.assertEquals(RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.stream().filter(metricSample -> serviceName.equals(metricSample.getTags().get("interface"))).count()); return null; } ); @@ -212,7 +212,7 @@ void testServiceSubscribeMetrics() { // push finish rt +1 List metricSamples = collector.collect(); // App(7) + rt(5) + service(total/success) = 14 - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + 2, metricSamples.size()); + Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); long c1 = subscribeEvent.getTimePair().calc(); subscribeEvent = RegistryEvent.toSsEvent(applicationModel, serviceName); @@ -233,7 +233,7 @@ void testServiceSubscribeMetrics() { metricSamples = collector.collect(); // App(7) + rt(5) + service(total/success/failed) = 15 - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + 3, metricSamples.size()); + Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 5 + RegistryMetricsConstants.SERVICE_LEVEL_KEYS.size(), metricSamples.size()); // calc rt for (MetricSample sample : metricSamples) { @@ -254,6 +254,7 @@ void testServiceSubscribeMetrics() { @Test public void testNotify() { + Map lastNumMap = new HashMap<>(); MetricsEventBus.post(RegistryEvent.toNotifyEvent(applicationModel), () -> { try { @@ -261,7 +262,6 @@ public void testNotify() { } catch (InterruptedException e) { e.printStackTrace(); } - Map lastNumMap = new HashMap<>(); // 1 different services lastNumMap.put("demo.service1", 3); lastNumMap.put("demo.service2", 4); @@ -271,7 +271,7 @@ public void testNotify() { ); List metricSamples = collector.collect(); // App(7) + num(service*3) + rt(5) = 9 - Assertions.assertEquals(RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 3 + 5, metricSamples.size()); + Assertions.assertEquals((RegistryMetricsConstants.APP_LEVEL_KEYS.size() + 3 + 5)*3, metricSamples.size()); } } From 16daf9527b1565a1a07f132067cf90e94e8d7add Mon Sep 17 00:00:00 2001 From: Albumen Kevin Date: Wed, 24 May 2023 15:09:39 +0800 Subject: [PATCH 15/18] Skip scan object class (#12388) * Skip scan object class * Skip scan object class --- .../common/utils/SerializeSecurityConfigurator.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/SerializeSecurityConfigurator.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/SerializeSecurityConfigurator.java index 629443142d1..ba7227a1ac9 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/SerializeSecurityConfigurator.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/SerializeSecurityConfigurator.java @@ -271,6 +271,15 @@ private void checkClass(Set markedClass, Class clazz) { addToAllow(clazz.getName()); + if (ClassUtils.isSimpleType(clazz) || clazz.isPrimitive() || clazz.isArray()) { + return; + } + String className = clazz.getName(); + if (className.startsWith("java.") || className.startsWith("javax.") || className.startsWith("com.sun.") || + className.startsWith("sun.") || className.startsWith("jdk.")) { + return; + } + Class[] interfaces = clazz.getInterfaces(); for (Class interfaceClass : interfaces) { checkClass(markedClass, interfaceClass); From da70e9097f8c4c796f4f3546f8f1e4699e9880b1 Mon Sep 17 00:00:00 2001 From: Albumen Kevin Date: Wed, 24 May 2023 20:19:25 +0800 Subject: [PATCH 16/18] Fix alibaba AbstractLoadBalance compact (#12394) --- .../loadbalance/AbstractLoadBalance.java | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java b/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java index 3ec1c785221..f4aa3394033 100644 --- a/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java +++ b/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java @@ -17,6 +17,45 @@ package com.alibaba.dubbo.rpc.cluster.loadbalance; +import com.alibaba.dubbo.common.Constants; +import com.alibaba.dubbo.common.URL; +import com.alibaba.dubbo.rpc.Invocation; +import com.alibaba.dubbo.rpc.Invoker; +import com.alibaba.dubbo.rpc.cluster.LoadBalance; + +import java.util.List; + @Deprecated -public abstract class AbstractLoadBalance extends org.apache.dubbo.rpc.cluster.loadbalance.AbstractLoadBalance { +public abstract class AbstractLoadBalance extends org.apache.dubbo.rpc.cluster.loadbalance.AbstractLoadBalance implements LoadBalance { + + @Override + public Invoker select(List> invokers, URL url, Invocation invocation) { + if (invokers == null || invokers.size() == 0) + return null; + if (invokers.size() == 1) + return invokers.get(0); + return doSelect(invokers, url, invocation); + } + + protected abstract Invoker doSelect(List> invokers, URL url, Invocation invocation); + + protected int getWeight(Invoker invoker, Invocation invocation) { + int weight = invoker.getUrl().getMethodParameter(invocation.getMethodName(), Constants.WEIGHT_KEY, Constants.DEFAULT_WEIGHT); + if (weight > 0) { + long timestamp = invoker.getUrl().getParameter(Constants.TIMESTAMP_KEY, 0L); + if (timestamp > 0L) { + int uptime = (int) (System.currentTimeMillis() - timestamp); + int warmup = invoker.getUrl().getParameter(Constants.WARMUP_KEY, Constants.DEFAULT_WARMUP); + if (uptime > 0 && uptime < warmup) { + weight = calculateWarmupWeight(uptime, warmup, weight); + } + } + } + return weight; + } + + static int calculateWarmupWeight(int uptime, int warmup, int weight) { + int ww = (int) ( (float) uptime / ( (float) warmup / (float) weight ) ); + return ww < 1 ? 1 : (ww > weight ? weight : ww); + } } From 2ffda4b23932b9dc4fe1ebdec2cc66d51a20a99c Mon Sep 17 00:00:00 2001 From: Albumen Kevin Date: Thu, 25 May 2023 09:24:19 +0800 Subject: [PATCH 17/18] Fix alibaba AbstractLoadBalance compact (#12396) --- .../dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java b/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java index f4aa3394033..a8a39062adc 100644 --- a/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java +++ b/dubbo-compatible/src/main/java/com/alibaba/dubbo/rpc/cluster/loadbalance/AbstractLoadBalance.java @@ -26,7 +26,7 @@ import java.util.List; @Deprecated -public abstract class AbstractLoadBalance extends org.apache.dubbo.rpc.cluster.loadbalance.AbstractLoadBalance implements LoadBalance { +public abstract class AbstractLoadBalance implements LoadBalance { @Override public Invoker select(List> invokers, URL url, Invocation invocation) { From c57118658c89da09412cbd695f60ea0c3bdc1ec1 Mon Sep 17 00:00:00 2001 From: jojocodeX <571943037@qq.com> Date: Thu, 25 May 2023 13:49:09 +0800 Subject: [PATCH 18/18] Metrics add sliding time window statistics (#12364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加Jackson序列化器 支持Jackson序列化器扩展配置 代码格式化调整 * Jackson module 注册错误忽略异常 * Jackson module 注册错误忽略异常 * add LdapJackson2Module Jackson module * add spi to dubbo-all pom config * remove unused code * undo remove code * Run CI * merge conflict * ignore codec error * Change log output * Add log error code * Prapare 3.2.2 release * Add sliding window stat * fix checkstyle * rename and version change * MetricsKey rename * MetricsKey rename * change ut --------- Co-authored-by: Albumen Kevin Co-authored-by: songxiaosheng --- .../aggregate/SampleAggregatedEntry.java | 68 ++++++++ .../aggregate/TimeWindowAggregator.java | 151 ++++++++++++++++++ .../dubbo/metrics/model/key/MetricsKey.java | 4 +- .../aggregate/TimeWindowAggregatorTest.java | 84 ++++++++++ .../collector/AggregateMetricsCollector.java | 17 ++ .../AggregateMetricsCollectorTest.java | 42 +++++ 6 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/SampleAggregatedEntry.java create mode 100644 dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregator.java create mode 100644 dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregatorTest.java diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/SampleAggregatedEntry.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/SampleAggregatedEntry.java new file mode 100644 index 00000000000..d4e222b9705 --- /dev/null +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/SampleAggregatedEntry.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.metrics.aggregate; + + +public class SampleAggregatedEntry { + + private Long count; + private double max; + private double min; + private double avg; + private double total; + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public double getMax() { + return max; + } + + public void setMax(double max) { + this.max = max; + } + + public double getMin() { + return min; + } + + public void setMin(double min) { + this.min = min; + } + + public double getAvg() { + return avg; + } + + public void setAvg(double avg) { + this.avg = avg; + } + + public double getTotal() { + return total; + } + + public void setTotal(double total) { + this.total = total; + } +} diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregator.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregator.java new file mode 100644 index 00000000000..082f563ebd6 --- /dev/null +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregator.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.metrics.aggregate; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.DoubleAccumulator; +import java.util.concurrent.atomic.LongAdder; + +public class TimeWindowAggregator { + + private final SnapshotSlidingWindow slidingWindow; + + public TimeWindowAggregator(int bucketNum, int timeWindowSeconds) { + this.slidingWindow = new SnapshotSlidingWindow(bucketNum, TimeUnit.SECONDS.toMillis(timeWindowSeconds)); + } + + public SnapshotSlidingWindow getSlidingWindow() { + return slidingWindow; + } + + public void add(double value) { + SnapshotObservation sample = this.slidingWindow.currentPane().getValue(); + sample.add(value); + } + + public SampleAggregatedEntry get() { + SampleAggregatedEntry aggregatedEntry = new SampleAggregatedEntry(); + + double total = 0L; + long count = 0; + double max = Double.MIN_VALUE; + double min = Double.MAX_VALUE; + + List windows = this.slidingWindow.values(); + + for (SnapshotObservation window : windows) { + total += window.getTotal(); + count += window.getCount(); + + max = Math.max(max, window.getMax()); + min = Math.min(min, window.getMin()); + } + + if (count > 0) { + double avg = total / count; + aggregatedEntry.setAvg(Math.round(avg * 100.0) / 100.0); + }else { + aggregatedEntry.setAvg(0); + } + + aggregatedEntry.setMax(max == Double.MIN_VALUE ? 0 : max); + aggregatedEntry.setMin(min == Double.MAX_VALUE ? 0 : min); + aggregatedEntry.setTotal(total); + aggregatedEntry.setCount(count); + + return aggregatedEntry; + } + + public static class SnapshotSlidingWindow extends SlidingWindow { + + public SnapshotSlidingWindow(int sampleCount, long intervalInMs) { + super(sampleCount, intervalInMs); + } + + @Override + public SnapshotObservation newEmptyValue(long timeMillis) { + return new SnapshotObservation(); + } + + @Override + protected Pane resetPaneTo(final Pane pane, long startTime) { + pane.setStartInMs(startTime); + pane.getValue().reset(); + return pane; + } + } + + public static class SnapshotObservation { + + private final AtomicReference min = new AtomicReference<>(Double.MAX_VALUE); + private final AtomicReference max = new AtomicReference<>(0d); + private final DoubleAccumulator total = new DoubleAccumulator((x, y) -> x + y, 0); + private final LongAdder count = new LongAdder(); + + public void add(double sample) { + total.accumulate(sample); + count.increment(); + updateMin(sample); + updateMax(sample); + } + + private void updateMin(double sample) { + Double curMin; + do { + curMin = min.get(); + } while (sample < curMin && !min.compareAndSet(curMin, sample)); + } + + private void updateMax(double sample) { + Double curMax; + do { + curMax = max.get(); + if (sample <= curMax) { + return; + } + + } while (!max.compareAndSet(curMax, sample)); + } + + public void reset() { + min.set(Double.MAX_VALUE); + max.set(0d); + count.reset(); + total.reset(); + } + + public double getMin() { + return min.get(); + } + + public double getMax() { + return max.get(); + } + + public Double getTotal() { + return total.get(); + } + + public long getCount() { + return count.sum(); + } + } + +} diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java index 09cb9881187..b4d7997a190 100644 --- a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java +++ b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/key/MetricsKey.java @@ -57,7 +57,9 @@ public enum MetricsKey { METRIC_RT_P95("dubbo.%s.rt.milliseconds.p95", "Response Time P95"), METRIC_RT_P90("dubbo.%s.rt.milliseconds.p90", "Response Time P90"), METRIC_RT_P50("dubbo.%s.rt.milliseconds.p50", "Response Time P50"), - + METRIC_RT_MIN_AGG("dubbo.%s.rt.min.milliseconds.aggregate", "Aggregated Min Response"), + METRIC_RT_MAX_AGG("dubbo.%s.rt.max.milliseconds.aggregate", "Aggregated Max Response"), + METRIC_RT_AVG_AGG("dubbo.%s.rt.avg.milliseconds.aggregate", "Aggregated Avg Response"), // register metrics key REGISTER_METRIC_REQUESTS("dubbo.registry.register.requests.total", "Total Register Requests"), diff --git a/dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregatorTest.java b/dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregatorTest.java new file mode 100644 index 00000000000..fc2d42c614e --- /dev/null +++ b/dubbo-metrics/dubbo-metrics-api/src/test/java/org/apache/dubbo/metrics/aggregate/TimeWindowAggregatorTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.metrics.aggregate; + +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.util.concurrent.TimeUnit; + +public class TimeWindowAggregatorTest { + @Test + public void testTimeWindowAggregator() { + TimeWindowAggregator aggregator = new TimeWindowAggregator(5, 5); + + // 第一个时间窗口，时间范围：0秒 - 5秒 + aggregator.add(10); + aggregator.add(20); + aggregator.add(30); + + SampleAggregatedEntry entry1 = aggregator.get(); + Assertions.assertEquals(20, entry1.getAvg()); + Assertions.assertEquals(60, entry1.getTotal()); + Assertions.assertEquals(3, entry1.getCount()); + Assertions.assertEquals(30, entry1.getMax()); + Assertions.assertEquals(10, entry1.getMin()); + + // 第二个时间窗口，时间范围：5秒 - 10秒 + try { + TimeUnit.SECONDS.sleep(5); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + aggregator.add(15); + aggregator.add(25); + aggregator.add(35); + + SampleAggregatedEntry entry2 = aggregator.get(); + Assertions.assertEquals(25, entry2.getAvg()); + Assertions.assertEquals(75, entry2.getTotal()); + Assertions.assertEquals(3, entry2.getCount()); + Assertions.assertEquals(35, entry2.getMax()); + Assertions.assertEquals(15, entry2.getMin()); + + // 第三个时间窗口，时间范围：10秒 - 15秒 + try { + TimeUnit.SECONDS.sleep(5); + } catch (InterruptedException e) { + e.printStackTrace(); + } + aggregator.add(12); + aggregator.add(22); + aggregator.add(32); + + SampleAggregatedEntry entry3 = aggregator.get(); + Assertions.assertEquals(22, entry3.getAvg()); + Assertions.assertEquals(66, entry3.getTotal()); + Assertions.assertEquals(3, entry3.getCount()); + Assertions.assertEquals(32, entry3.getMax()); + Assertions.assertEquals(12, entry3.getMin()); + } +} + + + + + + + diff --git a/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollector.java b/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollector.java index 765f33e71d3..171395c1577 100644 --- a/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollector.java +++ b/dubbo-metrics/dubbo-metrics-default/src/main/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollector.java @@ -22,6 +22,7 @@ import org.apache.dubbo.config.context.ConfigManager; import org.apache.dubbo.config.nested.AggregationConfig; import org.apache.dubbo.metrics.MetricsConstants; +import org.apache.dubbo.metrics.aggregate.TimeWindowAggregator; import org.apache.dubbo.metrics.aggregate.TimeWindowCounter; import org.apache.dubbo.metrics.aggregate.TimeWindowQuantile; import org.apache.dubbo.metrics.event.MetricsEvent; @@ -65,6 +66,8 @@ public class AggregateMetricsCollector implements MetricsCollector private static final Integer DEFAULT_BUCKET_NUM = 10; private static final Integer DEFAULT_TIME_WINDOW_SECONDS = 120; private Boolean collectEnabled = null; + private final ConcurrentMap rtAgr = new ConcurrentHashMap<>(); + public AggregateMetricsCollector(ApplicationModel applicationModel) { this.applicationModel = applicationModel; @@ -136,6 +139,9 @@ private void onRTEvent(RequestEvent event) { long responseTime = event.getTimePair().calc(); TimeWindowQuantile quantile = ConcurrentHashMapUtils.computeIfAbsent(rt, metric, k -> new TimeWindowQuantile(DEFAULT_COMPRESSION, bucketNum, timeWindowSeconds)); quantile.add(responseTime); + + TimeWindowAggregator timeWindowAggregator = ConcurrentHashMapUtils.computeIfAbsent(rtAgr, metric, methodMetric -> new TimeWindowAggregator(bucketNum, timeWindowSeconds)); + timeWindowAggregator.add(responseTime); } @@ -204,6 +210,17 @@ private void collectRT(List list) { list.add(new GaugeMetricSample<>(MetricsKey.METRIC_RT_P50.getNameByType(k.getSide()), MetricsKey.METRIC_RT_P50.getDescription(), k.getTags(), RT, v, value -> value.quantile(0.50))); }); + + rtAgr.forEach((k,v)->{ + list.add(new GaugeMetricSample<>(MetricsKey.METRIC_RT_MIN_AGG.getNameByType(k.getSide()), + MetricsKey.METRIC_RT_MIN_AGG.getDescription(), k.getTags(), RT, v, value -> v.get().getMin())); + + list.add(new GaugeMetricSample<>(MetricsKey.METRIC_RT_MAX_AGG.getNameByType(k.getSide()), + MetricsKey.METRIC_RT_MAX_AGG.getDescription(), k.getTags(), RT, v, value -> v.get().getMax())); + + list.add(new GaugeMetricSample<>(MetricsKey.METRIC_RT_AVG_AGG.getNameByType(k.getSide()), + MetricsKey.METRIC_RT_AVG_AGG.getDescription(), k.getTags(), RT, v, value -> v.get().getAvg())); + }); } private void registerListener() { diff --git a/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollectorTest.java b/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollectorTest.java index 08362c1bf67..383dac76f94 100644 --- a/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollectorTest.java +++ b/dubbo-metrics/dubbo-metrics-default/src/test/java/org/apache/dubbo/metrics/collector/AggregateMetricsCollectorTest.java @@ -232,6 +232,48 @@ public void testQPS() { Assertions.assertEquals(10000, ((TimeWindowCounter) ((GaugeMetricSample) sample).getValue()).get()); } + + @Test + public void testRtAggregation() { + metricsDispatcher.addListener(collector); + ConfigManager configManager = applicationModel.getApplicationConfigManager(); + MetricsConfig config = configManager.getMetrics().orElse(null); + AggregationConfig aggregationConfig = new AggregationConfig(); + aggregationConfig.setEnabled(true); + config.setAggregation(aggregationConfig); + + List rtList = new ArrayList<>(); + rtList.add(10L); + rtList.add(20L); + rtList.add(30L); + + for (Long requestTime: rtList) { + RequestEvent requestEvent = RequestEvent.toRequestEvent(applicationModel, invocation); + TestRequestEvent testRequestEvent = new TestRequestEvent(requestEvent.getSource(), requestEvent.getTypeWrapper()); + testRequestEvent.putAttachment(MetricsConstants.INVOCATION, invocation); + testRequestEvent.putAttachment(ATTACHMENT_KEY_SERVICE, MetricsSupport.getInterfaceName(invocation)); + testRequestEvent.putAttachment(MetricsConstants.INVOCATION_SIDE, MetricsSupport.getSide(invocation)); + testRequestEvent.setRt(requestTime); + MetricsEventBus.post(testRequestEvent, () -> null); + } + + List samples = collector.collect(); + for (MetricSample sample : samples) { + GaugeMetricSample gaugeMetricSample = (GaugeMetricSample) sample; + + if(gaugeMetricSample.getName().endsWith("max.milliseconds.aggregate")) { + Assertions.assertEquals(30, gaugeMetricSample.applyAsDouble()); + } + if (gaugeMetricSample.getName().endsWith("min.milliseconds.aggregate")) { + Assertions.assertEquals(10L, gaugeMetricSample.applyAsDouble()); + } + + if (gaugeMetricSample.getName().endsWith("avg.milliseconds.aggregate")) { + Assertions.assertEquals(20L, gaugeMetricSample.applyAsDouble()); + } + } + } + @Test void testP95AndP99() throws InterruptedException {