perf($Druid): set connection pool size equal to 2*processors

BREAKING CHANGE: set connection pool size equal to 2*processors
johnnymillergh · Aug 26, 2021 · a654323 · a654323
1 parent ea37259
commit a654323
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 10 deletions.
diff --git a/auth-center/src/main/resources/application.yml b/auth-center/src/main/resources/application.yml
@@ -44,9 +44,7 @@ spring:
     dynamic:
       primary: master_1
       druid:
-        initial-size: 5
-        max-active: 20
-        min-idle: 2
+        # connection pool size will be determined by DruidDataSourceCreatorPostProcessor.java
         max-wait: 60000
         min-evictable-idle-time-millis: 600000
         max-evictable-idle-time-millis: 900000

diff --git a/maf-mis/src/main/resources/application.yml b/maf-mis/src/main/resources/application.yml
@@ -46,9 +46,7 @@ spring:
     dynamic:
       primary: master_1
       druid:
-        initial-size: 5
-        max-active: 20
-        min-idle: 2
+        # connection pool size will be determined by DruidDataSourceCreatorPostProcessor.java
         max-wait: 60000
         min-evictable-idle-time-millis: 600000
         max-evictable-idle-time-millis: 900000

diff --git a/...a/com/jmsoftware/maf/springcloudstarter/database/DruidDataSourceCreatorPostProcessor.java b/...a/com/jmsoftware/maf/springcloudstarter/database/DruidDataSourceCreatorPostProcessor.java
@@ -24,11 +24,42 @@ public Object postProcessAfterInitialization(@NonNull Object bean, @NonNull Stri
         return bean;
     }
 
+    /**
+     * Post process dynamic data source properties. Enhance connection pool size by available processor count
+     * (logical processor count)
+     *
+     * <p>A formula which has held up pretty well across a lot of benchmarks for years is that for optimal throughput
+     * the number of active connections should be somewhere near ((<em>core_count</em> * 2) +
+     * <em>effective_spindle_count</em>). Core count should not include HT threads, even if hyperthreading is enabled
+     * . Effective spindle count is zero if the active data set is fully cached, and approaches the actual number of
+     * spindles as the cache hit rate falls. Benchmarks of WIP for version 9.2 suggest that this formula will need
+     * adjustment on that release. There hasn&#39;t been any analysis so far regarding how well the formula works
+     * with SSDs.</p>
+     * <p>However you choose a starting point for a connection pool size, you should probably try incremental
+     * adjustments with your production system to find the actual &quot;sweet spot&quot; for your hardware and
+     * workload.</p>
+     * <p>Remember that this &quot;sweet spot&quot; is for the number of connections that are <em>actively doing
+     * work</em>. Ignore mostly-idle connections used for system monitoring and control when working out an
+     * appropriate pool size. You should always make <strong>max_connections</strong> a bit bigger than the number of
+     * connections you enable in your connection pool. That way there are always a few slots available for direct
+     * connections for system maintenance and monitoring.</p>
+     *
+     * @param bean the bean
+     * @see
+     * <a href='https://wiki.postgresql.org/wiki/Number_Of_Database_Connections#How_to_Find_the_Optimal_Database_Connection_Pool_Size'>How to Find the Optimal Database Connection Pool Size</a>
+     * @see
+     * <a href='https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-usagenotes-j2ee-concepts-connection-pooling.html#idm46216069663472'>Sizing the Connection Pool</a>
+     */
     private void postProcessDynamicDataSourceProperties(DruidDataSourceCreator bean) {
         val cpuCoreCount = Runtime.getRuntime().availableProcessors();
-        val druidConfig = bean.getGConfig();
-        log.warn("Setting Druid connection pool by current cpuCoreCount: {}", cpuCoreCount);
-        druidConfig.setInitialSize(cpuCoreCount);
-        druidConfig.setMaxActive(cpuCoreCount * 2 + 1);
+        val minConnectionPoolSize = cpuCoreCount * 2 + 1;
+        val maxConnectionPoolSize = cpuCoreCount * 3;
+        bean.getGConfig()
+                .setInitialSize(minConnectionPoolSize)
+                .setMinIdle(minConnectionPoolSize)
+                .setMaxActive(maxConnectionPoolSize);
+        log.warn("Druid connection pool enhanced by current cpuCoreCount: {}, initial size: {}, min idle: {}" +
+                         ", max active: {}",
+                 cpuCoreCount, minConnectionPoolSize, minConnectionPoolSize, maxConnectionPoolSize);
     }
 }