[script] update warehouse jpa store metrics expire-time to 1h (#750)

[script] update warehouse jpa store metrics expire-time to 1h [script] update warehouse jpa store metrics expire-time to 1h [script] update warehouse jpa store metrics expire-time to 1h [script] update warehouse jpa store metrics expire-time to 1h [warehouse] support greptime db store metrics data
apache · Mar 21, 2023 · 157aee1 · 157aee1
1 parent ffd4de3
commit 157aee1
Show file tree

Hide file tree

Showing 16 changed files with 83 additions and 18 deletions.
diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/iotdb-init.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/iotdb-init.md
@@ -8,7 +8,7 @@ HertzBeat的历史数据存储依赖时序数据库 IoTDB 或 TDengine，任选
 
 Apache IoTDB是一体化收集、存储、管理与分析物联网时序数据的软件系统，我们使用其存储分析采集到的监控指标历史数据。支持V0.12 - V0.13版本，推荐使用V0.13.*版本。
 
-**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能和高性能**
+**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能，高性能和稳定性**
 
 > 如果您已有IoTDB环境，可直接跳到YML配置那一步。
 

diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/tdengine-init.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/tdengine-init.md
@@ -8,7 +8,7 @@ HertzBeat的历史数据存储依赖时序数据库 TDengine 或 IoTDB，任选
 
 TDengine是一款开源物联网时序型数据库，我们用其存储采集到的监控指标历史数据。 注意支持⚠️ 2.4.x版本。   
 
-**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能和高性能**
+**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能，高性能和稳定性**
 
 > 如果您已有TDengine环境，可直接跳到创建数据库实例那一步。
 

diff --git a/home/src/constants.js b/home/src/constants.js
@@ -222,6 +222,11 @@ export const friendsLink = [
     alt: 'hippo4j',
     url: 'https://hippo4j.cn/',
   },
+  {
+    img: 'bugstack_logo.png',
+    alt: 'bugStack',
+    url: 'https://bugstack.cn/',
+  }
 ]
 
 export const usersLink = [

diff --git a/home/src/pages/index.js b/home/src/pages/index.js
@@ -252,7 +252,8 @@ function autoRedirect() {
   let lang = global.navigator?.language || navigator?.userLanguage
   console.log('Current lang is ' + lang)
   if (lang != null && (lang.toLowerCase() === 'zh-cn' || lang.toLowerCase().indexOf('zh') > 0)) {
-    if (sessionStorage.getItem('auto_detect_redirect') !== 'true') {
+    console.log(window.location.pathname);
+    if (sessionStorage.getItem('auto_detect_redirect') !== 'true' && !window.location.pathname.startsWith('/zh-cn', false)) {
       console.log('current lang is zh-cn, redirect to zh-cn')
       sessionStorage.setItem('auto_detect_redirect', 'true')
       window.location.href = '/zh-cn'

diff --git a/home/static/img/icons/bugstack_logo.png b/home/static/img/icons/bugstack_logo.png
diff --git a/manager/src/main/resources/application-test.yml b/manager/src/main/resources/application-test.yml
@@ -41,6 +41,9 @@ spring:
 
 warehouse:
   store:
+    jpa:
+      enabled: true
+      expire-time: 1h
     td-engine:
       enabled: false
       driver-class-name: com.taosdata.jdbc.rs.RestfulDriver

diff --git a/script/application.yml b/script/application.yml
@@ -98,7 +98,7 @@ warehouse:
 # 存储历史数据方式, 下方只能enabled启用一种方式
     jpa:
       enabled: true
-      expire-time: 7D
+      expire-time: 1h
     td-engine:
       enabled: false
       driver-class-name: com.taosdata.jdbc.rs.RestfulDriver

diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml
@@ -96,7 +96,7 @@ warehouse:
 # 存储历史数据方式, 下方只能enabled启用一种方式
     jpa:
       enabled: false
-      expire-time: 7D
+      expire-time: 1h
     iot-db:
       enabled: true
       host: iotdb

diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml
@@ -96,7 +96,7 @@ warehouse:
 # 存储历史数据方式, 下方只能enabled启用一种方式
     jpa:
       enabled: false
-      expire-time: 7D
+      expire-time: 1h
     td-engine:
       enabled: true
       driver-class-name: com.taosdata.jdbc.rs.RestfulDriver

diff --git a/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java b/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java
@@ -27,6 +27,8 @@
 import com.usthe.common.util.CommonConstants;
 import com.usthe.warehouse.store.AbstractHistoryDataStorage;
 import com.usthe.warehouse.store.AbstractRealTimeDataStorage;
+import com.usthe.warehouse.store.HistoryJpaDatabaseDataStorage;
+import com.usthe.warehouse.store.RealTimeMemoryDataStorage;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
@@ -88,7 +90,17 @@ public ResponseEntity<Message<MetricsData>> getMetricsData(
             @PathVariable Long monitorId,
             @Parameter(description = "Metrics Name", example = "cpu")
             @PathVariable String metrics) {
-        AbstractRealTimeDataStorage realTimeDataStorage = realTimeDataStorages.stream().findFirst().orElse(null);
+        AbstractRealTimeDataStorage realTimeDataStorage = realTimeDataStorages.stream()
+                .filter(AbstractRealTimeDataStorage::isServerAvailable)
+                .max((o1, o2) -> {
+                    if (o1 instanceof RealTimeMemoryDataStorage) {
+                        return -1;
+                    } else if (o2 instanceof RealTimeMemoryDataStorage) {
+                        return 1;
+                    } else {
+                        return 0;
+                    }
+                }).orElse(null);
         if (realTimeDataStorage == null) {
             return ResponseEntity.ok().body(new Message<>(FAIL_CODE, "real time store not available"));
         }
@@ -131,7 +143,15 @@ public ResponseEntity<Message<MetricsHistoryData>> getMetricHistoryData(
             @RequestParam(required = false) Boolean interval
     ) {
         AbstractHistoryDataStorage historyDataStorage = historyDataStorages.stream()
-                .filter(AbstractHistoryDataStorage::isServerAvailable).findFirst().orElse(null);
+                .filter(AbstractHistoryDataStorage::isServerAvailable).max((o1, o2) -> {
+                    if (o1 instanceof HistoryJpaDatabaseDataStorage) {
+                        return -1;
+                    } else if (o2 instanceof HistoryJpaDatabaseDataStorage) {
+                        return 1;
+                    } else {
+                        return 0;
+                    }
+                }).orElse(null);
         if (historyDataStorage == null) {
             return ResponseEntity.ok().body(new Message<>(FAIL_CODE, "time series database not available"));
         }

diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/AbstractRealTimeDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/AbstractRealTimeDataStorage.java
@@ -30,6 +30,15 @@
 @Slf4j
 public abstract class AbstractRealTimeDataStorage implements DisposableBean {
 
+    protected boolean serverAvailable;
+
+    /**
+     * @return data storage是否可用
+     */
+    public boolean isServerAvailable() {
+        return serverAvailable;
+    }
+
     /**
      * save collect metrics data
      * @param metricsData metrics data

diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/DataStorageDispatch.java b/warehouse/src/main/java/com/usthe/warehouse/store/DataStorageDispatch.java
@@ -24,6 +24,7 @@
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * dispatch storage metrics data
@@ -46,15 +47,20 @@ public DataStorageDispatch(CommonDataQueue commonDataQueue,
                                List<AbstractRealTimeDataStorage> realTimeDataStorages) {
         this.commonDataQueue = commonDataQueue;
         this.workerPool = workerPool;
-        this.historyDataStorages = historyDataStorages;
-        this.realTimeDataStorages = realTimeDataStorages;
+        this.historyDataStorages = historyDataStorages.stream()
+                .filter(AbstractHistoryDataStorage::isServerAvailable).collect(Collectors.toList());
+        this.realTimeDataStorages = realTimeDataStorages.stream()
+                .filter(AbstractRealTimeDataStorage::isServerAvailable).collect(Collectors.toList());
         startStoragePersistentData();
         startStorageRealTimeData();
     }
 
     private void startStorageRealTimeData() {
         Runnable runnable = () -> {
             Thread.currentThread().setName("warehouse-realtime-data-storage");
+            if (realTimeDataStorages != null && realTimeDataStorages.size() > 1) {
+                realTimeDataStorages.removeIf(item -> item instanceof RealTimeMemoryDataStorage);
+            }
             while (!Thread.currentThread().isInterrupted()) {
                 try {
                     CollectRep.MetricsData metricsData = commonDataQueue.pollRealTimeStorageMetricsData();
@@ -74,6 +80,9 @@ private void startStorageRealTimeData() {
     protected void startStoragePersistentData() {
         Runnable runnable = () -> {
             Thread.currentThread().setName("warehouse-persistent-data-storage");
+            if (historyDataStorages != null && historyDataStorages.size() > 1) {
+                historyDataStorages.removeIf(item -> item instanceof HistoryJpaDatabaseDataStorage);
+            }
             while (!Thread.currentThread().isInterrupted()) {
                 try {
                     CollectRep.MetricsData metricsData = commonDataQueue.pollPersistentStorageMetricsData();

diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java
@@ -64,8 +64,10 @@ public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
         this.historyDao = historyDao;
     }
 
-    @Scheduled( fixedDelay = 60, timeUnit = TimeUnit.MINUTES)
+    @Scheduled( fixedDelay = 10, timeUnit = TimeUnit.MINUTES)
     public void expiredDataCleaner() {
+        log.warn("[jpa-metrics-store]-start running expired data cleaner." +
+                "Please use time series db instead of jpa for better performance");
         String expireTimeStr = jpaProperties.getExpireTime();
         long expireTime = 0;
         try {

diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeMemoryDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeMemoryDataStorage.java
@@ -18,6 +18,7 @@
 package com.usthe.warehouse.store;
 
 import com.usthe.common.entity.message.CollectRep;
+import com.usthe.warehouse.config.WarehouseProperties;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.lang.NonNull;
@@ -38,9 +39,16 @@
 public class RealTimeMemoryDataStorage extends AbstractRealTimeDataStorage {
 
     private final Map<String, CollectRep.MetricsData> metricsDataMap;
+    private static final Integer DEFAULT_INIT_SIZE = 1024;
 
-    public RealTimeMemoryDataStorage() {
-        metricsDataMap = new ConcurrentHashMap<>(1024);
+    public RealTimeMemoryDataStorage(WarehouseProperties properties) {
+        int initSize = DEFAULT_INIT_SIZE;
+        if (properties != null && properties.getStore() != null && properties.getStore().getMemory() != null
+                && properties.getStore().getMemory().getInitSize() != null) {
+            initSize = properties.getStore().getMemory().getInitSize();
+        }
+        metricsDataMap = new ConcurrentHashMap<>(initSize);
+        this.serverAvailable = true;
     }
 
     @Override

diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeRedisDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeRedisDataStorage.java
@@ -49,7 +49,7 @@ public class RealTimeRedisDataStorage extends AbstractRealTimeDataStorage {
     private StatefulRedisConnection<String, CollectRep.MetricsData> connection;
 
     public RealTimeRedisDataStorage(WarehouseProperties properties) {
-        initRedisClient(properties);
+        this.serverAvailable = initRedisClient(properties);
     }
 
     @Override
@@ -79,10 +79,10 @@ public void saveData(CollectRep.MetricsData metricsData) {
         });
     }
 
-    private void initRedisClient(WarehouseProperties properties) {
+    private boolean initRedisClient(WarehouseProperties properties) {
         if (properties == null || properties.getStore() == null || properties.getStore().getRedis() == null) {
             log.error("init error, please config Warehouse redis props in application.yml");
-            throw new IllegalArgumentException("please config Warehouse redis props");
+            return false;
         }
         WarehouseProperties.StoreProperties.RedisProperties redisProp = properties.getStore().getRedis();
         RedisURI.Builder uriBuilder = RedisURI.builder()
@@ -92,8 +92,14 @@ private void initRedisClient(WarehouseProperties properties) {
         if (redisProp.getPassword() != null && !"".equals(redisProp.getPassword())) {
             uriBuilder.withPassword(redisProp.getPassword().toCharArray());
         }
-        redisClient = RedisClient.create(uriBuilder.build());
-        connection = redisClient.connect(new MetricsDataRedisCodec());
+        try {
+            redisClient = RedisClient.create(uriBuilder.build());
+            connection = redisClient.connect(new MetricsDataRedisCodec());
+            return true;
+        } catch (Exception e) {
+            log.error("init redis error {}", e.getMessage(), e);
+        }
+        return false;
     }
 
     @Override

diff --git a/warehouse/src/test/java/com/usthe/warehouse/controller/MetricsDataControllerTest.java b/warehouse/src/test/java/com/usthe/warehouse/controller/MetricsDataControllerTest.java
@@ -83,6 +83,7 @@ void getMetricsData() throws Exception {
         final String getUrl = "/api/monitor/" + monitorId + "/metrics/" + metric;
 
         when(realTimeDataStorage.getCurrentMetricsData(eq(monitorId), eq(metric))).thenReturn(null);
+        when(realTimeDataStorage.isServerAvailable()).thenReturn(true);
         this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl))
                 .andExpect(status().isOk())
                 .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
@@ -97,6 +98,7 @@ void getMetricsData() throws Exception {
                 .setTime(time)
                 .build();
         when(realTimeDataStorage.getCurrentMetricsData(eq(monitorId), eq(metric))).thenReturn(metricsData);
+        when(realTimeDataStorage.isServerAvailable()).thenReturn(true);
         this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl))
                 .andExpect(status().isOk())
                 .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))