[manager]feature: support configuring the maximum retention count of …

…history records for JPA (#1005)
apache · Jun 1, 2023 · d3d14be · d3d14be
1 parent dac576a
commit d3d14be
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 5 deletions.
diff --git a/manager/src/main/resources/application.yml b/manager/src/main/resources/application.yml
@@ -115,7 +115,12 @@ warehouse:
     # 存储历史数据方式, 下方只能enabled启用一种方式
     jpa:
       enabled: true
+      # The maximum retention time for history records, after which records will be deleted
       expire-time: 1h
+      # The maximum number of history records retained, if this number is exceeded, half of the data in this configuration item will be deleted
+      # (please set this configuration reasonably as history records can affect performance when it is large)
+      # 历史数据的最大保留条数，超过此数量时，将会删除一半于此配量的数据（由于历史数据较大时会影响性能，请合理设置此配置）
+      max-history-record-num: 6000
     td-engine:
       enabled: false
       driver-class-name: com.taosdata.jdbc.rs.RestfulDriver

diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java
@@ -259,6 +259,11 @@ public static class JpaProperties {
              */
             private String expireTime = "1h";
 
+            /**
+             * The maximum number of history records retained
+             */
+            private Integer maxHistoryRecordNum = 20_000;
+
             public boolean isEnabled() {
                 return enabled;
             }
@@ -274,6 +279,14 @@ public String getExpireTime() {
             public void setExpireTime(String expireTime) {
                 this.expireTime = expireTime;
             }
+
+            public Integer getMaxHistoryRecordNum() {
+                return maxHistoryRecordNum;
+            }
+
+            public void setMaxHistoryRecordNum(Integer maxHistoryRecordNum) {
+                this.maxHistoryRecordNum = maxHistoryRecordNum;
+            }
         }
 
         public static class InfluxdbProperties {

diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/dao/HistoryDao.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/dao/HistoryDao.java
@@ -23,6 +23,7 @@
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 import org.springframework.transaction.annotation.Transactional;
 
 /**
@@ -43,12 +44,13 @@ public interface HistoryDao extends JpaRepository<History, Long>, JpaSpecificati
 
     /**
      * delete older history record
+     * @param delNum number to be deleted
      * @return rows deleted
      */
     @Modifying
     @Transactional(rollbackFor = Exception.class)
-    @Query(value = "delete from hzb_history limit 10000", nativeQuery = true)
-    int deleteOlderHistoriesRecord();
+    @Query(value = "delete from hzb_history limit :delNum", nativeQuery = true)
+    int deleteOlderHistoriesRecord(@Param(value = "delNum") int delNum);
 
     /**
      * truncateTable

diff --git a/...se/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryJpaDatabaseDataStorage.java b/...se/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryJpaDatabaseDataStorage.java
@@ -56,7 +56,6 @@ public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage {
 	private final WarehouseProperties.StoreProperties.JpaProperties jpaProperties;
 
 	private static final int STRING_MAX_LENGTH = 1024;
-	private static final int MAX_HISTORY_TABLE_RECORD = 20_000;
 
 	public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
 	                                     HistoryDao historyDao) {
@@ -89,8 +88,8 @@ public void expiredDataCleaner() {
 			int rows = historyDao.deleteHistoriesByTimeBefore(expireTime);
 			log.info("[jpa-metrics-store]-delete {} rows.", rows);
 			long total = historyDao.count();
-			if (total > MAX_HISTORY_TABLE_RECORD) {
-				rows = historyDao.deleteOlderHistoriesRecord();
+			if (total > jpaProperties.getMaxHistoryRecordNum()) {
+				rows = historyDao.deleteOlderHistoriesRecord(jpaProperties.getMaxHistoryRecordNum() / 2);
 				log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows);
 			}
 		} catch (Exception e) {