From 4c19f3411066664e6f15d65190f611d8beaaa74c Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Tue, 21 Mar 2023 17:17:00 +0800 Subject: [PATCH] [warehouse] bugfix too many data in h2 db cause oom --- manager/pom.xml | 2 +- .../com/usthe/warehouse/dao/HistoryDao.java | 25 +++++++++++++++- .../store/HistoryJpaDatabaseDataStorage.java | 29 ++++++++++++++----- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/manager/pom.xml b/manager/pom.xml index 5a50eb0e509..9cacab9827e 100644 --- a/manager/pom.xml +++ b/manager/pom.xml @@ -31,7 +31,7 @@ 2.9.3 1.0 1.0 - 2.1.212 + 2.1.214 3.2.0 3.3.0 8.0.28 diff --git a/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java b/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java index 294dbb02ee0..53ea09004f3 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java +++ b/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java @@ -21,6 +21,9 @@ import com.usthe.common.entity.warehouse.History; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.transaction.annotation.Transactional; /** * history entity dao @@ -32,6 +35,26 @@ public interface HistoryDao extends JpaRepository, JpaSpecificati /** * delete history before expireTime * @param expireTime expireTime + * @return rows deleted */ - void deleteHistoriesByTimeBefore(Long expireTime); + @Modifying + @Transactional(rollbackFor = Exception.class) + int deleteHistoriesByTimeBefore(Long expireTime); + + /** + * delete older history record + * @return rows deleted + */ + @Modifying + @Transactional(rollbackFor = Exception.class) + @Query(value = "delete from hzb_history limit 5000", nativeQuery = true) + int deleteOlderHistoriesRecord(); + + /** + * truncateTable + */ + @Modifying + @Transactional(rollbackFor = Exception.class) + @Query(value = "TRUNCATE TABLE hzb_history", nativeQuery = true) + void truncateTable(); } diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java index ae3c192e5b6..0cbc7340e45 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java @@ -56,6 +56,7 @@ public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage { private WarehouseProperties.StoreProperties.JpaProperties jpaProperties; private static final int STRING_MAX_LENGTH = 1024; + private static final int MAX_HISTORY_TABLE_RECORD = 200_000; public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, HistoryDao historyDao) { @@ -64,7 +65,7 @@ public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, this.historyDao = historyDao; } - @Scheduled( fixedDelay = 10, timeUnit = TimeUnit.MINUTES) + @Scheduled( fixedDelay = 1, timeUnit = TimeUnit.MINUTES) public void expiredDataCleaner() { log.warn("[jpa-metrics-store]-start running expired data cleaner." + "Please use time series db instead of jpa for better performance"); @@ -80,14 +81,22 @@ public void expiredDataCleaner() { expireTime = dateTime.toEpochSecond() * 1000; } } catch (Exception e) { - log.error("expiredDataCleaner time error: {}. use default expire time to clean: 7d", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(7)); + log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1)); expireTime = dateTime.toEpochSecond() * 1000; } try { - historyDao.deleteHistoriesByTimeBefore(expireTime); + int rows = historyDao.deleteHistoriesByTimeBefore(expireTime); + log.info("[jpa-metrics-store]-delete {} rows.", rows); + long total = historyDao.count(); + if (total > MAX_HISTORY_TABLE_RECORD) { + rows = historyDao.deleteOlderHistoriesRecord(); + log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows); + } } catch (Exception e) { log.error("expiredDataCleaner database error: {}.", e.getMessage()); + log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance."); + historyDao.truncateTable(); } } @@ -120,19 +129,23 @@ void saveData(CollectRep.MetricsData metricsData) { } for (int i = 0; i < fieldsList.size(); i++) { CollectRep.Field field = fieldsList.get(i); + // ignore string value store in db + if (field.getType() == CommonConstants.TYPE_STRING) { + continue; + } historyBuilder.metric(field.getName()); if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { - if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { + if (field.getType() == CommonConstants.TYPE_NUMBER) { historyBuilder.metricType(CommonConstants.TYPE_NUMBER) .dou(Double.parseDouble(valueRow.getColumns(i))); - } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { + } else if (field.getType() == CommonConstants.TYPE_STRING) { historyBuilder.metricType(CommonConstants.TYPE_STRING) .str(formatStrValue(valueRow.getColumns(i))); } } else { - if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { + if (field.getType() == CommonConstants.TYPE_NUMBER) { historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null); - } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { + } else if (field.getType() == CommonConstants.TYPE_STRING) { historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null); } }