diff --git a/manager/pom.xml b/manager/pom.xml
index 5a50eb0e509..9cacab9827e 100644
--- a/manager/pom.xml
+++ b/manager/pom.xml
@@ -31,7 +31,7 @@
2.9.3
1.0
1.0
-
2.1.212
+ 2.1.214
3.2.0
3.3.0
8.0.28
diff --git a/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java b/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java
index 294dbb02ee0..53ea09004f3 100644
--- a/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java
+++ b/warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java
@@ -21,6 +21,9 @@
import com.usthe.common.entity.warehouse.History;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.transaction.annotation.Transactional;
/**
* history entity dao
@@ -32,6 +35,26 @@ public interface HistoryDao extends JpaRepository, JpaSpecificati
/**
* delete history before expireTime
* @param expireTime expireTime
+ * @return rows deleted
*/
- void deleteHistoriesByTimeBefore(Long expireTime);
+ @Modifying
+ @Transactional(rollbackFor = Exception.class)
+ int deleteHistoriesByTimeBefore(Long expireTime);
+
+ /**
+ * delete older history record
+ * @return rows deleted
+ */
+ @Modifying
+ @Transactional(rollbackFor = Exception.class)
+ @Query(value = "delete from hzb_history limit 5000", nativeQuery = true)
+ int deleteOlderHistoriesRecord();
+
+ /**
+ * truncateTable
+ */
+ @Modifying
+ @Transactional(rollbackFor = Exception.class)
+ @Query(value = "TRUNCATE TABLE hzb_history", nativeQuery = true)
+ void truncateTable();
}
diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java
index ae3c192e5b6..0cbc7340e45 100644
--- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java
+++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java
@@ -56,6 +56,7 @@ public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage {
private WarehouseProperties.StoreProperties.JpaProperties jpaProperties;
private static final int STRING_MAX_LENGTH = 1024;
+ private static final int MAX_HISTORY_TABLE_RECORD = 200_000;
public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
HistoryDao historyDao) {
@@ -64,7 +65,7 @@ public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
this.historyDao = historyDao;
}
- @Scheduled( fixedDelay = 10, timeUnit = TimeUnit.MINUTES)
+ @Scheduled( fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
public void expiredDataCleaner() {
log.warn("[jpa-metrics-store]-start running expired data cleaner." +
"Please use time series db instead of jpa for better performance");
@@ -80,14 +81,22 @@ public void expiredDataCleaner() {
expireTime = dateTime.toEpochSecond() * 1000;
}
} catch (Exception e) {
- log.error("expiredDataCleaner time error: {}. use default expire time to clean: 7d", e.getMessage());
- ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(7));
+ log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage());
+ ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1));
expireTime = dateTime.toEpochSecond() * 1000;
}
try {
- historyDao.deleteHistoriesByTimeBefore(expireTime);
+ int rows = historyDao.deleteHistoriesByTimeBefore(expireTime);
+ log.info("[jpa-metrics-store]-delete {} rows.", rows);
+ long total = historyDao.count();
+ if (total > MAX_HISTORY_TABLE_RECORD) {
+ rows = historyDao.deleteOlderHistoriesRecord();
+ log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows);
+ }
} catch (Exception e) {
log.error("expiredDataCleaner database error: {}.", e.getMessage());
+ log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance.");
+ historyDao.truncateTable();
}
}
@@ -120,19 +129,23 @@ void saveData(CollectRep.MetricsData metricsData) {
}
for (int i = 0; i < fieldsList.size(); i++) {
CollectRep.Field field = fieldsList.get(i);
+ // ignore string value store in db
+ if (field.getType() == CommonConstants.TYPE_STRING) {
+ continue;
+ }
historyBuilder.metric(field.getName());
if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) {
- if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) {
+ if (field.getType() == CommonConstants.TYPE_NUMBER) {
historyBuilder.metricType(CommonConstants.TYPE_NUMBER)
.dou(Double.parseDouble(valueRow.getColumns(i)));
- } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) {
+ } else if (field.getType() == CommonConstants.TYPE_STRING) {
historyBuilder.metricType(CommonConstants.TYPE_STRING)
.str(formatStrValue(valueRow.getColumns(i)));
}
} else {
- if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) {
+ if (field.getType() == CommonConstants.TYPE_NUMBER) {
historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null);
- } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) {
+ } else if (field.getType() == CommonConstants.TYPE_STRING) {
historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null);
}
}