Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[manager]feature: support configuring the maximum retention count of history records for JPA #1005

Merged
merged 1 commit into from
Jun 1, 2023
Merged

[manager]feature: support configuring the maximum retention count of history records for JPA #1005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions manager/src/main/resources/application.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,12 @@ warehouse:
# 存储历史数据方式, 下方只能enabled启用一种方式
jpa:
enabled: true
# The maximum retention time for history records, after which records will be deleted
expire-time: 1h
# The maximum number of history records retained, if this number is exceeded, half of the data in this configuration item will be deleted
# (please set this configuration reasonably as history records can affect performance when it is large)
# 历史数据的最大保留条数，超过此数量时，将会删除一半于此配量的数据（由于历史数据较大时会影响性能，请合理设置此配置）
max-history-record-num: 6000
td-engine:
enabled: false
driver-class-name: com.taosdata.jdbc.rs.RestfulDriver
Expand Down
13 changes: 13 additions & 0 deletions warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,11 @@ public static class JpaProperties {
*/
private String expireTime = "1h";

/**
* The maximum number of history records retained
*/
private Integer maxHistoryRecordNum = 20_000;

public boolean isEnabled() {
return enabled;
}
Expand All @@ -274,6 +279,14 @@ public String getExpireTime() {
public void setExpireTime(String expireTime) {
this.expireTime = expireTime;
}

public Integer getMaxHistoryRecordNum() {
return maxHistoryRecordNum;
}

public void setMaxHistoryRecordNum(Integer maxHistoryRecordNum) {
this.maxHistoryRecordNum = maxHistoryRecordNum;
}
}

public static class InfluxdbProperties {
Expand Down
6 changes: 4 additions & 2 deletions warehouse/src/main/java/org/dromara/hertzbeat/warehouse/dao/HistoryDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;

/**
Expand All @@ -43,12 +44,13 @@ public interface HistoryDao extends JpaRepository<History, Long>, JpaSpecificati

/**
* delete older history record
* @param delNum number to be deleted
* @return rows deleted
*/
@Modifying
@Transactional(rollbackFor = Exception.class)
@Query(value = "delete from hzb_history limit 10000", nativeQuery = true)
int deleteOlderHistoriesRecord();
@Query(value = "delete from hzb_history limit :delNum", nativeQuery = true)
int deleteOlderHistoriesRecord(@Param(value = "delNum") int delNum);

/**
* truncateTable
Expand Down
5 changes: 2 additions & 3 deletions ...se/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryJpaDatabaseDataStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage {
private final WarehouseProperties.StoreProperties.JpaProperties jpaProperties;

private static final int STRING_MAX_LENGTH = 1024;
private static final int MAX_HISTORY_TABLE_RECORD = 20_000;

public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
HistoryDao historyDao) {
Expand Down Expand Up @@ -89,8 +88,8 @@ public void expiredDataCleaner() {
int rows = historyDao.deleteHistoriesByTimeBefore(expireTime);
log.info("[jpa-metrics-store]-delete {} rows.", rows);
long total = historyDao.count();
if (total > MAX_HISTORY_TABLE_RECORD) {
rows = historyDao.deleteOlderHistoriesRecord();
if (total > jpaProperties.getMaxHistoryRecordNum()) {
rows = historyDao.deleteOlderHistoriesRecord(jpaProperties.getMaxHistoryRecordNum() / 2);
log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows);
}
} catch (Exception e) {
Expand Down