Skip to content

Commit

Permalink
[warehouse] bugfix too many data in h2 db cause oom (#754)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomsun28
tomsun28 committed Jan 16, 2024
1 parent 3093e9c commit cfeb76d
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 10 deletions.
2 changes: 1 addition & 1 deletion manager/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<caffeine.version>2.9.3</caffeine.version>
<collector.version>1.0</collector.version>
<common.version>1.0</common.version>
<h2.version>2.1.212</h2.version>
<h2.version>2.1.214</h2.version>
<maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
<mysql.version>8.0.28</mysql.version>
Expand Down
25 changes: 24 additions & 1 deletion warehouse/src/main/java/com/usthe/warehouse/dao/HistoryDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import com.usthe.common.entity.warehouse.History;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.transaction.annotation.Transactional;

/**
* history entity dao
Expand All @@ -32,6 +35,26 @@ public interface HistoryDao extends JpaRepository<History, Long>, JpaSpecificati
/**
* delete history before expireTime
* @param expireTime expireTime
* @return rows deleted
*/
void deleteHistoriesByTimeBefore(Long expireTime);
@Modifying
@Transactional(rollbackFor = Exception.class)
int deleteHistoriesByTimeBefore(Long expireTime);

/**
* delete older history record
* @return rows deleted
*/
@Modifying
@Transactional(rollbackFor = Exception.class)
@Query(value = "delete from hzb_history limit 5000", nativeQuery = true)
int deleteOlderHistoriesRecord();

/**
* truncateTable
*/
@Modifying
@Transactional(rollbackFor = Exception.class)
@Query(value = "TRUNCATE TABLE hzb_history", nativeQuery = true)
void truncateTable();
}
29 changes: 21 additions & 8 deletions warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage {
private WarehouseProperties.StoreProperties.JpaProperties jpaProperties;

private static final int STRING_MAX_LENGTH = 1024;
private static final int MAX_HISTORY_TABLE_RECORD = 200_000;

public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
HistoryDao historyDao) {
Expand All @@ -64,7 +65,7 @@ public HistoryJpaDatabaseDataStorage(WarehouseProperties properties,
this.historyDao = historyDao;
}

@Scheduled( fixedDelay = 10, timeUnit = TimeUnit.MINUTES)
@Scheduled( fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
public void expiredDataCleaner() {
log.warn("[jpa-metrics-store]-start running expired data cleaner." +
"Please use time series db instead of jpa for better performance");
Expand All @@ -80,14 +81,22 @@ public void expiredDataCleaner() {
expireTime = dateTime.toEpochSecond() * 1000;
}
} catch (Exception e) {
log.error("expiredDataCleaner time error: {}. use default expire time to clean: 7d", e.getMessage());
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(7));
log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage());
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1));
expireTime = dateTime.toEpochSecond() * 1000;
}
try {
historyDao.deleteHistoriesByTimeBefore(expireTime);
int rows = historyDao.deleteHistoriesByTimeBefore(expireTime);
log.info("[jpa-metrics-store]-delete {} rows.", rows);
long total = historyDao.count();
if (total > MAX_HISTORY_TABLE_RECORD) {
rows = historyDao.deleteOlderHistoriesRecord();
log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows);
}
} catch (Exception e) {
log.error("expiredDataCleaner database error: {}.", e.getMessage());
log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance.");
historyDao.truncateTable();
}
}

Expand Down Expand Up @@ -120,19 +129,23 @@ void saveData(CollectRep.MetricsData metricsData) {
}
for (int i = 0; i < fieldsList.size(); i++) {
CollectRep.Field field = fieldsList.get(i);
// ignore string value store in db
if (field.getType() == CommonConstants.TYPE_STRING) {
continue;
}
historyBuilder.metric(field.getName());
if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) {
if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) {
if (field.getType() == CommonConstants.TYPE_NUMBER) {
historyBuilder.metricType(CommonConstants.TYPE_NUMBER)
.dou(Double.parseDouble(valueRow.getColumns(i)));
} else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) {
} else if (field.getType() == CommonConstants.TYPE_STRING) {
historyBuilder.metricType(CommonConstants.TYPE_STRING)
.str(formatStrValue(valueRow.getColumns(i)));
}
} else {
if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) {
if (field.getType() == CommonConstants.TYPE_NUMBER) {
historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null);
} else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) {
} else if (field.getType() == CommonConstants.TYPE_STRING) {
historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null);
}
}
Expand Down

0 comments on commit cfeb76d

Please sign in to comment.