Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fix](timezone) make TimeUtils formatter use correct time_zone #37465

Merged
merged 2 commits into from
Jul 11, 2024
Merged

[Fix](timezone) make TimeUtils formatter use correct time_zone #37465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
15a4fbe
1
zclllyybb Jul 8, 2024
48e3a3c
fix feut
zclllyybb Jul 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,7 @@ public static TQueryGlobals createQueryGlobals() {
Calendar currentDate = Calendar.getInstance();
LocalDateTime localDateTime = LocalDateTime.ofInstant(currentDate.toInstant(),
currentDate.getTimeZone().toZoneId());
String nowStr = localDateTime.format(TimeUtils.DATETIME_NS_FORMAT);
String nowStr = localDateTime.format(TimeUtils.getDatetimeNsFormatWithTimeZone());
queryGlobals.setNowString(nowStr);
queryGlobals.setNanoSeconds(LocalDateTime.now().getNano());
return queryGlobals;
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,8 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw
try {
BackupMeta backupMeta = BackupMeta.read(dataInputStream);
String backupTimestamp =
TimeUtils.longToTimeString(jobInfo.getBackupTime(), TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
TimeUtils.longToTimeString(jobInfo.getBackupTime(),
TimeUtils.getDatetimeFormatWithHyphenWithTimeZone());
restoreJob = new RestoreJob(stmt.getLabel(), backupTimestamp,
db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(),
stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(),
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,8 @@ private void waitingAllUploadingFinished() {
}

private void saveMetaInfo() {
String createTimeStr = TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
String createTimeStr = TimeUtils.longToTimeString(createTime,
TimeUtils.getDatetimeFormatWithHyphenWithTimeZone());
// local job dir: backup/repo__repo_id/label__createtime/
// Add repo_id to isolate jobs from different repos.
localJobDirPath = Paths.get(BackupHandler.BACKUP_ROOT_DIR.toString(),
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ private static String jobInfoFileNameWithTimestamp(long createTime) {
return PREFIX_JOB_INFO;
} else {
return PREFIX_JOB_INFO
+ TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
+ TimeUtils.longToTimeString(createTime, TimeUtils.getDatetimeFormatWithHyphenWithTimeZone());
}
}

Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,9 +401,9 @@ private static void checkStorageMedium(String storageMedium) throws DdlException

private static DateTimeFormatter getDateTimeFormatter(String timeUnit) {
if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return TimeUtils.DATETIME_FORMAT;
return TimeUtils.getDatetimeFormatWithTimeZone();
} else {
return TimeUtils.DATE_FORMAT;
return TimeUtils.getDateFormatWithTimeZone();
}
}

Expand Down Expand Up @@ -834,9 +834,9 @@ public static String getHistoryPartitionRangeString(DynamicPartitionProperty dyn

private static LocalDateTime getDateTimeByTimeUnit(String time, String timeUnit) {
if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return LocalDateTime.parse(time, TimeUtils.DATETIME_FORMAT);
return LocalDateTime.parse(time, TimeUtils.getDatetimeFormatWithTimeZone());
} else {
return LocalDate.from(TimeUtils.DATE_FORMAT.parse(time)).atStartOfDay();
return LocalDate.from(TimeUtils.getDateFormatWithTimeZone().parse(time)).atStartOfDay();
}
}

Expand Down
120 changes: 62 additions & 58 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand All @@ -47,6 +46,7 @@
import java.util.Date;
import java.util.Map;
import java.util.TimeZone;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand All @@ -55,13 +55,7 @@
public class TimeUtils {
public static final String UTC_TIME_ZONE = "UTC"; // This is just a Country to represent UTC offset +00:00
public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
public static final ZoneId TIME_ZONE;
public static final ImmutableMap<String, String> timeZoneAliasMap;
// NOTICE: Date formats are not synchronized.
// it must be used as synchronized externally.
public static final DateTimeFormatter DATE_FORMAT;
public static final DateTimeFormatter DATETIME_FORMAT;
public static final DateTimeFormatter TIME_FORMAT;
public static final Pattern DATETIME_FORMAT_REG =
Pattern.compile("^((\\d{2}(([02468][048])|([13579][26]))[\\-\\/\\s]?((((0?[13578])|(1[02]))[\\-\\/\\s]?"
+ "((0?[1-9])|([1-2][0-9])|(3[01])))|(((0?[469])|(11))[\\-\\/\\s]?"
Expand All @@ -70,23 +64,49 @@ public class TimeUtils {
+ "[\\-\\/\\s]?((0?[1-9])|([1-2][0-9])|(3[01])))|(((0?[469])|(11))[\\-\\/\\s]?"
+ "((0?[1-9])|([1-2][0-9])|(30)))|(0?2[\\-\\/\\s]?((0?[1-9])|(1[0-9])|(2[0-8]))))))"
+ "(\\s(((0?[0-9])|([1][0-9])|([2][0-3]))\\:([0-5]?[0-9])((\\s)|(\\:([0-5]?[0-9])))))?$");
public static final DateTimeFormatter DATETIME_MS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
.withZone(ZoneId.systemDefault());
public static final DateTimeFormatter DATETIME_NS_FORMAT = DateTimeFormatter.ofPattern(
"yyyy-MM-dd HH:mm:ss.SSSSSSSSS")
.withZone(ZoneId.systemDefault());
public static final DateTimeFormatter DATETIME_FORMAT_WITH_HYPHEN = DateTimeFormatter.ofPattern(
"yyyy-MM-dd-HH-mm-ss")
.withZone(ZoneId.systemDefault());

// these formatters must be visited by getter to make sure they have right
// timezone.
// NOTICE: Date formats are not synchronized.
// it must be used as synchronized externally.
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("HH");
private static final DateTimeFormatter DATETIME_MS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
private static final DateTimeFormatter DATETIME_NS_FORMAT = DateTimeFormatter.ofPattern(
"yyyy-MM-dd HH:mm:ss.SSSSSSSSS");
private static final DateTimeFormatter DATETIME_FORMAT_WITH_HYPHEN = DateTimeFormatter.ofPattern(
"yyyy-MM-dd-HH-mm-ss");

public static DateTimeFormatter getDateFormatWithTimeZone() {
return DATE_FORMAT.withZone(getDorisZoneId());
}

public static DateTimeFormatter getDatetimeFormatWithTimeZone() {
return DATETIME_FORMAT.withZone(getDorisZoneId());
}

public static DateTimeFormatter getTimeFormatWithTimeZone() {
return TIME_FORMAT.withZone(getDorisZoneId());
}

public static DateTimeFormatter getDatetimeMsFormatWithTimeZone() {
return DATETIME_MS_FORMAT.withZone(getDorisZoneId());
}

public static DateTimeFormatter getDatetimeNsFormatWithTimeZone() {
return DATETIME_NS_FORMAT.withZone(getDorisZoneId());
}

public static DateTimeFormatter getDatetimeFormatWithHyphenWithTimeZone() {
return DATETIME_FORMAT_WITH_HYPHEN.withZone(getDorisZoneId());
}

private static final Logger LOG = LogManager.getLogger(TimeUtils.class);
private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]?\\d{1,2}:\\d{2}$");
public static Date MIN_DATE = null;
public static Date MAX_DATE = null;
public static Date MIN_DATETIME = null;
public static Date MAX_DATETIME = null;

static {
Map<String, String> timeZoneMap = Maps.newHashMap();
Map<String, String> timeZoneMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
timeZoneMap.putAll(ZoneId.SHORT_IDS);

// set CST to +08:00 instead of America/Chicago
Expand All @@ -96,32 +116,6 @@ public class TimeUtils {
timeZoneMap.put("GMT", UTC_TIME_ZONE);

timeZoneAliasMap = ImmutableMap.copyOf(timeZoneMap);
TIME_ZONE = getSystemTimeZone().toZoneId();
DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
DATE_FORMAT.withZone(TIME_ZONE);

DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
DATETIME_FORMAT.withZone(TIME_ZONE);

TIME_FORMAT = DateTimeFormatter.ofPattern("HH");
TIME_FORMAT.withZone(TIME_ZONE);

try {

MIN_DATE = Date.from(
LocalDate.parse("0001-01-01", DATE_FORMAT).atStartOfDay().atZone(TIME_ZONE).toInstant());
MAX_DATE = Date.from(
LocalDate.parse("9999-12-31", DATE_FORMAT).atStartOfDay().atZone(TIME_ZONE).toInstant());

MIN_DATETIME = Date.from(
LocalDateTime.parse("0001-01-01 00:00:00", DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
MAX_DATETIME = Date.from(
LocalDateTime.parse("9999-12-31 23:59:59", DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());

} catch (DateTimeParseException e) {
LOG.error("invalid date format", e);
System.exit(-1);
}
}

public static long getStartTimeMs() {
Expand All @@ -133,7 +127,7 @@ public static long getElapsedTimeMs(long startTime) {
}

public static String getCurrentFormatTime() {
return LocalDateTime.now().format(DATETIME_FORMAT);
return LocalDateTime.now().format(getDatetimeFormatWithTimeZone());
}

public static TimeZone getTimeZone() {
Expand All @@ -146,13 +140,17 @@ public static TimeZone getTimeZone() {
return TimeZone.getTimeZone(ZoneId.of(timezone, timeZoneAliasMap));
}

public static ZoneId getDorisZoneId() {
return getTimeZone().toZoneId();
}

public static TimeZone getUTCTimeZone() {
return TimeZone.getTimeZone(UTC_TIME_ZONE);
}

// return the time zone of current system
public static TimeZone getSystemTimeZone() {
return TimeZone.getTimeZone(ZoneId.of(ZoneId.systemDefault().getId(), timeZoneAliasMap));
return TimeZone.getTimeZone(ZoneId.of(TimeZone.getDefault().getID(), timeZoneAliasMap));
}

// get time zone of given zone name, or return system time zone if name is null.
Expand All @@ -167,7 +165,7 @@ public static String longToTimeString(Long timeStamp, DateTimeFormatter dateForm
if (timeStamp == null || timeStamp <= 0L) {
return FeConstants.null_string;
}
return dateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), ZoneId.systemDefault()));
return dateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), getDorisZoneId()));
}

public static String longToTimeStringWithFormat(Long timeStamp, DateTimeFormatter datetimeFormatTimeZone) {
Expand All @@ -177,11 +175,11 @@ public static String longToTimeStringWithFormat(Long timeStamp, DateTimeFormatte
}

public static String longToTimeString(Long timeStamp) {
return longToTimeStringWithFormat(timeStamp, DATETIME_FORMAT);
return longToTimeStringWithFormat(timeStamp, getDatetimeFormatWithTimeZone());
}

public static String longToTimeStringWithms(Long timeStamp) {
return longToTimeStringWithFormat(timeStamp, DATETIME_MS_FORMAT);
return longToTimeStringWithFormat(timeStamp, getDatetimeMsFormatWithTimeZone());
}

public static Date getHourAsDate(String hour) {
Expand All @@ -191,7 +189,8 @@ public static Date getHourAsDate(String hour) {
}
try {
return Date.from(
LocalTime.parse(fullHour, TIME_FORMAT).atDate(LocalDate.now()).atZone(TIME_ZONE).toInstant());
LocalTime.parse(fullHour, getTimeFormatWithTimeZone()).atDate(LocalDate.now())
.atZone(getDorisZoneId()).toInstant());
} catch (DateTimeParseException e) {
LOG.warn("invalid time format: {}", fullHour);
return null;
Expand All @@ -208,13 +207,15 @@ public static Date parseDate(String dateStr, PrimitiveType type) throws Analysis
if (type == PrimitiveType.DATE) {
ParsePosition pos = new ParsePosition(0);
date = Date.from(
LocalDate.from(DATE_FORMAT.parse(dateStr, pos)).atStartOfDay().atZone(TIME_ZONE).toInstant());
LocalDate.from(getDateFormatWithTimeZone().parse(dateStr, pos)).atStartOfDay()
.atZone(getDorisZoneId()).toInstant());
if (pos.getIndex() != dateStr.length() || date == null) {
throw new AnalysisException("Invalid date string: " + dateStr);
}
} else if (type == PrimitiveType.DATETIME) {
try {
date = Date.from(LocalDateTime.parse(dateStr, DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
date = Date.from(LocalDateTime.parse(dateStr, getDatetimeFormatWithTimeZone())
.atZone(getDorisZoneId()).toInstant());
} catch (DateTimeParseException e) {
throw new AnalysisException("Invalid date string: " + dateStr);
}
Expand All @@ -231,9 +232,11 @@ public static Date parseDate(String dateStr, Type type) throws AnalysisException

public static String format(Date date, PrimitiveType type) {
if (type == PrimitiveType.DATE) {
return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).format(DATE_FORMAT);
return LocalDateTime.ofInstant(date.toInstant(), getDorisZoneId())
.format(getDateFormatWithTimeZone());
} else if (type == PrimitiveType.DATETIME) {
return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).format(DATETIME_FORMAT);
return LocalDateTime.ofInstant(date.toInstant(), getDorisZoneId())
.format(getDatetimeFormatWithTimeZone());
} else {
return "INVALID";
}
Expand All @@ -246,15 +249,16 @@ public static String format(Date date, Type type) {
public static long timeStringToLong(String timeStr) {
Date d;
try {
d = Date.from(LocalDateTime.parse(timeStr, DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
d = Date.from(LocalDateTime.parse(timeStr, getDatetimeFormatWithTimeZone())
.atZone(getDorisZoneId()).toInstant());
} catch (DateTimeParseException e) {
return -1;
}
return d.getTime();
}

public static long timeStringToLong(String timeStr, TimeZone timeZone) {
DateTimeFormatter dateFormatTimeZone = DATETIME_FORMAT;
DateTimeFormatter dateFormatTimeZone = getDatetimeFormatWithTimeZone();
dateFormatTimeZone.withZone(timeZone.toZoneId());
LocalDateTime d;
try {
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,9 +260,9 @@ protected void runAfterCatalogReady() {
for (Map.Entry<String, TStreamLoadRecord> entry : streamLoadRecordBatch.entrySet()) {
TStreamLoadRecord streamLoadItem = entry.getValue();
String startTime = TimeUtils.longToTimeString(streamLoadItem.getStartTime(),
TimeUtils.DATETIME_MS_FORMAT);
TimeUtils.getDatetimeMsFormatWithTimeZone());
String finishTime = TimeUtils.longToTimeString(streamLoadItem.getFinishTime(),
TimeUtils.DATETIME_MS_FORMAT);
TimeUtils.getDatetimeMsFormatWithTimeZone());
if (LOG.isDebugEnabled()) {
LOG.debug("receive stream load record info from backend: {}."
+ " label: {}, db: {}, tbl: {}, user: {}, user_ip: {},"
Expand Down
20 changes: 11 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public static void printSummary(Events<CanalEntry.Entry, EntryPosition> dataEven
String startPosition = buildPositionForDump(entries.get(0));
String endPosition = buildPositionForDump(entries.get(entries.size() - 1));
logger.info(context_format, dataEvents.getId(), entries.size(), dataEvents.getMemSize(),
TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()), startPosition, endPosition);
TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now()), startPosition, endPosition);
}

public static void printSummary(Message message, int size, long memsize) {
Expand All @@ -80,7 +80,7 @@ public static void printSummary(Message message, int size, long memsize) {
String startPosition = buildPositionForDump(message.getEntries().get(0));
String endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
logger.info(context_format, message.getId(), size, memsize,
TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()), startPosition, endPosition);
TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now()), startPosition, endPosition);
}

public static String buildPositionForDump(CanalEntry.Entry entry) {
Expand All @@ -94,7 +94,7 @@ public static String buildPositionForDump(CanalEntry.Entry entry) {
.append(":")
.append(header.getExecuteTime())
.append("(")
.append(TimeUtils.DATETIME_FORMAT.format(date))
.append(TimeUtils.getDatetimeFormatWithTimeZone().format(date))
.append(")");
if (StringUtils.isNotEmpty(entry.getHeader().getGtid())) {
sb.append(" gtid(").append(entry.getHeader().getGtid())
Expand All @@ -120,8 +120,8 @@ public static void printRow(CanalEntry.RowChange rowChange, CanalEntry.Header he
logger.info(row_format, header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), header.getSchemaName(),
header.getTableName(), eventType,
String.valueOf(header.getExecuteTime()), TimeUtils.DATETIME_FORMAT.format(date),
header.getGtid(), String.valueOf(delayTime));
String.valueOf(header.getExecuteTime()), TimeUtils.getDatetimeFormatWithTimeZone().format(date),
header.getGtid(), String.valueOf(delayTime));
if (eventType == CanalEntry.EventType.QUERY || rowChange.getIsDdl()) {
logger.info(" sql ----> " + rowChange.getSql() + SEP);
return;
Expand Down Expand Up @@ -197,8 +197,9 @@ public static void transactionBegin(CanalEntry.Entry entry) {
// print transaction begin info, thread ID, time consumption
logger.info(transaction_format, entry.getHeader().getLogfileName(),
String.valueOf(entry.getHeader().getLogfileOffset()),
String.valueOf(entry.getHeader().getExecuteTime()), TimeUtils.DATETIME_FORMAT.format(date),
entry.getHeader().getGtid(), String.valueOf(delayTime));
String.valueOf(entry.getHeader().getExecuteTime()),
TimeUtils.getDatetimeFormatWithTimeZone().format(date),
entry.getHeader().getGtid(), String.valueOf(delayTime));
logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
printXAInfo(begin.getPropsList());
}
Expand All @@ -219,8 +220,9 @@ public static void transactionEnd(CanalEntry.Entry entry) {
printXAInfo(end.getPropsList());
logger.info(transaction_format, entry.getHeader().getLogfileName(),
String.valueOf(entry.getHeader().getLogfileOffset()),
String.valueOf(entry.getHeader().getExecuteTime()), TimeUtils.DATETIME_FORMAT.format(date),
entry.getHeader().getGtid(), String.valueOf(delayTime));
String.valueOf(entry.getHeader().getExecuteTime()),
TimeUtils.getDatetimeFormatWithTimeZone().format(date),
entry.getHeader().getGtid(), String.valueOf(delayTime));
}

public static boolean isDML(CanalEntry.EventType eventType) {
Expand Down
Loading
Loading