diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 4e9ff9037b343c..b5ae0096f5ad2a 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -50,6 +50,7 @@ #include "exprs/utility_functions.h" #include "exprs/json_functions.h" #include "exprs/hll_hash_function.h" +#include "exprs/timezone_db.h" #include "geo/geo_functions.h" #include "olap/options.h" #include "util/time.h" @@ -268,6 +269,7 @@ void init_daemon(int argc, char** argv, const std::vector& paths) { HllHashFunctions::init(); ESFunctions::init(); GeoFunctions::init(); + TimezoneDatabase::init(); pthread_t tc_malloc_pid; pthread_create(&tc_malloc_pid, NULL, tcmalloc_gc_thread, NULL); diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index d07c63bf7d16d5..ed321e50a968e6 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -306,7 +306,7 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, case TYPE_DATE: case TYPE_DATETIME: { if (col.IsNumber()) { - if (!reinterpret_cast(slot)->from_unixtime(col.GetInt64())) { + if (!reinterpret_cast(slot)->from_unixtime(col.GetInt64(), "+08:00")) { return Status::InternalError(strings::Substitute(ERROR_INVALID_COL_DATA, type_to_string(type))); } diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index 1e5741f6bbca40..d70fbfc0084f79 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -846,14 +846,14 @@ Status EsScanNode::materialize_row(MemPool* tuple_pool, Tuple* tuple, break; case TYPE_DATE: if (val_idx >= col.long_vals.size() || - !reinterpret_cast(slot)->from_unixtime(col.long_vals[val_idx])) { + !reinterpret_cast(slot)->from_unixtime(col.long_vals[val_idx], "+08:00")) { return Status::InternalError(strings::Substitute(ERROR_INVALID_COL_DATA, "TYPE_DATE")); } reinterpret_cast(slot)->cast_to_date(); break; case TYPE_DATETIME: { if (val_idx >= col.long_vals.size() || - !reinterpret_cast(slot)->from_unixtime(col.long_vals[val_idx])) { + !reinterpret_cast(slot)->from_unixtime(col.long_vals[val_idx], "+08:00")) { return Status::InternalError(strings::Substitute(ERROR_INVALID_COL_DATA, "TYPE_DATETIME")); } reinterpret_cast(slot)->set_type(TIME_DATETIME); diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index 65d7f7053095b5..9b90979dab7ed2 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -17,13 +17,9 @@ #include "exprs/timestamp_functions.h" -#include -#include -#include -#include - #include "exprs/expr.h" #include "exprs/anyval_util.h" +#include "exprs/timezone_db.h" #include "runtime/tuple_row.h" #include "runtime/datetime_value.h" #include "runtime/runtime_state.h" @@ -31,78 +27,11 @@ #include "runtime/string_value.hpp" #include "util/debug_util.h" -#define TIMEZONE_DATABASE "be/files/date_time_zonespec.csv" - namespace doris { -boost::local_time::tz_database TimezoneDatabase::_s_tz_database; -std::vector TimezoneDatabase::_s_tz_region_list; - void TimestampFunctions::init() { } -StringVal TimestampFunctions::from_unix( - FunctionContext* context, const IntVal& unix_time) { - if (unix_time.is_null) { - return StringVal::null(); - } - DateTimeValue t; - if (!t.from_unixtime(unix_time.val)) { - return StringVal::null(); - } - // default fmt means return DATE_TIME - t.set_type(TIME_DATETIME); - char buf[64]; - t.to_string(buf); - return AnyValUtil::from_string_temp(context, buf); -} - -StringVal TimestampFunctions::from_unix( - FunctionContext* context, const IntVal& unix_time, const StringVal& fmt) { - if (unix_time.is_null || fmt.is_null) { - return StringVal::null(); - } - DateTimeValue t; - if (!t.from_unixtime(unix_time.val)) { - return StringVal::null(); - } - - if (!check_format(fmt, t)) { - return StringVal::null(); - } - - char buf[64]; - t.to_string(buf); - return AnyValUtil::from_string_temp(context, buf); -} - -IntVal TimestampFunctions::to_unix(FunctionContext* context) { - return IntVal(context->impl()->state()->now()->unix_timestamp()); -} - -IntVal TimestampFunctions::to_unix( - FunctionContext* context, const StringVal& string_val, const StringVal& fmt) { - if (string_val.is_null || fmt.is_null) { - return IntVal::null(); - } - // const DateTimeVal& tv_val = ToTimestamp(context, string_val, fmt); - DateTimeValue tv_val; - if (!tv_val.from_date_format_str( - (const char*)fmt.ptr, fmt.len, (const char*)string_val.ptr, string_val.len)) { - return IntVal::null(); - } - return IntVal(tv_val.unix_timestamp()); -} - -IntVal TimestampFunctions::to_unix( - FunctionContext* context, const DateTimeVal& ts_val) { - if (ts_val.is_null) { - return IntVal::null(); - } - const DateTimeValue& tv = DateTimeValue::from_datetime_val(ts_val); - return IntVal(tv.unix_timestamp()); -} - // TODO: accept Java data/time format strings: // http://docs.oracle.com/javase/1.4.2/docs/api/java/text/SimpleDateFormat.html // Convert them to boost format strings. @@ -228,33 +157,6 @@ IntVal TimestampFunctions::second( return IntVal(ts_value.second()); } -DateTimeVal TimestampFunctions::now(FunctionContext* context) { - const DateTimeValue* now = context->impl()->state()->now(); - DateTimeVal return_val; - now->to_datetime_val(&return_val); - return return_val; -} - -DateTimeVal TimestampFunctions::curtime(FunctionContext* context) { - DateTimeValue now = *context->impl()->state()->now(); - now.cast_to_time(); - DateTimeVal return_val; - now.to_datetime_val(&return_val); - return return_val; -} - -DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { - TimeInterval interval; - // TODO(liuhy): we only support Beijing Timezone, so minus 28800 - interval.second = -28800; - DateTimeValue dtv = *(context->impl()->state()->now()); - dtv.date_add_interval(interval, SECOND); - - DateTimeVal return_val; - dtv.to_datetime_val(&return_val); - return return_val; -} - DateTimeVal TimestampFunctions::to_date( FunctionContext* ctx, const DateTimeVal& ts_val) { if (ts_val.is_null) { @@ -462,17 +364,15 @@ IntVal TimestampFunctions::to_days( return IntVal(ts_value.daynr()); } -// TODO(dhc): implement this funciton really DoubleVal TimestampFunctions::time_diff( FunctionContext* ctx, const DateTimeVal& ts_val1, const DateTimeVal& ts_val2) { if (ts_val1.is_null || ts_val2.is_null) { return DoubleVal::null(); } + const DateTimeValue& ts_value1 = DateTimeValue::from_datetime_val(ts_val1); const DateTimeValue& ts_value2 = DateTimeValue::from_datetime_val(ts_val2); - int64_t timediff = ts_value1.unix_timestamp() - ts_value2.unix_timestamp(); - - return DoubleVal(timediff); + return DoubleVal(ts_value1.second_diff(ts_value2)); } IntVal TimestampFunctions::date_diff( @@ -485,142 +385,134 @@ IntVal TimestampFunctions::date_diff( return IntVal(ts_value1.daynr() - ts_value2.daynr()); } +// TimeZone correlation functions. DateTimeVal TimestampFunctions::timestamp( FunctionContext* ctx, const DateTimeVal& val) { return val; } -void* TimestampFunctions::from_utc(Expr* e, TupleRow* row) { - return NULL; - // DCHECK_EQ(e->get_num_children(), 2); - // Expr* op1 = e->children()[0]; - // Expr* op2 = e->children()[1]; - // DateTimeValue* tv = reinterpret_cast(op1->get_value(row)); - // StringValue* tz = reinterpret_cast(op2->get_value(row)); - - // if (tv == NULL || tz == NULL) { - // return NULL; - // } - - // if (tv->not_a_date_time()) { - // return NULL; - // } - - // boost::local_time::time_zone_ptr timezone = TimezoneDatabase::find_timezone(tz->debug_string()); - - // This should raise some sort of error or at least null. Hive just ignores it. - // if (timezone == NULL) { - // LOG(ERROR) << "Unknown timezone '" << *tz << "'" << std::endl; - // e->_result.timestamp_val = *tv; - // return &e->_result.timestamp_val; - // } - - // boost::posix_time::ptime temp; - // tv->to_ptime(&temp); - // boost::local_time::local_date_time lt(temp, timezone); - // e->_result.timestamp_val = lt.local_time(); - // return &e->_result.timestamp_val; +StringVal TimestampFunctions::from_unix( + FunctionContext* context, const IntVal& unix_time) { + if (unix_time.is_null) { + return StringVal::null(); + } + DateTimeValue dtv; + if (!dtv.from_unixtime(unix_time.val, context->impl()->state()->timezone())) { + return StringVal::null(); + } + char buf[64]; + dtv.to_string(buf); + return AnyValUtil::from_string_temp(context, buf); } -void* TimestampFunctions::to_utc(Expr* e, TupleRow* row) { - return NULL; - // DCHECK_EQ(e->get_num_children(), 2); - // Expr* op1 = e->children()[0]; - // Expr* op2 = e->children()[1]; - // DateTimeValue* tv = reinterpret_cast(op1->get_value(row)); - // StringValue* tz = reinterpret_cast(op2->get_value(row)); - - // if (tv == NULL || tz == NULL) { - // return NULL; - // } - - // if (tv->not_a_date_time()) { - // return NULL; - // } - - // boost::local_time::time_zone_ptr timezone = TimezoneDatabase::find_timezone(tz->debug_string()); - - // This should raise some sort of error or at least null. Hive just ignores it. - // if (timezone == NULL) { - // LOG(ERROR) << "Unknown timezone '" << *tz << "'" << std::endl; - // e->_result.timestamp_val = *tv; - // return &e->_result.timestamp_val; - // } +StringVal TimestampFunctions::from_unix( + FunctionContext* context, const IntVal& unix_time, const StringVal& fmt) { + if (unix_time.is_null || fmt.is_null) { + return StringVal::null(); + } + DateTimeValue dtv; + if (!dtv.from_unixtime(unix_time.val, context->impl()->state()->timezone())) { + return StringVal::null(); + } - // boost::local_time::local_date_time lt(tv->date(), tv->time_of_day(), - // timezone, boost::local_time::local_date_time::NOT_DATE_TIME_ON_ERROR); - // e->_result.timestamp_val = DateTimeValue(lt.utc_time()); - // return &e->_result.timestamp_val; + char buf[128]; + if (!dtv.to_format_string((const char*)fmt.ptr, fmt.len, buf)) { + return StringVal::null(); + } + return AnyValUtil::from_string_temp(context, buf); } -TimezoneDatabase::TimezoneDatabase() { - // Create a temporary file and write the timezone information. The boost - // interface only loads this format from a file. We don't want to raise - // an error here since this is done when the backend is created and this - // information might not actually get used by any queries. - char filestr[] = "/tmp/doris.tzdb.XXXXXXX"; - FILE* file = NULL; - int fd = -1; +IntVal TimestampFunctions::to_unix(FunctionContext* context) { + return IntVal(context->impl()->state()->timestamp() / 1000); +} - if ((fd = mkstemp(filestr)) == -1) { - LOG(ERROR) << "Could not create temporary timezone file: " << filestr; - return; +IntVal TimestampFunctions::to_unix( + FunctionContext* context, const StringVal& string_val, const StringVal& fmt) { + if (string_val.is_null || fmt.is_null) { + return IntVal::null(); } - - if ((file = fopen(filestr, "w")) == NULL) { - unlink(filestr); - close(fd); - LOG(ERROR) << "Could not open temporary timezone file: " << filestr; - return; + DateTimeValue tv; + if (!tv.from_date_format_str( + (const char *)fmt.ptr, fmt.len, (const char *)string_val.ptr, string_val.len)) { + return IntVal::null(); } - if (fputs(_s_timezone_database_str, file) == EOF) { - unlink(filestr); - close(fd); - fclose(file); - LOG(ERROR) << "Could not load temporary timezone file: " << filestr; - return; + int64_t timestamp; + if(!tv.unix_timestamp(×tamp, context->impl()->state()->timezone())) { + return IntVal::null(); + } else { + return IntVal(timestamp); } - - fclose(file); - _s_tz_database.load_from_file(std::string(filestr)); - _s_tz_region_list = _s_tz_database.region_list(); - unlink(filestr); - close(fd); } -TimezoneDatabase::~TimezoneDatabase() { } - -boost::local_time::time_zone_ptr TimezoneDatabase::find_timezone(const std::string& tz) { - // See if they specified a zone id - if (tz.find_first_of('/') != std::string::npos) { - return _s_tz_database.time_zone_from_region(tz); +IntVal TimestampFunctions::to_unix( + FunctionContext* context, const DateTimeVal& ts_val) { + if (ts_val.is_null) { + return IntVal::null(); } + const DateTimeValue &tv = DateTimeValue::from_datetime_val(ts_val); + + int64_t timestamp; + if(!tv.unix_timestamp(×tamp, context->impl()->state()->timezone())) { + return IntVal::null(); + } else { + return IntVal(timestamp); + } +} - for (std::vector::const_iterator iter = _s_tz_region_list.begin(); - iter != _s_tz_region_list.end(); ++iter) { - boost::local_time::time_zone_ptr tzp = _s_tz_database.time_zone_from_region(*iter); - DCHECK(tzp != NULL); +DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { + DateTimeValue dtv; + if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, "+00:00")) { + return DateTimeVal::null(); + } - if (tzp->dst_zone_abbrev() == tz) { - return tzp; - } + DateTimeVal return_val; + dtv.to_datetime_val(&return_val); + return return_val; +} - if (tzp->std_zone_abbrev() == tz) { - return tzp; - } +DateTimeVal TimestampFunctions::now(FunctionContext* context) { + DateTimeValue dtv; + if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, + context->impl()->state()->timezone())) { + return DateTimeVal::null(); + } - if (tzp->dst_zone_name() == tz) { - return tzp; - } + DateTimeVal return_val; + dtv.to_datetime_val(&return_val); + return return_val; +} - if (tzp->std_zone_name() == tz) { - return tzp; - } +DoubleVal TimestampFunctions::curtime(FunctionContext* context) { + DateTimeValue dtv; + if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, + context->impl()->state()->timezone())) { + return DoubleVal::null(); } - return boost::local_time::time_zone_ptr(); + return dtv.hour() * 3600 + dtv.minute() * 60 + dtv.second(); +} +DateTimeVal TimestampFunctions::convert_tz(FunctionContext* ctx, const DateTimeVal& ts_val, + const StringVal& from_tz, const StringVal& to_tz) { + if (TimezoneDatabase::find_timezone(std::string((char *)from_tz.ptr, from_tz.len)) == nullptr || + TimezoneDatabase::find_timezone(std::string((char *)to_tz.ptr, to_tz.len)) == nullptr + ) { + return DateTimeVal::null(); + } + const DateTimeValue &ts_value = DateTimeValue::from_datetime_val(ts_val); + int64_t timestamp; + if(!ts_value.unix_timestamp(×tamp, std::string((char *)from_tz.ptr, from_tz.len))) { + return DateTimeVal::null(); + } + DateTimeValue ts_value2; + if (!ts_value2.from_unixtime(timestamp, std::string((char *)to_tz.ptr, to_tz.len))) { + return DateTimeVal::null(); + } + + DateTimeVal return_val; + ts_value2.to_datetime_val(&return_val); + return return_val; } } diff --git a/be/src/exprs/timestamp_functions.h b/be/src/exprs/timestamp_functions.h index d8df9c9ac10cc3..fbc62b385bc7cf 100644 --- a/be/src/exprs/timestamp_functions.h +++ b/be/src/exprs/timestamp_functions.h @@ -35,23 +35,6 @@ class TupleRow; class TimestampFunctions { public: static void init(); - /// Returns the current time. - static doris_udf::IntVal to_unix(doris_udf::FunctionContext* context); - /// Converts 'tv_val' to a unix time_t - static doris_udf::IntVal to_unix( - doris_udf::FunctionContext* context, const doris_udf::DateTimeVal& tv_val); - /// Parses 'string_val' based on the format 'fmt'. - static doris_udf::IntVal to_unix( - doris_udf::FunctionContext* context, const doris_udf::StringVal& string_val, - const doris_udf::StringVal& fmt); - /// Return a timestamp string from a unix time_t - /// Optional second argument is the format of the string. - /// TIME is the integer type of the unix time argument. - static doris_udf::StringVal from_unix( - doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time); - static doris_udf::StringVal from_unix( - doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time, - const doris_udf::StringVal& fmt); // Functions to extract parts of the timestamp, return integers. static doris_udf::IntVal year( @@ -76,9 +59,6 @@ class TimestampFunctions { doris_udf::FunctionContext* context, const doris_udf::DateTimeVal& ts_val); // Date/time functions. - static doris_udf::DateTimeVal now(doris_udf::FunctionContext* context); - static doris_udf::DateTimeVal curtime(doris_udf::FunctionContext* context); - static doris_udf::DateTimeVal utc_timestamp(doris_udf::FunctionContext* context); static doris_udf::DateTimeVal to_date( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val); static doris_udf::IntVal date_diff( @@ -87,7 +67,6 @@ class TimestampFunctions { static doris_udf::DoubleVal time_diff( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val1, const doris_udf::DateTimeVal& ts_val2); - static doris_udf::DateTimeVal years_add( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::IntVal& count); @@ -136,7 +115,6 @@ class TimestampFunctions { static doris_udf::DateTimeVal micros_sub( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::IntVal& count); - static doris_udf::StringVal date_format( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::StringVal& format); @@ -152,18 +130,37 @@ class TimestampFunctions { static doris_udf::StringVal day_name( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val); + // TimeZone correlation functions. static doris_udf::DateTimeVal timestamp( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& val); - // Helper for add/sub functions on the time portion. template static doris_udf::DateTimeVal timestamp_time_op( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::IntVal& count, bool is_add); - - // Convert a timestamp to or from a particular timezone based time. - static void* from_utc(Expr* e, TupleRow* row); - static void* to_utc(Expr* e, TupleRow* row); + static doris_udf::DateTimeVal now(doris_udf::FunctionContext* context); + static doris_udf::DoubleVal curtime(doris_udf::FunctionContext* context); + static doris_udf::DateTimeVal utc_timestamp(doris_udf::FunctionContext* context); + /// Returns the current time. + static doris_udf::IntVal to_unix(doris_udf::FunctionContext* context); + /// Converts 'tv_val' to a unix time_t + static doris_udf::IntVal to_unix( + doris_udf::FunctionContext* context, const doris_udf::DateTimeVal& tv_val); + /// Parses 'string_val' based on the format 'fmt'. + static doris_udf::IntVal to_unix( + doris_udf::FunctionContext* context, const doris_udf::StringVal& string_val, + const doris_udf::StringVal& fmt); + /// Return a timestamp string from a unix time_t + /// Optional second argument is the format of the string. + /// TIME is the integer type of the unix time argument. + static doris_udf::StringVal from_unix( + doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time); + static doris_udf::StringVal from_unix( + doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time, + const doris_udf::StringVal& fmt); + static doris_udf::DateTimeVal convert_tz(doris_udf::FunctionContext* ctx, + const doris_udf::DateTimeVal& ts_val, const doris_udf::StringVal& from_tz, + const doris_udf::StringVal& to_tz); // Helper function to check date/time format strings. // TODO: eventually return format converted from Java to Boost. @@ -171,23 +168,7 @@ class TimestampFunctions { // Issue a warning for a bad format string. static void report_bad_format(const StringVal* format); - }; - -// Functions to load and access the timestamp database. -class TimezoneDatabase { -public: - TimezoneDatabase(); - ~TimezoneDatabase(); - - static boost::local_time::time_zone_ptr find_timezone(const std::string& tz); - -private: - static const char* _s_timezone_database_str; - static boost::local_time::tz_database _s_tz_database; - static std::vector _s_tz_region_list; -}; - } #endif diff --git a/be/src/exprs/timezone_db.cpp b/be/src/exprs/timezone_db.cpp index 3ba36736b901c8..448a0d18ad6a39 100644 --- a/be/src/exprs/timezone_db.cpp +++ b/be/src/exprs/timezone_db.cpp @@ -14,10 +14,65 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -#include "timestamp_functions.h" +#include "exprs/timezone_db.h" namespace doris { +boost::local_time::tz_database TimezoneDatabase::_s_tz_database; +void TimezoneDatabase::init() { + // Create a temporary file and write the timezone information. The boost + // interface only loads this format from a file. We don't want to raise + // an error here since this is done when the backend is created and this + // information might not actually get used by any queries. + char filestr[] = "/tmp/doris.tzdb.XXXXXXX"; + FILE *file = NULL; + int fd = -1; + + if ((fd = mkstemp(filestr)) == -1) { + LOG(ERROR) << "Could not create temporary timezone file: " << filestr; + return; + } + + if ((file = fopen(filestr, "w")) == NULL) { + unlink(filestr); + close(fd); + LOG(ERROR) << "Could not open temporary timezone file: " << filestr; + return; + } + + if (fputs(_s_timezone_database_str, file) == EOF) { + unlink(filestr); + close(fd); + fclose(file); + LOG(ERROR) << "Could not load temporary timezone file: " << filestr; + return; + } + + fclose(file); + _s_tz_database.load_from_file(std::string(filestr)); + unlink(filestr); + close(fd); +} + +boost::local_time::time_zone_ptr TimezoneDatabase::find_timezone(const std::string &tz) { + try { + // See if they specified a zone id + if (tz.find_first_of('/') != std::string::npos) { + return _s_tz_database.time_zone_from_region(tz); + } else if (tz == "CST") { + boost::local_time::time_zone_ptr tzp(new boost::local_time::posix_time_zone(std::string("TMP") + "+08:00")); + return tzp; + } else { + //eg. +08:00 + boost::local_time::time_zone_ptr tzp(new boost::local_time::posix_time_zone(std::string("TMP") + tz)); + return tzp; + } + } catch (boost::exception& e) { + return nullptr; + } +} + +const std::string TimezoneDatabase::default_time_zone = "+08:00"; + const char* TimezoneDatabase::_s_timezone_database_str = "\"ID\",\"STD ABBR\",\"STD NAME\",\"DST ABBR\",\"DST NAME\",\"GMT offset\",\"DST adjustment\",\ \"DST Start Date rule\",\"Start time\",\"DST End date rule\",\"End time\"\n\ diff --git a/be/src/exprs/timezone_db.h b/be/src/exprs/timezone_db.h new file mode 100644 index 00000000000000..d16d1d43b29430 --- /dev/null +++ b/be/src/exprs/timezone_db.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +#ifndef DORIS_BE_EXPRS_TIMEZONE_DB_H +#define DORIS_BE_EXPRS_TIMEZONE_DB_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" + +namespace doris { + +class TimezoneDatabase { +public: + static void init(); + static boost::local_time::time_zone_ptr find_timezone(const std::string &tz); + static const std::string default_time_zone; +private: + static const char *_s_timezone_database_str; + static boost::local_time::tz_database _s_tz_database; +}; +} +#endif diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp index b45e9bc578afb0..b11d7f8c0592af 100644 --- a/be/src/runtime/datetime_value.cpp +++ b/be/src/runtime/datetime_value.cpp @@ -1524,43 +1524,43 @@ bool DateTimeValue::date_add_interval(const TimeInterval& interval, TimeUnit uni return true; } -int DateTimeValue::unix_timestamp() const { - int64_t days = daynr() - calc_daynr(1970, 1, 1); - if (days < 0) { - return 0; - } - int64_t seconds = days * 86400 + _hour * 3600 + _minute * 60 + _second; - if (seconds > std::numeric_limits::max()) { - return 0; - } - // TODO(zc): we only support Beijing Timezone, so minus 28800 - seconds -= 28800; - if (seconds < 0) { - return 0; - } - return seconds; -} - -bool DateTimeValue::from_unixtime(int64_t seconds) { - if (seconds < 0) { +bool DateTimeValue::unix_timestamp(int64_t* timestamp, const std::string& timezone) const{ + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { return false; } - // TODO(zc): we only support Beijing Timezone, so add 28800 - seconds += 28800; - int64_t days = seconds / 86400 + calc_daynr(1970, 1, 1); + char buf[64]; + char* to = to_datetime_string(buf); + boost::posix_time::ptime pt = boost::posix_time::time_from_string(std::string(buf, to - buf -1)); - _neg = false; - get_date_from_daynr(days); - seconds %= 86400; - if (seconds == 0) { - _type = TIME_DATE; - return true; + boost::local_time::local_date_time lt(pt.date(), pt.time_of_day(), local_time_zone, + boost::local_time::local_date_time::NOT_DATE_TIME_ON_ERROR); + + boost::posix_time::ptime utc_ptime = lt.utc_time(); + boost::posix_time::ptime utc_start(boost::gregorian::date(1970, 1, 1)); + boost::posix_time::time_duration dur = utc_ptime - utc_start; + *timestamp = dur.total_milliseconds() / 1000; + return true; +} + +bool DateTimeValue::from_unixtime(int64_t timestamp, const std::string& timezone) { + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { + return false; } + boost::local_time::local_date_time lt(boost::posix_time::from_time_t(timestamp), local_time_zone); + boost::posix_time::ptime local_ptime = lt.local_time(); + + _neg = 0; _type = TIME_DATETIME; - _hour = seconds / 3600; - seconds %= 3600; - _minute = seconds / 60; - _second = seconds % 60; + _year = local_ptime.date().year(); + _month = local_ptime.date().month(); + _day = local_ptime.date().day(); + _hour = local_ptime.time_of_day().hours(); + _minute = local_ptime.time_of_day().minutes(); + _second = local_ptime.time_of_day().seconds(); + _microsecond = 0; + return true; } @@ -1581,7 +1581,7 @@ const char* DateTimeValue::day_name() const { DateTimeValue DateTimeValue::local_time() { DateTimeValue value; - value.from_unixtime(time(NULL)); + value.from_unixtime(time(NULL), TimezoneDatabase::default_time_zone); return value; } diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index 3ae5c5af4ad50b..b6d04af4bd0e3d 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -25,6 +25,7 @@ #include "udf/udf.h" #include "util/hash_util.hpp" +#include "exprs/timezone_db.h" namespace doris { @@ -331,10 +332,14 @@ class DateTimeValue { // Add interval bool date_add_interval(const TimeInterval& interval, TimeUnit unit); - - int unix_timestamp() const; - - bool from_unixtime(int64_t); + + //unix_timestamp is called with a timezone argument, + //it returns seconds of the value of date literal since '1970-01-01 00:00:00' UTC + bool unix_timestamp(int64_t* timestamp, const std::string& timezone) const; + + //construct datetime_value from timestamp and timezone + //timestamp is an internal timestamp value representing seconds since '1970-01-01 00:00:00' UTC + bool from_unixtime(int64_t, const std::string& timezone); bool operator==(const DateTimeValue& other) const { // NOTE: This is not same with MySQL. @@ -435,7 +440,10 @@ class DateTimeValue { } int64_t second_diff(const DateTimeValue& rhs) const { - return unix_timestamp() - rhs.unix_timestamp(); + int day_diff = daynr() - rhs.daynr(); + int time_diff = (hour() * 3600 + minute() * 60 + second()) + - (rhs.hour() * 3600 + rhs.minute() * 60 + rhs.second()); + return day_diff * 3600 * 24 + time_diff; } void set_type(int type); diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 7ecb716ec03ad1..b9663f91943daa 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -77,7 +77,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request) { // VLOG(2) << "request:\n" << apache::thrift::ThriftDebugString(request); _runtime_state.reset(new RuntimeState( - request, request.query_options, request.query_globals.now_string, _exec_env)); + request, request.query_options, request.query_globals, _exec_env)); RETURN_IF_ERROR(_runtime_state->init_mem_trackers(_query_id)); _runtime_state->set_be_number(request.backend_num); diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 257c6f90364176..172d80193a6857 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -26,6 +26,7 @@ #include "common/object_pool.h" #include "common/status.h" #include "exprs/expr.h" +#include "exprs/timezone_db.h" #include "runtime/buffered_block_mgr.h" #include "runtime/buffered_block_mgr2.h" #include "runtime/bufferpool/reservation_util.h" @@ -50,7 +51,7 @@ namespace doris { RuntimeState::RuntimeState( const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env) : + const TQueryGlobals& query_globals, ExecEnv* exec_env) : _obj_pool(new ObjectPool()), _data_stream_recvrs_pool(new ObjectPool()), _unreported_error_idx(0), @@ -68,14 +69,14 @@ RuntimeState::RuntimeState( _error_log_file_path(""), _error_log_file(nullptr), _instance_buffer_reservation(new ReservationTracker) { - Status status = init(fragment_instance_id, query_options, now, exec_env); + Status status = init(fragment_instance_id, query_options, query_globals, exec_env); DCHECK(status.ok()); } RuntimeState::RuntimeState( const TExecPlanFragmentParams& fragment_params, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env) : + const TQueryGlobals& query_globals, ExecEnv* exec_env) : _obj_pool(new ObjectPool()), _data_stream_recvrs_pool(new ObjectPool()), _unreported_error_idx(0), @@ -95,19 +96,23 @@ RuntimeState::RuntimeState( _error_log_file_path(""), _error_log_file(nullptr), _instance_buffer_reservation(new ReservationTracker) { - Status status = init(fragment_params.params.fragment_instance_id, query_options, now, exec_env); + Status status = init(fragment_params.params.fragment_instance_id, query_options, query_globals, exec_env); DCHECK(status.ok()); } -RuntimeState::RuntimeState(const std::string& now) +RuntimeState::RuntimeState(const TQueryGlobals& query_globals) : _obj_pool(new ObjectPool()), _data_stream_recvrs_pool(new ObjectPool()), _unreported_error_idx(0), _profile(_obj_pool.get(), ""), _per_fragment_instance_idx(0) { _query_options.batch_size = DEFAULT_BATCH_SIZE; - _now.reset(new DateTimeValue()); - _now->from_date_str(now.c_str(), now.size()); + _timestamp = atol(query_globals.now_string.c_str()); + if (query_globals.__isset.time_zone) { + _timezone = query_globals.time_zone; + } else { + _timezone = TimezoneDatabase::default_time_zone; + } } RuntimeState::~RuntimeState() { @@ -161,11 +166,15 @@ RuntimeState::~RuntimeState() { Status RuntimeState::init( const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env) { + const TQueryGlobals& query_globals, ExecEnv* exec_env) { _fragment_instance_id = fragment_instance_id; _query_options = query_options; - _now.reset(new DateTimeValue()); - _now->from_date_str(now.c_str(), now.size()); + _timestamp = atol(query_globals.now_string.c_str()); + if (query_globals.__isset.time_zone) { + _timezone = query_globals.time_zone; + } else { + _timezone = TimezoneDatabase::default_time_zone; + } _exec_env = exec_env; if (!query_options.disable_codegen) { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 24aedd6a30f0de..64216e355602ee 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -68,15 +68,15 @@ class RuntimeState { // for ut only RuntimeState(const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env); + const TQueryGlobals& query_globals, ExecEnv* exec_env); RuntimeState( const TExecPlanFragmentParams& fragment_params, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env); + const TQueryGlobals& query_globals, ExecEnv* exec_env); // RuntimeState for executing expr in fe-support. - RuntimeState(const std::string& now); + RuntimeState(const TQueryGlobals& query_globals); // Empty d'tor to avoid issues with scoped_ptr. ~RuntimeState(); @@ -84,7 +84,7 @@ class RuntimeState { // Set per-query state. Status init(const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env); + const TQueryGlobals& query_globals, ExecEnv* exec_env); // Set up four-level hierarchy of mem trackers: process, query, fragment instance. // The instance tracker is tied to our profile. @@ -136,8 +136,11 @@ class RuntimeState { int num_scanner_threads() const { return _query_options.num_scanner_threads; } - const DateTimeValue* now() const { - return _now.get(); + int64_t timestamp() const { + return _timestamp; + } + const std::string& timezone() const { + return _timezone; } const std::string& user() const { return _user; @@ -532,9 +535,10 @@ class RuntimeState { // Username of user that is executing the query to which this RuntimeState belongs. std::string _user; - // Query-global timestamp, e.g., for implementing now(). - // Use pointer to avoid inclusion of timestampvalue.h and avoid clang issues. - boost::scoped_ptr _now; + + //Query-global timestamp + int64_t _timestamp; + std::string _timezone; TUniqueId _query_id; TUniqueId _fragment_instance_id; diff --git a/be/src/runtime/test_env.cc b/be/src/runtime/test_env.cc index 691a08ee3d35ba..b0d5048d3b09b7 100644 --- a/be/src/runtime/test_env.cc +++ b/be/src/runtime/test_env.cc @@ -66,7 +66,7 @@ RuntimeState* TestEnv::create_runtime_state(int64_t query_id) { TExecPlanFragmentParams plan_params = TExecPlanFragmentParams(); plan_params.params.query_id.hi = 0; plan_params.params.query_id.lo = query_id; - return new RuntimeState(plan_params, TQueryOptions(), "", _exec_env.get()); + return new RuntimeState(plan_params, TQueryOptions(), TQueryGlobals(), _exec_env.get()); } Status TestEnv::create_query_state(int64_t query_id, int max_buffers, int block_size, diff --git a/be/src/testutil/function_utils.cpp b/be/src/testutil/function_utils.cpp index d6572b103a93bd..28ac6c1d152f3f 100644 --- a/be/src/testutil/function_utils.cpp +++ b/be/src/testutil/function_utils.cpp @@ -34,6 +34,15 @@ FunctionUtils::FunctionUtils() { _fn_ctx = FunctionContextImpl::create_context( _state, _memory_pool, return_type, arg_types, 0, false); } +FunctionUtils::FunctionUtils(RuntimeState* state) { + _state = state; + doris_udf::FunctionContext::TypeDesc return_type; + std::vector arg_types; + _mem_tracker = new MemTracker(); + _memory_pool = new MemPool(_mem_tracker); + _fn_ctx = FunctionContextImpl::create_context( + _state, _memory_pool, return_type, arg_types, 0, false); +} FunctionUtils::~FunctionUtils() { _fn_ctx->impl()->close(); diff --git a/be/src/testutil/function_utils.h b/be/src/testutil/function_utils.h index c47a5d1df12402..d777aedb0c7408 100644 --- a/be/src/testutil/function_utils.h +++ b/be/src/testutil/function_utils.h @@ -28,6 +28,7 @@ class RuntimeState; class FunctionUtils { public: FunctionUtils(); + FunctionUtils(RuntimeState* state); ~FunctionUtils(); doris_udf::FunctionContext* get_fn_ctx() { diff --git a/be/test/exec/broker_scan_node_test.cpp b/be/test/exec/broker_scan_node_test.cpp index 066f468dfa99cb..5ccb5506e3c1d7 100644 --- a/be/test/exec/broker_scan_node_test.cpp +++ b/be/test/exec/broker_scan_node_test.cpp @@ -38,7 +38,7 @@ namespace doris { class BrokerScanNodeTest : public testing::Test { public: - BrokerScanNodeTest() : _runtime_state("BrokerScanNodeTest") { + BrokerScanNodeTest() : _runtime_state(TQueryGlobals()) { init(); _runtime_state._instance_mem_tracker.reset(new MemTracker()); } diff --git a/be/test/exec/broker_scanner_test.cpp b/be/test/exec/broker_scanner_test.cpp index 81c0cd1a8309b7..1c59c10d790a4f 100644 --- a/be/test/exec/broker_scanner_test.cpp +++ b/be/test/exec/broker_scanner_test.cpp @@ -38,7 +38,7 @@ namespace doris { class BrokerScannerTest : public testing::Test { public: - BrokerScannerTest() : _runtime_state("BrokerScannerTest") { + BrokerScannerTest() : _runtime_state(TQueryGlobals()) { init(); _profile = _runtime_state.runtime_profile(); _runtime_state._instance_mem_tracker.reset(new MemTracker()); diff --git a/be/test/exec/es_http_scan_node_test.cpp b/be/test/exec/es_http_scan_node_test.cpp index e3fd63a9d2e64f..0c8f73a15d52af 100644 --- a/be/test/exec/es_http_scan_node_test.cpp +++ b/be/test/exec/es_http_scan_node_test.cpp @@ -38,7 +38,7 @@ namespace doris { // mock class EsHttpScanNodeTest : public testing::Test { public: - EsHttpScanNodeTest() : _runtime_state("EsHttpScanNodeTest") { + EsHttpScanNodeTest() : _runtime_state(TQueryGlobals()) { _runtime_state._instance_mem_tracker.reset(new MemTracker()); TDescriptorTable t_desc_table; diff --git a/be/test/exec/es_predicate_test.cpp b/be/test/exec/es_predicate_test.cpp index 3c18bf1af4c454..0fd5abe1c30f88 100644 --- a/be/test/exec/es_predicate_test.cpp +++ b/be/test/exec/es_predicate_test.cpp @@ -40,7 +40,7 @@ class RuntimeState; class EsPredicateTest : public testing::Test { public: - EsPredicateTest() : _runtime_state("EsPredicateTest") { + EsPredicateTest() : _runtime_state(TQueryGlobals()) { _runtime_state._instance_mem_tracker.reset(new MemTracker()); TDescriptorTable t_desc_table; diff --git a/be/test/exec/es_scan_node_test.cpp b/be/test/exec/es_scan_node_test.cpp index adc2c0d8e3e9bd..77f2cb7cf69454 100644 --- a/be/test/exec/es_scan_node_test.cpp +++ b/be/test/exec/es_scan_node_test.cpp @@ -37,7 +37,7 @@ namespace doris { // mock class EsScanNodeTest : public testing::Test { public: - EsScanNodeTest() : _runtime_state("EsScanNodeTest") { + EsScanNodeTest() : _runtime_state(TQueryGlobals()) { _runtime_state._instance_mem_tracker.reset(new MemTracker()); TDescriptorTable t_desc_table; diff --git a/be/test/exec/parquet_scanner_test.cpp b/be/test/exec/parquet_scanner_test.cpp index 4bea0130c33672..2d96e44cdd5b95 100644 --- a/be/test/exec/parquet_scanner_test.cpp +++ b/be/test/exec/parquet_scanner_test.cpp @@ -38,7 +38,7 @@ namespace doris { class ParquetSannerTest : public testing::Test { public: - ParquetSannerTest() : _runtime_state("ParquetSannerTest") { + ParquetSannerTest() : _runtime_state(TQueryGlobals()) { init(); _runtime_state._instance_mem_tracker.reset(new MemTracker()); } diff --git a/be/test/exec/tablet_sink_test.cpp b/be/test/exec/tablet_sink_test.cpp index 5c9c8365a12aa8..c1bf8c602b0bc5 100644 --- a/be/test/exec/tablet_sink_test.cpp +++ b/be/test/exec/tablet_sink_test.cpp @@ -335,7 +335,7 @@ TEST_F(OlapTableSinkTest, normal) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -441,7 +441,7 @@ TEST_F(OlapTableSinkTest, convert) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1024; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -568,7 +568,7 @@ TEST_F(OlapTableSinkTest, init_fail1) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -626,7 +626,7 @@ TEST_F(OlapTableSinkTest, init_fail3) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -685,7 +685,7 @@ TEST_F(OlapTableSinkTest, init_fail4) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -752,7 +752,7 @@ TEST_F(OlapTableSinkTest, add_batch_failed) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -847,7 +847,7 @@ TEST_F(OlapTableSinkTest, decimal) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; diff --git a/be/test/exprs/timestamp_functions_test.cpp b/be/test/exprs/timestamp_functions_test.cpp index 792f72c5e0dda8..4fc33ccbe9e0f6 100644 --- a/be/test/exprs/timestamp_functions_test.cpp +++ b/be/test/exprs/timestamp_functions_test.cpp @@ -18,12 +18,42 @@ #include "exprs/timestamp_functions.h" #include +#include + +#include "testutil/function_utils.h" +#include "exprs/timezone_db.h" +#include "udf/udf.h" +#include "udf/udf_internal.h" +#include "runtime/runtime_state.h" +#include "runtime/exec_env.h" +#include "runtime/test_env.h" + namespace doris { +class FunctionContextImpl; class TimestampFunctionsTest : public testing::Test { public: TimestampFunctionsTest() { } + + void SetUp() { + TimezoneDatabase::init(); + + TQueryGlobals globals; + globals.__set_now_string("1565080737805"); + globals.__set_time_zone("America/Los_Angeles"); + state = new RuntimeState(globals); + utils = new FunctionUtils(state); + ctx = utils->get_fn_ctx(); + } + void TearDown() { + delete state; + delete utils; + } +private: + RuntimeState* state; + FunctionUtils* utils; + FunctionContext* ctx; }; TEST_F(TimestampFunctionsTest, day_of_week_test) { @@ -38,8 +68,6 @@ TEST_F(TimestampFunctionsTest, day_of_week_test) { } TEST_F(TimestampFunctionsTest, time_diff_test) { - doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); - DateTimeValue dt1(20190718120000); dt1.set_type(TIME_DATETIME); doris_udf::DateTimeVal tv1; @@ -50,11 +78,50 @@ TEST_F(TimestampFunctionsTest, time_diff_test) { doris_udf::DateTimeVal tv2; dt2.to_datetime_val(&tv2); - ASSERT_EQ(-3662, TimestampFunctions::time_diff(context, tv1, tv2).val); + ASSERT_EQ(-3662, TimestampFunctions::time_diff(ctx, tv1, tv2).val); +} + +TEST_F(TimestampFunctionsTest, now) { + DateTimeVal now = TimestampFunctions::now(ctx); + DateTimeValue dt = DateTimeValue::from_datetime_val(now); + ASSERT_EQ(20190806013857, dt.to_int64()); +} + +TEST_F(TimestampFunctionsTest, from_unix) { + IntVal unixtimestamp(1565080737); + StringVal sval = TimestampFunctions::from_unix(ctx, unixtimestamp); + ASSERT_EQ("2019-08-06 01:38:57", std::string((char*) sval.ptr, sval.len)); } +TEST_F(TimestampFunctionsTest, to_unix) { + DateTimeVal dt_val ; + dt_val.packed_time = 1847544683002068992; + dt_val.type = TIME_DATETIME; + ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx).val); + ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx, dt_val).val); + ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx, StringVal("2019-08-06 01:38:57"), "%Y-%m-%d %H:%i:%S").val); } +TEST_F(TimestampFunctionsTest, curtime) { + ASSERT_EQ(3600 + 38*60 + 57, TimestampFunctions::curtime(ctx).val); +} + +TEST_F(TimestampFunctionsTest, convert_tz_test) { + doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); + DateTimeValue dt1(20190806163857); + dt1.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv1; + dt1.to_datetime_val(&tv1); + DateTimeVal t = TimestampFunctions::convert_tz(context, tv1, StringVal("Asia/Shanghai"), StringVal("America/Los_Angeles")); + DateTimeValue dt2 = DateTimeValue::from_datetime_val(t); + ASSERT_EQ(20190806013857, dt2.to_int64()); + + t = TimestampFunctions::convert_tz(context, tv1, StringVal("CST"), StringVal("America/Los_Angeles")); + DateTimeValue dt3 = DateTimeValue::from_datetime_val(t); + ASSERT_EQ(20190806013857, dt3.to_int64()); +} + +} int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/be/test/runtime/buffered_block_mgr2_test.cpp b/be/test/runtime/buffered_block_mgr2_test.cpp index eb4824840d361f..0b4cfa196d3320 100644 --- a/be/test/runtime/buffered_block_mgr2_test.cpp +++ b/be/test/runtime/buffered_block_mgr2_test.cpp @@ -557,7 +557,7 @@ class BufferedBlockMgrTest : public ::testing::Test { // RuntimeState* shared_state = new RuntimeState(TExecPlanFragmentParams(), "", // _test_env->exec_env()); RuntimeState* shared_state = new RuntimeState( - TUniqueId(), TQueryOptions(), "", _test_env->exec_env()); + TUniqueId(), TQueryOptions(), TQueryGlobals(), _test_env->exec_env()); for (int i = 0; i < num_threads; ++i) { thread* t = new boost::thread(boost::bind( &BufferedBlockMgrTest::CreateDestroyThread, this, i, shared_state)); diff --git a/be/test/runtime/datetime_value_test.cpp b/be/test/runtime/datetime_value_test.cpp index 852eb57a1471dd..83effa66dfdaef 100644 --- a/be/test/runtime/datetime_value_test.cpp +++ b/be/test/runtime/datetime_value_test.cpp @@ -29,6 +29,7 @@ namespace doris { class DateTimeValueTest : public testing::Test { public: DateTimeValueTest() { + TimezoneDatabase::init(); } protected: @@ -294,7 +295,7 @@ TEST_F(DateTimeValueTest, from_unixtime) { char str[MAX_DTVALUE_STR_LEN]; DateTimeValue value; - value.from_unixtime(570672000); + value.from_unixtime(570672000, TimezoneDatabase::default_time_zone); value.to_string(str); ASSERT_STREQ("1988-02-01 08:00:00", str); } @@ -302,19 +303,25 @@ TEST_F(DateTimeValueTest, from_unixtime) { // Calculate format TEST_F(DateTimeValueTest, unix_timestamp) { DateTimeValue value; - + int64_t timestamp; value.from_date_int64(19691231); - ASSERT_EQ(0, value.unix_timestamp()); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(-115200, timestamp); value.from_date_int64(19700101); - ASSERT_EQ(0, value.unix_timestamp()); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(-28800, timestamp); value.from_date_int64(19700102); - ASSERT_EQ(86400 - 28800, value.unix_timestamp()); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(86400 - 28800, timestamp); value.from_date_int64(19880201000000); - ASSERT_EQ(570672000 - 28800, value.unix_timestamp()); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(570672000 - 28800, timestamp); value.from_date_int64(20380119); - ASSERT_EQ(2147472000 - 28800, value.unix_timestamp()); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(2147472000 - 28800, timestamp); value.from_date_int64(20380120); - ASSERT_EQ(0, value.unix_timestamp()); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(2147529600, timestamp); } // Calculate format diff --git a/docs/documentation/cn/administrator-guide/time-zone.md b/docs/documentation/cn/administrator-guide/time-zone.md new file mode 100644 index 00000000000000..a4e42b0365bbea --- /dev/null +++ b/docs/documentation/cn/administrator-guide/time-zone.md @@ -0,0 +1,53 @@ +# 时区 + +Doris 支持多时区设置 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 + +## 基本概念 + +Doris 内部存在多个时区相关参数 + +* system_time_zone : + 当服务器启动时，会根据机器设置时区自动设置，设置后不可修改。 + +* time_zone : + 服务器当前时区，区分session级别和global级别 + +## 具体操作 + +1. show variables like '%time_zone%' + + 查看当前时区相关配置 + +2. SET time_zone = 'Asia/Shanghai' + + 该命令可以设置session级别的时区，连接断开后失效 + +3. SET global time_zone = 'Asia/Shanghai' + + 该命令可以设置global级别的时区参数，fe会将参数持久化，连接断开后不失效 + +### 时区的影响 + +时区设置会影响对时区敏感的时间值的显示和存储。 + +包括NOW()或CURTIME()等时间函数显示的值，也包括show load, show backends中的时间值。 + +但不会影响create table 中时间类型分区列的less than值，也不会影响存储为date/datetime类型的值的显示。 + +## 使用限制 + +时区值可以使用几种格式给出，不区分大小写: + +* 表示UTC偏移量的字符串，如'+10:00'或'-6:00' + +* 标准时区格式，如"Asia/Shanghai"、"America/Los_Angeles" + +* 不支持缩写时区格式，如"MET"、"CTT"。因为缩写时区在不同场景下存在歧义，不建议使用。 + +* 为了兼容Doris，支持CST缩写时区，内部会将CST转移为"Asia/Shanghai"的中国标准时区 + diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md new file mode 100644 index 00000000000000..ece25505aa3c09 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md @@ -0,0 +1,27 @@ +# convert_tz + +## Syntax + +`DATETIME CONVERT_TZ(DATETIME dt, VARCHAR from_tz, VARCHAR to_tz)` + +## Description + +转换datetime值dt，从 from_tz 由给定转到 to_tz 时区给出的时区，并返回的结果值。 如果参数无效该函数返回NULL。 + +## Examples + +``` +mysql> select convert_tz('2019-08-01 13:21:03', 'Asia/Shanghai', 'America/Los_Angeles'); ++---------------------------------------------------------------------------+ +| convert_tz('2019-08-01 13:21:03', 'Asia/Shanghai', 'America/Los_Angeles') | ++---------------------------------------------------------------------------+ +| 2019-07-31 22:21:03 | ++---------------------------------------------------------------------------+ + +mysql> select convert_tz('2019-08-01 13:21:03', '+08:00', 'America/Los_Angeles'); ++--------------------------------------------------------------------+ +| convert_tz('2019-08-01 13:21:03', '+08:00', 'America/Los_Angeles') | ++--------------------------------------------------------------------+ +| 2019-07-31 22:21:03 | ++--------------------------------------------------------------------+ +``` \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md new file mode 100644 index 00000000000000..83b9b05f3a6fc8 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md @@ -0,0 +1,20 @@ +# curtime,current_time + +## Syntax + +`TIME CURTIME()` + +## Description + +õǰʱ䣬TIMEͷ + +## Examples + +``` +mysql> select current_time(); ++----------------+ +| current_time() | ++----------------+ +| 15:25:47 | ++----------------+ +``` diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java index 7adae22a674572..490cdd03d2c3d2 100644 --- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -66,6 +66,8 @@ public class TimeUtils { public static int MIN_TIME; public static int MAX_TIME; + public static String DEFAULT_TIME_ZONE = "Asia/Shanghai"; + static { TIME_ZONE = new SimpleTimeZone(8 * 3600 * 1000, ""); diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index f510eec943f15a..8cded734b13dd8 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -29,6 +29,7 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.ListUtil; import org.apache.doris.common.util.RuntimeProfile; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.LoadErrorHub; import org.apache.doris.planner.DataPartition; import org.apache.doris.planner.DataSink; @@ -84,8 +85,6 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -106,8 +105,6 @@ public class Coordinator { private static final Logger LOG = LogManager.getLogger(Coordinator.class); - private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private static String localIP = FrontendOptions.getLocalHostAddress(); // Overall status of the entire query; set to the first reported fragment error @@ -193,7 +190,12 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) { this.descTable = analyzer.getDescTbl().toThrift(); this.returnedAllResults = false; this.queryOptions = context.getSessionVariable().toThrift(); - this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); + this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); + if (context.getSessionVariable().getTimeZone().equals("CST")) { + this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); + } else { + this.queryGlobals.setTime_zone(context.getSessionVariable().getTimeZone()); + } this.tResourceInfo = new TResourceInfo(context.getQualifiedUser(), context.getSessionVariable().getResourceGroup()); this.needReport = context.getSessionVariable().isReportSucc(); @@ -213,7 +215,8 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, this.fragments = fragments; this.scanNodes = scanNodes; this.queryOptions = new TQueryOptions(); - this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); + this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); + this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); this.tResourceInfo = new TResourceInfo("", ""); this.needReport = true; this.clusterName = cluster; diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 91bbf531b00510..3ed1d73d68fa98 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -108,9 +108,9 @@ [['from_unixtime'], 'VARCHAR', ['INT', 'VARCHAR'], '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf' '15FunctionContextERKNS1_6IntValERKNS1_9StringValE'], - [['now', 'current_timestamp'], 'DATETIME', [], + [['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIME', [], '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE'], - [['curtime', 'current_time'], 'DATETIME', [], + [['curtime', 'current_time'], 'TIME', [], '_ZN5doris18TimestampFunctions7curtimeEPN9doris_udf15FunctionContextE'], [['utc_timestamp'], 'DATETIME', [], '_ZN5doris18TimestampFunctions13utc_timestampEPN9doris_udf15FunctionContextE'], @@ -218,6 +218,9 @@ '_ZN5doris18TimestampFunctions10month_nameEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeValE'], + [['convert_tz'], 'DATETIME', ['DATETIME', 'VARCHAR', 'VARCHAR'], + '_ZN5doris18TimestampFunctions10convert_tzEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValES9_'], + # Math builtin functions [['pi'], 'DOUBLE', [], '_ZN5doris13MathFunctions2piEPN9doris_udf15FunctionContextE'], diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 25428d880589a7..f8d373e5a3085d 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -182,6 +182,7 @@ struct TPlanFragmentExecParams { struct TQueryGlobals { // String containing a timestamp set as the current time. 1: required string now_string + 2: optional string time_zone }