diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 9afd78d1cc88..a0f68c76e4a8 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -873,9 +873,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.35" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", diff --git a/datafusion/core/tests/optimizer_integration.rs b/datafusion/core/tests/optimizer_integration.rs new file mode 100644 index 000000000000..f9696955769e --- /dev/null +++ b/datafusion/core/tests/optimizer_integration.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::{plan_err, Result}; +use datafusion_expr::{AggregateUDF, LogicalPlan, ScalarUDF, TableSource, WindowUDF}; +use datafusion_optimizer::analyzer::Analyzer; +use datafusion_optimizer::optimizer::Optimizer; +use datafusion_optimizer::{OptimizerConfig, OptimizerContext}; +use datafusion_sql::planner::{ContextProvider, SqlToRel}; +use datafusion_sql::sqlparser::ast::Statement; +use datafusion_sql::sqlparser::dialect::GenericDialect; +use datafusion_sql::sqlparser::parser::Parser; +use datafusion_sql::TableReference; + +use chrono::DateTime; +use datafusion_functions::datetime; + +#[cfg(test)] +#[ctor::ctor] +fn init() { + // enable logging so RUST_LOG works + let _ = env_logger::try_init(); +} + +#[test] +fn timestamp_nano_ts_none_predicates() -> Result<()> { + let sql = "SELECT col_int32 + FROM test + WHERE col_ts_nano_none < (now() - interval '1 hour')"; + let plan = test_sql(sql)?; + // a scan should have the now()... predicate folded to a single + // constant and compared to the column without a cast so it can be + // pushed down / pruned + let expected = + "Projection: test.col_int32\ + \n Filter: test.col_ts_nano_none < TimestampNanosecond(1666612093000000000, None)\ + \n TableScan: test projection=[col_int32, col_ts_nano_none]"; + assert_eq!(expected, format!("{plan:?}")); + Ok(()) +} + +#[test] +fn timestamp_nano_ts_utc_predicates() { + let sql = "SELECT col_int32 + FROM test + WHERE col_ts_nano_utc < (now() - interval '1 hour')"; + let plan = test_sql(sql).unwrap(); + // a scan should have the now()... predicate folded to a single + // constant and compared to the column without a cast so it can be + // pushed down / pruned + let expected = + "Projection: test.col_int32\n Filter: test.col_ts_nano_utc < TimestampNanosecond(1666612093000000000, Some(\"+00:00\"))\ + \n TableScan: test projection=[col_int32, col_ts_nano_utc]"; + assert_eq!(expected, format!("{plan:?}")); +} + +fn test_sql(sql: &str) -> Result { + // parse the SQL + let dialect = GenericDialect {}; // or AnsiDialect, or your own dialect ... + let ast: Vec = Parser::parse_sql(&dialect, sql).unwrap(); + let statement = &ast[0]; + + // create a logical query plan + let now_udf = datetime::functions() + .iter() + .find(|f| f.name() == "now") + .unwrap() + .to_owned(); + let context_provider = MyContextProvider::default().with_udf(now_udf); + let sql_to_rel = SqlToRel::new(&context_provider); + let plan = sql_to_rel.sql_statement_to_plan(statement.clone()).unwrap(); + + // hard code the return value of now() + let now_time = DateTime::from_timestamp(1666615693, 0).unwrap(); + let config = OptimizerContext::new() + .with_skip_failing_rules(false) + .with_query_execution_start_time(now_time); + let analyzer = Analyzer::new(); + let optimizer = Optimizer::new(); + // analyze and optimize the logical plan + let plan = analyzer.execute_and_check(&plan, config.options(), |_, _| {})?; + optimizer.optimize(&plan, &config, |_, _| {}) +} + +#[derive(Default)] +struct MyContextProvider { + options: ConfigOptions, + udfs: HashMap>, +} + +impl MyContextProvider { + fn with_udf(mut self, udf: Arc) -> Self { + self.udfs.insert(udf.name().to_string(), udf); + self + } +} + +impl ContextProvider for MyContextProvider { + fn get_table_source(&self, name: TableReference) -> Result> { + let table_name = name.table(); + if table_name.starts_with("test") { + let schema = Schema::new_with_metadata( + vec![ + Field::new("col_int32", DataType::Int32, true), + Field::new("col_uint32", DataType::UInt32, true), + Field::new("col_utf8", DataType::Utf8, true), + Field::new("col_date32", DataType::Date32, true), + Field::new("col_date64", DataType::Date64, true), + // timestamp with no timezone + Field::new( + "col_ts_nano_none", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + ), + // timestamp with UTC timezone + Field::new( + "col_ts_nano_utc", + DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())), + true, + ), + ], + HashMap::new(), + ); + + Ok(Arc::new(MyTableSource { + schema: Arc::new(schema), + })) + } else { + plan_err!("table does not exist") + } + } + + fn get_function_meta(&self, name: &str) -> Option> { + self.udfs.get(name).cloned() + } + + fn get_aggregate_meta(&self, _name: &str) -> Option> { + None + } + + fn get_variable_type(&self, _variable_names: &[String]) -> Option { + None + } + + fn get_window_meta(&self, _name: &str) -> Option> { + None + } + + fn options(&self) -> &ConfigOptions { + &self.options + } + + fn udfs_names(&self) -> Vec { + Vec::new() + } + + fn udafs_names(&self) -> Vec { + Vec::new() + } + + fn udwfs_names(&self) -> Vec { + Vec::new() + } +} + +struct MyTableSource { + schema: SchemaRef, +} + +impl TableSource for MyTableSource { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } +} diff --git a/datafusion/core/tests/simplification.rs b/datafusion/core/tests/simplification.rs index 41457df02cfc..25f994d320c1 100644 --- a/datafusion/core/tests/simplification.rs +++ b/datafusion/core/tests/simplification.rs @@ -185,10 +185,6 @@ fn make_udf_add(volatility: Volatility) -> Arc { )) } -fn now_expr() -> Expr { - call_fn("now", vec![]).unwrap() -} - fn cast_to_int64_expr(expr: Expr) -> Expr { Expr::Cast(Cast::new(expr.into(), DataType::Int64)) } @@ -255,7 +251,7 @@ fn now_less_than_timestamp() -> Result<()> { // cast(now() as int) < cast(to_timestamp(...) as int) + 50000_i64 let plan = LogicalPlanBuilder::from(table_scan) .filter( - cast_to_int64_expr(now_expr()) + cast_to_int64_expr(now()) .lt(cast_to_int64_expr(to_timestamp_expr(ts_string)) + lit(50000_i64)), )? .build()?; @@ -368,14 +364,14 @@ fn test_const_evaluator_now() { let time = chrono::Utc.timestamp_nanos(ts_nanos); let ts_string = "2020-09-08T12:05:00+00:00"; // now() --> ts - test_evaluate_with_start_time(now_expr(), lit_timestamp_nano(ts_nanos), &time); + test_evaluate_with_start_time(now(), lit_timestamp_nano(ts_nanos), &time); // CAST(now() as int64) + 100_i64 --> ts + 100_i64 - let expr = cast_to_int64_expr(now_expr()) + lit(100_i64); + let expr = cast_to_int64_expr(now()) + lit(100_i64); test_evaluate_with_start_time(expr, lit(ts_nanos + 100), &time); // CAST(now() as int64) < cast(to_timestamp(...) as int64) + 50000_i64 ---> true - let expr = cast_to_int64_expr(now_expr()) + let expr = cast_to_int64_expr(now()) .lt(cast_to_int64_expr(to_timestamp_expr(ts_string)) + lit(50000i64)); test_evaluate_with_start_time(expr, lit(true), &time); } @@ -413,3 +409,25 @@ fn test_evaluator_udfs() { )); test_evaluate(expr, expected_expr); } + +#[test] +fn multiple_now() -> Result<()> { + let table_scan = test_table_scan(); + let time = Utc::now(); + let proj = vec![now(), now().alias("t2")]; + let plan = LogicalPlanBuilder::from(table_scan) + .project(proj)? + .build()?; + + // expect the same timestamp appears in both exprs + let actual = get_optimized_plan_formatted(&plan, &time); + let expected = format!( + "Projection: TimestampNanosecond({}, Some(\"+00:00\")) AS now(), TimestampNanosecond({}, Some(\"+00:00\")) AS t2\ + \n TableScan: test", + time.timestamp_nanos_opt().unwrap(), + time.timestamp_nanos_opt().unwrap() + ); + + assert_eq!(expected, actual); + Ok(()) +} diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index f9725a5514d3..0593ed4703bf 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -204,14 +204,6 @@ pub enum BuiltinScalarFunction { Substr, /// to_hex ToHex, - /// from_unixtime - FromUnixtime, - ///now - Now, - ///current_date - CurrentDate, - /// current_time - CurrentTime, /// make_date MakeDate, /// translate @@ -369,17 +361,11 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::Translate => Volatility::Immutable, BuiltinScalarFunction::Trim => Volatility::Immutable, BuiltinScalarFunction::Upper => Volatility::Immutable, - BuiltinScalarFunction::FromUnixtime => Volatility::Immutable, BuiltinScalarFunction::OverLay => Volatility::Immutable, BuiltinScalarFunction::Levenshtein => Volatility::Immutable, BuiltinScalarFunction::SubstrIndex => Volatility::Immutable, BuiltinScalarFunction::FindInSet => Volatility::Immutable, - // Stable builtin functions - BuiltinScalarFunction::Now => Volatility::Stable, - BuiltinScalarFunction::CurrentDate => Volatility::Stable, - BuiltinScalarFunction::CurrentTime => Volatility::Stable, - // Volatile builtin functions BuiltinScalarFunction::Random => Volatility::Volatile, BuiltinScalarFunction::Uuid => Volatility::Volatile, @@ -396,7 +382,6 @@ impl BuiltinScalarFunction { /// 2. Deduce the output `DataType` based on the provided `input_expr_types`. pub fn return_type(self, input_expr_types: &[DataType]) -> Result { use DataType::*; - use TimeUnit::*; // Note that this function *must* return the same type that the respective physical expression returns // or the execution panics. @@ -544,12 +529,6 @@ impl BuiltinScalarFunction { utf8_to_int_type(&input_expr_types[0], "find_in_set") } BuiltinScalarFunction::ToChar => Ok(Utf8), - BuiltinScalarFunction::FromUnixtime => Ok(Timestamp(Second, None)), - BuiltinScalarFunction::Now => { - Ok(Timestamp(Nanosecond, Some("+00:00".into()))) - } - BuiltinScalarFunction::CurrentDate => Ok(Date32), - BuiltinScalarFunction::CurrentTime => Ok(Time64(Nanosecond)), BuiltinScalarFunction::MakeDate => Ok(Date32), BuiltinScalarFunction::Translate => { utf8_to_str_type(&input_expr_types[0], "translate") @@ -757,9 +736,6 @@ impl BuiltinScalarFunction { ], self.volatility(), ), - BuiltinScalarFunction::FromUnixtime => { - Signature::uniform(1, vec![Int64], self.volatility()) - } BuiltinScalarFunction::Digest => Signature::one_of( vec![ Exact(vec![Utf8, Utf8]), @@ -904,11 +880,6 @@ impl BuiltinScalarFunction { // will be as good as the number of digits in the number Signature::uniform(1, vec![Float64, Float32], self.volatility()) } - BuiltinScalarFunction::Now - | BuiltinScalarFunction::CurrentDate - | BuiltinScalarFunction::CurrentTime => { - Signature::uniform(0, vec![], self.volatility()) - } BuiltinScalarFunction::MakeDate => Signature::uniform( 3, vec![Int32, Int64, UInt32, UInt64, Utf8], @@ -1032,12 +1003,8 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::FindInSet => &["find_in_set"], // time/date functions - BuiltinScalarFunction::Now => &["now"], - BuiltinScalarFunction::CurrentDate => &["current_date", "today"], - BuiltinScalarFunction::CurrentTime => &["current_time"], BuiltinScalarFunction::MakeDate => &["make_date"], BuiltinScalarFunction::ToChar => &["to_char", "date_format"], - BuiltinScalarFunction::FromUnixtime => &["from_unixtime"], // hashing functions BuiltinScalarFunction::Digest => &["digest"], diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index 56833d452f19..8212f75583ea 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -788,15 +788,6 @@ scalar_expr!( datetime format, "converts a date, time, timestamp or duration to a string based on the provided format" ); -scalar_expr!( - FromUnixtime, - from_unixtime, - unixtime, - "returns the unix time in format" -); -scalar_expr!(CurrentDate, current_date, ,"returns current UTC date as a [`DataType::Date32`] value"); -scalar_expr!(Now, now, ,"returns current timestamp in nanoseconds, using the same value for all instances of now() in same statement"); -scalar_expr!(CurrentTime, current_time, , "returns current UTC time as a [`DataType::Time64`] value"); scalar_expr!(MakeDate, make_date, year month day, "make a date from year, month and day component parts"); scalar_expr!(Nanvl, nanvl, x y, "returns x if x is not NaN otherwise returns y"); scalar_expr!( @@ -1258,8 +1249,6 @@ mod test { test_scalar_expr!(Trim, trim, string); test_scalar_expr!(Upper, upper, string); - test_scalar_expr!(FromUnixtime, from_unixtime, unixtime); - test_scalar_expr!(ArrayPopFront, array_pop_front, array); test_scalar_expr!(ArrayPopBack, array_pop_back, array); test_scalar_expr!(ArrayPosition, array_position, array, element, index); diff --git a/datafusion/expr/src/signature.rs b/datafusion/expr/src/signature.rs index ad29fe0724a4..0d65c068c4a0 100644 --- a/datafusion/expr/src/signature.rs +++ b/datafusion/expr/src/signature.rs @@ -44,7 +44,7 @@ pub enum Volatility { Immutable, /// A stable function may return different values given the same input across different /// queries but must return the same value for a given input within a query. An example of - /// this is [super::BuiltinScalarFunction::Now]. DataFusion + /// this is the `Now` function. DataFusion /// will attempt to inline `Stable` functions during planning, when possible. /// For query `select col1, now() from t1`, it might take a while to execute but /// `now()` column will be the same for each output row, which is evaluated diff --git a/datafusion/functions/src/datetime/current_date.rs b/datafusion/functions/src/datetime/current_date.rs new file mode 100644 index 000000000000..5338234a8e49 --- /dev/null +++ b/datafusion/functions/src/datetime/current_date.rs @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use arrow::datatypes::DataType; +use arrow::datatypes::DataType::Date32; +use chrono::{Datelike, NaiveDate}; + +use datafusion_common::{internal_err, Result, ScalarValue}; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; +use datafusion_expr::{ColumnarValue, Expr, ScalarUDFImpl, Signature, Volatility}; + +#[derive(Debug)] +pub(super) struct CurrentDateFunc { + signature: Signature, + aliases: Vec, +} + +impl CurrentDateFunc { + pub fn new() -> Self { + Self { + signature: Signature::uniform(0, vec![], Volatility::Stable), + aliases: vec![String::from("today")], + } + } +} + +/// Create an implementation of `current_date()` that always returns the +/// specified current date. +/// +/// The semantics of `current_date()` require it to return the same value +/// wherever it appears within a single statement. This value is +/// chosen during planning time. +impl ScalarUDFImpl for CurrentDateFunc { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "current_date" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(Date32) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> Result { + internal_err!( + "invoke should not be called on a simplified current_date() function" + ) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn simplify( + &self, + _args: Vec, + info: &dyn SimplifyInfo, + ) -> Result { + let now_ts = info.execution_props().query_execution_start_time; + let days = Some( + now_ts.num_days_from_ce() + - NaiveDate::from_ymd_opt(1970, 1, 1) + .unwrap() + .num_days_from_ce(), + ); + Ok(ExprSimplifyResult::Simplified(Expr::Literal( + ScalarValue::Date32(days), + ))) + } +} diff --git a/datafusion/functions/src/datetime/current_time.rs b/datafusion/functions/src/datetime/current_time.rs new file mode 100644 index 000000000000..b8a8aa2acb53 --- /dev/null +++ b/datafusion/functions/src/datetime/current_time.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use arrow::datatypes::DataType; +use arrow::datatypes::DataType::Time64; +use arrow::datatypes::TimeUnit::Nanosecond; + +use datafusion_common::{internal_err, Result, ScalarValue}; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; +use datafusion_expr::{ColumnarValue, Expr, ScalarUDFImpl, Signature, Volatility}; + +#[derive(Debug)] +pub(super) struct CurrentTimeFunc { + signature: Signature, +} + +impl CurrentTimeFunc { + pub fn new() -> Self { + Self { + signature: Signature::uniform(0, vec![], Volatility::Stable), + } + } +} + +/// Create an implementation of `current_time()` that always returns the +/// specified current time. +/// +/// The semantics of `current_time()` require it to return the same value +/// wherever it appears within a single statement. This value is +/// chosen during planning time. +impl ScalarUDFImpl for CurrentTimeFunc { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "current_time" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(Time64(Nanosecond)) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> Result { + internal_err!( + "invoke should not be called on a simplified current_time() function" + ) + } + + fn simplify( + &self, + _args: Vec, + info: &dyn SimplifyInfo, + ) -> Result { + let now_ts = info.execution_props().query_execution_start_time; + let nano = now_ts.timestamp_nanos_opt().map(|ts| ts % 86400000000000); + Ok(ExprSimplifyResult::Simplified(Expr::Literal( + ScalarValue::Time64Nanosecond(nano), + ))) + } +} diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs new file mode 100644 index 000000000000..f0d5016c0db9 --- /dev/null +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use arrow::datatypes::DataType; +use arrow::datatypes::DataType::{Int64, Timestamp}; +use arrow::datatypes::TimeUnit::Second; + +use datafusion_common::{exec_err, Result}; +use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; + +#[derive(Debug)] +pub(super) struct FromUnixtimeFunc { + signature: Signature, +} + +impl FromUnixtimeFunc { + pub fn new() -> Self { + Self { + signature: Signature::uniform(1, vec![Int64], Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for FromUnixtimeFunc { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "from_unixtime" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(Timestamp(Second, None)) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { + if args.len() != 1 { + return exec_err!( + "from_unixtime function requires 1 argument, got {}", + args.len() + ); + } + + match args[0].data_type() { + Int64 => args[0].cast_to(&Timestamp(Second, None), None), + other => { + exec_err!( + "Unsupported data type {:?} for function from_unixtime", + other + ) + } + } + } +} diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index 7792174a7684..4702820782c6 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -22,17 +22,29 @@ use std::sync::Arc; use datafusion_expr::ScalarUDF; mod common; +mod current_date; +mod current_time; mod date_bin; mod date_part; mod date_trunc; +mod from_unixtime; +mod now; mod to_date; mod to_timestamp; mod to_unixtime; // create UDFs +make_udf_function!(current_date::CurrentDateFunc, CURRENT_DATE, current_date); +make_udf_function!(current_time::CurrentTimeFunc, CURRENT_TIME, current_time); make_udf_function!(date_bin::DateBinFunc, DATE_BIN, date_bin); make_udf_function!(date_part::DatePartFunc, DATE_PART, date_part); make_udf_function!(date_trunc::DateTruncFunc, DATE_TRUNC, date_trunc); +make_udf_function!( + from_unixtime::FromUnixtimeFunc, + FROM_UNIXTIME, + from_unixtime +); +make_udf_function!(now::NowFunc, NOW, now); make_udf_function!(to_date::ToDateFunc, TO_DATE, to_date); make_udf_function!(to_unixtime::ToUnixtimeFunc, TO_UNIXTIME, to_unixtime); make_udf_function!(to_timestamp::ToTimestampFunc, TO_TIMESTAMP, to_timestamp); @@ -63,6 +75,16 @@ make_udf_function!( pub mod expr_fn { use datafusion_expr::Expr; + #[doc = "returns current UTC date as a Date32 value"] + pub fn current_date() -> Expr { + super::current_date().call(vec![]) + } + + #[doc = "returns current UTC time as a Time64 value"] + pub fn current_time() -> Expr { + super::current_time().call(vec![]) + } + #[doc = "coerces an arbitrary timestamp to the start of the nearest specified interval"] pub fn date_bin(stride: Expr, source: Expr, origin: Expr) -> Expr { super::date_bin().call(vec![stride, source, origin]) @@ -78,6 +100,16 @@ pub mod expr_fn { super::date_trunc().call(vec![part, date]) } + #[doc = "converts an integer to RFC3339 timestamp format string"] + pub fn from_unixtime(unixtime: Expr) -> Expr { + super::from_unixtime().call(vec![unixtime]) + } + + #[doc = "returns the current timestamp in nanoseconds, using the same value for all instances of now() in same statement"] + pub fn now() -> Expr { + super::now().call(vec![]) + } + /// ```ignore /// # use std::sync::Arc; /// @@ -162,9 +194,13 @@ pub mod expr_fn { /// Return a list of all functions in this package pub fn functions() -> Vec> { vec![ + current_date(), + current_time(), date_bin(), date_part(), date_trunc(), + from_unixtime(), + now(), to_date(), to_unixtime(), to_timestamp(), diff --git a/datafusion/functions/src/datetime/now.rs b/datafusion/functions/src/datetime/now.rs new file mode 100644 index 000000000000..cc7979df0d86 --- /dev/null +++ b/datafusion/functions/src/datetime/now.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use arrow::datatypes::DataType; +use arrow::datatypes::DataType::Timestamp; +use arrow::datatypes::TimeUnit::Nanosecond; + +use datafusion_common::{internal_err, Result, ScalarValue}; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; +use datafusion_expr::{ColumnarValue, Expr, ScalarUDFImpl, Signature, Volatility}; + +#[derive(Debug)] +pub(super) struct NowFunc { + signature: Signature, +} + +impl NowFunc { + pub fn new() -> Self { + Self { + signature: Signature::uniform(0, vec![], Volatility::Stable), + } + } +} + +/// Create an implementation of `now()` that always returns the +/// specified timestamp. +/// +/// The semantics of `now()` require it to return the same value +/// wherever it appears within a single statement. This value is +/// chosen during planning time. +impl ScalarUDFImpl for NowFunc { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "now" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(Timestamp(Nanosecond, Some("+00:00".into()))) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> Result { + internal_err!("invoke should not be called on a simplified now() function") + } + + fn simplify( + &self, + _args: Vec, + info: &dyn SimplifyInfo, + ) -> Result { + let now_ts = info + .execution_props() + .query_execution_start_time + .timestamp_nanos_opt(); + Ok(ExprSimplifyResult::Simplified(Expr::Literal( + ScalarValue::TimestampNanosecond(now_ts, Some("+00:00".into())), + ))) + } +} diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs index 00d60d0a80dc..70b163acc208 100644 --- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs +++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs @@ -146,7 +146,6 @@ mod tests { }; use datafusion_expr::{call_fn, or, BinaryExpr, Cast, Operator}; - use crate::simplify_expressions::utils::for_test::now_expr; use crate::test::{assert_fields_eq, test_table_scan_with_name}; use crate::OptimizerContext; @@ -446,28 +445,6 @@ mod tests { Ok(()) } - #[test] - fn multiple_now_expr() -> Result<()> { - let table_scan = test_table_scan(); - let time = Utc::now(); - let proj = vec![now_expr(), now_expr().alias("t2")]; - let plan = LogicalPlanBuilder::from(table_scan) - .project(proj)? - .build()?; - - // expect the same timestamp appears in both exprs - let actual = get_optimized_plan_formatted(&plan, &time); - let expected = format!( - "Projection: TimestampNanosecond({}, Some(\"+00:00\")) AS now(), TimestampNanosecond({}, Some(\"+00:00\")) AS t2\ - \n TableScan: test", - time.timestamp_nanos_opt().unwrap(), - time.timestamp_nanos_opt().unwrap() - ); - - assert_eq!(expected, actual); - Ok(()) - } - #[test] fn simplify_and_eval() -> Result<()> { // demonstrate a case where the evaluation needs to run prior diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index 8952d5d79856..1dd3a6162894 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -530,12 +530,3 @@ pub fn simpl_concat_ws(delimiter: &Expr, args: &[Expr]) -> Result { )), } } - -#[cfg(test)] -pub mod for_test { - use datafusion_expr::{call_fn, Expr}; - - pub fn now_expr() -> Expr { - call_fn("now", vec![]).unwrap() - } -} diff --git a/datafusion/optimizer/tests/optimizer_integration.rs b/datafusion/optimizer/tests/optimizer_integration.rs index b02623854b8a..acafc0bafaf4 100644 --- a/datafusion/optimizer/tests/optimizer_integration.rs +++ b/datafusion/optimizer/tests/optimizer_integration.rs @@ -32,8 +32,6 @@ use datafusion_sql::sqlparser::dialect::GenericDialect; use datafusion_sql::sqlparser::parser::Parser; use datafusion_sql::TableReference; -use chrono::DateTime; - #[cfg(test)] #[ctor::ctor] fn init() { @@ -234,38 +232,6 @@ fn concat_ws_literals() -> Result<()> { Ok(()) } -#[test] -fn timestamp_nano_ts_none_predicates() -> Result<()> { - let sql = "SELECT col_int32 - FROM test - WHERE col_ts_nano_none < (now() - interval '1 hour')"; - let plan = test_sql(sql)?; - // a scan should have the now()... predicate folded to a single - // constant and compared to the column without a cast so it can be - // pushed down / pruned - let expected = - "Projection: test.col_int32\ - \n Filter: test.col_ts_nano_none < TimestampNanosecond(1666612093000000000, None)\ - \n TableScan: test projection=[col_int32, col_ts_nano_none]"; - assert_eq!(expected, format!("{plan:?}")); - Ok(()) -} - -#[test] -fn timestamp_nano_ts_utc_predicates() { - let sql = "SELECT col_int32 - FROM test - WHERE col_ts_nano_utc < (now() - interval '1 hour')"; - let plan = test_sql(sql).unwrap(); - // a scan should have the now()... predicate folded to a single - // constant and compared to the column without a cast so it can be - // pushed down / pruned - let expected = - "Projection: test.col_int32\n Filter: test.col_ts_nano_utc < TimestampNanosecond(1666612093000000000, Some(\"+00:00\"))\ - \n TableScan: test projection=[col_int32, col_ts_nano_utc]"; - assert_eq!(expected, format!("{plan:?}")); -} - #[test] fn propagate_empty_relation() { let sql = "SELECT test.col_int32 FROM test JOIN ( SELECT col_int32 FROM test WHERE false ) AS ta1 ON test.col_int32 = ta1.col_int32;"; @@ -340,17 +306,11 @@ fn test_sql(sql: &str) -> Result { let dialect = GenericDialect {}; // or AnsiDialect, or your own dialect ... let ast: Vec = Parser::parse_sql(&dialect, sql).unwrap(); let statement = &ast[0]; - - // create a logical query plan let context_provider = MyContextProvider::default(); let sql_to_rel = SqlToRel::new(&context_provider); let plan = sql_to_rel.sql_statement_to_plan(statement.clone()).unwrap(); - // hard code the return value of now() - let now_time = DateTime::from_timestamp(1666615693, 0).unwrap(); - let config = OptimizerContext::new() - .with_skip_failing_rules(false) - .with_query_execution_start_time(now_time); + let config = OptimizerContext::new().with_skip_failing_rules(false); let analyzer = Analyzer::new(); let optimizer = Optimizer::new(); // analyze and optimize the logical plan diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs index 6a9a03bec071..e0e86e7bd44b 100644 --- a/datafusion/physical-expr/src/datetime_expressions.rs +++ b/datafusion/physical-expr/src/datetime_expressions.rs @@ -19,7 +19,6 @@ use std::sync::Arc; -use arrow::datatypes::TimeUnit; use arrow::util::display::{ArrayFormatter, DurationFormat, FormatOptions}; use arrow::{ array::{Array, ArrayRef, PrimitiveArray}, @@ -29,59 +28,12 @@ use arrow_array::builder::PrimitiveBuilder; use arrow_array::cast::AsArray; use arrow_array::types::{Date32Type, Int32Type}; use arrow_array::StringArray; -use chrono::{DateTime, Datelike, NaiveDate, Utc}; +use chrono::prelude::*; +use chrono::NaiveDate; + use datafusion_common::{exec_err, Result, ScalarValue}; use datafusion_expr::ColumnarValue; -/// Create an implementation of `now()` that always returns the -/// specified timestamp. -/// -/// The semantics of `now()` require it to return the same value -/// wherever it appears within a single statement. This value is -/// chosen during planning time. -pub fn make_now( - now_ts: DateTime, -) -> impl Fn(&[ColumnarValue]) -> Result { - let now_ts = now_ts.timestamp_nanos_opt(); - move |_arg| { - Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - now_ts, - Some("+00:00".into()), - ))) - } -} - -/// Create an implementation of `current_date()` that always returns the -/// specified current date. -/// -/// The semantics of `current_date()` require it to return the same value -/// wherever it appears within a single statement. This value is -/// chosen during planning time. -pub fn make_current_date( - now_ts: DateTime, -) -> impl Fn(&[ColumnarValue]) -> Result { - let days = Some( - now_ts.num_days_from_ce() - - NaiveDate::from_ymd_opt(1970, 1, 1) - .unwrap() - .num_days_from_ce(), - ); - move |_arg| Ok(ColumnarValue::Scalar(ScalarValue::Date32(days))) -} - -/// Create an implementation of `current_time()` that always returns the -/// specified current time. -/// -/// The semantics of `current_time()` require it to return the same value -/// wherever it appears within a single statement. This value is -/// chosen during planning time. -pub fn make_current_time( - now_ts: DateTime, -) -> impl Fn(&[ColumnarValue]) -> Result { - let nano = now_ts.timestamp_nanos_opt().map(|ts| ts % 86400000000000); - move |_arg| Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(nano))) -} - /// Returns a string representation of a date, time, timestamp or duration based /// on a Chrono pattern. /// @@ -399,28 +351,6 @@ pub fn make_date(args: &[ColumnarValue]) -> Result { } } -/// from_unixtime() SQL function implementation -pub fn from_unixtime_invoke(args: &[ColumnarValue]) -> Result { - if args.len() != 1 { - return exec_err!( - "from_unixtime function requires 1 argument, got {}", - args.len() - ); - } - - match args[0].data_type() { - DataType::Int64 => { - args[0].cast_to(&DataType::Timestamp(TimeUnit::Second, None), None) - } - other => { - exec_err!( - "Unsupported data type {:?} for function from_unixtime", - other - ) - } - } -} - #[cfg(test)] mod tests { use std::sync::Arc; @@ -432,7 +362,6 @@ mod tests { TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt32Array, }; - use chrono::{NaiveDateTime, Timelike}; use datafusion_common::ScalarValue; diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs index 2ac98a155322..072e4ba47e24 100644 --- a/datafusion/physical-expr/src/functions.rs +++ b/datafusion/physical-expr/src/functions.rs @@ -240,7 +240,7 @@ where /// Create a physical scalar function. pub fn create_physical_fun( fun: &BuiltinScalarFunction, - execution_props: &ExecutionProps, + _execution_props: &ExecutionProps, ) -> Result { Ok(match fun { // math functions @@ -414,29 +414,8 @@ pub fn create_physical_fun( BuiltinScalarFunction::ConcatWithSeparator => Arc::new(|args| { make_scalar_function_inner(string_expressions::concat_ws)(args) }), - BuiltinScalarFunction::Now => { - // bind value for now at plan time - Arc::new(datetime_expressions::make_now( - execution_props.query_execution_start_time, - )) - } - BuiltinScalarFunction::CurrentDate => { - // bind value for current_date at plan time - Arc::new(datetime_expressions::make_current_date( - execution_props.query_execution_start_time, - )) - } - BuiltinScalarFunction::CurrentTime => { - // bind value for current_time at plan time - Arc::new(datetime_expressions::make_current_time( - execution_props.query_execution_start_time, - )) - } BuiltinScalarFunction::MakeDate => Arc::new(datetime_expressions::make_date), BuiltinScalarFunction::ToChar => Arc::new(datetime_expressions::to_char), - BuiltinScalarFunction::FromUnixtime => { - Arc::new(datetime_expressions::from_unixtime_invoke) - } BuiltinScalarFunction::InitCap => Arc::new(|args| match args[0].data_type() { DataType::Utf8 => { make_scalar_function_inner(string_expressions::initcap::)(args) @@ -2622,7 +2601,6 @@ mod tests { let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); let funs = [ - BuiltinScalarFunction::Now, BuiltinScalarFunction::Pi, BuiltinScalarFunction::Random, BuiltinScalarFunction::Uuid, diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index 2378ff524309..e6ee41fadb9f 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -600,19 +600,19 @@ enum ScalarFunction { // 56 was ToTimestampMillis // 57 was ToTimestampMicros // 58 was ToTimestampSeconds - Now = 59; + // 59 was Now Translate = 60; Trim = 61; Upper = 62; Coalesce = 63; Power = 64; // 65 was StructFun - FromUnixtime = 66; + // 66 was FromUnixtime Atan2 = 67; // 68 was DateBin // 69 was ArrowTypeof - CurrentDate = 70; - CurrentTime = 71; + // 70 was CurrentDate + // 71 was CurrentTime Uuid = 72; Cbrt = 73; Acosh = 74; diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 37cc1a45785b..be26ccee18c4 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -23157,16 +23157,12 @@ impl serde::Serialize for ScalarFunction { Self::Strpos => "Strpos", Self::Substr => "Substr", Self::ToHex => "ToHex", - Self::Now => "Now", Self::Translate => "Translate", Self::Trim => "Trim", Self::Upper => "Upper", Self::Coalesce => "Coalesce", Self::Power => "Power", - Self::FromUnixtime => "FromUnixtime", Self::Atan2 => "Atan2", - Self::CurrentDate => "CurrentDate", - Self::CurrentTime => "CurrentTime", Self::Uuid => "Uuid", Self::Cbrt => "Cbrt", Self::Acosh => "Acosh", @@ -23264,16 +23260,12 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Strpos", "Substr", "ToHex", - "Now", "Translate", "Trim", "Upper", "Coalesce", "Power", - "FromUnixtime", "Atan2", - "CurrentDate", - "CurrentTime", "Uuid", "Cbrt", "Acosh", @@ -23400,16 +23392,12 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Strpos" => Ok(ScalarFunction::Strpos), "Substr" => Ok(ScalarFunction::Substr), "ToHex" => Ok(ScalarFunction::ToHex), - "Now" => Ok(ScalarFunction::Now), "Translate" => Ok(ScalarFunction::Translate), "Trim" => Ok(ScalarFunction::Trim), "Upper" => Ok(ScalarFunction::Upper), "Coalesce" => Ok(ScalarFunction::Coalesce), "Power" => Ok(ScalarFunction::Power), - "FromUnixtime" => Ok(ScalarFunction::FromUnixtime), "Atan2" => Ok(ScalarFunction::Atan2), - "CurrentDate" => Ok(ScalarFunction::CurrentDate), - "CurrentTime" => Ok(ScalarFunction::CurrentTime), "Uuid" => Ok(ScalarFunction::Uuid), "Cbrt" => Ok(ScalarFunction::Cbrt), "Acosh" => Ok(ScalarFunction::Acosh), diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index c557fb48b191..54d3bffae198 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -2902,19 +2902,19 @@ pub enum ScalarFunction { /// 56 was ToTimestampMillis /// 57 was ToTimestampMicros /// 58 was ToTimestampSeconds - Now = 59, + /// 59 was Now Translate = 60, Trim = 61, Upper = 62, Coalesce = 63, Power = 64, /// 65 was StructFun - FromUnixtime = 66, + /// 66 was FromUnixtime Atan2 = 67, /// 68 was DateBin /// 69 was ArrowTypeof - CurrentDate = 70, - CurrentTime = 71, + /// 70 was CurrentDate + /// 71 was CurrentTime Uuid = 72, Cbrt = 73, Acosh = 74, @@ -3035,16 +3035,12 @@ impl ScalarFunction { ScalarFunction::Strpos => "Strpos", ScalarFunction::Substr => "Substr", ScalarFunction::ToHex => "ToHex", - ScalarFunction::Now => "Now", ScalarFunction::Translate => "Translate", ScalarFunction::Trim => "Trim", ScalarFunction::Upper => "Upper", ScalarFunction::Coalesce => "Coalesce", ScalarFunction::Power => "Power", - ScalarFunction::FromUnixtime => "FromUnixtime", ScalarFunction::Atan2 => "Atan2", - ScalarFunction::CurrentDate => "CurrentDate", - ScalarFunction::CurrentTime => "CurrentTime", ScalarFunction::Uuid => "Uuid", ScalarFunction::Cbrt => "Cbrt", ScalarFunction::Acosh => "Acosh", @@ -3136,16 +3132,12 @@ impl ScalarFunction { "Strpos" => Some(Self::Strpos), "Substr" => Some(Self::Substr), "ToHex" => Some(Self::ToHex), - "Now" => Some(Self::Now), "Translate" => Some(Self::Translate), "Trim" => Some(Self::Trim), "Upper" => Some(Self::Upper), "Coalesce" => Some(Self::Coalesce), "Power" => Some(Self::Power), - "FromUnixtime" => Some(Self::FromUnixtime), "Atan2" => Some(Self::Atan2), - "CurrentDate" => Some(Self::CurrentDate), - "CurrentTime" => Some(Self::CurrentTime), "Uuid" => Some(Self::Uuid), "Cbrt" => Some(Self::Cbrt), "Acosh" => Some(Self::Acosh), diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index e2a1b219d09a..fb7e82d51c52 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -52,17 +52,17 @@ use datafusion_expr::{ array_replace, array_replace_all, array_replace_n, array_resize, array_slice, array_union, ascii, asinh, atan, atan2, atanh, bit_length, btrim, cbrt, ceil, character_length, chr, coalesce, concat_expr, concat_ws_expr, cos, cosh, cot, - current_date, current_time, degrees, digest, ends_with, exp, + degrees, digest, ends_with, exp, expr::{self, InList, Sort, WindowFunction}, - factorial, find_in_set, floor, from_unixtime, gcd, initcap, iszero, lcm, left, - levenshtein, ln, log, log10, log2, + factorial, find_in_set, floor, gcd, initcap, iszero, lcm, left, levenshtein, ln, log, + log10, log2, logical_plan::{PlanType, StringifiedPlan}, - lower, lpad, ltrim, md5, nanvl, now, octet_length, overlay, pi, power, radians, - random, repeat, replace, reverse, right, round, rpad, rtrim, sha224, sha256, sha384, - sha512, signum, sin, sinh, split_part, sqrt, starts_with, strpos, substr, - substr_index, substring, to_hex, translate, trim, trunc, upper, uuid, - AggregateFunction, Between, BinaryExpr, BuiltInWindowFunction, BuiltinScalarFunction, - Case, Cast, Expr, GetFieldAccess, GetIndexedField, GroupingSet, + lower, lpad, ltrim, md5, nanvl, octet_length, overlay, pi, power, radians, random, + repeat, replace, reverse, right, round, rpad, rtrim, sha224, sha256, sha384, sha512, + signum, sin, sinh, split_part, sqrt, starts_with, strpos, substr, substr_index, + substring, to_hex, translate, trim, trunc, upper, uuid, AggregateFunction, Between, + BinaryExpr, BuiltInWindowFunction, BuiltinScalarFunction, Case, Cast, Expr, + GetFieldAccess, GetIndexedField, GroupingSet, GroupingSet::GroupingSets, JoinConstraint, JoinType, Like, Operator, TryCast, WindowFrame, WindowFrameBound, WindowFrameUnits, @@ -519,16 +519,12 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction { ScalarFunction::Substr => Self::Substr, ScalarFunction::ToHex => Self::ToHex, ScalarFunction::ToChar => Self::ToChar, - ScalarFunction::Now => Self::Now, - ScalarFunction::CurrentDate => Self::CurrentDate, - ScalarFunction::CurrentTime => Self::CurrentTime, ScalarFunction::MakeDate => Self::MakeDate, ScalarFunction::Uuid => Self::Uuid, ScalarFunction::Translate => Self::Translate, ScalarFunction::Coalesce => Self::Coalesce, ScalarFunction::Pi => Self::Pi, ScalarFunction::Power => Self::Power, - ScalarFunction::FromUnixtime => Self::FromUnixtime, ScalarFunction::Atan2 => Self::Atan2, ScalarFunction::Nanvl => Self::Nanvl, ScalarFunction::Iszero => Self::Iszero, @@ -1677,7 +1673,6 @@ pub fn parse_expr( args, ))) } - ScalarFunction::Now => Ok(now()), ScalarFunction::Translate => Ok(translate( parse_expr(&args[0], registry, codec)?, parse_expr(&args[1], registry, codec)?, @@ -1698,15 +1693,10 @@ pub fn parse_expr( parse_expr(&args[0], registry, codec)?, parse_expr(&args[1], registry, codec)?, )), - ScalarFunction::FromUnixtime => { - Ok(from_unixtime(parse_expr(&args[0], registry, codec)?)) - } ScalarFunction::Atan2 => Ok(atan2( parse_expr(&args[0], registry, codec)?, parse_expr(&args[1], registry, codec)?, )), - ScalarFunction::CurrentDate => Ok(current_date()), - ScalarFunction::CurrentTime => Ok(current_time()), ScalarFunction::Cot => Ok(cot(parse_expr(&args[0], registry, codec)?)), ScalarFunction::Nanvl => Ok(nanvl( parse_expr(&args[0], registry, codec)?, diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index b89e89570d89..65b4c8ba0445 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -1501,15 +1501,11 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction { BuiltinScalarFunction::Strpos => Self::Strpos, BuiltinScalarFunction::Substr => Self::Substr, BuiltinScalarFunction::ToHex => Self::ToHex, - BuiltinScalarFunction::Now => Self::Now, - BuiltinScalarFunction::CurrentDate => Self::CurrentDate, - BuiltinScalarFunction::CurrentTime => Self::CurrentTime, BuiltinScalarFunction::MakeDate => Self::MakeDate, BuiltinScalarFunction::Translate => Self::Translate, BuiltinScalarFunction::Coalesce => Self::Coalesce, BuiltinScalarFunction::Pi => Self::Pi, BuiltinScalarFunction::Power => Self::Power, - BuiltinScalarFunction::FromUnixtime => Self::FromUnixtime, BuiltinScalarFunction::Atan2 => Self::Atan2, BuiltinScalarFunction::Nanvl => Self::Nanvl, BuiltinScalarFunction::Iszero => Self::Iszero,