fix compilation issues

apache · Dec 27, 2023 · bb49765 · bb49765
1 parent fec0d43
commit bb49765
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 85 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -49,7 +49,7 @@ uuid = { version = "1.3", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] }
 async-trait = "0.1"
 futures = "0.3"
-object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] }
+object_store = { version = "0.8.0", features = ["aws", "gcp", "azure"] }
 parking_lot = "0.12"
 regex-syntax = "0.8.1"
 syn = "2.0.37"

diff --git a/src/common/data_type.rs b/src/common/data_type.rs
@@ -291,10 +291,14 @@ impl DataTypeMap {
             ScalarValue::List(arr) => Ok(arr.data_type().to_owned()),
             ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())),
             ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)),
-            ScalarValue::FixedSizeList(array_ref) => Ok(DataType::FixedSizeList(
-                array_ref.to_owned(),
-                array_ref.len() as i32,
-            )),
+            ScalarValue::FixedSizeList(_array_ref) => {
+                // Ok(DataType::FixedSizeList(
+                //     array_ref.to_owned(),
+                //     array_ref.len() as i32,
+                // ))
+                todo!()
+            }
+            ScalarValue::LargeList(_) => todo!(),
             ScalarValue::DurationSecond(_) => Ok(DataType::Duration(TimeUnit::Second)),
             ScalarValue::DurationMillisecond(_) => Ok(DataType::Duration(TimeUnit::Millisecond)),
             ScalarValue::DurationMicrosecond(_) => Ok(DataType::Duration(TimeUnit::Microsecond)),

diff --git a/src/dataframe.rs b/src/dataframe.rs
@@ -36,7 +36,7 @@ use std::sync::Arc;
 /// The actual execution of a plan runs natively on Rust and Arrow on a multi-threaded environment.
 #[pyclass(name = "DataFrame", module = "datafusion", subclass)]
 #[derive(Clone)]
-pub(crate) struct PyDataFrame {
+pub struct PyDataFrame {
     df: Arc<DataFrame>,
 }
 

diff --git a/src/expr.rs b/src/expr.rs
@@ -346,7 +346,8 @@ impl PyExpr {
                 ScalarValue::DurationMillisecond(v) => v.into_py(py),
                 ScalarValue::Struct(_, _) => todo!(),
                 ScalarValue::Dictionary(_, _) => todo!(),
-                ScalarValue::FixedSizeList(_, _, _) => todo!(),
+                ScalarValue::FixedSizeList(_) => todo!(),
+                ScalarValue::LargeList(_) => todo!(),
             }),
             _ => Err(py_type_err(format!(
                 "Non Expr::Literal encountered in types: {:?}",

diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs
@@ -44,12 +44,22 @@ impl From<Distinct> for PyDistinct {
 
 impl Display for PyDistinct {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(
-            f,
-            "Distinct
+        match &self.distinct {
+            Distinct::All(input) => write!(
+                f,
+                "Distinct ALL
             \nInput: {:?}",
-            self.distinct.input,
-        )
+                input,
+            ),
+            Distinct::On(distinct_on) => {
+                write!(
+                    f,
+                    "Distinct ON
+            \nInput: {:?}",
+                    distinct_on.input,
+                )
+            }
+        }
     }
 }
 
@@ -71,7 +81,12 @@ impl PyDistinct {
 
 impl LogicalNode for PyDistinct {
     fn inputs(&self) -> Vec<PyLogicalPlan> {
-        vec![PyLogicalPlan::from((*self.distinct.input).clone())]
+        match &self.distinct {
+            Distinct::All(input) => vec![PyLogicalPlan::from(input.as_ref().clone())],
+            Distinct::On(distinct_on) => {
+                vec![PyLogicalPlan::from(distinct_on.input.as_ref().clone())]
+            }
+        }
     }
 
     fn to_variant(&self, py: Python) -> PyResult<PyObject> {

diff --git a/src/functions.rs b/src/functions.rs
@@ -23,11 +23,11 @@ use crate::expr::conditional_expr::PyCaseBuilder;
 use crate::expr::window::PyWindowFrame;
 use crate::expr::PyExpr;
 use datafusion::execution::FunctionRegistry;
-use datafusion_common::Column;
+use datafusion_common::{Column, TableReference};
 use datafusion_expr::expr::Alias;
 use datafusion_expr::{
     aggregate_function,
-    expr::{AggregateFunction, ScalarFunction, Sort, WindowFunction},
+    expr::{AggregateFunction, AggregateFunctionDefinition, ScalarFunction, Sort, WindowFunction},
     lit,
     window_function::find_df_window_func,
     BuiltinScalarFunction, Expr,
@@ -88,8 +88,9 @@ fn order_by(expr: PyExpr, asc: Option<bool>, nulls_first: Option<bool>) -> PyRes
 /// Creates a new Alias Expr
 #[pyfunction]
 fn alias(expr: PyExpr, name: &str) -> PyResult<PyExpr> {
+    let relation: Option<TableReference> = None;
     Ok(PyExpr {
-        expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, name)),
+        expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, relation, name)),
     })
 }
 
@@ -109,7 +110,9 @@ fn col(name: &str) -> PyResult<PyExpr> {
 fn count_star() -> PyResult<PyExpr> {
     Ok(PyExpr {
         expr: Expr::AggregateFunction(AggregateFunction {
-            fun: aggregate_function::AggregateFunction::Count,
+            func_def: datafusion_expr::expr::AggregateFunctionDefinition::BuiltIn(
+                aggregate_function::AggregateFunction::Count,
+            ),
             args: vec![lit(1)],
             distinct: false,
             filter: None,
@@ -181,7 +184,9 @@ macro_rules! scalar_function {
         #[pyo3(signature = (*args))]
         fn $NAME(args: Vec<PyExpr>) -> PyExpr {
             let expr = datafusion_expr::Expr::ScalarFunction(ScalarFunction {
-                fun: BuiltinScalarFunction::$FUNC,
+                func_def: datafusion_expr::ScalarFunctionDefinition::BuiltIn(
+                    BuiltinScalarFunction::$FUNC,
+                ),
                 args: args.into_iter().map(|e| e.into()).collect(),
             });
             expr.into()
@@ -199,7 +204,9 @@ macro_rules! aggregate_function {
         #[pyo3(signature = (*args, distinct=false))]
         fn $NAME(args: Vec<PyExpr>, distinct: bool) -> PyExpr {
             let expr = datafusion_expr::Expr::AggregateFunction(AggregateFunction {
-                fun: datafusion_expr::aggregate_function::AggregateFunction::$FUNC,
+                func_def: AggregateFunctionDefinition::BuiltIn(
+                    datafusion_expr::aggregate_function::AggregateFunction::$FUNC,
+                ),
                 args: args.into_iter().map(|e| e.into()).collect(),
                 distinct,
                 filter: None,