ref: Move ScalarValue & downcast to sparrow-arrow (#407)

There is more cleanup to be done here -- using error_stack, getting rid
of the (generally undesired) string parsing, etc., but splitting here in
the interest of smaller / easier to review changes.

crates/sparrow-api/Cargo.toml

@@ -25,7 +25,7 @@ prost-wkt.workspace = true
 prost-wkt-types.workspace = true
 serde.workspace = true
 serde_yaml.workspace = true
-sparrow-core = { path = "../sparrow-core" }
+sparrow-arrow = { path = "../sparrow-arrow" }
 sparrow-syntax = { path = "../sparrow-syntax" }
 tempfile.workspace = true
 thiserror.workspace = true

crates/sparrow-api/src/kaskada/v1alpha/plan_impl.rs

@@ -1,7 +1,7 @@
 use anyhow::Context;
 use arrow::datatypes::{DataType, TimeUnit};
 use itertools::Itertools;
-use sparrow_core::{
+use sparrow_arrow::scalar_value::{
     timeunit_from_suffix, timeunit_suffix, ScalarRecord, ScalarTimestamp, ScalarValue,
 };
 

crates/sparrow-arrow/Cargo.toml

@@ -15,12 +15,16 @@ avro = ["avro-schema"]
 
 # This crate should not depend on any other Sparrow crates.
 [dependencies]
+anyhow.workspace = true
 arrow.workspace = true
 avro-schema = { workspace = true, optional = true }
 avro-rs = { workspace = true }
+chrono.workspace = true
+decorum.workspace = true
 derive_more.workspace = true
 error-stack.workspace = true
 itertools.workspace = true
+num.workspace = true
 serde.workspace = true
 tracing.workspace = true
 

crates/sparrow-arrow/src/lib.rs

@@ -13,3 +13,7 @@ pub mod attachments;
 pub mod avro;
 
 pub mod serde;
+
+pub mod downcast;
+pub mod scalar_value;
+pub mod utils;

crates/sparrow-core/src/scalar_value.rs → crates/sparrow-arrow/src/scalar_value.rs

@@ -8,10 +8,10 @@ use decorum::Total;
 use itertools::izip;
 use num::{One, Signed, Zero};
 
-use crate::utils::make_struct_array;
-use crate::{
+use crate::downcast::{
     downcast_boolean_array, downcast_primitive_array, downcast_string_array, downcast_struct_array,
 };
+use crate::utils::make_struct_array;
 
 /// Represents a single value of a given data type.
 ///

crates/sparrow-catalog/Cargo.toml

@@ -26,7 +26,6 @@ similar.workspace = true
 static_init.workspace = true
 sparrow-api = { path = "../sparrow-api" }
 sparrow-compiler = { path = "../sparrow-compiler" }
-sparrow-core = { path = "../sparrow-core" }
 sparrow-main = { path = "../sparrow-main" }
 sparrow-syntax = { path = "../sparrow-syntax" }
 sparrow-qfr = { path = "../sparrow-qfr" }

crates/sparrow-compiler/Cargo.toml

@@ -38,6 +38,7 @@ serde_yaml.workspace = true
 sha2.workspace = true
 smallvec.workspace = true
 sparrow-api = { path = "../sparrow-api" }
+sparrow-arrow = { path = "../sparrow-arrow" }
 sparrow-core = { path = "../sparrow-core" }
 sparrow-instructions = { path = "../sparrow-instructions" }
 sparrow-kernels = { path = "../sparrow-kernels" }

crates/sparrow-compiler/src/ast_to_dfg.rs

@@ -15,7 +15,7 @@ use egg::Id;
 use itertools::{izip, Itertools};
 use record_ops_to_dfg::*;
 use smallvec::{smallvec, SmallVec};
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_instructions::CastEvaluator;
 use sparrow_plan::{GroupId, InstKind, InstOp};
 use sparrow_syntax::{

crates/sparrow-compiler/src/ast_to_dfg/record_ops_to_dfg.rs

@@ -5,7 +5,7 @@ use arrow::datatypes::{DataType, Field};
 use hashbrown::HashSet;
 use itertools::{izip, Itertools};
 use smallvec::{smallvec, SmallVec};
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_plan::InstKind;
 use sparrow_syntax::{ArgVec, FenlType, Located, Location, Resolved};
 

crates/sparrow-compiler/src/dfg.rs

@@ -36,7 +36,7 @@ use egg::{AstSize, Extractor, Id, Language};
 use hashbrown::HashMap;
 use itertools::{izip, Itertools};
 pub(crate) use language::ChildrenVec;
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_plan::{InstKind, InstOp};
 use sparrow_syntax::{FenlType, Location};
 pub(crate) use step_kind::*;

crates/sparrow-compiler/src/dfg/analysis.rs

@@ -8,7 +8,7 @@ use egg::{Analysis, Id, Language};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use smallvec::smallvec;
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_plan::{InstKind, InstOp};
 
 use super::{DfgGraph, DfgLang, StepKind};

crates/sparrow-compiler/src/dfg/const_eval.rs

@@ -1,6 +1,6 @@
 use anyhow::Context;
 use itertools::Itertools;
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_instructions::{
     ColumnarValue, ComputeStore, Evaluator, GroupingIndices, RuntimeInfo, StaticArg, StaticInfo,
 };
@@ -204,7 +204,7 @@ impl RuntimeInfo for ConstEvaluator {
 
 #[cfg(test)]
 mod tests {
-    use sparrow_core::ScalarValue;
+    use sparrow_arrow::scalar_value::ScalarValue;
     use sparrow_plan::{InstKind, InstOp};
     use strum::IntoEnumIterator;
 

crates/sparrow-compiler/src/dfg/language.rs

@@ -77,7 +77,7 @@ mod tests {
 
     use egg::{EGraph, Pattern, Searcher, Var};
     use smallvec::smallvec;
-    use sparrow_core::ScalarValue;
+    use sparrow_arrow::scalar_value::ScalarValue;
     use sparrow_plan::{InstKind, InstOp};
 
     use super::*;

crates/sparrow-compiler/src/dfg/simplification.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 
 use egg::{rewrite, Applier, Id, Rewrite, Runner, Subst};
 use smallvec::smallvec;
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_syntax::LiteralValue;
 use static_init::dynamic;
 use tracing::{info, info_span, warn};

crates/sparrow-compiler/src/dfg/step_kind.rs

@@ -6,7 +6,7 @@ use std::str::FromStr;
 use anyhow::Context;
 use sparrow_api::kaskada::v1alpha::operation_plan::tick_operation::TickBehavior;
 use sparrow_api::kaskada::v1alpha::{slice_plan, LateBoundValue};
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_plan::{InstKind, InstOp, TableId};
 use sparrow_syntax::WindowBehavior;
 use uuid::Uuid;

crates/sparrow-compiler/src/plan/expression_to_plan.rs

@@ -1,7 +1,7 @@
 use anyhow::Context;
 use sparrow_api::kaskada::v1alpha::DataType;
 use sparrow_api::kaskada::v1alpha::{expression_plan, ExpressionPlan};
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_plan::{InstKind, Mode};
 use sparrow_syntax::{ArgVec, FenlType};
 

crates/sparrow-compiler/src/plan/operation_to_plan.rs

@@ -11,7 +11,7 @@ use sparrow_api::kaskada::v1alpha::{
     expression_plan, operation_plan, OperationInputRef, OperationPlan, SlicePlan,
 };
 use sparrow_api::kaskada::v1alpha::{DataType, Schema};
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 
 use super::transform_to_plan::TransformToPlan;
 use crate::dfg::{DfgExpr, Expression, Operation, StepKind};

crates/sparrow-compiler/src/types/instruction.rs

@@ -3,7 +3,7 @@ use std::borrow::Cow;
 use anyhow::{anyhow, Context};
 use arrow::datatypes::{DataType, Field};
 use itertools::{izip, Itertools};
-use sparrow_core::ScalarValue;
+use sparrow_arrow::scalar_value::ScalarValue;
 use sparrow_instructions::CastEvaluator;
 use sparrow_plan::{InstKind, Mode};
 use sparrow_syntax::{ArgVec, FenlType, Resolved};

crates/sparrow-core/Cargo.toml

@@ -20,6 +20,7 @@ num.workspace = true
 owning_ref.workspace = true
 parquet.workspace = true
 serde.workspace = true
+sparrow-arrow = { path = "../sparrow-arrow" }
 static_init.workspace = true
 tonic.workspace = true
 tracing.workspace = true

crates/sparrow-core/src/key_triple.rs

@@ -8,8 +8,8 @@ use arrow::datatypes::{ArrowPrimitiveType, TimestampNanosecondType, UInt64Type};
 use arrow::record_batch::RecordBatch;
 use owning_ref::ArcRef;
 
-use crate::arrow_downcast::downcast_primitive_array;
 use crate::TableSchema;
+use sparrow_arrow::downcast::downcast_primitive_array;
 
 /// The key columns for a given batch.
 ///

crates/sparrow-core/src/lib.rs

@@ -9,19 +9,14 @@
 )]
 use std::pin::Pin;
 
-mod arrow_downcast;
 mod context_code;
 mod key_triple;
-mod scalar_value;
 mod table_schema;
-pub mod utils;
 
 use arrow::record_batch::RecordBatch;
-pub use arrow_downcast::*;
 pub use context_code::*;
 use futures::Stream;
 pub use key_triple::{KeyTriple, KeyTriples};
-pub use scalar_value::*;
 pub use table_schema::TableSchema;
 
 // A stream of record batches (wrapped in anyhow::Result, boxed and pinned).

crates/sparrow-instructions/Cargo.toml

@@ -28,7 +28,7 @@ serde.workspace = true
 serde_json.workspace = true
 smallvec.workspace = true
 sparrow-api = { path = "../sparrow-api" }
-sparrow-core = { path = "../sparrow-core" }
+sparrow-arrow = { path = "../sparrow-arrow" }
 sparrow-kernels = { path = "../sparrow-kernels" }
 sparrow-plan = { path = "../sparrow-plan" }
 sparrow-syntax = { path = "../sparrow-syntax" }

crates/sparrow-instructions/src/columnar_value.rs

@@ -4,10 +4,10 @@ use arrow::array::{
 };
 use arrow::datatypes::*;
 use owning_ref::ArcRef;
-use sparrow_core::{
+use sparrow_arrow::downcast::{
     downcast_boolean_array, downcast_primitive_array, downcast_string_array, downcast_struct_array,
-    NativeFromScalar, ScalarValue,
 };
+use sparrow_arrow::scalar_value::{NativeFromScalar, ScalarValue};
 
 /// The input to an instruction.
 #[derive(Debug, Clone)]

crates/sparrow-instructions/src/evaluators/aggregation/boolean/first_boolean_evaluator.rs

@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use arrow::array::{ArrayRef, BooleanArray, UInt32Array};
 use itertools::izip;
-use sparrow_core::downcast_boolean_array;
+use sparrow_arrow::downcast::downcast_boolean_array;
 use sparrow_kernels::BitBufferIterator;
 use sparrow_plan::ValueRef;
 

crates/sparrow-instructions/src/evaluators/aggregation/boolean/last_boolean_evaluator.rs

@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use arrow::array::{ArrayRef, BooleanArray, UInt32Array};
 use itertools::izip;
-use sparrow_core::downcast_boolean_array;
+use sparrow_arrow::downcast::downcast_boolean_array;
 use sparrow_kernels::BitBufferIterator;
 use sparrow_plan::ValueRef;
 

crates/sparrow-instructions/src/evaluators/aggregation/boolean/two_stacks_first_boolean_evaluator.rs

@@ -4,7 +4,7 @@ use anyhow::anyhow;
 use arrow::array::{ArrayRef, BooleanArray, UInt32Array};
 use arrow::datatypes::Int64Type;
 use itertools::izip;
-use sparrow_core::downcast_boolean_array;
+use sparrow_arrow::downcast::downcast_boolean_array;
 use sparrow_kernels::BitBufferIterator;
 use sparrow_plan::ValueRef;
 

crates/sparrow-instructions/src/evaluators/aggregation/boolean/two_stacks_last_boolean_evaluator.rs

@@ -4,7 +4,7 @@ use anyhow::anyhow;
 use arrow::array::{ArrayRef, BooleanArray, UInt32Array};
 use arrow::datatypes::Int64Type;
 use itertools::izip;
-use sparrow_core::downcast_boolean_array;
+use sparrow_arrow::downcast::downcast_boolean_array;
 use sparrow_kernels::BitBufferIterator;
 use sparrow_plan::ValueRef;
 

crates/sparrow-instructions/src/evaluators/aggregation/generic/count_evaluator.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use arrow::array::{ArrayRef, BooleanArray, PrimitiveArray, UInt32Array};
 use arrow::datatypes::UInt32Type;
 use itertools::izip;
-use sparrow_core::downcast_boolean_array;
+use sparrow_arrow::downcast::downcast_boolean_array;
 use sparrow_kernels::BitBufferIterator;
 use sparrow_plan::ValueRef;
 
@@ -315,7 +315,7 @@ impl CountIfEvaluator {
 mod tests {
     use arrow::array::UInt32Array;
     use arrow::datatypes::UInt32Type;
-    use sparrow_core::downcast_primitive_array;
+    use sparrow_arrow::downcast::downcast_primitive_array;
 
     use super::*;