first draft

Cargo.toml

@@ -18,7 +18,6 @@ log = "0.4" # to log events
 byteorder = "1.4" # for bytes conversions
 binrw = "0.13" # to efficiently read blocks
 num = { version = "0.4", features = ["serde"] } # for complex numbers
-num-traits = "0.2"
 half = "2" # for f16 handling
 encoding_rs = "0.8" # for endian management and bytes to text conversion (utf8, SBC, UTF16)
 codepage = "0.1" # to convert code page into encoding
@@ -36,25 +35,29 @@ roxmltree = "0.19" # for xml parsing
 yazi = "0.2" # for DZ block data deflate
 md-5 = "0.10" # md5sum of attachments
 transpose = "0.2" # for DZBlock transpose
-numpy = "0.20" # export rust ndarrays into numpy
 fasteval = "0.2" # for algebraic conversion
 itertools = "0.12"
 serde = { version = "1.0", features = ["derive"] } # for serialization
 whoami = "1.2" # to get user nam for writing file
 rand = "0.8" # for random numbers
-parquet2 = "0.17" # for export to parquet file format
-arrow2 = { version = "0.18", features = [
-    "compute",
-    "io_parquet",
-    "io_parquet_compression",
+libc = "0.2" # for the C api
+env_logger = "*"
+arrow = { version = "50.0.0", features = [
+    "pyarrow",
+    "prettyprint",
+    "ffi",
 ] } # for efficient data storing in memory
-polars = { version = "0.36", features = [
+parquet = "50.0" # for export to parquet file format
+polars = { version = "0.37", features = [
     "dtype-full",
     "object",
     "fmt",
 ] } # for python dataframe
-libc = "0.2" # for the C api
-env_logger = "*"
+numpy = "0.20" # export rust ndarrays into numpy
+
+[dependencies.pyo3]
+version = "0.20"
+features = ["extension-module", "num-complex", "anyhow"]
 
 [dev-dependencies]
 criterion = "0.5" # for benchmark
@@ -67,10 +70,6 @@ cbindgen = "0.26" # to generate C api headers
 name = "mdfr"
 crate-type = ["rlib", "cdylib"]
 
-[dependencies.pyo3]
-version = "0.20"
-features = ["extension-module", "num-complex", "anyhow"]
-
 [[bench]]
 name = "mdf_benchmark"
 harness = false

src/c_api.rs

@@ -1,6 +1,6 @@
 //! C API
 use crate::mdfreader::Mdf;
-use arrow2::ffi::{export_array_to_c, ArrowArray};
+use arrow::ffi::{to_ffi, FFI_ArrowArray};
 use libc::c_char;
 use std::ffi::{c_uchar, c_ushort, CStr, CString};
 
@@ -172,18 +172,19 @@ pub unsafe extern "C" fn load_all_channels_data_in_memory(mdf: *mut Mdf) {
 pub unsafe extern "C" fn get_channel_array(
     mdf: *const Mdf,
     channel_name: *const libc::c_char,
-) -> *const ArrowArray {
+) -> *const FFI_ArrowArray {
     let name = CStr::from_ptr(channel_name)
         .to_str()
         .expect("Could not convert into utf8 the file name string");
     if let Some(mdf) = mdf.as_ref() {
         match mdf.get_channel_data(name) {
             Some(data) => {
-                let array = Box::new(export_array_to_c(data.clone()));
-                let array_ptr: *const ArrowArray = &*array;
+                let (array, _) =
+                    to_ffi(&data.to_data()).expect("ffi failed converting arrow array into C");
+                let array_ptr: *const FFI_ArrowArray = &*array;
                 array_ptr
             }
-            None => std::ptr::null::<ArrowArray>(), // null pointers
+            None => std::ptr::null::<FFI_ArrowArray>(), // null pointers
         }
     } else {
         panic!("Null pointer given for Mdf Rust object")