Skip to content

Commit

Permalink
ARROW-11561: [Rust][DataFusion] Add Send + Sync to MemTable::load
Browse files Browse the repository at this point in the history
This PR adds Send + Sync to the MemTable::load method to allow implementation of a `persist` method like Spark's Dataframe in an async function.

Closes #9448 from seddonm1/send-sync

Authored-by: Mike Seddon <seddonm1@gmail.com>
Signed-off-by: Andrew Lamb <andrew@nerdnetworks.org>
  • Loading branch information
seddonm1 authored and alamb committed Feb 11, 2021
1 parent 5e3fcfa commit 7c5cf92
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 8 deletions.
9 changes: 3 additions & 6 deletions rust/benchmarks/src/bin/tpch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,9 @@ async fn benchmark(opt: BenchmarkOpt) -> Result<Vec<arrow::record_batch::RecordB
println!("Loading table '{}' into memory", table);
let start = Instant::now();

let memtable = MemTable::load(
table_provider.as_ref(),
opt.batch_size,
Some(opt.partitions),
)
.await?;
let memtable =
MemTable::load(table_provider, opt.batch_size, Some(opt.partitions))
.await?;
println!(
"Loaded table '{}' into memory in {} ms",
table,
Expand Down
2 changes: 1 addition & 1 deletion rust/datafusion/benches/sort_limit_query_sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ fn create_context() -> Arc<Mutex<ExecutionContext>> {
let partitions = 16;

rt.block_on(async {
let mem_table = MemTable::load(&csv, 16 * 1024, Some(partitions))
let mem_table = MemTable::load(Box::new(csv), 16 * 1024, Some(partitions))
.await
.unwrap();

Expand Down
2 changes: 1 addition & 1 deletion rust/datafusion/src/datasource/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ impl MemTable {

/// Create a mem table by reading from another data source
pub async fn load(
t: &dyn TableProvider,
t: Box<dyn TableProvider + Send + Sync>,
batch_size: usize,
output_partitions: Option<usize>,
) -> Result<Self> {
Expand Down

0 comments on commit 7c5cf92

Please sign in to comment.