From d48425ecbc4e07b44a6af4e3334be51dc2919490 Mon Sep 17 00:00:00 2001 From: Michael Maletich Date: Fri, 21 Jun 2024 07:22:38 -0500 Subject: [PATCH] Use the correct schema for schema adapter. --- crates/core/src/delta_datafusion/mod.rs | 6 +----- crates/core/tests/integration_datafusion.rs | 15 ++++++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index 132fea0750..4f4a504d0d 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -507,11 +507,7 @@ impl<'a> DeltaScanBuilder<'a> { let config = self.config; let schema = match self.schema { Some(schema) => schema, - None => { - self.snapshot - .physical_arrow_schema(self.log_store.object_store()) - .await? - } + None => self.snapshot.arrow_schema()?, }; let logical_schema = df_logical_schema(self.snapshot, &config)?; diff --git a/crates/core/tests/integration_datafusion.rs b/crates/core/tests/integration_datafusion.rs index cb3cc41edb..81cbe9ef72 100644 --- a/crates/core/tests/integration_datafusion.rs +++ b/crates/core/tests/integration_datafusion.rs @@ -900,13 +900,14 @@ mod local { let batches = ctx.sql("SELECT * FROM demo").await?.collect().await?; + // Without defining a schema of the select the default for a timestamp is ms UTC let expected = vec![ - "+-------------------------------+---------------------+------------+", - "| BIG_DATE | NORMAL_DATE | SOME_VALUE |", - "+-------------------------------+---------------------+------------+", - "| 1816-03-28T05:56:08.066277376 | 2022-02-01T00:00:00 | 2 |", - "| 1816-03-29T05:56:08.066277376 | 2022-01-01T00:00:00 | 1 |", - "+-------------------------------+---------------------+------------+", + "+-----------------------------+----------------------+------------+", + "| BIG_DATE | NORMAL_DATE | SOME_VALUE |", + "+-----------------------------+----------------------+------------+", + "| 1816-03-28T05:56:08.066278Z | 2022-02-01T00:00:00Z | 2 |", + "| 1816-03-29T05:56:08.066278Z | 2022-01-01T00:00:00Z | 1 |", + "+-----------------------------+----------------------+------------+", ]; assert_batches_sorted_eq!(&expected, &batches); @@ -1113,7 +1114,7 @@ mod local { .unwrap(); let batch = batches.pop().unwrap(); - let expected_schema = Schema::new(vec![Field::new("id", ArrowDataType::Int32, true)]); + let expected_schema = Schema::new(vec![Field::new("id", ArrowDataType::Int64, false)]); assert_eq!(batch.schema().as_ref(), &expected_schema); Ok(()) }