[NSE-843] ArrowDataSouce: Arrow dataset inspect() is called every tim…

…e a file is read
oap-project · Apr 13, 2022 · 91b1a4e · 91b1a4e
1 parent e329253
commit 91b1a4e
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/.../src/main/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowFileFormat.scala b/.../src/main/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowFileFormat.scala
@@ -125,7 +125,7 @@ class ArrowFileFormat extends FileFormat with DataSourceRegister with Serializab
             options.asJava).asScala.toMap))
 
       // todo predicate validation / pushdown
-      val dataset = factory.finish();
+      val dataset = factory.finish(ArrowUtils.toArrowSchema(requiredSchema));
 
       val filter = if (enableFilterPushDown) {
         // disable filter pushdown by arrow 7.0.0

diff --git a/.../com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowPartitionReaderFactory.scala b/.../com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowPartitionReaderFactory.scala
@@ -59,7 +59,7 @@ case class ArrowPartitionReaderFactory(
     val path = partitionedFile.filePath
     val factory = ArrowUtils.makeArrowDiscovery(URLDecoder.decode(path, "UTF-8"),
       partitionedFile.start, partitionedFile.length, options)
-    val dataset = factory.finish()
+    val dataset = factory.finish(ArrowUtils.toArrowSchema(readDataSchema))
     val filter = if (enableFilterPushDown) {
       // disable filter pushdown by arrow 7.0.0
       org.apache.arrow.dataset.filter.Filter.EMPTY