From 1fce7d839d71382b66f033daa931d5a49ecbd1cf Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Thu, 24 Nov 2022 13:19:11 +0800 Subject: [PATCH] Setting correct row number in batch scan w/ partition columns This patch fixes the row number in batch scan w/ partition columns Signed-off-by: Yuan Zhou --- .../spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala index ad117e3ea..b42dd70be 100644 --- a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala @@ -117,6 +117,7 @@ object ArrowUtils { vectors.map(_.asInstanceOf[ColumnVector]) ++ partitionVectors .map { vector => + vector.setValueCount(rowCount) // The vector should call retain() whenever reuse it. vector.retain() vector.asInstanceOf[ColumnVector]