Rebase onto 0.8.0 and release

rymurr · Apr 26, 2021 · 4fddd56 · 4fddd56
1 parent 99cdec7
commit 4fddd56
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 6 deletions.
diff --git a/...e/spark/sql/delta/LogFileMetaParser.scala → ...e/spark/sql/delta/LogFileMetaParser.scala b/...e/spark/sql/delta/LogFileMetaParser.scala → ...e/spark/sql/delta/LogFileMetaParser.scala
diff --git a/core/src/main/scala/org/apache/spark/sql/delta/util/DeltaFileOperations.scala b/core/src/main/scala/org/apache/spark/sql/delta/util/DeltaFileOperations.scala
@@ -213,7 +213,8 @@ object DeltaFileOperations extends DeltaLogging {
     val dirsAndFiles = spark.sparkContext.parallelize(subDirs).mapPartitions { dirs =>
       val logStore = LogStore(SparkEnv.get.conf, hadoopConf.value.value)
       val logFileHandler = LogFileMetaParser(SparkEnv.get.conf, hadoopConf.value.value, logStore)
-      listUsingLogStore(logFileHandler, dirs, recurse = false, hiddenFileNameFilter, listAsDirectories)
+      listUsingLogStore(logFileHandler, dirs, recurse = false, hiddenFileNameFilter,
+        listAsDirectories)
     }.repartition(listParallelism) // Initial list of subDirs may be small
 
     val allDirsAndFiles = dirsAndFiles.mapPartitions { firstLevelDirsAndFiles =>
@@ -265,8 +266,11 @@ object DeltaFileOperations extends DeltaLogging {
       dirs: Seq[String],
       recursive: Boolean = true,
       fileFilter: String => Boolean = defaultHiddenFileFilter): Iterator[SerializableFileStatus] = {
-    val logStore = LogStore(SparkEnv.get.conf, spark.sessionState.newHadoopConf)
-    listUsingLogStore(logStore, dirs.toIterator, recurse = recursive, fileFilter)
+    val hadoopConfig = spark.sessionState.newHadoopConf
+    val sparkConf = SparkEnv.get.conf
+    val logStore = LogStore(sparkConf, hadoopConfig)
+    val logFileHandler = LogFileMetaParser(sparkConf, hadoopConfig, logStore)
+    listUsingLogStore(logFileHandler, dirs.toIterator, recurse = recursive, fileFilter)
   }
 
   /**
@@ -281,10 +285,11 @@ object DeltaFileOperations extends DeltaLogging {
     recursive: Boolean = true,
     fileFilter: String => Boolean = defaultHiddenFileFilter): Iterator[SerializableFileStatus] = {
     val hadoopConfig = spark.sessionState.newHadoopConf
-    val logStore = LogStore(SparkEnv.get.conf, hadoopConfig)
+    val sparkConf = SparkEnv.get.conf
+    val logStore = LogStore(sparkConf, hadoopConfig)
     val logFileHandler = LogFileMetaParser(sparkConf, hadoopConfig, logStore)
     val listDirs = getAllTopComponents(new Path(listFilename), new Path(topDir))
-    listUsingLogStore(logStore, listDirs.toIterator, recurse = recursive, fileFilter,
+    listUsingLogStore(logFileHandler, listDirs.toIterator, recurse = recursive, fileFilter,
       listAsDirectories = false)
   }
 

diff --git a/version.sbt b/version.sbt
@@ -1 +1 @@
-version in ThisBuild := "0.9.0-nessie-SNAPSHOT"
+version in ThisBuild := "0.8.0-nessie"