[REMOTE-SHUFFLE-66] Enhance shuffle removing logic (#69)

Signed-off-by: Jiafu Zhang <jiafu.zhang@intel.com>
oap-project · Apr 26, 2023 · 4b33318 · 4b33318
1 parent 50b79bc
commit 4b33318
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/shuffle-daos/pom.xml b/shuffle-daos/pom.xml
@@ -242,7 +242,7 @@
         <dependency>
             <groupId>io.daos</groupId>
             <artifactId>daos-java</artifactId>
-            <version>2.2.1</version>
+            <version>2.4.1</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

diff --git a/shuffle-daos/src/main/scala/org/apache/spark/shuffle/daos/DaosShuffleManager.scala b/shuffle-daos/src/main/scala/org/apache/spark/shuffle/daos/DaosShuffleManager.scala
@@ -184,6 +184,9 @@ class DaosShuffleManager(conf: SparkConf) extends ShuffleManager with Logging {
   override def unregisterShuffle(shuffleId: Int): Boolean = {
     if (SparkContext.DRIVER_IDENTIFIER.equals(SparkEnv.get.executorId) && shuffleIdSet.remove(shuffleId)) {
       removeShuffle(shuffleId)
+      true
+    } else {
+      false
     }
   }