From 691ee0004883824a81c8534199d243af2f15db39 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Sat, 22 Mar 2014 21:15:48 -0700 Subject: [PATCH] It works ish --- .../apache/spark/util/ParentClassLoader.java | 9 +++-- .../org/apache/spark/executor/Executor.scala | 9 +++-- .../executor/ExecutorURLClassLoader.scala | 39 ++++++++++--------- .../ExecutorURLClassLoaderSuite.scala | 6 +-- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/org/apache/spark/util/ParentClassLoader.java b/core/src/main/java/org/apache/spark/util/ParentClassLoader.java index e12b0641592a9..e0eb8429e139f 100644 --- a/core/src/main/java/org/apache/spark/util/ParentClassLoader.java +++ b/core/src/main/java/org/apache/spark/util/ParentClassLoader.java @@ -18,14 +18,15 @@ package org.apache.spark.util; /** - * A class loader which makes findClass accesiable to the child + * A class loader which makes findClass accesible to the child */ public class ParentClassLoader extends ClassLoader { - public ParentClassLoader(ClassLoader parent) { - super(parent); + public ParentClassLoader(ClassLoader myParent) { + super(myParent); } + @Override public Class findClass(String name) throws ClassNotFoundException { - return super.loadClass(name); + return super.findClass(name); } } diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 74b5ab07135c9..1a0cd2e4b7198 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -291,8 +291,8 @@ private[spark] class Executor( * Create a ClassLoader for use in tasks, adding any JARs specified by the user or any classes * created by the interpreter to the search path */ - private def createClassLoader(): ExecutorURLClassLoader = { - val loader = Thread.currentThread().getContextClassLoader + private def createClassLoader(): AddableURLClassLoader = { + val loader = this.getClass.getClassLoader // For each of the jars in the jarSet, add them to the class loader. // We assume each of the files has already been fetched. @@ -300,7 +300,10 @@ private[spark] class Executor( new File(uri.split("/").last).toURI.toURL }.toArray val userClassPathFirst = conf.getBoolean("spark.classpath.userClassPathFirst", false) - new ExecutorURLClassLoader(urls, loader, userClassPathFirst) + userClassPathFirst match { + case true => new ChildExecutorURLClassLoader(urls, loader) + case false => new ExecutorURLClassLoader(urls, loader) + } } /** diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala index e2fa252e4e7a2..f78986747db17 100644 --- a/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala +++ b/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala @@ -26,8 +26,13 @@ import org.apache.spark.util.ParentClassLoader * We also make changes so user classes can come before the default classes. */ -private[spark] class ExecutorURLClassLoader(urls: Array[URL], parent: ClassLoader, userFirst: Boolean) - extends ClassLoader { +trait AddableURLClassLoader extends ClassLoader { + def addURL(url: URL) + def getURLs(): Array[URL] +} + +private[spark] class ChildExecutorURLClassLoader(urls: Array[URL], parent: ClassLoader) + extends ClassLoader with AddableURLClassLoader { object userClassLoader extends URLClassLoader(urls, null){ override def addURL(url: URL) { @@ -41,22 +46,11 @@ private[spark] class ExecutorURLClassLoader(urls: Array[URL], parent: ClassLoade val parentClassLoader = new ParentClassLoader(parent) override def findClass(name: String): Class[_] = { - if (!userFirst) { - try { - val c = parentClassLoader.findClass(name) - c - } catch { - case e: ClassNotFoundException => { - userClassLoader.findClass(name) - } - } - } else { - try { - userClassLoader.findClass(name) - } catch { - case e: ClassNotFoundException => { - parentClassLoader.findClass(name) - } + try { + userClassLoader.findClass(name) + } catch { + case e: ClassNotFoundException => { + parentClassLoader.findClass(name) } } } @@ -69,3 +63,12 @@ private[spark] class ExecutorURLClassLoader(urls: Array[URL], parent: ClassLoade userClassLoader.getURLs() } } + +private[spark] class ExecutorURLClassLoader(urls: Array[URL], parent: ClassLoader) + extends URLClassLoader(urls, parent) with AddableURLClassLoader { + + override def addURL(url: URL) { + super.addURL(url) + } +} + diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorURLClassLoaderSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorURLClassLoaderSuite.scala index 9dedfa46c4168..793b8b5db3e7c 100644 --- a/core/src/test/scala/org/apache/spark/executor/ExecutorURLClassLoaderSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/ExecutorURLClassLoaderSuite.scala @@ -28,8 +28,8 @@ class ExecutorURLClassLoaderSuite extends FunSuite { val urls = List(new File(spark_home + "/core/src/test/resources/fake-spark-class.jar").toURI.toURL).toArray val urls2 = List(new File(spark_home + "/core/src/test/resources/fake-spark-class-2.jar").toURI.toURL).toArray test("child first") { - val parentLoader = new ExecutorURLClassLoader(urls2, null, false) - val classLoader = new ExecutorURLClassLoader(urls, parentLoader, true) + val parentLoader = new URLClassLoader(urls2, null) + val classLoader = new ChildExecutorURLClassLoader(urls, parentLoader) val fakeClass = classLoader.loadClass("org.apache.spark.test.FakeClass2").newInstance() val fakeClassVersion = fakeClass.toString assert(fakeClassVersion === "1") @@ -37,7 +37,7 @@ class ExecutorURLClassLoaderSuite extends FunSuite { test("parent first") { val parentLoader = new URLClassLoader(urls2, null) - val classLoader = new ExecutorURLClassLoader(urls, parentLoader, false) + val classLoader = new ExecutorURLClassLoader(urls, parentLoader) val fakeClass = classLoader.loadClass("org.apache.spark.test.FakeClass1").newInstance() val fakeClassVersion = fakeClass.toString assert(fakeClassVersion === "2")