fix #7512 by defining sc

twosigma · Jun 11, 2018 · 7f32566 · 7f32566
1 parent 2f3c212
commit 7f32566
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 5 deletions.
diff --git a/doc/scala/SparkUI.ipynb b/doc/scala/SparkUI.ipynb
@@ -11,11 +11,36 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {
     "scrolled": false
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "method": "display_data"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c0436b7c-ca2b-4bf8-8690-6f7a80b0da26",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "method": "display_data"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "%%classpath add mvn\n",
     "org.apache.spark spark-sql_2.11 2.2.1"
@@ -65,14 +90,34 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c168a154-3319-4284-967d-5e58bfb1a5e9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "method": "display_data"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "%%spark --connect\n",
     "SparkSession.builder().master(\"local[100]\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java
@@ -55,6 +55,7 @@
 import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER;
 import static com.twosigma.beakerx.widget.SparkUI.SPARK_REPL_CLASS_OUTPUT_DIR;
 import static com.twosigma.beakerx.widget.SparkUI.SPARK_SESSION_NAME;
+import static com.twosigma.beakerx.widget.SparkUI.SPARK_CONTEXT_NAME;
 import static com.twosigma.beakerx.widget.SparkUI.STANDARD_SETTINGS;
 import static com.twosigma.beakerx.widget.StartStopSparkListener.START_STOP_SPARK_LISTENER;
 
@@ -140,11 +141,12 @@ public String sparkVersion() {
   private TryResult initSparkContextInShell(KernelFunctionality kernel, Message parent) {
     String addSc = String.format(("import com.twosigma.beakerx.widget.SparkVariable\n" +
                     "val %s = SparkVariable.getSparkSession()\n" +
+                    "val %s = %s.sparkContext\n" +
                     "import org.apache.spark.SparkContext._\n" +
                     "import %s.implicits._\n" +
                     "import %s.sql\n" +
                     "import org.apache.spark.sql.functions._\n"),
-            SPARK_SESSION_NAME, SPARK_SESSION_NAME, SPARK_SESSION_NAME);
+            SPARK_SESSION_NAME, SPARK_CONTEXT_NAME, SPARK_SESSION_NAME, SPARK_SESSION_NAME, SPARK_SESSION_NAME);
 
     SimpleEvaluationObject seo = createSimpleEvaluationObject(addSc, kernel, new Message(new Header(JupyterMessages.COMM_MSG, parent.getHeader().getSession())), 1);
     return kernel.executeCode(addSc, seo);

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java
@@ -31,6 +31,7 @@ public interface SparkUIApi {
   String BEAKERX_ID = "beakerx.id";
   List<String> STANDARD_SETTINGS = Arrays.asList(SPARK_MASTER, SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_CORES, SPARK_APP_NAME, BEAKERX_ID, SPARK_EXTRA_LISTENERS, SPARK_REPL_CLASS_OUTPUT_DIR);
   String SPARK_SESSION_NAME = "spark";
+  String SPARK_CONTEXT_NAME = "sc";
 
   List<SparkConfiguration.Configuration> getAdvancedOptions();