From 7c4bd57d1c7614e254d3911f9e99f0677aadaa3c Mon Sep 17 00:00:00 2001 From: jarek Date: Wed, 23 May 2018 14:44:12 +0200 Subject: [PATCH] #7326: send sparkAppId --- .../java/com/twosigma/beakerx/widget/SparkManager.java | 2 ++ .../com/twosigma/beakerx/widget/SparkManagerImpl.java | 9 ++++++++- .../java/com/twosigma/beakerx/widget/SparkUIManager.java | 4 +--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java index c41843e63b..a083a4b13e 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java @@ -34,6 +34,8 @@ public interface SparkManager { SparkSession.Builder getBuilder(); + String getSparkAppId(); + interface SparkManagerFactory { SparkManager create(SparkSession.Builder sparkSessionBuilder); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java index 5499333c2f..284be6bfd6 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java @@ -31,6 +31,7 @@ import org.apache.spark.scheduler.SparkListenerStageSubmitted; import org.apache.spark.scheduler.SparkListenerTaskEnd; import org.apache.spark.scheduler.SparkListenerTaskStart; +import org.apache.spark.sql.RuntimeConfig; import org.apache.spark.sql.SparkSession; import scala.Tuple2; import scala.collection.Iterator; @@ -76,6 +77,12 @@ public SparkSession getOrCreate() { return sparkSessionBuilder.getOrCreate(); } + @Override + public String getSparkAppId() { + RuntimeConfig conf = getOrCreate().conf(); + return conf.getAll().get("spark.app.id").get(); + } + @Override public SparkContext sparkContext() { return getOrCreate().sparkContext(); @@ -191,7 +198,7 @@ private static boolean isLocalSpark(SparkConf sparkConf) { } - public static class SparkManagerFactoryImpl implements SparkManagerFactory{ + public static class SparkManagerFactoryImpl implements SparkManagerFactory { @Override public SparkManager create(SparkSession.Builder sparkSessionBuilder) { diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index 565bbe21d6..85c547cc0e 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -21,8 +21,6 @@ import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.message.Message; import org.apache.spark.sql.SparkSession; - -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -132,10 +130,10 @@ private void sendError(Message parentMessage, KernelFunctionality kernel, String seo.error(message); } - public void applicationStart() { sparkUI.clearView(); statusPanel = createStatusPanel(); + sparkUI.sendUpdate("sparkAppId", sparkManager.getSparkAppId()); } public void applicationEnd() {