jarek/7331: save/load spark UI defaults (#7438)

* #7331: save spark UI defaults * #7331: remove sparkContext * #7331: remove getSparkContext
twosigma · May 29, 2018 · 8893d41 · 8893d41
1 parent d5169cd
commit 8893d41
Show file tree

Hide file tree

Showing 18 changed files with 965 additions and 458 deletions.
diff --git a/beakerx/beakerx/environment.py b/beakerx/beakerx/environment.py
@@ -34,7 +34,8 @@
             "auto_save": true,
             "use_data_grid": true,
             "show_catalog": false
-        }
+        },
+        "spark_options":{}
     }
 }
 """

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommand.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommand.java
@@ -28,12 +28,14 @@
 import com.twosigma.beakerx.kernel.msg.JupyterMessages;
 import com.twosigma.beakerx.message.Header;
 import com.twosigma.beakerx.message.Message;
-import com.twosigma.beakerx.widget.SparkManager;
-import com.twosigma.beakerx.widget.SparkManagerImpl;
+import com.twosigma.beakerx.widget.SparkEngineImpl;
 import com.twosigma.beakerx.widget.SparkUI;
+import com.twosigma.beakerx.widget.SparkUiDefaultsImpl;
 import org.apache.spark.SparkConf;
 import org.apache.spark.sql.SparkSession;
 
+import java.io.File;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -47,19 +49,23 @@ public class SparkMagicCommand implements MagicCommandFunctionality {
   public static final String SPARK = "%%sparkRunner";
   private KernelFunctionality kernel;
   private SparkUI.SparkUIFactory sparkUIFactory;
-  private SparkManager.SparkManagerFactory sparkManagerFactory;
   private SparkUI sparkUI;
   private Map<String, SparkOption> sparkOptions;
 
   public SparkMagicCommand(KernelFunctionality kernel) {
     //constructor for reflection in LoadMagicMagicCommand
-    this(kernel, new SparkUI.SparkUIFactoryImpl(), new SparkManagerImpl.SparkManagerFactoryImpl());
+    this(
+            kernel,
+            new SparkUI.SparkUIFactoryImpl(
+                    new SparkEngineImpl.SparkEngineFactoryImpl(),
+                    new SparkUiDefaultsImpl(
+                            Paths.get(System.getProperty("user.home") + File.separator + ".jupyter" + File.separator + "beakerx.json"))));
+
   }
 
-  SparkMagicCommand(KernelFunctionality kernel, SparkUI.SparkUIFactory sparkUIFactory, SparkManager.SparkManagerFactory sparkManagerFactory) {
+  SparkMagicCommand(KernelFunctionality kernel, SparkUI.SparkUIFactory sparkUIFactory) {
     this.kernel = kernel;
     this.sparkUIFactory = sparkUIFactory;
-    this.sparkManagerFactory = sparkManagerFactory;
     configureOptions();
   }
 
@@ -76,7 +82,7 @@ public String getMagicCommandName() {
 
   @Override
   public MagicCommandOutcomeItem execute(MagicCommandExecutionParam param) {
-    if (sparkUI != null && sparkUI.isSparkSessionIsActive()) {
+    if (sparkUI != null && sparkUI.isActive()) {
       return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, "Active spark session exists. If you want to close it run 'spark.close()'");
     }
     List<String> options = getOptions(param);
@@ -135,16 +141,13 @@ private MagicCommandOutcomeItem createSparkUIBasedOnUserSparkConfiguration(Magic
   }
 
   private MagicCommandOutcomeItem createSparkUI(SparkSession.Builder builder) {
-    SparkManager sparkManager = sparkManagerFactory.create(builder);
-    this.sparkUI = sparkUIFactory.create(sparkManager);
+    this.sparkUI = sparkUIFactory.create(builder);
     return displaySparkUI(sparkUI);
   }
 
   private MagicCommandOutcomeItem createSparkUI(SparkConf sparkConf) {
     SparkSession.Builder builder = SparkSession.builder().config(sparkConf);
-    SparkManager sparkManager = sparkManagerFactory.create(builder);
-    this.sparkUI = sparkUIFactory.create(sparkManager);
-    return displaySparkUI(sparkUI);
+    return createSparkUI(builder);
   }
 
   private MagicCommandOutcomeItem displaySparkUI(SparkUI sparkUI) {

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java
@@ -18,28 +18,35 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
-
 import static java.util.Collections.EMPTY_LIST;
 import static java.util.Collections.singletonList;
 
 public class SparkConfiguration extends VBox {
 
-  public static final String VIEW_NAME_VALUE = "SparkConfigurationView";
-  public static final String MODEL_NAME_VALUE = "SparkConfigurationModel";
+  static final String VIEW_NAME_VALUE = "SparkConfigurationView";
+  static final String MODEL_NAME_VALUE = "SparkConfigurationModel";
 
   private Button add;
   private HBox header;
   private Properties properties;
 
-  public SparkConfiguration() {
+  SparkConfiguration(Map<String, String> advancedSettings) {
     super(new ArrayList<>());
     this.add = createAddButton();
     this.header = new HBox(singletonList(this.add));
-    this.properties = new Properties(new ArrayList<>());
+    List<PropertyItem> propertyItems = createPropertyItems(advancedSettings);
+    this.properties = new Properties(propertyItems);
     add(new VBox(Arrays.asList(this.header, this.properties.getWidget())));
   }
 
+  private List<PropertyItem> createPropertyItems(Map<String, String> advancedSettings) {
+    return advancedSettings.entrySet().stream()
+            .map(x -> createPropertyItem(x.getKey(), x.getValue()))
+            .collect(Collectors.toList());
+  }
+
   private Button createAddButton() {
     Button add = new Button();
     add.setDescription("+");
@@ -48,15 +55,26 @@ private Button createAddButton() {
   }
 
   private void addProperty() {
-    Text name = new Text();
-    name.setPlaceholder("name");
-    Text value = new Text();
-    value.setPlaceholder("value");
+    PropertyItem propertyItem = createPropertyItem(new Text(), new Text());
+    this.properties.add(propertyItem);
+  }
+
+  private PropertyItem createPropertyItem(String name, String value) {
+    Text nameWidget = new Text();
+    nameWidget.setValue(name);
+    Text valueWidget = new Text();
+    valueWidget.setValue(value);
+    return createPropertyItem(nameWidget, valueWidget);
+  }
+
+  private PropertyItem createPropertyItem(Text nameWidget, Text valueWidget) {
+    nameWidget.setPlaceholder("name");
+    valueWidget.setPlaceholder("value");
     Button remove = new Button();
     remove.setDescription("-");
-    PropertyItem propertyItem = new PropertyItem(name, value, remove);
+    PropertyItem propertyItem = new PropertyItem(nameWidget, valueWidget, remove);
     remove.registerOnClick((content, message) -> this.properties.getWidget().removeDOMWidget(propertyItem));
-    this.properties.add(propertyItem);
+    return propertyItem;
   }
 
   @Override

diff --git a/...twosigma/beakerx/widget/SparkManager.java → .../twosigma/beakerx/widget/SparkEngine.java b/...twosigma/beakerx/widget/SparkManager.java → .../twosigma/beakerx/widget/SparkEngine.java
@@ -22,27 +22,25 @@
 import org.apache.spark.SparkContext;
 import org.apache.spark.sql.SparkSession;
 
-import java.util.List;
+import java.util.Map;
 
-public interface SparkManager {
+public interface SparkEngine {
 
-  TryResult configure(KernelFunctionality kernel, SparkUIManager sparkContextManager, Message parentMessage);
+  TryResult configure(KernelFunctionality kernel, SparkUIApi sparkUI, Message parentMessage);
 
   SparkSession getOrCreate();
 
-  SparkConf getSparkConf(List<SparkConfiguration.Configuration> configurations);
-
-  SparkContext sparkContext();
-
-  SparkSession.Builder getBuilder();
+  SparkConf getSparkConf();
 
   String getSparkAppId();
 
+  Map<String, String> getAdvanceSettings();
+
   String getSparkUiWebUrl();
 
   String getSparkMasterUrl();
 
-  interface SparkManagerFactory {
-    SparkManager create(SparkSession.Builder sparkSessionBuilder);
+  interface SparkEngineFactory {
+    SparkEngine create(SparkSession.Builder sparkSessionBuilder);
   }
 }