#7332 SparkUI - saving and loading configuration profiles (#7528)

* SparkUI - saving and loading configuration profiles

* SparkUI - changed config structure, restored default config

* #7332 SparkUI - pr changes:
-dropdown + modal instead editable combobox
-sparkUI buttons tooltips
-saving default profile name
-changed spark config structure
-spark tests

* #7332 SparkUI - bugfix for config structure

* #7332 changed default config name to empty string

* #7332 SparkUI - fix for changing profile value after stopping session

* #7332 SparkUI - disable/enable form on start/stop spark session

* #7332 SparkUI - fixed saving default parameters

* #7332 SparkUI - changed profile config structure

js/notebook/src/shared/style/spark.scss

@@ -203,6 +203,11 @@
   }
 }
 
+.bx-spark-save-button {
+    margin-left: 10px;
+    width: 80px;
+}
+
 .bx-toolbar-spark-widget {
   display: inline-block;
   overflow: hidden;
@@ -326,7 +331,7 @@
       -webkit-transform: rotate(300deg);
       transform: rotate(300deg);
       -webkit-animation-delay: -0.083333333333333s;
-      animation-delay: -0.083333333333333s;
+      animation-delay: -0.083333333333AAA333s;
     }
     &:nth-child(12) {
       -webkit-transform: rotate(330deg);
@@ -336,3 +341,9 @@
     }
   }
 }
+
+.bx-spark-profile {
+  .widget-label {
+    width: 140px !important
+  }
+}

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java

@@ -39,4 +39,12 @@ public VBox getWidget() {
   public void add(PropertyItem propertyItem) {
     this.widget.add(propertyItem);
   }
+
+  public void disable() {
+    getItems().forEach(x -> x.disable());
+  }
+
+  public void enable() {
+    getItems().forEach(x -> x.enable());
+  }
 }

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java

@@ -38,4 +38,16 @@ public String getNameAsString() {
   public String getValueAsString() {
     return value.getValue();
   }
+
+  public void disable() {
+    name.setDisabled(true);
+    value.setDisabled(true);
+    remove.setDisabled(true);
+  }
+
+  public void enable() {
+    name.setDisabled(false);
+    value.setDisabled(false);
+    remove.setDisabled(false);
+  }
 }

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java

@@ -118,6 +118,23 @@ public List<Configuration> getConfiguration() {
             .collect(Collectors.toList());
   }
 
+  public void setConfiguration(Map<String, String> advancedSettings) {
+    List<PropertyItem> propertyItems = createPropertyItems(advancedSettings);
+    this.properties = new PropertiesWidget(propertyItems);
+    this.remove(this.getChildren().get(0));
+    add(new VBox(asList(this.header, this.properties.getWidget())));
+  }
+
+  public void setDisabledToAll() {
+    this.add.setDisabled(true);
+    this.properties.disable();
+  }
+
+  public void setEnabledToAll() {
+    this.add.setDisabled(false);
+    this.properties.enable();
+  }
+
   public static class Configuration {
 
     private String name;

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java

@@ -156,6 +156,9 @@ private SparkConf createSparkConf(List<SparkConfiguration.Configuration> configu
     SparkConf sparkConf = new SparkConf();
     sparkConf.set(SPARK_EXTRA_LISTENERS, old.get(SPARK_EXTRA_LISTENERS));
     sparkConf.set(BEAKERX_ID, old.get(BEAKERX_ID));
+    if (old.contains(SPARK_APP_NAME)) {
+      sparkConf.set(SPARK_APP_NAME, old.get(SPARK_APP_NAME));
+    }
     configurations.forEach(x -> {
       if (x.getName() != null) {
         sparkConf.set(x.getName(), (x.getValue() != null) ? x.getValue() : "");

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java

@@ -21,7 +21,6 @@
 import com.twosigma.beakerx.kernel.KernelManager;
 import com.twosigma.beakerx.kernel.msg.StacktraceHtmlPrinter;
 import com.twosigma.beakerx.message.Message;
-import org.apache.spark.SparkConf;
 import org.apache.spark.sql.SparkSession;
 
 import java.util.ArrayList;
@@ -40,6 +39,10 @@ public class SparkUI extends VBox implements SparkUIApi {
   public static final String SPARK_MASTER_DEFAULT = "local[*]";
   private static final String SPARK_APP_ID = "sparkAppId";
   public static final String ERROR_CREATING_SPARK_SESSION = "Error creating SparkSession, see the console log for more explanation";
+  public static final String SPARK_EXECUTOR_CORES_DEFAULT = "10";
+  public static final String SPARK_EXECUTOR_MEMORY_DEFAULT = "8g";
+  public static final Map<String, String> SPARK_ADVANCED_OPTIONS_DEFAULT = new HashMap<>();
+
 
   private final SparkUIForm sparkUIForm;
   private VBox sparkUIFormPanel;
@@ -60,7 +63,7 @@ public class SparkUI extends VBox implements SparkUIApi {
     this.sparkUIFormPanel = new VBox(new ArrayList<>());
     add(sparkUIFormPanel);
     SparkVariable.putSparkUI(this);
-    this.sparkUIForm = new SparkUIForm(sparkEngine, this::initSparkContext);
+    this.sparkUIForm = new SparkUIForm(sparkEngine, sparkUiDefaults, this::initSparkContext);
     this.sparkUIFormPanel.add(sparkUIForm);
   }
 
@@ -113,7 +116,8 @@ private void configureSparkContext(Message parentMessage, KernelFunctionality ke
         this.sparkUIForm.sendError(StacktraceHtmlPrinter.printRedBold(ERROR_CREATING_SPARK_SESSION));
       } else {
         singleSparkSession.active();
-        saveSparkConf(sparkEngine.getSparkConf());
+        sparkUIForm.saveDefaults();
+        sparkUiDefaults.saveProfileName(sparkUIForm.getProfileName());
         applicationStart();
       }
     } catch (Exception e) {
@@ -128,6 +132,7 @@ private SparkSession getSparkSession() {
   private void applicationStart() {
     this.statusPanel = new SparkUIStatus(message -> getSparkSession().sparkContext().stop());
     this.sparkUIForm.setDomClasses(new ArrayList<>(asList("bx-disabled")));
+    this.sparkUIForm.setAllToDisabled();
     add(0, this.statusPanel);
     sendUpdate(SPARK_APP_ID, sparkEngine.getSparkAppId());
     sendUpdate("sparkUiWebUrl", sparkEngine.getSparkUiWebUrl());
@@ -137,6 +142,7 @@ private void applicationStart() {
   @Override
   public void applicationEnd() {
     this.sparkUIForm.setDomClasses(new ArrayList<>());
+    this.sparkUIForm.setAllToEnabled();
     removeStatusPanel();
     singleSparkSession.inActive();
   }
@@ -245,10 +251,6 @@ public Button getConnectButton() {
     return this.sparkUIForm.getConnectButton();
   }
 
-  void saveSparkConf(SparkConf sparkConf) {
-    sparkUiDefaults.saveSparkConf(sparkConf);
-  }
-
   public interface SparkUIFactory {
     SparkUI create(SparkSession.Builder builder);
   }

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java

@@ -28,6 +28,7 @@ public interface SparkUIApi {
   String SPARK_EXECUTOR_MEMORY = "spark.executor.memory";
   String SPARK_EXECUTOR_CORES = "spark.executor.cores";
   String SPARK_EXTRA_LISTENERS = "spark.extraListeners";
+  String SPARK_ADVANCED_OPTIONS = "properties";
   String BEAKERX_ID = "beakerx.id";
   List<String> STANDARD_SETTINGS = Arrays.asList(SPARK_MASTER, SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_CORES, SPARK_APP_NAME, BEAKERX_ID, SPARK_EXTRA_LISTENERS, SPARK_REPL_CLASS_OUTPUT_DIR);
   String SPARK_SESSION_NAME = "spark";