#7332 SparkUI - pr changes:

-dropdown + modal instead editable combobox -sparkUI buttons tooltips -saving default profile name -changed spark config structure -spark tests
twosigma · Jun 18, 2018 · e0b0361 · e0b0361
1 parent c608461
commit e0b0361
Show file tree

Hide file tree

Showing 9 changed files with 152 additions and 24 deletions.
diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss
@@ -204,8 +204,8 @@
 }
 
 .bx-spark-save-button {
-    margin-left: 40px;
-    width: 60px;
+    margin-left: 10px;
+    width: 80px;
 }
 
 .bx-toolbar-spark-widget {
@@ -231,3 +231,9 @@
     line-height: 14px;
   }
 }
+
+.bx-spark-profile {
+  .widget-label {
+    width: 140px !important
+  }
+}
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java
@@ -117,6 +117,7 @@ private void configureSparkContext(Message parentMessage, KernelFunctionality ke
       } else {
         singleSparkSession.active();
         sparkUIForm.saveDefaults();
+        sparkUiDefaults.saveProfileName(sparkUIForm.getProfileName());
         applicationStart();
       }
     } catch (Exception e) {

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java
@@ -29,18 +29,26 @@
 import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_MEMORY;
 import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER;
 import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER_DEFAULT;
-import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_ADVANCED_OPTIONS;
 import static com.twosigma.beakerx.widget.SparkUiDefaults.DEFAULT_PROFILE;
 import static java.util.Arrays.asList;
 
 public class SparkUIForm extends VBox {
 
   static final String CONNECT = "Start";
   static final String PROFILE_DESC = "Profile";
+  private static final String CONFIG = "config";
+  private static final String OK_BUTTON_DESC = "Create";
+  private static final String CANCEL_BUTTON_DESC = "Cancel";
+  private static final String PROFILE_NAME_PLC_HOLD = "Enter profile name...";
+  private static final String NEW_PROFILE_DESC = "New profile";
+  private static final String SAVE_PROFILE_TOOLTIP = "Save profile";
+  private static final String NEW_PROFILE_TOOLTIP = "Create new profile";
+  private static final String REMOVE_PROFILE_TOOLTIP = "Delete this profile";
 
   private HBox errors;
   private HBox profileManagement;
-  private ComboBox profile;
+  private Dropdown profile;
+  private Text newProfileName;
   private Text masterURL;
   private Text executorMemory;
   private Text executorCores;
@@ -51,6 +59,7 @@ public class SparkUIForm extends VBox {
   private Spinner spinner;
   private HBox spinnerPanel;
   private SparkUiDefaults sparkUiDefaults;
+  private HBox profileModal;
 
   public SparkUIForm(SparkEngine sparkEngine, SparkUiDefaults sparkUiDefaults, SparkUI.OnSparkButtonAction onStartAction) {
     super(new ArrayList<>());
@@ -61,13 +70,15 @@ public SparkUIForm(SparkEngine sparkEngine, SparkUiDefaults sparkUiDefaults, Spa
   }
 
   private void createSparkView() {
+    this.profileModal = createProfileModal();
     this.profileManagement = createProfileManagement();
     this.masterURL = createMasterURL();
     this.executorMemory = createExecutorMemory();
     this.executorCores = createExecutorCores();
     this.errors = new HBox(new ArrayList<>());
     this.errors.setDomClasses(asList("bx-spark-connect-error"));
     this.addConnectButton(createConnectButton(), this.errors);
+    this.add(profileModal);
     this.add(profileManagement);
     this.add(masterURL);
     this.add(executorCores);
@@ -76,27 +87,55 @@ private void createSparkView() {
     this.add(advancedOption);
   }
 
+  private HBox createProfileModal() {
+    newProfileName = new Text();
+    newProfileName.setPlaceholder(PROFILE_NAME_PLC_HOLD);
+    newProfileName.setDescription(NEW_PROFILE_DESC);
+    newProfileName.setDomClasses(asList("bx-spark-config"));
+
+    Button okButton = new Button();
+    okButton.setDescription(OK_BUTTON_DESC);
+    okButton.registerOnClick(this::createProfileOK);
+    okButton.setDomClasses(asList("bx-button", "bx-spark-save-button"));
+    okButton.setTooltip(NEW_PROFILE_TOOLTIP);
+
+    Button cancelButton = new Button();
+    cancelButton.setDescription(CANCEL_BUTTON_DESC);
+    cancelButton.registerOnClick(this::createProfileCancel);
+    cancelButton.setDomClasses(asList("bx-button", "bx-spark-save-button"));
+
+    HBox modal = new HBox(asList(newProfileName, okButton, cancelButton));
+    modal.setDomClasses(asList("hidden"));
+    return modal;
+  }
+
   private HBox createProfileManagement() {
     List profiles = sparkUiDefaults.getProfiles();
-    ComboBox profileComboBox = new ComboBox(true);
-    profileComboBox.setEditable(true);
-    profileComboBox.setOptions(this.sparkUiDefaults.getProfileNames());
-    profileComboBox.setValue(profiles == null || profiles.size() ==0 ? "" : sparkUiDefaults.getProfiles().get(0).get("name"));
-    profileComboBox.setDescription(PROFILE_DESC);
-    profileComboBox.register(this::loadProfile);
-    profileComboBox.setDomClasses(asList("bx-spark-config"));
-    this.profile = profileComboBox;
+    Dropdown profileDropdown = new Dropdown();
+    profileDropdown.setOptions(this.sparkUiDefaults.getProfileNames());
+    profileDropdown.setValue(this.sparkUiDefaults.getCurrentProfileName());
+    profileDropdown.setDescription(PROFILE_DESC);
+    profileDropdown.register(this::loadProfile);
+    profileDropdown.setDomClasses(asList("bx-spark-config", "bx-spark-profile"));
+    this.profile = profileDropdown;
 
     Button saveButton = new Button();
     saveButton.setDescription("Save");
     saveButton.registerOnClick((content, message) -> saveProfile());
     saveButton.setDomClasses(asList("bx-button", "bx-spark-save-button"));
+    saveButton.setTooltip(SAVE_PROFILE_TOOLTIP);
+
+    Button newButton = new Button();
+    newButton.registerOnClick((content, message) -> newProfile());
+    newButton.setDomClasses(asList("bx-button", "icon-add"));
+    newButton.setTooltip(NEW_PROFILE_TOOLTIP);
 
     Button removeButton = new Button();
     removeButton.registerOnClick((content, message) -> removeProfile());
     removeButton.setDomClasses(asList("bx-button", "icon-close"));
+    removeButton.setTooltip(REMOVE_PROFILE_TOOLTIP);
 
-    return new HBox(Arrays.asList(profileComboBox, saveButton, removeButton));
+    return new HBox(Arrays.asList(profileDropdown, saveButton,  newButton, removeButton));
   }
 
   private void loadProfile() {
@@ -108,7 +147,7 @@ private void loadProfile() {
     Map<String, Object> profileData =
             (Map<String, Object>) sparkUiDefaults
                     .getProfileByName(profileName)
-                    .getOrDefault("spark_options", new HashMap<>());
+                    .getOrDefault(CONFIG, new HashMap<>());
     if (profileData.size() > 0) {
       this.masterURL.setValue(profileData.getOrDefault(SparkUI.SPARK_MASTER, SparkUI.SPARK_MASTER_DEFAULT));
       this.executorCores.setValue(profileData.getOrDefault(SparkUI.SPARK_EXECUTOR_CORES, SparkUI.SPARK_EXECUTOR_CORES_DEFAULT));
@@ -130,9 +169,28 @@ private void saveProfile() {
     HashMap sparkConfig = getCurrentConfig();
     HashMap<String, Object> sparkProfile = new HashMap<>();
     sparkProfile.put("name", profile.getValue());
-    sparkProfile.put("spark_options", sparkConfig);
+    sparkProfile.put(CONFIG, sparkConfig);
     sparkUiDefaults.saveProfile(sparkProfile);
     profile.setOptions(sparkUiDefaults.getProfileNames());
+    profile.setValue(sparkProfile.get("name"));
+  }
+
+  private void newProfile() {
+    profileManagement.setDomClasses(asList("hidden"));
+    profileModal.setDomClasses(new ArrayList<>(0));
+  }
+
+  private void createProfileOK(HashMap hashMap, Message message) {
+    profileModal.setDomClasses(asList("hidden"));
+    profileManagement.setDomClasses(new ArrayList<>(0));
+    profile.setValue(newProfileName.getValue());
+    saveProfile();
+    newProfileName.setValue("");
+  }
+
+  private void createProfileCancel(HashMap hashMap, Message message) {
+    profileModal.setDomClasses(asList("hidden"));
+    profileManagement.setDomClasses(new ArrayList<>(0));
   }
 
   private HashMap<String, Object> getCurrentConfig() {
@@ -149,7 +207,7 @@ public void saveDefaults() {
     HashMap sparkConfig = getCurrentConfig();
     HashMap<String, Object> sparkProfile = new HashMap<>();
     sparkProfile.put("name", DEFAULT_PROFILE);
-    sparkProfile.put("spark_options", sparkConfig);
+    sparkProfile.put(CONFIG, sparkConfig);
     sparkUiDefaults.saveProfile(sparkProfile);
     profile.setOptions(sparkUiDefaults.getProfileNames());
   }
@@ -257,4 +315,8 @@ public Button getConnectButton() {
     return connectButton;
   }
 
+  public String getProfileName() {
+    return profile.getValue();
+  }
+
 }
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java
@@ -39,4 +39,8 @@ public interface SparkUiDefaults {
   void saveProfile(Map<String, Object> profile);
 
   List<String> getProfileNames();
+
+  void saveProfileName(String profileName);
+
+  String getCurrentProfileName();
 }
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java
@@ -17,6 +17,7 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.internal.LinkedTreeMap;
 import org.apache.spark.SparkConf;
 import org.apache.spark.sql.SparkSession;
 import scala.Tuple2;
@@ -49,11 +50,14 @@ public class SparkUiDefaultsImpl implements SparkUiDefaults {
   public static final String PROPERTIES = "properties";
   public static final String SPARK_OPTIONS = "spark_options";
   public static final String BEAKERX = "beakerx";
-  private static final Object SPARK_PROFILES = "spark_profiles";
+  private static final String SPARK_PROFILES = "profiles";
+  private static final String CURRENT_PROFILE = "current_profile";
+  private static final String CONFIG = "config";
 
   private List<Map<String, Object>> profiles = new ArrayList<>();
   private Gson gson = new GsonBuilder().setPrettyPrinting().create();
   private Path path;
+  private String currentProfile = DEFAULT_PROFILE;
 
   public SparkUiDefaultsImpl(Path path) {
     this.path = path;
@@ -62,7 +66,8 @@ public SparkUiDefaultsImpl(Path path) {
   public void saveSparkConf(List<Map<String, Object>> profiles) {
     try {
       Map<String, Map> map = beakerxJsonAsMap(path);
-      map.get(BEAKERX).put(SPARK_PROFILES, profiles == null ? new ArrayList<>() : profiles);
+      Map<String, Object> sparkOptions = (Map<String, Object>) map.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>());
+      sparkOptions.put(SPARK_PROFILES, profiles == null ? new ArrayList<>() : profiles);
       String content = gson.toJson(map);
       Files.write(path, content.getBytes(StandardCharsets.UTF_8));
       this.profiles = profiles;
@@ -75,7 +80,7 @@ public void saveSparkConf(List<Map<String, Object>> profiles) {
   public void loadDefaults(SparkSession.Builder builder) {
     SparkConf sparkConf = SparkEngineImpl.getSparkConfBasedOn(builder);
     loadProfiles();
-    Map<String, Object> map = (Map<String, Object>) getProfileByName(DEFAULT_PROFILE).get(SPARK_OPTIONS);
+    Map<String, Object> map = (Map<String, Object>) getProfileByName(currentProfile).get(CONFIG);
     if (map != null) {
       map.entrySet().stream()
               .filter(x -> !sparkConf.contains(x.getKey()))
@@ -96,11 +101,13 @@ public Map<String, Object> getProfileByName(String name) {
   @Override
   public void loadProfiles() {
     Map<String, Map> beakerxJson = beakerxJsonAsMap(path);
-    List<Map<String, Object>> profiles = (List<Map<String, Object>>) beakerxJson.get(BEAKERX).get(SPARK_PROFILES);
+    Map sparkOptions = (Map) beakerxJson.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>());
+    List<Map<String, Object>> profiles = (List<Map<String, Object>>) sparkOptions.get(SPARK_PROFILES);
+    currentProfile = (String) sparkOptions.getOrDefault(CURRENT_PROFILE, DEFAULT_PROFILE);
     if (profiles == null) {
       Map<String, Object> defaultProfile = new HashMap<>();
       defaultProfile.put("name", DEFAULT_PROFILE);
-      defaultProfile.put(SPARK_OPTIONS, new HashMap<>());
+      defaultProfile.put(CONFIG, new HashMap<>());
       profiles = new ArrayList<>();
       profiles.add(defaultProfile);
     }
@@ -126,6 +133,24 @@ public List<String> getProfileNames() {
     return profiles.stream().map(x -> (String) x.get("name")).collect(Collectors.toList());
   }
 
+  @Override
+  public void saveProfileName(String profileName) {
+    try {
+      Map<String, Map> beakerxJson = beakerxJsonAsMap(path);
+      beakerxJson.get(BEAKERX).put(CURRENT_PROFILE, profileName);
+      String content = gson.toJson(beakerxJson);
+      Files.write(path, content.getBytes(StandardCharsets.UTF_8));
+      currentProfile = profileName;
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Override
+  public String getCurrentProfileName() {
+    return currentProfile;
+  }
+
   @Override
   public void removeSparkConf(String profileName) {
     profiles.removeIf(x -> x.get("name").equals(profileName));

diff --git a/.../test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java b/.../test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java
@@ -128,6 +128,16 @@ public List<String> getProfileNames() {
           return null;
         }
 
+        @Override
+        public void saveProfileName(String profileName) {
+
+        }
+
+        @Override
+        public String getCurrentProfileName() {
+          return null;
+        }
+
       }, singleSparkSession);
 
       @Override

diff --git a/...sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/...sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java
@@ -202,6 +202,16 @@ public List<String> getProfileNames() {
       return null;
     }
 
+    @Override
+    public void saveProfileName(String profileName) {
+
+    }
+
+    @Override
+    public String getCurrentProfileName() {
+      return null;
+    }
+
   }
 
 

diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java
@@ -108,6 +108,16 @@ public void saveProfile(Map<String, Object> profile) {
     public List<String> getProfileNames() {
       return null;
     }
+
+    @Override
+    public void saveProfileName(String profileName) {
+
+    }
+
+    @Override
+    public String getCurrentProfileName() {
+      return null;
+    }
   }
 
   static class SparkManagerImplTest implements SparkEngine {

diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java
@@ -47,7 +47,7 @@ public class SparkUiDefaultsImplTest {
   private final String PROFILE1 = "profile_1";
   private final String PROFILE2 = "profile_2";
   private final String NAME = "name";
-  private final String SPARK_OPT = "spark_options";
+  private final String SPARK_OPT = "config";
 
   @Before
   public void setUp() {
@@ -136,8 +136,8 @@ private Map getOptions(String profileName) {
 
   private Map getOptions(String profileName, Path path) {
     Map<String, Map> beakerxTestJson = sut.beakerxJsonAsMap(path).get(BEAKERX);
-    List<Map<String, Object>> profiles = (List<Map<String, Object>>) beakerxTestJson.get("spark_profiles");
-    return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).map(x -> (Map) x.get(SPARK_OPTIONS)).findFirst().orElse(null);
+    List<Map<String, Object>> profiles = (List<Map<String, Object>>) beakerxTestJson.get("spark_options").get("profiles");
+    return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).map(x -> (Map) x.get(SPARK_OPT)).findFirst().orElse(null);
   }
 
   @Test