From e0b03617dd4d4176a27d572c421921e7724e155a Mon Sep 17 00:00:00 2001 From: Lukasz Mitusinski Date: Mon, 18 Jun 2018 14:17:12 +0200 Subject: [PATCH] #7332 SparkUI - pr changes: -dropdown + modal instead editable combobox -sparkUI buttons tooltips -saving default profile name -changed spark config structure -spark tests --- js/notebook/src/shared/style/spark.scss | 10 ++- .../com/twosigma/beakerx/widget/SparkUI.java | 1 + .../twosigma/beakerx/widget/SparkUIForm.java | 90 ++++++++++++++++--- .../beakerx/widget/SparkUiDefaults.java | 4 + .../beakerx/widget/SparkUiDefaultsImpl.java | 35 ++++++-- .../SparkMagicCommandAutoConnectTest.java | 10 +++ .../magic/command/SparkMagicCommandTest.java | 10 +++ .../twosigma/beakerx/widget/SparkUITest.java | 10 +++ .../widget/SparkUiDefaultsImplTest.java | 6 +- 9 files changed, 152 insertions(+), 24 deletions(-) diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss index 764df24a7f..2a9c947e96 100644 --- a/js/notebook/src/shared/style/spark.scss +++ b/js/notebook/src/shared/style/spark.scss @@ -204,8 +204,8 @@ } .bx-spark-save-button { - margin-left: 40px; - width: 60px; + margin-left: 10px; + width: 80px; } .bx-toolbar-spark-widget { @@ -231,3 +231,9 @@ line-height: 14px; } } + +.bx-spark-profile { + .widget-label { + width: 140px !important + } +} diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java index 78b6ce531e..1992f5c7d2 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java @@ -117,6 +117,7 @@ private void configureSparkContext(Message parentMessage, KernelFunctionality ke } else { singleSparkSession.active(); sparkUIForm.saveDefaults(); + sparkUiDefaults.saveProfileName(sparkUIForm.getProfileName()); applicationStart(); } } catch (Exception e) { diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java index 28731f4cec..546a53fa8e 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java @@ -29,7 +29,6 @@ import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_MEMORY; import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER; import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER_DEFAULT; -import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_ADVANCED_OPTIONS; import static com.twosigma.beakerx.widget.SparkUiDefaults.DEFAULT_PROFILE; import static java.util.Arrays.asList; @@ -37,10 +36,19 @@ public class SparkUIForm extends VBox { static final String CONNECT = "Start"; static final String PROFILE_DESC = "Profile"; + private static final String CONFIG = "config"; + private static final String OK_BUTTON_DESC = "Create"; + private static final String CANCEL_BUTTON_DESC = "Cancel"; + private static final String PROFILE_NAME_PLC_HOLD = "Enter profile name..."; + private static final String NEW_PROFILE_DESC = "New profile"; + private static final String SAVE_PROFILE_TOOLTIP = "Save profile"; + private static final String NEW_PROFILE_TOOLTIP = "Create new profile"; + private static final String REMOVE_PROFILE_TOOLTIP = "Delete this profile"; private HBox errors; private HBox profileManagement; - private ComboBox profile; + private Dropdown profile; + private Text newProfileName; private Text masterURL; private Text executorMemory; private Text executorCores; @@ -51,6 +59,7 @@ public class SparkUIForm extends VBox { private Spinner spinner; private HBox spinnerPanel; private SparkUiDefaults sparkUiDefaults; + private HBox profileModal; public SparkUIForm(SparkEngine sparkEngine, SparkUiDefaults sparkUiDefaults, SparkUI.OnSparkButtonAction onStartAction) { super(new ArrayList<>()); @@ -61,6 +70,7 @@ public SparkUIForm(SparkEngine sparkEngine, SparkUiDefaults sparkUiDefaults, Spa } private void createSparkView() { + this.profileModal = createProfileModal(); this.profileManagement = createProfileManagement(); this.masterURL = createMasterURL(); this.executorMemory = createExecutorMemory(); @@ -68,6 +78,7 @@ private void createSparkView() { this.errors = new HBox(new ArrayList<>()); this.errors.setDomClasses(asList("bx-spark-connect-error")); this.addConnectButton(createConnectButton(), this.errors); + this.add(profileModal); this.add(profileManagement); this.add(masterURL); this.add(executorCores); @@ -76,27 +87,55 @@ private void createSparkView() { this.add(advancedOption); } + private HBox createProfileModal() { + newProfileName = new Text(); + newProfileName.setPlaceholder(PROFILE_NAME_PLC_HOLD); + newProfileName.setDescription(NEW_PROFILE_DESC); + newProfileName.setDomClasses(asList("bx-spark-config")); + + Button okButton = new Button(); + okButton.setDescription(OK_BUTTON_DESC); + okButton.registerOnClick(this::createProfileOK); + okButton.setDomClasses(asList("bx-button", "bx-spark-save-button")); + okButton.setTooltip(NEW_PROFILE_TOOLTIP); + + Button cancelButton = new Button(); + cancelButton.setDescription(CANCEL_BUTTON_DESC); + cancelButton.registerOnClick(this::createProfileCancel); + cancelButton.setDomClasses(asList("bx-button", "bx-spark-save-button")); + + HBox modal = new HBox(asList(newProfileName, okButton, cancelButton)); + modal.setDomClasses(asList("hidden")); + return modal; + } + private HBox createProfileManagement() { List profiles = sparkUiDefaults.getProfiles(); - ComboBox profileComboBox = new ComboBox(true); - profileComboBox.setEditable(true); - profileComboBox.setOptions(this.sparkUiDefaults.getProfileNames()); - profileComboBox.setValue(profiles == null || profiles.size() ==0 ? "" : sparkUiDefaults.getProfiles().get(0).get("name")); - profileComboBox.setDescription(PROFILE_DESC); - profileComboBox.register(this::loadProfile); - profileComboBox.setDomClasses(asList("bx-spark-config")); - this.profile = profileComboBox; + Dropdown profileDropdown = new Dropdown(); + profileDropdown.setOptions(this.sparkUiDefaults.getProfileNames()); + profileDropdown.setValue(this.sparkUiDefaults.getCurrentProfileName()); + profileDropdown.setDescription(PROFILE_DESC); + profileDropdown.register(this::loadProfile); + profileDropdown.setDomClasses(asList("bx-spark-config", "bx-spark-profile")); + this.profile = profileDropdown; Button saveButton = new Button(); saveButton.setDescription("Save"); saveButton.registerOnClick((content, message) -> saveProfile()); saveButton.setDomClasses(asList("bx-button", "bx-spark-save-button")); + saveButton.setTooltip(SAVE_PROFILE_TOOLTIP); + + Button newButton = new Button(); + newButton.registerOnClick((content, message) -> newProfile()); + newButton.setDomClasses(asList("bx-button", "icon-add")); + newButton.setTooltip(NEW_PROFILE_TOOLTIP); Button removeButton = new Button(); removeButton.registerOnClick((content, message) -> removeProfile()); removeButton.setDomClasses(asList("bx-button", "icon-close")); + removeButton.setTooltip(REMOVE_PROFILE_TOOLTIP); - return new HBox(Arrays.asList(profileComboBox, saveButton, removeButton)); + return new HBox(Arrays.asList(profileDropdown, saveButton, newButton, removeButton)); } private void loadProfile() { @@ -108,7 +147,7 @@ private void loadProfile() { Map profileData = (Map) sparkUiDefaults .getProfileByName(profileName) - .getOrDefault("spark_options", new HashMap<>()); + .getOrDefault(CONFIG, new HashMap<>()); if (profileData.size() > 0) { this.masterURL.setValue(profileData.getOrDefault(SparkUI.SPARK_MASTER, SparkUI.SPARK_MASTER_DEFAULT)); this.executorCores.setValue(profileData.getOrDefault(SparkUI.SPARK_EXECUTOR_CORES, SparkUI.SPARK_EXECUTOR_CORES_DEFAULT)); @@ -130,9 +169,28 @@ private void saveProfile() { HashMap sparkConfig = getCurrentConfig(); HashMap sparkProfile = new HashMap<>(); sparkProfile.put("name", profile.getValue()); - sparkProfile.put("spark_options", sparkConfig); + sparkProfile.put(CONFIG, sparkConfig); sparkUiDefaults.saveProfile(sparkProfile); profile.setOptions(sparkUiDefaults.getProfileNames()); + profile.setValue(sparkProfile.get("name")); + } + + private void newProfile() { + profileManagement.setDomClasses(asList("hidden")); + profileModal.setDomClasses(new ArrayList<>(0)); + } + + private void createProfileOK(HashMap hashMap, Message message) { + profileModal.setDomClasses(asList("hidden")); + profileManagement.setDomClasses(new ArrayList<>(0)); + profile.setValue(newProfileName.getValue()); + saveProfile(); + newProfileName.setValue(""); + } + + private void createProfileCancel(HashMap hashMap, Message message) { + profileModal.setDomClasses(asList("hidden")); + profileManagement.setDomClasses(new ArrayList<>(0)); } private HashMap getCurrentConfig() { @@ -149,7 +207,7 @@ public void saveDefaults() { HashMap sparkConfig = getCurrentConfig(); HashMap sparkProfile = new HashMap<>(); sparkProfile.put("name", DEFAULT_PROFILE); - sparkProfile.put("spark_options", sparkConfig); + sparkProfile.put(CONFIG, sparkConfig); sparkUiDefaults.saveProfile(sparkProfile); profile.setOptions(sparkUiDefaults.getProfileNames()); } @@ -257,4 +315,8 @@ public Button getConnectButton() { return connectButton; } + public String getProfileName() { + return profile.getValue(); + } + } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java index 1809522ed6..2097e276f8 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java @@ -39,4 +39,8 @@ public interface SparkUiDefaults { void saveProfile(Map profile); List getProfileNames(); + + void saveProfileName(String profileName); + + String getCurrentProfileName(); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java index 6df05ae6c8..57fe67da1d 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java @@ -17,6 +17,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.internal.LinkedTreeMap; import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import scala.Tuple2; @@ -49,11 +50,14 @@ public class SparkUiDefaultsImpl implements SparkUiDefaults { public static final String PROPERTIES = "properties"; public static final String SPARK_OPTIONS = "spark_options"; public static final String BEAKERX = "beakerx"; - private static final Object SPARK_PROFILES = "spark_profiles"; + private static final String SPARK_PROFILES = "profiles"; + private static final String CURRENT_PROFILE = "current_profile"; + private static final String CONFIG = "config"; private List> profiles = new ArrayList<>(); private Gson gson = new GsonBuilder().setPrettyPrinting().create(); private Path path; + private String currentProfile = DEFAULT_PROFILE; public SparkUiDefaultsImpl(Path path) { this.path = path; @@ -62,7 +66,8 @@ public SparkUiDefaultsImpl(Path path) { public void saveSparkConf(List> profiles) { try { Map map = beakerxJsonAsMap(path); - map.get(BEAKERX).put(SPARK_PROFILES, profiles == null ? new ArrayList<>() : profiles); + Map sparkOptions = (Map) map.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>()); + sparkOptions.put(SPARK_PROFILES, profiles == null ? new ArrayList<>() : profiles); String content = gson.toJson(map); Files.write(path, content.getBytes(StandardCharsets.UTF_8)); this.profiles = profiles; @@ -75,7 +80,7 @@ public void saveSparkConf(List> profiles) { public void loadDefaults(SparkSession.Builder builder) { SparkConf sparkConf = SparkEngineImpl.getSparkConfBasedOn(builder); loadProfiles(); - Map map = (Map) getProfileByName(DEFAULT_PROFILE).get(SPARK_OPTIONS); + Map map = (Map) getProfileByName(currentProfile).get(CONFIG); if (map != null) { map.entrySet().stream() .filter(x -> !sparkConf.contains(x.getKey())) @@ -96,11 +101,13 @@ public Map getProfileByName(String name) { @Override public void loadProfiles() { Map beakerxJson = beakerxJsonAsMap(path); - List> profiles = (List>) beakerxJson.get(BEAKERX).get(SPARK_PROFILES); + Map sparkOptions = (Map) beakerxJson.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>()); + List> profiles = (List>) sparkOptions.get(SPARK_PROFILES); + currentProfile = (String) sparkOptions.getOrDefault(CURRENT_PROFILE, DEFAULT_PROFILE); if (profiles == null) { Map defaultProfile = new HashMap<>(); defaultProfile.put("name", DEFAULT_PROFILE); - defaultProfile.put(SPARK_OPTIONS, new HashMap<>()); + defaultProfile.put(CONFIG, new HashMap<>()); profiles = new ArrayList<>(); profiles.add(defaultProfile); } @@ -126,6 +133,24 @@ public List getProfileNames() { return profiles.stream().map(x -> (String) x.get("name")).collect(Collectors.toList()); } + @Override + public void saveProfileName(String profileName) { + try { + Map beakerxJson = beakerxJsonAsMap(path); + beakerxJson.get(BEAKERX).put(CURRENT_PROFILE, profileName); + String content = gson.toJson(beakerxJson); + Files.write(path, content.getBytes(StandardCharsets.UTF_8)); + currentProfile = profileName; + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public String getCurrentProfileName() { + return currentProfile; + } + @Override public void removeSparkConf(String profileName) { profiles.removeIf(x -> x.get("name").equals(profileName)); diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java index f46246775e..e7c39dcd74 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java @@ -128,6 +128,16 @@ public List getProfileNames() { return null; } + @Override + public void saveProfileName(String profileName) { + + } + + @Override + public String getCurrentProfileName() { + return null; + } + }, singleSparkSession); @Override diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java index 3bc2086ab7..c3c7e7cee5 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java @@ -202,6 +202,16 @@ public List getProfileNames() { return null; } + @Override + public void saveProfileName(String profileName) { + + } + + @Override + public String getCurrentProfileName() { + return null; + } + } diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java index 5b55cb2e3e..cbc1a0003d 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java @@ -108,6 +108,16 @@ public void saveProfile(Map profile) { public List getProfileNames() { return null; } + + @Override + public void saveProfileName(String profileName) { + + } + + @Override + public String getCurrentProfileName() { + return null; + } } static class SparkManagerImplTest implements SparkEngine { diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java index dcb949d14e..e11344a909 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java @@ -47,7 +47,7 @@ public class SparkUiDefaultsImplTest { private final String PROFILE1 = "profile_1"; private final String PROFILE2 = "profile_2"; private final String NAME = "name"; - private final String SPARK_OPT = "spark_options"; + private final String SPARK_OPT = "config"; @Before public void setUp() { @@ -136,8 +136,8 @@ private Map getOptions(String profileName) { private Map getOptions(String profileName, Path path) { Map beakerxTestJson = sut.beakerxJsonAsMap(path).get(BEAKERX); - List> profiles = (List>) beakerxTestJson.get("spark_profiles"); - return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).map(x -> (Map) x.get(SPARK_OPTIONS)).findFirst().orElse(null); + List> profiles = (List>) beakerxTestJson.get("spark_options").get("profiles"); + return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).map(x -> (Map) x.get(SPARK_OPT)).findFirst().orElse(null); } @Test