diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java index 7abbbd3e99..c22bdbc513 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java @@ -36,7 +36,6 @@ public class SparkUIForm extends VBox { static final String CONNECT = "Start"; static final String PROFILE_DESC = "Profile"; - private static final String CONFIG = "config"; private static final String OK_BUTTON_DESC = "Create"; private static final String CANCEL_BUTTON_DESC = "Cancel"; private static final String PROFILE_NAME_PLC_HOLD = "Enter profile name..."; @@ -145,8 +144,7 @@ private void loadProfile() { sparkUiDefaults.loadProfiles(); Map profileData = (Map) sparkUiDefaults - .getProfileByName(profileName) - .getOrDefault(CONFIG, new HashMap<>()); + .getProfileByName(profileName); if (profileData.size() > 0) { this.masterURL.setValue(profileData.getOrDefault(SparkUI.SPARK_MASTER, SparkUI.SPARK_MASTER_DEFAULT)); this.executorCores.setValue(profileData.getOrDefault(SparkUI.SPARK_EXECUTOR_CORES, SparkUI.SPARK_EXECUTOR_CORES_DEFAULT)); @@ -165,10 +163,8 @@ private void saveProfile() { if (profile.getValue().equals(DEFAULT_PROFILE)) { return; } - HashMap sparkConfig = getCurrentConfig(); - HashMap sparkProfile = new HashMap<>(); + HashMap sparkProfile = getCurrentConfig(); sparkProfile.put("name", profile.getValue()); - sparkProfile.put(CONFIG, sparkConfig); sparkUiDefaults.saveProfile(sparkProfile); profile.setOptions(sparkUiDefaults.getProfileNames()); profile.setValue(sparkProfile.get("name")); @@ -203,10 +199,8 @@ private HashMap getCurrentConfig() { public void saveDefaults() { - HashMap sparkConfig = getCurrentConfig(); - HashMap sparkProfile = new HashMap<>(); + HashMap sparkProfile = getCurrentConfig(); sparkProfile.put("name", DEFAULT_PROFILE); - sparkProfile.put(CONFIG, sparkConfig); sparkUiDefaults.saveProfile(sparkProfile); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java index 911da1bdd5..40c3f5d32a 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java @@ -37,6 +37,7 @@ import java.util.stream.IntStream; import static com.twosigma.beakerx.widget.SparkUI.BEAKERX_ID; +import static com.twosigma.beakerx.widget.SparkUI.SPARK_ADVANCED_OPTIONS_DEFAULT; import static com.twosigma.beakerx.widget.SparkUI.SPARK_APP_NAME; import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_CORES_DEFAULT; import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_MEMORY_DEFAULT; @@ -56,7 +57,6 @@ public class SparkUiDefaultsImpl implements SparkUiDefaults { public static final String BEAKERX = "beakerx"; private static final String SPARK_PROFILES = "profiles"; private static final String CURRENT_PROFILE = "current_profile"; - private static final String CONFIG = "config"; private List> profiles = new ArrayList<>(); private Gson gson = new GsonBuilder().setPrettyPrinting().create(); @@ -85,7 +85,7 @@ public void saveSparkConf(List> profiles) { public void loadDefaults(SparkSession.Builder builder) { SparkConf sparkConf = SparkEngineImpl.getSparkConfBasedOn(builder); loadProfiles(); - Map map = (Map) getProfileByName(currentProfile).get(CONFIG); + Map map = (Map) getProfileByName(currentProfile); if (map != null) { map.entrySet().stream() .filter(x -> !sparkConf.contains(x.getKey())) @@ -110,15 +110,13 @@ public void loadProfiles() { List> profiles = (List>) sparkOptions.get(SPARK_PROFILES); currentProfile = (String) sparkOptions.getOrDefault(CURRENT_PROFILE, DEFAULT_PROFILE); if (profiles == null) { - //save default config if it doesn't exist + //save default config if doesn't exist Map defaultProfile = new HashMap<>(); defaultProfile.put("name", DEFAULT_PROFILE); - Map config = new HashMap(); - config.put(SPARK_MASTER, SPARK_MASTER_DEFAULT); - config.put(SPARK_EXECUTOR_CORES, SPARK_EXECUTOR_CORES_DEFAULT); - config.put(SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_MEMORY_DEFAULT); - config.put(SPARK_ADVANCED_OPTIONS, new ArrayList<>()); - defaultProfile.put(CONFIG, config); + defaultProfile.put(SPARK_MASTER, SPARK_MASTER_DEFAULT); + defaultProfile.put(SPARK_EXECUTOR_CORES, SPARK_EXECUTOR_CORES_DEFAULT); + defaultProfile.put(SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_MEMORY_DEFAULT); + defaultProfile.put(SPARK_ADVANCED_OPTIONS, new ArrayList<>()); saveProfile(defaultProfile); } else { this.profiles = profiles; diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java index e11344a909..ba190a2bcb 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java @@ -61,11 +61,9 @@ public void saveMasterURL() { //given HashMap profileConfig = new HashMap<>(); profileConfig.put(SPARK_MASTER, "local[4]"); - Map profile = new HashMap<>(); - profile.put(NAME, PROFILE1); - profile.put(SPARK_OPT, profileConfig); + profileConfig.put(NAME, PROFILE1); //when - sut.saveProfile(profile); + sut.saveProfile(profileConfig); //then Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_MASTER); @@ -77,11 +75,9 @@ public void saveExecutorMemory() { //given HashMap profileConfig = new HashMap<>(); profileConfig.put(SPARK_EXECUTOR_MEMORY, "8g"); - Map profile = new HashMap<>(); - profile.put(NAME, PROFILE1); - profile.put(SPARK_OPT, profileConfig); + profileConfig.put(NAME, PROFILE1); //when - sut.saveProfile(profile); + sut.saveProfile(profileConfig); //then Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_EXECUTOR_MEMORY); @@ -93,11 +89,9 @@ public void saveCores() { //given HashMap profileConfig = new HashMap<>(); profileConfig.put(SPARK_EXECUTOR_CORES, "10"); - Map profile = new HashMap<>(); - profile.put(NAME, PROFILE1); - profile.put(SPARK_OPT, profileConfig); + profileConfig.put(NAME, PROFILE1); //when - sut.saveProfile(profile); + sut.saveProfile(profileConfig); //then Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_EXECUTOR_CORES); @@ -110,11 +104,9 @@ public void saveAsProp() { HashMap profileConfig = new HashMap<>(); profileConfig.put(SPARK_ADVANCED_OPTIONS, Arrays.asList( new SparkConfiguration.Configuration("sparkOption2", "sp2"))); - Map profile = new HashMap<>(); - profile.put(NAME, PROFILE1); - profile.put(SPARK_OPT, profileConfig); + profileConfig.put(NAME, PROFILE1); //when - sut.saveProfile(profile); + sut.saveProfile(profileConfig); //then List props = getProps(); assertThat(props).isNotEmpty(); @@ -137,7 +129,7 @@ private Map getOptions(String profileName) { private Map getOptions(String profileName, Path path) { Map beakerxTestJson = sut.beakerxJsonAsMap(path).get(BEAKERX); List> profiles = (List>) beakerxTestJson.get("spark_options").get("profiles"); - return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).map(x -> (Map) x.get(SPARK_OPT)).findFirst().orElse(null); + return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).findFirst().orElse(new HashMap<>()); } @Test @@ -147,14 +139,12 @@ public void saveAndLoadDefaults() { profileConfig.put(SPARK_ADVANCED_OPTIONS, Arrays.asList( new SparkConfiguration.Configuration("sparkOption2", "sp2"))); profileConfig.put(SPARK_MASTER, "local[4]"); - Map profile = new HashMap<>(); - profile.put(NAME, DEFAULT_PROFILE); - profile.put(SPARK_OPT, profileConfig); + profileConfig.put(NAME, DEFAULT_PROFILE); List config = new ArrayList(); - config.add(profile); + config.add(profileConfig); //when - sut.saveProfile(profile); + sut.saveProfile(profileConfig); //then SparkSession.Builder builder = SparkSession.builder(); sut.loadDefaults(builder); @@ -166,20 +156,15 @@ public void saveAndLoadDefaults() { @Test public void createTwoProfiles() { //given - Map profile1 = new HashMap<>(); HashMap profileConfig1 = new HashMap<>(); profileConfig1.put(SPARK_MASTER, "local[4]"); - profile1.put(NAME, PROFILE1); - profile1.put(SPARK_OPT, profileConfig1); + profileConfig1.put(NAME, PROFILE1); - - Map profile2 = new HashMap<>(); HashMap profileConfig2 = new HashMap<>(); profileConfig2.put(SPARK_MASTER, "local[8]"); - profile2.put(NAME, PROFILE2); - profile2.put(SPARK_OPT, profileConfig2); + profileConfig2.put(NAME, PROFILE2); //when - sut.saveSparkConf(Arrays.asList(profile1, profile2)); + sut.saveSparkConf(Arrays.asList(profileConfig1, profileConfig2)); //then Map options1 = getOptions(PROFILE1); Map options2 = getOptions(PROFILE2); @@ -193,20 +178,16 @@ public void createTwoProfiles() { @Test public void removeProfile() { //given - Map profile1 = new HashMap<>(); HashMap profileConfig1 = new HashMap<>(); profileConfig1.put(SPARK_MASTER, "local[4]"); - profile1.put(NAME, PROFILE1); - profile1.put(SPARK_OPT, profileConfig1); + profileConfig1.put(NAME, PROFILE1); - Map profile2 = new HashMap<>(); HashMap profileConfig2 = new HashMap<>(); profileConfig2.put(SPARK_MASTER, "local[8]"); - profile2.put(NAME, PROFILE2); - profile2.put(SPARK_OPT, profileConfig2); + profileConfig2.put(NAME, PROFILE2); //when - sut.saveProfile(profile1); - sut.saveProfile(profile2); + sut.saveProfile(profileConfig1); + sut.saveProfile(profileConfig2); sut.removeSparkConf(PROFILE1); //then Map options2 = getOptions(PROFILE2); @@ -218,20 +199,16 @@ public void removeProfile() { @Test public void overwriteProfile() { //given - Map profile1 = new HashMap<>(); HashMap profileConfig1 = new HashMap<>(); profileConfig1.put(SPARK_MASTER, "local[4]"); - profile1.put(NAME, PROFILE1); - profile1.put(SPARK_OPT, profileConfig1); + profileConfig1.put(NAME, PROFILE1); - Map profile2 = new HashMap<>(); HashMap profileConfig2 = new HashMap<>(); profileConfig2.put(SPARK_MASTER, "local[8]"); - profile2.put(NAME, PROFILE1); - profile2.put(SPARK_OPT, profileConfig2); + profileConfig2.put(NAME, PROFILE1); //when - sut.saveProfile(profile1); - sut.saveProfile(profile2); + sut.saveProfile(profileConfig1); + sut.saveProfile(profileConfig2); //then Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_MASTER);