Skip to content

Commit

Permalink
#7332 SparkUI - changed profile config structure
Browse files Browse the repository at this point in the history
  • Loading branch information
Lukasz Mitusinski committed Jun 19, 2018
1 parent 6acfa4a commit 210e970
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 64 deletions.
12 changes: 3 additions & 9 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public class SparkUIForm extends VBox {

static final String CONNECT = "Start";
static final String PROFILE_DESC = "Profile";
private static final String CONFIG = "config";
private static final String OK_BUTTON_DESC = "Create";
private static final String CANCEL_BUTTON_DESC = "Cancel";
private static final String PROFILE_NAME_PLC_HOLD = "Enter profile name...";
Expand Down Expand Up @@ -145,8 +144,7 @@ private void loadProfile() {
sparkUiDefaults.loadProfiles();
Map<String, Object> profileData =
(Map<String, Object>) sparkUiDefaults
.getProfileByName(profileName)
.getOrDefault(CONFIG, new HashMap<>());
.getProfileByName(profileName);
if (profileData.size() > 0) {
this.masterURL.setValue(profileData.getOrDefault(SparkUI.SPARK_MASTER, SparkUI.SPARK_MASTER_DEFAULT));
this.executorCores.setValue(profileData.getOrDefault(SparkUI.SPARK_EXECUTOR_CORES, SparkUI.SPARK_EXECUTOR_CORES_DEFAULT));
Expand All @@ -165,10 +163,8 @@ private void saveProfile() {
if (profile.getValue().equals(DEFAULT_PROFILE)) {
return;
}
HashMap sparkConfig = getCurrentConfig();
HashMap<String, Object> sparkProfile = new HashMap<>();
HashMap sparkProfile = getCurrentConfig();
sparkProfile.put("name", profile.getValue());
sparkProfile.put(CONFIG, sparkConfig);
sparkUiDefaults.saveProfile(sparkProfile);
profile.setOptions(sparkUiDefaults.getProfileNames());
profile.setValue(sparkProfile.get("name"));
Expand Down Expand Up @@ -203,10 +199,8 @@ private HashMap<String, Object> getCurrentConfig() {


public void saveDefaults() {
HashMap sparkConfig = getCurrentConfig();
HashMap<String, Object> sparkProfile = new HashMap<>();
HashMap sparkProfile = getCurrentConfig();
sparkProfile.put("name", DEFAULT_PROFILE);
sparkProfile.put(CONFIG, sparkConfig);
sparkUiDefaults.saveProfile(sparkProfile);
}

Expand Down
16 changes: 7 additions & 9 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.util.stream.IntStream;

import static com.twosigma.beakerx.widget.SparkUI.BEAKERX_ID;
import static com.twosigma.beakerx.widget.SparkUI.SPARK_ADVANCED_OPTIONS_DEFAULT;
import static com.twosigma.beakerx.widget.SparkUI.SPARK_APP_NAME;
import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_CORES_DEFAULT;
import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_MEMORY_DEFAULT;
Expand All @@ -56,7 +57,6 @@ public class SparkUiDefaultsImpl implements SparkUiDefaults {
public static final String BEAKERX = "beakerx";
private static final String SPARK_PROFILES = "profiles";
private static final String CURRENT_PROFILE = "current_profile";
private static final String CONFIG = "config";

private List<Map<String, Object>> profiles = new ArrayList<>();
private Gson gson = new GsonBuilder().setPrettyPrinting().create();
Expand Down Expand Up @@ -85,7 +85,7 @@ public void saveSparkConf(List<Map<String, Object>> profiles) {
public void loadDefaults(SparkSession.Builder builder) {
SparkConf sparkConf = SparkEngineImpl.getSparkConfBasedOn(builder);
loadProfiles();
Map<String, Object> map = (Map<String, Object>) getProfileByName(currentProfile).get(CONFIG);
Map<String, Object> map = (Map<String, Object>) getProfileByName(currentProfile);
if (map != null) {
map.entrySet().stream()
.filter(x -> !sparkConf.contains(x.getKey()))
Expand All @@ -110,15 +110,13 @@ public void loadProfiles() {
List<Map<String, Object>> profiles = (List<Map<String, Object>>) sparkOptions.get(SPARK_PROFILES);
currentProfile = (String) sparkOptions.getOrDefault(CURRENT_PROFILE, DEFAULT_PROFILE);
if (profiles == null) {
//save default config if it doesn't exist
//save default config if doesn't exist
Map<String, Object> defaultProfile = new HashMap<>();
defaultProfile.put("name", DEFAULT_PROFILE);
Map config = new HashMap();
config.put(SPARK_MASTER, SPARK_MASTER_DEFAULT);
config.put(SPARK_EXECUTOR_CORES, SPARK_EXECUTOR_CORES_DEFAULT);
config.put(SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_MEMORY_DEFAULT);
config.put(SPARK_ADVANCED_OPTIONS, new ArrayList<>());
defaultProfile.put(CONFIG, config);
defaultProfile.put(SPARK_MASTER, SPARK_MASTER_DEFAULT);
defaultProfile.put(SPARK_EXECUTOR_CORES, SPARK_EXECUTOR_CORES_DEFAULT);
defaultProfile.put(SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_MEMORY_DEFAULT);
defaultProfile.put(SPARK_ADVANCED_OPTIONS, new ArrayList<>());
saveProfile(defaultProfile);
} else {
this.profiles = profiles;
Expand Down
69 changes: 23 additions & 46 deletions kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,9 @@ public void saveMasterURL() {
//given
HashMap<String, Object> profileConfig = new HashMap<>();
profileConfig.put(SPARK_MASTER, "local[4]");
Map<String, Object> profile = new HashMap<>();
profile.put(NAME, PROFILE1);
profile.put(SPARK_OPT, profileConfig);
profileConfig.put(NAME, PROFILE1);
//when
sut.saveProfile(profile);
sut.saveProfile(profileConfig);
//then
Map options = getOptions(PROFILE1);
String prop = (String) options.get(SPARK_MASTER);
Expand All @@ -77,11 +75,9 @@ public void saveExecutorMemory() {
//given
HashMap<String, Object> profileConfig = new HashMap<>();
profileConfig.put(SPARK_EXECUTOR_MEMORY, "8g");
Map<String, Object> profile = new HashMap<>();
profile.put(NAME, PROFILE1);
profile.put(SPARK_OPT, profileConfig);
profileConfig.put(NAME, PROFILE1);
//when
sut.saveProfile(profile);
sut.saveProfile(profileConfig);
//then
Map options = getOptions(PROFILE1);
String prop = (String) options.get(SPARK_EXECUTOR_MEMORY);
Expand All @@ -93,11 +89,9 @@ public void saveCores() {
//given
HashMap<String, Object> profileConfig = new HashMap<>();
profileConfig.put(SPARK_EXECUTOR_CORES, "10");
Map<String, Object> profile = new HashMap<>();
profile.put(NAME, PROFILE1);
profile.put(SPARK_OPT, profileConfig);
profileConfig.put(NAME, PROFILE1);
//when
sut.saveProfile(profile);
sut.saveProfile(profileConfig);
//then
Map options = getOptions(PROFILE1);
String prop = (String) options.get(SPARK_EXECUTOR_CORES);
Expand All @@ -110,11 +104,9 @@ public void saveAsProp() {
HashMap<String, Object> profileConfig = new HashMap<>();
profileConfig.put(SPARK_ADVANCED_OPTIONS, Arrays.asList(
new SparkConfiguration.Configuration("sparkOption2", "sp2")));
Map<String, Object> profile = new HashMap<>();
profile.put(NAME, PROFILE1);
profile.put(SPARK_OPT, profileConfig);
profileConfig.put(NAME, PROFILE1);
//when
sut.saveProfile(profile);
sut.saveProfile(profileConfig);
//then
List<Object> props = getProps();
assertThat(props).isNotEmpty();
Expand All @@ -137,7 +129,7 @@ private Map getOptions(String profileName) {
private Map getOptions(String profileName, Path path) {
Map<String, Map> beakerxTestJson = sut.beakerxJsonAsMap(path).get(BEAKERX);
List<Map<String, Object>> profiles = (List<Map<String, Object>>) beakerxTestJson.get("spark_options").get("profiles");
return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).map(x -> (Map) x.get(SPARK_OPT)).findFirst().orElse(null);
return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).findFirst().orElse(new HashMap<>());
}

@Test
Expand All @@ -147,14 +139,12 @@ public void saveAndLoadDefaults() {
profileConfig.put(SPARK_ADVANCED_OPTIONS, Arrays.asList(
new SparkConfiguration.Configuration("sparkOption2", "sp2")));
profileConfig.put(SPARK_MASTER, "local[4]");
Map<String, Object> profile = new HashMap<>();
profile.put(NAME, DEFAULT_PROFILE);
profile.put(SPARK_OPT, profileConfig);
profileConfig.put(NAME, DEFAULT_PROFILE);

List config = new ArrayList();
config.add(profile);
config.add(profileConfig);
//when
sut.saveProfile(profile);
sut.saveProfile(profileConfig);
//then
SparkSession.Builder builder = SparkSession.builder();
sut.loadDefaults(builder);
Expand All @@ -166,20 +156,15 @@ public void saveAndLoadDefaults() {
@Test
public void createTwoProfiles() {
//given
Map<String, Object> profile1 = new HashMap<>();
HashMap<String, Object> profileConfig1 = new HashMap<>();
profileConfig1.put(SPARK_MASTER, "local[4]");
profile1.put(NAME, PROFILE1);
profile1.put(SPARK_OPT, profileConfig1);
profileConfig1.put(NAME, PROFILE1);


Map<String, Object> profile2 = new HashMap<>();
HashMap<String, Object> profileConfig2 = new HashMap<>();
profileConfig2.put(SPARK_MASTER, "local[8]");
profile2.put(NAME, PROFILE2);
profile2.put(SPARK_OPT, profileConfig2);
profileConfig2.put(NAME, PROFILE2);
//when
sut.saveSparkConf(Arrays.asList(profile1, profile2));
sut.saveSparkConf(Arrays.asList(profileConfig1, profileConfig2));
//then
Map options1 = getOptions(PROFILE1);
Map options2 = getOptions(PROFILE2);
Expand All @@ -193,20 +178,16 @@ public void createTwoProfiles() {
@Test
public void removeProfile() {
//given
Map<String, Object> profile1 = new HashMap<>();
HashMap<String, Object> profileConfig1 = new HashMap<>();
profileConfig1.put(SPARK_MASTER, "local[4]");
profile1.put(NAME, PROFILE1);
profile1.put(SPARK_OPT, profileConfig1);
profileConfig1.put(NAME, PROFILE1);

Map<String, Object> profile2 = new HashMap<>();
HashMap<String, Object> profileConfig2 = new HashMap<>();
profileConfig2.put(SPARK_MASTER, "local[8]");
profile2.put(NAME, PROFILE2);
profile2.put(SPARK_OPT, profileConfig2);
profileConfig2.put(NAME, PROFILE2);
//when
sut.saveProfile(profile1);
sut.saveProfile(profile2);
sut.saveProfile(profileConfig1);
sut.saveProfile(profileConfig2);
sut.removeSparkConf(PROFILE1);
//then
Map options2 = getOptions(PROFILE2);
Expand All @@ -218,20 +199,16 @@ public void removeProfile() {
@Test
public void overwriteProfile() {
//given
Map<String, Object> profile1 = new HashMap<>();
HashMap<String, Object> profileConfig1 = new HashMap<>();
profileConfig1.put(SPARK_MASTER, "local[4]");
profile1.put(NAME, PROFILE1);
profile1.put(SPARK_OPT, profileConfig1);
profileConfig1.put(NAME, PROFILE1);

Map<String, Object> profile2 = new HashMap<>();
HashMap<String, Object> profileConfig2 = new HashMap<>();
profileConfig2.put(SPARK_MASTER, "local[8]");
profile2.put(NAME, PROFILE1);
profile2.put(SPARK_OPT, profileConfig2);
profileConfig2.put(NAME, PROFILE1);
//when
sut.saveProfile(profile1);
sut.saveProfile(profile2);
sut.saveProfile(profileConfig1);
sut.saveProfile(profileConfig2);
//then
Map options = getOptions(PROFILE1);
String prop = (String) options.get(SPARK_MASTER);
Expand Down

0 comments on commit 210e970

Please sign in to comment.