From 8b907d0c3d7540c9e66e4fa0435e25ac83db6d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mitusi=C5=84ski?= <33416713+lmitusinski@users.noreply.github.com> Date: Tue, 19 Jun 2018 19:04:11 +0200 Subject: [PATCH] #7332 SparkUI - saving and loading configuration profiles (#7528) * SparkUI - saving and loading configuration profiles * SparkUI - changed config structure, restored default config * #7332 SparkUI - pr changes: -dropdown + modal instead editable combobox -sparkUI buttons tooltips -saving default profile name -changed spark config structure -spark tests * #7332 SparkUI - bugfix for config structure * #7332 changed default config name to empty string * #7332 SparkUI - fix for changing profile value after stopping session * #7332 SparkUI - disable/enable form on start/stop spark session * #7332 SparkUI - fixed saving default parameters * #7332 SparkUI - changed profile config structure --- js/notebook/src/shared/style/spark.scss | 13 +- .../beakerx/widget/PropertiesWidget.java | 8 + .../twosigma/beakerx/widget/PropertyItem.java | 12 ++ .../beakerx/widget/SparkConfiguration.java | 17 ++ .../beakerx/widget/SparkEngineImpl.java | 3 + .../com/twosigma/beakerx/widget/SparkUI.java | 16 +- .../twosigma/beakerx/widget/SparkUIApi.java | 1 + .../twosigma/beakerx/widget/SparkUIForm.java | 173 +++++++++++++++++- .../beakerx/widget/SparkUiDefaults.java | 24 ++- .../beakerx/widget/SparkUiDefaultsImpl.java | 109 ++++++++++- .../SparkMagicCommandAutoConnectTest.java | 47 ++++- .../magic/command/SparkMagicCommandTest.java | 47 ++++- .../twosigma/beakerx/widget/SparkUITest.java | 53 ++++-- .../widget/SparkUiDefaultsImplTest.java | 135 +++++++++++--- 14 files changed, 597 insertions(+), 61 deletions(-) diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss index 8c993ba1c3..5915823989 100644 --- a/js/notebook/src/shared/style/spark.scss +++ b/js/notebook/src/shared/style/spark.scss @@ -203,6 +203,11 @@ } } +.bx-spark-save-button { + margin-left: 10px; + width: 80px; +} + .bx-toolbar-spark-widget { display: inline-block; overflow: hidden; @@ -326,7 +331,7 @@ -webkit-transform: rotate(300deg); transform: rotate(300deg); -webkit-animation-delay: -0.083333333333333s; - animation-delay: -0.083333333333333s; + animation-delay: -0.083333333333AAA333s; } &:nth-child(12) { -webkit-transform: rotate(330deg); @@ -336,3 +341,9 @@ } } } + +.bx-spark-profile { + .widget-label { + width: 140px !important + } +} diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java index 41bb3a3121..b454b95d6f 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java @@ -39,4 +39,12 @@ public VBox getWidget() { public void add(PropertyItem propertyItem) { this.widget.add(propertyItem); } + + public void disable() { + getItems().forEach(x -> x.disable()); + } + + public void enable() { + getItems().forEach(x -> x.enable()); + } } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java index 3150e1dcc7..7a66dbed75 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java @@ -38,4 +38,16 @@ public String getNameAsString() { public String getValueAsString() { return value.getValue(); } + + public void disable() { + name.setDisabled(true); + value.setDisabled(true); + remove.setDisabled(true); + } + + public void enable() { + name.setDisabled(false); + value.setDisabled(false); + remove.setDisabled(false); + } } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java index be82247ddc..2c294831eb 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java @@ -118,6 +118,23 @@ public List getConfiguration() { .collect(Collectors.toList()); } + public void setConfiguration(Map advancedSettings) { + List propertyItems = createPropertyItems(advancedSettings); + this.properties = new PropertiesWidget(propertyItems); + this.remove(this.getChildren().get(0)); + add(new VBox(asList(this.header, this.properties.getWidget()))); + } + + public void setDisabledToAll() { + this.add.setDisabled(true); + this.properties.disable(); + } + + public void setEnabledToAll() { + this.add.setDisabled(false); + this.properties.enable(); + } + public static class Configuration { private String name; diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java index ba1e463dee..74fba0d8b7 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java @@ -156,6 +156,9 @@ private SparkConf createSparkConf(List configu SparkConf sparkConf = new SparkConf(); sparkConf.set(SPARK_EXTRA_LISTENERS, old.get(SPARK_EXTRA_LISTENERS)); sparkConf.set(BEAKERX_ID, old.get(BEAKERX_ID)); + if (old.contains(SPARK_APP_NAME)) { + sparkConf.set(SPARK_APP_NAME, old.get(SPARK_APP_NAME)); + } configurations.forEach(x -> { if (x.getName() != null) { sparkConf.set(x.getName(), (x.getValue() != null) ? x.getValue() : ""); diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java index 430720136c..7dc5704923 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java @@ -21,7 +21,6 @@ import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.kernel.msg.StacktraceHtmlPrinter; import com.twosigma.beakerx.message.Message; -import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import java.util.ArrayList; @@ -40,6 +39,10 @@ public class SparkUI extends VBox implements SparkUIApi { public static final String SPARK_MASTER_DEFAULT = "local[*]"; private static final String SPARK_APP_ID = "sparkAppId"; public static final String ERROR_CREATING_SPARK_SESSION = "Error creating SparkSession, see the console log for more explanation"; + public static final String SPARK_EXECUTOR_CORES_DEFAULT = "10"; + public static final String SPARK_EXECUTOR_MEMORY_DEFAULT = "8g"; + public static final Map SPARK_ADVANCED_OPTIONS_DEFAULT = new HashMap<>(); + private final SparkUIForm sparkUIForm; private VBox sparkUIFormPanel; @@ -60,7 +63,7 @@ public class SparkUI extends VBox implements SparkUIApi { this.sparkUIFormPanel = new VBox(new ArrayList<>()); add(sparkUIFormPanel); SparkVariable.putSparkUI(this); - this.sparkUIForm = new SparkUIForm(sparkEngine, this::initSparkContext); + this.sparkUIForm = new SparkUIForm(sparkEngine, sparkUiDefaults, this::initSparkContext); this.sparkUIFormPanel.add(sparkUIForm); } @@ -113,7 +116,8 @@ private void configureSparkContext(Message parentMessage, KernelFunctionality ke this.sparkUIForm.sendError(StacktraceHtmlPrinter.printRedBold(ERROR_CREATING_SPARK_SESSION)); } else { singleSparkSession.active(); - saveSparkConf(sparkEngine.getSparkConf()); + sparkUIForm.saveDefaults(); + sparkUiDefaults.saveProfileName(sparkUIForm.getProfileName()); applicationStart(); } } catch (Exception e) { @@ -128,6 +132,7 @@ private SparkSession getSparkSession() { private void applicationStart() { this.statusPanel = new SparkUIStatus(message -> getSparkSession().sparkContext().stop()); this.sparkUIForm.setDomClasses(new ArrayList<>(asList("bx-disabled"))); + this.sparkUIForm.setAllToDisabled(); add(0, this.statusPanel); sendUpdate(SPARK_APP_ID, sparkEngine.getSparkAppId()); sendUpdate("sparkUiWebUrl", sparkEngine.getSparkUiWebUrl()); @@ -137,6 +142,7 @@ private void applicationStart() { @Override public void applicationEnd() { this.sparkUIForm.setDomClasses(new ArrayList<>()); + this.sparkUIForm.setAllToEnabled(); removeStatusPanel(); singleSparkSession.inActive(); } @@ -245,10 +251,6 @@ public Button getConnectButton() { return this.sparkUIForm.getConnectButton(); } - void saveSparkConf(SparkConf sparkConf) { - sparkUiDefaults.saveSparkConf(sparkConf); - } - public interface SparkUIFactory { SparkUI create(SparkSession.Builder builder); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java index f59be73d2e..ad697f4d5c 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java @@ -28,6 +28,7 @@ public interface SparkUIApi { String SPARK_EXECUTOR_MEMORY = "spark.executor.memory"; String SPARK_EXECUTOR_CORES = "spark.executor.cores"; String SPARK_EXTRA_LISTENERS = "spark.extraListeners"; + String SPARK_ADVANCED_OPTIONS = "properties"; String BEAKERX_ID = "beakerx.id"; List STANDARD_SETTINGS = Arrays.asList(SPARK_MASTER, SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_CORES, SPARK_APP_NAME, BEAKERX_ID, SPARK_EXTRA_LISTENERS, SPARK_REPL_CLASS_OUTPUT_DIR); String SPARK_SESSION_NAME = "spark"; diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java index ff08c6433e..c22bdbc513 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIForm.java @@ -15,23 +15,39 @@ */ package com.twosigma.beakerx.widget; +import com.google.gson.internal.LinkedTreeMap; import com.twosigma.beakerx.message.Message; import org.apache.spark.SparkConf; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_CORES; import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_MEMORY; import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER; import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER_DEFAULT; +import static com.twosigma.beakerx.widget.SparkUiDefaults.DEFAULT_PROFILE; import static java.util.Arrays.asList; public class SparkUIForm extends VBox { static final String CONNECT = "Start"; + static final String PROFILE_DESC = "Profile"; + private static final String OK_BUTTON_DESC = "Create"; + private static final String CANCEL_BUTTON_DESC = "Cancel"; + private static final String PROFILE_NAME_PLC_HOLD = "Enter profile name..."; + private static final String NEW_PROFILE_DESC = "New profile"; + private static final String SAVE_PROFILE_TOOLTIP = "Save profile"; + private static final String NEW_PROFILE_TOOLTIP = "Create new profile"; + private static final String REMOVE_PROFILE_TOOLTIP = "Delete this profile"; private HBox errors; + private HBox profileManagement; + private Dropdown profile; + private Text newProfileName; private Text masterURL; private Text executorMemory; private Text executorCores; @@ -41,21 +57,28 @@ public class SparkUIForm extends VBox { private Button connectButton; private Spinner spinner; private HBox spinnerPanel; + private SparkUiDefaults sparkUiDefaults; + private HBox profileModal; - public SparkUIForm(SparkEngine sparkEngine, SparkUI.OnSparkButtonAction onStartAction) { + public SparkUIForm(SparkEngine sparkEngine, SparkUiDefaults sparkUiDefaults, SparkUI.OnSparkButtonAction onStartAction) { super(new ArrayList<>()); this.sparkEngine = sparkEngine; this.onStartAction = onStartAction; + this.sparkUiDefaults = sparkUiDefaults; createSparkView(); } private void createSparkView() { + this.profileModal = createProfileModal(); + this.profileManagement = createProfileManagement(); this.masterURL = createMasterURL(); this.executorMemory = createExecutorMemory(); this.executorCores = createExecutorCores(); this.errors = new HBox(new ArrayList<>()); this.errors.setDomClasses(asList("bx-spark-connect-error")); this.addConnectButton(createConnectButton(), this.errors); + this.add(profileModal); + this.add(profileManagement); this.add(masterURL); this.add(executorCores); this.add(executorMemory); @@ -63,6 +86,134 @@ private void createSparkView() { this.add(advancedOption); } + private HBox createProfileModal() { + newProfileName = new Text(); + newProfileName.setPlaceholder(PROFILE_NAME_PLC_HOLD); + newProfileName.setDescription(NEW_PROFILE_DESC); + newProfileName.setDomClasses(asList("bx-spark-config")); + + Button okButton = new Button(); + okButton.setDescription(OK_BUTTON_DESC); + okButton.registerOnClick(this::createProfileOK); + okButton.setDomClasses(asList("bx-button", "bx-spark-save-button")); + okButton.setTooltip(NEW_PROFILE_TOOLTIP); + + Button cancelButton = new Button(); + cancelButton.setDescription(CANCEL_BUTTON_DESC); + cancelButton.registerOnClick(this::createProfileCancel); + cancelButton.setDomClasses(asList("bx-button", "bx-spark-save-button")); + + HBox modal = new HBox(asList(newProfileName, okButton, cancelButton)); + modal.setDomClasses(asList("hidden")); + return modal; + } + + private HBox createProfileManagement() { + Dropdown profileDropdown = new Dropdown(); + profileDropdown.setOptions(this.sparkUiDefaults.getProfileNames()); + profileDropdown.setValue(this.sparkUiDefaults.getCurrentProfileName()); + profileDropdown.setDescription(PROFILE_DESC); + profileDropdown.register(this::loadProfile); + profileDropdown.setDomClasses(asList("bx-spark-config", "bx-spark-profile")); + this.profile = profileDropdown; + + Button saveButton = new Button(); + saveButton.setDescription("Save"); + saveButton.registerOnClick((content, message) -> saveProfile()); + saveButton.setDomClasses(asList("bx-button", "bx-spark-save-button")); + saveButton.setTooltip(SAVE_PROFILE_TOOLTIP); + + Button newButton = new Button(); + newButton.registerOnClick((content, message) -> newProfile()); + newButton.setDomClasses(asList("bx-button", "icon-add")); + newButton.setTooltip(NEW_PROFILE_TOOLTIP); + + Button removeButton = new Button(); + removeButton.registerOnClick((content, message) -> removeProfile()); + removeButton.setDomClasses(asList("bx-button", "icon-close")); + removeButton.setTooltip(REMOVE_PROFILE_TOOLTIP); + + return new HBox(Arrays.asList(profileDropdown, saveButton, newButton, removeButton)); + } + + private void loadProfile() { + String profileName = this.profile.getValue(); + if (!this.sparkUiDefaults.getProfileNames().contains(profileName)) { + return; + } + sparkUiDefaults.loadProfiles(); + Map profileData = + (Map) sparkUiDefaults + .getProfileByName(profileName); + if (profileData.size() > 0) { + this.masterURL.setValue(profileData.getOrDefault(SparkUI.SPARK_MASTER, SparkUI.SPARK_MASTER_DEFAULT)); + this.executorCores.setValue(profileData.getOrDefault(SparkUI.SPARK_EXECUTOR_CORES, SparkUI.SPARK_EXECUTOR_CORES_DEFAULT)); + this.executorMemory.setValue(profileData.getOrDefault(SparkUI.SPARK_EXECUTOR_MEMORY, SparkUI.SPARK_EXECUTOR_MEMORY_DEFAULT)); + Map advancedSettings = new HashMap<>(); + ((ArrayList)profileData.getOrDefault(SparkUI.SPARK_ADVANCED_OPTIONS, SparkUI.SPARK_ADVANCED_OPTIONS_DEFAULT)) + .stream() + .forEach(x -> { + advancedSettings.put(x.get("name").toString(), x.get("value").toString()); + }); + this.advancedOption.setConfiguration(advancedSettings); + } + } + + private void saveProfile() { + if (profile.getValue().equals(DEFAULT_PROFILE)) { + return; + } + HashMap sparkProfile = getCurrentConfig(); + sparkProfile.put("name", profile.getValue()); + sparkUiDefaults.saveProfile(sparkProfile); + profile.setOptions(sparkUiDefaults.getProfileNames()); + profile.setValue(sparkProfile.get("name")); + } + + private void newProfile() { + profileManagement.setDomClasses(asList("hidden")); + profileModal.setDomClasses(new ArrayList<>(0)); + } + + private void createProfileOK(HashMap hashMap, Message message) { + profileModal.setDomClasses(asList("hidden")); + profileManagement.setDomClasses(new ArrayList<>(0)); + profile.setValue(newProfileName.getValue()); + saveProfile(); + newProfileName.setValue(""); + } + + private void createProfileCancel(HashMap hashMap, Message message) { + profileModal.setDomClasses(asList("hidden")); + profileManagement.setDomClasses(new ArrayList<>(0)); + } + + private HashMap getCurrentConfig() { + HashMap sparkConfig = new HashMap<>(); + sparkConfig.put(SparkUI.SPARK_MASTER, getMasterURL().getValue()); + sparkConfig.put(SparkUI.SPARK_EXECUTOR_CORES, getExecutorCores().getValue()); + sparkConfig.put(SparkUI.SPARK_EXECUTOR_MEMORY, getExecutorMemory().getValue()); + sparkConfig.put(SparkUI.SPARK_ADVANCED_OPTIONS, getAdvancedOptions()); + return sparkConfig; + } + + + public void saveDefaults() { + HashMap sparkProfile = getCurrentConfig(); + sparkProfile.put("name", DEFAULT_PROFILE); + sparkUiDefaults.saveProfile(sparkProfile); + } + + private void removeProfile() { + if (profile.getValue().equals(DEFAULT_PROFILE)) { + return; + } + sparkUiDefaults.removeSparkConf(profile.getValue()); + profile.setOptions(sparkUiDefaults.getProfileNames()); + profile.setValue(DEFAULT_PROFILE); + loadProfile(); + } + private void addConnectButton(Button connect, HBox errors) { this.connectButton = connect; this.spinnerPanel = new HBox(); @@ -109,7 +260,7 @@ private Text createExecutorMemory() { if (getSparkConf().contains(SPARK_EXECUTOR_MEMORY)) { memory.setValue(getSparkConf().get(SPARK_EXECUTOR_MEMORY)); } else { - memory.setValue("8g"); + memory.setValue(SparkUI.SPARK_EXECUTOR_MEMORY_DEFAULT); } return memory; } @@ -121,7 +272,7 @@ private Text createExecutorCores() { if (getSparkConf().contains(SPARK_EXECUTOR_CORES)) { cores.setValue(getSparkConf().get(SPARK_EXECUTOR_CORES)); } else { - cores.setValue("10"); + cores.setValue(SparkUI.SPARK_EXECUTOR_CORES_DEFAULT); } return cores; } @@ -156,4 +307,20 @@ public void clearErrors() { public Button getConnectButton() { return connectButton; } + + public String getProfileName() { + return profile.getValue(); + } + + + public void setAllToDisabled() { + this.profileManagement.getChildren().stream().map(x -> (ValueWidget) x).forEach(x -> x.setDisabled(true)); + this.advancedOption.setDisabledToAll(); + } + + public void setAllToEnabled() { + this.profileManagement.getChildren().stream().map(x -> (ValueWidget) x).forEach(x -> x.setDisabled(false)); + this.advancedOption.setEnabledToAll(); + } + } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java index e20249df35..f3a8a88b3c 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaults.java @@ -15,12 +15,32 @@ */ package com.twosigma.beakerx.widget; -import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; +import java.util.List; +import java.util.Map; + public interface SparkUiDefaults { - void saveSparkConf(SparkConf sparkConf); + String DEFAULT_PROFILE = ""; + + void saveSparkConf(List> sparkConf); void loadDefaults(SparkSession.Builder builder); + + List> getProfiles(); + + Map getProfileByName(String name); + + void removeSparkConf(String profileName); + + void loadProfiles(); + + void saveProfile(Map profile); + + List getProfileNames(); + + void saveProfileName(String profileName); + + String getCurrentProfileName(); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java index b70ccb76ea..40c3f5d32a 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUiDefaultsImpl.java @@ -17,22 +17,33 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.internal.LinkedTreeMap; import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import scala.Tuple2; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.twosigma.beakerx.widget.SparkUI.BEAKERX_ID; +import static com.twosigma.beakerx.widget.SparkUI.SPARK_ADVANCED_OPTIONS_DEFAULT; import static com.twosigma.beakerx.widget.SparkUI.SPARK_APP_NAME; +import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_CORES_DEFAULT; +import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXECUTOR_MEMORY_DEFAULT; import static com.twosigma.beakerx.widget.SparkUI.SPARK_EXTRA_LISTENERS; +import static com.twosigma.beakerx.widget.SparkUI.SPARK_MASTER_DEFAULT; +import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_ADVANCED_OPTIONS; import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_EXECUTOR_CORES; import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_EXECUTOR_MEMORY; import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_MASTER; @@ -44,32 +55,37 @@ public class SparkUiDefaultsImpl implements SparkUiDefaults { public static final String PROPERTIES = "properties"; public static final String SPARK_OPTIONS = "spark_options"; public static final String BEAKERX = "beakerx"; + private static final String SPARK_PROFILES = "profiles"; + private static final String CURRENT_PROFILE = "current_profile"; + + private List> profiles = new ArrayList<>(); private Gson gson = new GsonBuilder().setPrettyPrinting().create(); private Path path; + private String currentProfile = DEFAULT_PROFILE; public SparkUiDefaultsImpl(Path path) { this.path = path; } - @Override - @SuppressWarnings("unchecked") - public void saveSparkConf(SparkConf sparkConf) { - Map newSparkConf = toMap(sparkConf); + public void saveSparkConf(List> profiles) { try { Map map = beakerxJsonAsMap(path); - map.get(BEAKERX).put(SPARK_OPTIONS, newSparkConf); + Map sparkOptions = (Map) map.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>()); + sparkOptions.put(SPARK_PROFILES, profiles == null ? new ArrayList<>() : profiles); + map.get(BEAKERX).put(SPARK_OPTIONS, sparkOptions); String content = gson.toJson(map); Files.write(path, content.getBytes(StandardCharsets.UTF_8)); - } catch (Exception e) { - throw new RuntimeException(e); + this.profiles = profiles; + } catch (IOException e) { + e.printStackTrace(); } } @Override public void loadDefaults(SparkSession.Builder builder) { SparkConf sparkConf = SparkEngineImpl.getSparkConfBasedOn(builder); - Map beakerxJson = beakerxJsonAsMap(path); - Map map = getOptions(beakerxJson); + loadProfiles(); + Map map = (Map) getProfileByName(currentProfile); if (map != null) { map.entrySet().stream() .filter(x -> !sparkConf.contains(x.getKey())) @@ -77,6 +93,81 @@ public void loadDefaults(SparkSession.Builder builder) { } } + @Override + public List> getProfiles() { + return profiles; + } + + public Map getProfileByName(String name) { + Map profile = new HashMap<>(); + return profiles.stream().filter(x -> x.get("name").equals(name)).findFirst().orElse(profile); + } + + @Override + public void loadProfiles() { + Map beakerxJson = beakerxJsonAsMap(path); + Map sparkOptions = (Map) beakerxJson.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>()); + List> profiles = (List>) sparkOptions.get(SPARK_PROFILES); + currentProfile = (String) sparkOptions.getOrDefault(CURRENT_PROFILE, DEFAULT_PROFILE); + if (profiles == null) { + //save default config if doesn't exist + Map defaultProfile = new HashMap<>(); + defaultProfile.put("name", DEFAULT_PROFILE); + defaultProfile.put(SPARK_MASTER, SPARK_MASTER_DEFAULT); + defaultProfile.put(SPARK_EXECUTOR_CORES, SPARK_EXECUTOR_CORES_DEFAULT); + defaultProfile.put(SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_MEMORY_DEFAULT); + defaultProfile.put(SPARK_ADVANCED_OPTIONS, new ArrayList<>()); + saveProfile(defaultProfile); + } else { + this.profiles = profiles; + } + } + + @Override + public void saveProfile(Map profile) { + int idx = IntStream.range(0, profiles.size()) + .filter(i -> profile.get("name").equals(profiles.get(i).get("name"))) + .findFirst().orElse(-1); + if (idx == -1) { + profiles.add(profile); + } else { + profiles.set(idx, profile); + } + saveSparkConf(profiles); + + } + + @Override + public List getProfileNames() { + return profiles.stream().map(x -> (String) x.get("name")).collect(Collectors.toList()); + } + + @Override + public void saveProfileName(String profileName) { + try { + Map map = beakerxJsonAsMap(path); + Map sparkOptions = (Map) map.get(BEAKERX).getOrDefault(SPARK_OPTIONS, new HashMap<>()); + sparkOptions.put(CURRENT_PROFILE, profileName); + map.get(BEAKERX).put(SPARK_OPTIONS, sparkOptions); + String content = gson.toJson(map); + Files.write(path, content.getBytes(StandardCharsets.UTF_8)); + currentProfile = profileName; + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public String getCurrentProfileName() { + return currentProfile; + } + + @Override + public void removeSparkConf(String profileName) { + profiles.removeIf(x -> x.get("name").equals(profileName)); + saveSparkConf(profiles); + } + @SuppressWarnings("unchecked") private void addToBuilder(SparkSession.Builder builder, String key, Object value) { if (isOneOfMainProp(key)) { diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java index 7fc95b5256..e7c39dcd74 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandAutoConnectTest.java @@ -23,12 +23,13 @@ import com.twosigma.beakerx.widget.SparkEngine; import com.twosigma.beakerx.widget.SparkUI; import com.twosigma.beakerx.widget.SparkUiDefaults; -import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import org.junit.Before; import org.junit.Test; import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static com.twosigma.beakerx.MessageFactorTest.commMsg; import static org.assertj.core.api.Assertions.assertThat; @@ -86,8 +87,9 @@ private MagicCommandOutcomeItem createSparkUi(String option) { private SparkUI.SparkUIFactory createSparkUIFactory(SparkEngine.SparkEngineFactory sparkManagerFactory, SingleSparkSession singleSparkSession) { return new SparkUI.SparkUIFactory() { private SparkUI.SparkUIFactoryImpl factory = new SparkUI.SparkUIFactoryImpl(sparkManagerFactory, new SparkUiDefaults() { + @Override - public void saveSparkConf(SparkConf sparkConf) { + public void saveSparkConf(List> sparkConf) { } @@ -95,6 +97,47 @@ public void saveSparkConf(SparkConf sparkConf) { public void loadDefaults(SparkSession.Builder builder) { } + + @Override + public List> getProfiles() { + return null; + } + + @Override + public Map getProfileByName(String name) { + return null; + } + + @Override + public void removeSparkConf(String profileName) { + + } + + @Override + public void loadProfiles() { + + } + + @Override + public void saveProfile(Map profile) { + + } + + @Override + public List getProfileNames() { + return null; + } + + @Override + public void saveProfileName(String profileName) { + + } + + @Override + public String getCurrentProfileName() { + return null; + } + }, singleSparkSession); @Override diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java index aa31b161aa..c3c7e7cee5 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java @@ -23,9 +23,9 @@ import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutcomeItem; import com.twosigma.beakerx.message.Message; import com.twosigma.beakerx.widget.SingleSparkSession; -import com.twosigma.beakerx.widget.SparkUIApi; import com.twosigma.beakerx.widget.SparkEngine; import com.twosigma.beakerx.widget.SparkUI; +import com.twosigma.beakerx.widget.SparkUIApi; import com.twosigma.beakerx.widget.SparkUiDefaults; import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; @@ -160,8 +160,10 @@ public String sparkVersion() { } static class SparkUiDefaultsImplTest implements SparkUiDefaults { + + @Override - public void saveSparkConf(SparkConf sparkConf) { + public void saveSparkConf(List> sparkConf) { } @@ -169,6 +171,47 @@ public void saveSparkConf(SparkConf sparkConf) { public void loadDefaults(SparkSession.Builder builder) { } + + @Override + public List> getProfiles() { + return null; + } + + @Override + public Map getProfileByName(String name) { + return null; + } + + @Override + public void removeSparkConf(String profileName) { + + } + + @Override + public void loadProfiles() { + + } + + @Override + public void saveProfile(Map profile) { + + } + + @Override + public List getProfileNames() { + return null; + } + + @Override + public void saveProfileName(String profileName) { + + } + + @Override + public String getCurrentProfileName() { + return null; + } + } diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java index 7b36ffa72b..cbc1a0003d 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java @@ -28,9 +28,9 @@ import org.junit.Test; import java.util.HashMap; +import java.util.List; import java.util.Map; -import static com.twosigma.beakerx.MessageFactorTest.commMsg; import static org.assertj.core.api.Assertions.assertThat; public class SparkUITest { @@ -64,22 +64,13 @@ public void loadDefaultsWhenCreateSparkUI() { assertThat(sparkUiDefaults.loaded).isTrue(); } - @Test - public void saveDefaultsWhenConnectToSparkSession() { - //given - //when - sparkUI.getConnectButton().onClick(new HashMap(), commMsg()); - //then - assertThat(sparkUiDefaults.saved).isTrue(); - } - static class SparkUiDefaultsImplMock implements SparkUiDefaults { public boolean saved = false; public boolean loaded = false; @Override - public void saveSparkConf(SparkConf sparkConf) { + public void saveSparkConf(List> sparkConf) { saved = true; } @@ -87,6 +78,46 @@ public void saveSparkConf(SparkConf sparkConf) { public void loadDefaults(SparkSession.Builder builder) { loaded = true; } + + @Override + public List> getProfiles() { + return null; + } + + @Override + public Map getProfileByName(String name) { + return null; + } + + @Override + public void removeSparkConf(String profileName) { + + } + + @Override + public void loadProfiles() { + + } + + @Override + public void saveProfile(Map profile) { + + } + + @Override + public List getProfileNames() { + return null; + } + + @Override + public void saveProfileName(String profileName) { + + } + + @Override + public String getCurrentProfileName() { + return null; + } } static class SparkManagerImplTest implements SparkEngine { diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java index fa781fa7e6..ba190a2bcb 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUiDefaultsImplTest.java @@ -20,16 +20,21 @@ import org.apache.spark.sql.SparkSession; import org.junit.Before; import org.junit.Test; + import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; +import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_ADVANCED_OPTIONS; import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_EXECUTOR_CORES; import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_EXECUTOR_MEMORY; import static com.twosigma.beakerx.widget.SparkUIApi.SPARK_MASTER; +import static com.twosigma.beakerx.widget.SparkUiDefaults.DEFAULT_PROFILE; import static com.twosigma.beakerx.widget.SparkUiDefaultsImpl.BEAKERX; -import static com.twosigma.beakerx.widget.SparkUiDefaultsImpl.NAME; import static com.twosigma.beakerx.widget.SparkUiDefaultsImpl.PROPERTIES; import static com.twosigma.beakerx.widget.SparkUiDefaultsImpl.SPARK_OPTIONS; import static com.twosigma.beakerx.widget.SparkUiDefaultsImpl.VALUE; @@ -39,6 +44,10 @@ public class SparkUiDefaultsImplTest { private SparkUiDefaultsImpl sut; private Path pathToBeakerxTestJson; + private final String PROFILE1 = "profile_1"; + private final String PROFILE2 = "profile_2"; + private final String NAME = "name"; + private final String SPARK_OPT = "config"; @Before public void setUp() { @@ -50,12 +59,13 @@ public void setUp() { @Test public void saveMasterURL() { //given - SparkConf sparkConf = new SparkConf(); - sparkConf.set(SPARK_MASTER, "local[4]"); + HashMap profileConfig = new HashMap<>(); + profileConfig.put(SPARK_MASTER, "local[4]"); + profileConfig.put(NAME, PROFILE1); //when - sut.saveSparkConf(sparkConf); + sut.saveProfile(profileConfig); //then - Map options = getOptions(); + Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_MASTER); assertThat(prop).isEqualTo("local[4]"); } @@ -63,12 +73,13 @@ public void saveMasterURL() { @Test public void saveExecutorMemory() { //given - SparkConf sparkConf = new SparkConf(); - sparkConf.set(SPARK_EXECUTOR_MEMORY, "8g"); + HashMap profileConfig = new HashMap<>(); + profileConfig.put(SPARK_EXECUTOR_MEMORY, "8g"); + profileConfig.put(NAME, PROFILE1); //when - sut.saveSparkConf(sparkConf); + sut.saveProfile(profileConfig); //then - Map options = getOptions(); + Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_EXECUTOR_MEMORY); assertThat(prop).isEqualTo("8g"); } @@ -76,12 +87,13 @@ public void saveExecutorMemory() { @Test public void saveCores() { //given - SparkConf sparkConf = new SparkConf(); - sparkConf.set(SPARK_EXECUTOR_CORES, "10"); + HashMap profileConfig = new HashMap<>(); + profileConfig.put(SPARK_EXECUTOR_CORES, "10"); + profileConfig.put(NAME, PROFILE1); //when - sut.saveSparkConf(sparkConf); + sut.saveProfile(profileConfig); //then - Map options = getOptions(); + Map options = getOptions(PROFILE1); String prop = (String) options.get(SPARK_EXECUTOR_CORES); assertThat(prop).isEqualTo("10"); } @@ -89,10 +101,12 @@ public void saveCores() { @Test public void saveAsProp() { //given - SparkConf sparkConf = new SparkConf(); - sparkConf.set("sparkOption2", "sp2"); + HashMap profileConfig = new HashMap<>(); + profileConfig.put(SPARK_ADVANCED_OPTIONS, Arrays.asList( + new SparkConfiguration.Configuration("sparkOption2", "sp2"))); + profileConfig.put(NAME, PROFILE1); //when - sut.saveSparkConf(sparkConf); + sut.saveProfile(profileConfig); //then List props = getProps(); assertThat(props).isNotEmpty(); @@ -103,24 +117,34 @@ public void saveAsProp() { @SuppressWarnings("unchecked") private List getProps() { - Map options = getOptions(); + Map options = getOptions(PROFILE1); return (List) options.get(PROPERTIES); } @SuppressWarnings("unchecked") - private Map getOptions() { - Map beakerxTestJson = sut.beakerxJsonAsMap(this.pathToBeakerxTestJson).get(BEAKERX); - return beakerxTestJson.get(SPARK_OPTIONS); + private Map getOptions(String profileName) { + return getOptions(profileName, this.pathToBeakerxTestJson); + } + + private Map getOptions(String profileName, Path path) { + Map beakerxTestJson = sut.beakerxJsonAsMap(path).get(BEAKERX); + List> profiles = (List>) beakerxTestJson.get("spark_options").get("profiles"); + return profiles.stream().filter(x -> x.get(NAME).equals(profileName)).findFirst().orElse(new HashMap<>()); } @Test public void saveAndLoadDefaults() { //given - SparkConf sparkConf = new SparkConf(); - sparkConf.set("sparkOption2", "sp2"); - sparkConf.set(SPARK_MASTER, "local[4]"); + HashMap profileConfig = new HashMap<>(); + profileConfig.put(SPARK_ADVANCED_OPTIONS, Arrays.asList( + new SparkConfiguration.Configuration("sparkOption2", "sp2"))); + profileConfig.put(SPARK_MASTER, "local[4]"); + profileConfig.put(NAME, DEFAULT_PROFILE); + + List config = new ArrayList(); + config.add(profileConfig); //when - sut.saveSparkConf(sparkConf); + sut.saveProfile(profileConfig); //then SparkSession.Builder builder = SparkSession.builder(); sut.loadDefaults(builder); @@ -128,4 +152,67 @@ public void saveAndLoadDefaults() { assertThat(sparkConfBasedOn.get("sparkOption2")).isEqualTo("sp2"); assertThat(sparkConfBasedOn.get(SPARK_MASTER)).isEqualTo("local[4]"); } + + @Test + public void createTwoProfiles() { + //given + HashMap profileConfig1 = new HashMap<>(); + profileConfig1.put(SPARK_MASTER, "local[4]"); + profileConfig1.put(NAME, PROFILE1); + + HashMap profileConfig2 = new HashMap<>(); + profileConfig2.put(SPARK_MASTER, "local[8]"); + profileConfig2.put(NAME, PROFILE2); + //when + sut.saveSparkConf(Arrays.asList(profileConfig1, profileConfig2)); + //then + Map options1 = getOptions(PROFILE1); + Map options2 = getOptions(PROFILE2); + String prop1 = (String) options1.get(SPARK_MASTER); + String prop2 = (String) options2.get(SPARK_MASTER); + assertThat(prop1).isEqualTo("local[4]"); + assertThat(prop2).isEqualTo("local[8]"); + assertThat(sut.getProfiles().size()).isEqualTo(2); + } + + @Test + public void removeProfile() { + //given + HashMap profileConfig1 = new HashMap<>(); + profileConfig1.put(SPARK_MASTER, "local[4]"); + profileConfig1.put(NAME, PROFILE1); + + HashMap profileConfig2 = new HashMap<>(); + profileConfig2.put(SPARK_MASTER, "local[8]"); + profileConfig2.put(NAME, PROFILE2); + //when + sut.saveProfile(profileConfig1); + sut.saveProfile(profileConfig2); + sut.removeSparkConf(PROFILE1); + //then + Map options2 = getOptions(PROFILE2); + String prop2 = (String) options2.get(SPARK_MASTER); + assertThat(prop2).isEqualTo("local[8]"); + assertThat(sut.getProfiles().size()).isEqualTo(1); + } + + @Test + public void overwriteProfile() { + //given + HashMap profileConfig1 = new HashMap<>(); + profileConfig1.put(SPARK_MASTER, "local[4]"); + profileConfig1.put(NAME, PROFILE1); + + HashMap profileConfig2 = new HashMap<>(); + profileConfig2.put(SPARK_MASTER, "local[8]"); + profileConfig2.put(NAME, PROFILE1); + //when + sut.saveProfile(profileConfig1); + sut.saveProfile(profileConfig2); + //then + Map options = getOptions(PROFILE1); + String prop = (String) options.get(SPARK_MASTER); + assertThat(prop).isEqualTo("local[8]"); + assertThat(sut.getProfiles().size()).isEqualTo(1); + } } \ No newline at end of file