From 97c3e0856910f10e3af9bc900909221e2a0d4b40 Mon Sep 17 00:00:00 2001 From: jarek Date: Wed, 16 May 2018 12:56:37 +0200 Subject: [PATCH] #7330: add spark configuration widget --- js/notebook/src/SparkConfiguration.ts | 27 +++++ js/notebook/src/embed.js | 1 + js/notebook/src/index.js | 1 + .../java/com/twosigma/beakerx/widget/Box.java | 6 +- .../twosigma/beakerx/widget/StringWidget.java | 16 ++- .../twosigma/beakerx/widget/Properties.java | 42 +++++++ .../twosigma/beakerx/widget/PropertyItem.java | 41 +++++++ .../beakerx/widget/SparkConfiguration.java | 110 ++++++++++++++++++ .../twosigma/beakerx/widget/SparkManager.java | 4 +- .../beakerx/widget/SparkManagerImpl.java | 12 +- .../com/twosigma/beakerx/widget/SparkUI.java | 45 +++++-- .../beakerx/widget/SparkUIManager.java | 46 +++++--- .../magic/command/SparkMagicCommandTest.java | 4 +- 13 files changed, 314 insertions(+), 41 deletions(-) create mode 100644 js/notebook/src/SparkConfiguration.ts create mode 100644 kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/Properties.java create mode 100644 kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java create mode 100644 kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java diff --git a/js/notebook/src/SparkConfiguration.ts b/js/notebook/src/SparkConfiguration.ts new file mode 100644 index 0000000000..eb2bad950a --- /dev/null +++ b/js/notebook/src/SparkConfiguration.ts @@ -0,0 +1,27 @@ + +const widgets = require('./widgets'); + + +class SparkConfigurationModel extends widgets.VBoxModel { + + defaults() { + return { + ...super.defaults(), + _view_name: "SparkConfigurationView", + _model_name: "SparkConfigurationModel", + _model_module: 'beakerx', + _view_module: 'beakerx', + _model_module_version: BEAKERX_MODULE_VERSION, + _view_module_version: BEAKERX_MODULE_VERSION, + }; + } +} + +class SparkConfigurationView extends widgets.VBoxView { + +} + +export default { + SparkConfigurationModel, + SparkConfigurationView +}; \ No newline at end of file diff --git a/js/notebook/src/embed.js b/js/notebook/src/embed.js index 6a0898f44a..32f1987edb 100644 --- a/js/notebook/src/embed.js +++ b/js/notebook/src/embed.js @@ -21,6 +21,7 @@ var loadedModules = [ require("./CyclingDisplayBox"), require("./SparkUI").default, require("./SparkStateProgress").default, + require("./SparkConfiguration").default, require("./HTMLPre").default, require("./BxHTML").default, require("./Foldout").default, diff --git a/js/notebook/src/index.js b/js/notebook/src/index.js index b090c72d91..343b0bdeb1 100644 --- a/js/notebook/src/index.js +++ b/js/notebook/src/index.js @@ -35,6 +35,7 @@ var loadedModules = [ require("./CyclingDisplayBox"), require("./SparkUI").default, require("./SparkStateProgress").default, + require("./SparkConfiguration").default, require("./HTMLPre").default, require("./BxHTML").default, require("./Foldout").default, diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/widget/Box.java b/kernel/base/src/main/java/com/twosigma/beakerx/widget/Box.java index f0b687f706..de604eadaf 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/widget/Box.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/widget/Box.java @@ -32,12 +32,16 @@ public abstract class Box extends ValueWidget { public static final String VIEW_NAME_VALUE = "BoxView"; public static final String MODEL_NAME_VALUE = "BoxModel"; - List children; + private List children; public Box(List children) { this.children = children; } + public List getChildren() { + return children; + } + @Override protected HashMap content(HashMap content) { List commIds = comIds(); diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/widget/StringWidget.java b/kernel/base/src/main/java/com/twosigma/beakerx/widget/StringWidget.java index 06666f9521..7b9da5e99b 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/widget/StringWidget.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/widget/StringWidget.java @@ -18,15 +18,16 @@ import java.io.Serializable; import java.util.HashMap; -import com.twosigma.beakerx.widget.ValueWidget; - public abstract class StringWidget extends ValueWidget { + public static final String PLACE_HOLDER = "placeholder"; + private String placeholder = ""; + @Override protected HashMap content(HashMap content) { super.content(content); content.put(VALUE, this.value); - content.put("placeholder", ""); + content.put("placeholder", this.placeholder); return content; } @@ -35,4 +36,13 @@ public String getValueFromObject(Object input) { return getString(input); } + public String getPlaceholder() { + return placeholder; + } + + public void setPlaceholder(String placeholder) { + this.placeholder = placeholder; + sendUpdate(PLACE_HOLDER, this.placeholder); + } + } \ No newline at end of file diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/Properties.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/Properties.java new file mode 100644 index 0000000000..8701ef2d2d --- /dev/null +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/Properties.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.widget; + +import java.util.List; +import java.util.stream.Collectors; + +public class Properties { + + private final VBox widget; + + public Properties(List children) { + this.widget = new VBox(children.stream().map(x -> (Widget) x).collect(Collectors.toList())); + } + + public List getItems() { + return this.widget.getChildren().stream() + .map(x -> (PropertyItem) x) + .collect(Collectors.toList()); + } + + public VBox getWidget() { + return widget; + } + + public void add(PropertyItem propertyItem) { + this.widget.add(propertyItem); + } +} diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java new file mode 100644 index 0000000000..3150e1dcc7 --- /dev/null +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.widget; + +import static com.twosigma.beakerx.util.Preconditions.checkNotNull; +import static java.util.Arrays.asList; + +public class PropertyItem extends HBox { + + private Text name; + private Text value; + private Button remove; + + public PropertyItem(Text name, Text value, Button remove) { + super(asList(checkNotNull(name), checkNotNull(value), checkNotNull(remove))); + this.name = name; + this.value = value; + this.remove = remove; + } + + public String getNameAsString() { + return name.getValue(); + } + + public String getValueAsString() { + return value.getValue(); + } +} diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java new file mode 100644 index 0000000000..d173627495 --- /dev/null +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java @@ -0,0 +1,110 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.widget; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Collections.EMPTY_LIST; +import static java.util.Collections.singletonList; + +public class SparkConfiguration extends VBox { + + public static final String VIEW_NAME_VALUE = "SparkConfigurationView"; + public static final String MODEL_NAME_VALUE = "SparkConfigurationModel"; + + private Button add; + private HBox header; + private Properties properties; + + public SparkConfiguration() { + super(new ArrayList<>()); + this.add = createAddButton(); + this.header = new HBox(singletonList(this.add)); + add(this.header); + this.properties = new Properties(new ArrayList<>()); + add(this.properties.getWidget()); + } + + private Button createAddButton() { + Button add = new Button(); + add.setDescription("+"); + add.registerOnClick((content, message) -> addProperty()); + return add; + } + + private void addProperty() { + Text name = new Text(); + name.setPlaceholder("name"); + Text value = new Text(); + value.setPlaceholder("value"); + Button remove = new Button(); + remove.setDescription("-"); + PropertyItem propertyItem = new PropertyItem(name, value, remove); + remove.registerOnClick((content, message) -> this.properties.getWidget().removeDOMWidget(propertyItem)); + this.properties.add(propertyItem); + } + + @Override + public String getModelNameValue() { + return MODEL_NAME_VALUE; + } + + @Override + public String getViewNameValue() { + return VIEW_NAME_VALUE; + } + + @Override + public String getModelModuleValue() { + return BeakerxWidget.MODEL_MODULE_VALUE; + } + + @Override + public String getViewModuleValue() { + return BeakerxWidget.VIEW_MODULE_VALUE; + } + + public List getConfiguration() { + if (this.properties == null) { + return EMPTY_LIST; + } + List children = this.properties.getItems(); + return children.stream() + .map(x -> new Configuration(x.getNameAsString(), x.getValueAsString())) + .collect(Collectors.toList()); + } + + public static class Configuration { + + private String name; + private String value; + + public Configuration(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java index 5bb0739f3a..8c918fb460 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java @@ -21,13 +21,15 @@ import org.apache.spark.SparkContext; import org.apache.spark.sql.SparkSession; +import java.util.List; + public interface SparkManager { TryResult configure(KernelFunctionality kernel, SparkUIManager sparkContextManager); SparkSession getOrCreate(); - SparkConf getSparkConf(); + SparkConf getSparkConf(List configurations); SparkContext sparkContext(); diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java index cccc4301dd..0708f16162 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java @@ -34,6 +34,7 @@ import scala.collection.Iterator; import java.lang.reflect.Field; +import java.util.List; import java.util.UUID; import static com.twosigma.beakerx.kernel.PlainCode.createSimpleEvaluationObject; @@ -56,7 +57,7 @@ private SparkManagerImpl(SparkSession.Builder sparkSessionBuilder) { @Override public TryResult configure(KernelFunctionality kernel, SparkUIManager sparkContextManager) { this.sparkContextManager = sparkContextManager; - SparkConf sparkConf = configureSparkConf(getSparkConf()); + SparkConf sparkConf = configureSparkConf(getSparkConf(sparkContextManager.getAdvancedOptions())); sparkSessionBuilder.config(sparkConf); SparkSession sparkSession = getOrCreate(); addListener(getOrCreate().sparkContext()); @@ -97,7 +98,7 @@ private TryResult initSparkContextInShell(KernelFunctionality kernel) { return kernel.executeCode(addSc, seo); } - private SparkConf createSparkConf() { + private SparkConf createSparkConf(List configurations) { SparkConf sparkConf = new SparkConf(); try { Field options = this.sparkSessionBuilder.getClass().getDeclaredField("org$apache$spark$sql$SparkSession$Builder$$options"); @@ -107,14 +108,15 @@ private SparkConf createSparkConf() { Tuple2 x = (Tuple2) iterator.next(); sparkConf.set((String) (x)._1, (String) (x)._2); } + configurations.forEach(x -> sparkConf.set(x.getName(), x.getValue())); } catch (Exception e) { throw new RuntimeException(e); } return sparkConf; } - public SparkConf getSparkConf() { - return createSparkConf(); + public SparkConf getSparkConf(List configurations) { + return createSparkConf(configurations); } private SparkConf configureSparkConf(SparkConf sparkConf) { @@ -189,7 +191,7 @@ private static boolean isLocalSpark(SparkConf sparkConf) { } - public static class SparkManagerFactoryImpl implements SparkManagerFactory{ + public static class SparkManagerFactoryImpl implements SparkManagerFactory { @Override public SparkManager create(SparkSession.Builder sparkSessionBuilder) { diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java index 687953c248..0f4816ab3b 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java @@ -18,6 +18,7 @@ import org.apache.spark.sql.SparkSession; import java.util.ArrayList; +import java.util.Arrays; import java.util.UUID; import static com.twosigma.beakerx.widget.StartStopSparkListener.START_STOP_SPARK_LISTENER; @@ -31,8 +32,10 @@ public class SparkUI extends VBox { private final SparkUIManager sparkUIManager; - private VBox vBox; + private VBox sparkConfig; + private VBox sparkConfigPanel; private Button connectButton; + private HBox statusPanel; private static SparkUI create(SparkManager sparkManager) { return new SparkUI(sparkManager); @@ -41,8 +44,9 @@ private static SparkUI create(SparkManager sparkManager) { private SparkUI(SparkManager sparkManager) { super(new ArrayList<>()); configureSparkSessionBuilder(sparkManager.getBuilder()); - this.vBox = new VBox(new ArrayList<>()); - add(vBox); + this.sparkConfig = new VBox(new ArrayList<>()); + this.sparkConfigPanel = new VBox(Arrays.asList(sparkConfig)); + add(sparkConfigPanel); this.sparkUIManager = new SparkUIManager(this, sparkManager); } @@ -77,33 +81,52 @@ public boolean isSparkSessionIsActive() { } public void addMasterUrl(Text masterURL) { - vBox.add(masterURL); + sparkConfig.add(masterURL); } public void addExecutorCores(Text executorCores) { - vBox.add(executorCores); + sparkConfig.add(executorCores); } public void addExecutorMemory(Text executorMemory) { - vBox.add(executorMemory); + sparkConfig.add(executorMemory); } public void addConnectButton(Button connect) { this.connectButton = connect; - vBox.add(connectButton); + sparkConfig.add(connectButton); } public void clearView() { - removeDOMWidget(vBox); - connectButton = null; - this.vBox = new VBox(new ArrayList<>()); - add(vBox); + sparkConfigPanel.getLayout().setDisplayNone(); + sparkConfigPanel = null; + } + + public void addView() { + this.sparkConfigPanel = new VBox(Arrays.asList(sparkConfig)); + add(sparkConfigPanel); } public Button getConnectButton() { return connectButton; } + public void addAdvanceOptions(SparkConfiguration advancedOption) { + this.sparkConfig.add(advancedOption); + } + + public void addStatusPanel(HBox statusPanel) { + this.statusPanel = statusPanel; + add(statusPanel); + } + + public void removeStatusPanel() { + if (statusPanel != null) { + removeDOMWidget(statusPanel); + statusPanel = null; + } + } + public interface SparkUIFactory { SparkUI create(SparkManager sparkManager); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index 143445f74d..b831a3711f 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -20,11 +20,14 @@ import com.twosigma.beakerx.kernel.KernelFunctionality; import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.message.Message; +import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static com.twosigma.beakerx.kernel.PlainCode.createSimpleEvaluationObject; @@ -41,10 +44,10 @@ public class SparkUIManager { private final SparkUI sparkUI; private Map progressBars = new HashMap<>(); - private HBox statusPanel; private Text masterURL; private Text executorMemory; private Text executorCores; + private SparkConfiguration advancedOption; private boolean active = false; @@ -67,24 +70,31 @@ private void createSparkView() { this.sparkUI.addExecutorCores(executorCores); this.sparkUI.addExecutorMemory(executorMemory); this.sparkUI.addConnectButton(createConnectButton()); + this.advancedOption = new SparkConfiguration(); + this.sparkUI.addAdvanceOptions(advancedOption); } private Text createExecutorCores() { Text cores = new Text(); cores.setDescription("Executor cores"); - if (sparkManager.getSparkConf().contains(SPARK_EXECUTOR_CORES)) { - cores.setValue(sparkManager.getSparkConf().get(SPARK_EXECUTOR_CORES)); + if (getSparkConf().contains(SPARK_EXECUTOR_CORES)) { + cores.setValue(getSparkConf().get(SPARK_EXECUTOR_CORES)); } else { cores.setValue("10"); } return cores; } + private SparkConf getSparkConf() { + List list = (this.advancedOption != null) ? this.advancedOption.getConfiguration() : Collections.EMPTY_LIST; + return sparkManager.getSparkConf(list); + } + private Text createExecutorMemory() { Text masterURL = new Text(); masterURL.setDescription("Executor Memory"); - if (sparkManager.getSparkConf().contains(SPARK_EXECUTOR_MEMORY)) { - masterURL.setValue(sparkManager.getSparkConf().get(SPARK_EXECUTOR_MEMORY)); + if (getSparkConf().contains(SPARK_EXECUTOR_MEMORY)) { + masterURL.setValue(getSparkConf().get(SPARK_EXECUTOR_MEMORY)); } else { masterURL.setValue("8g"); } @@ -94,8 +104,8 @@ private Text createExecutorMemory() { private Text createMasterURL() { Text masterURL = new Text(); masterURL.setDescription("Master URL"); - if (sparkManager.getSparkConf().contains(SPARK_MASTER)) { - masterURL.setValue(sparkManager.getSparkConf().get(SPARK_MASTER)); + if (getSparkConf().contains(SPARK_MASTER)) { + masterURL.setValue(getSparkConf().get(SPARK_MASTER)); } else { masterURL.setValue(SPARK_MASTER_DEFAULT); } @@ -132,32 +142,26 @@ private void sendError(Message parentMessage, KernelFunctionality kernel, String seo.error(message); } - public void applicationStart() { sparkUI.clearView(); - statusPanel = createStatusPanel(); + sparkUI.addStatusPanel(createStatusPanel()); } public void applicationEnd() { - if (statusPanel != null) { - sparkUI.removeDOMWidget(statusPanel); - statusPanel = null; - active = false; - createSparkView(); - } + sparkUI.removeStatusPanel(); + active = false; + sparkUI.addView(); } private HBox createStatusPanel() { Label appStatus = createAppStatus(); Button disconnect = createDisconnectButton(); - HBox statusPanel = new HBox(Arrays.asList(uiLink(), disconnect, appStatus)); - sparkUI.add(statusPanel); - return statusPanel; + return new HBox(Arrays.asList(uiLink(), disconnect, appStatus)); } private Label createAppStatus() { Label appStatus = new Label(); - appStatus.setValue("Connected to " + this.sparkManager.getSparkConf().get("spark.master")); + appStatus.setValue("Connected to " + getSparkConf().get("spark.master")); return appStatus; } @@ -248,4 +252,8 @@ public Text getExecutorMemory() { public Text getExecutorCores() { return executorCores; } + + public List getAdvancedOptions() { + return this.advancedOption.getConfiguration(); + } } diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java index ec7e39bd15..8abf67e910 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java @@ -22,6 +22,7 @@ import com.twosigma.beakerx.kernel.magic.command.MagicCommandExecutionParam; import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutcomeItem; import com.twosigma.beakerx.message.Message; +import com.twosigma.beakerx.widget.SparkConfiguration; import com.twosigma.beakerx.widget.SparkUIManager; import com.twosigma.beakerx.widget.SparkManager; import com.twosigma.beakerx.widget.SparkUI; @@ -33,6 +34,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -111,7 +113,7 @@ public SparkSession getOrCreate() { } @Override - public SparkConf getSparkConf() { + public SparkConf getSparkConf(List configurations) { return sparkConf; }