#7330: add spark configuration widget

js/notebook/src/SparkConfiguration.ts

@@ -0,0 +1,27 @@
+
+const widgets = require('./widgets');
+
+
+class SparkConfigurationModel extends widgets.VBoxModel {
+
+    defaults() {
+        return {
+            ...super.defaults(),
+            _view_name: "SparkConfigurationView",
+            _model_name: "SparkConfigurationModel",
+            _model_module: 'beakerx',
+            _view_module: 'beakerx',
+            _model_module_version: BEAKERX_MODULE_VERSION,
+            _view_module_version: BEAKERX_MODULE_VERSION,
+        };
+    }
+}
+
+class SparkConfigurationView extends widgets.VBoxView {
+
+}
+
+export default {
+    SparkConfigurationModel,
+    SparkConfigurationView
+};

js/notebook/src/embed.js

@@ -21,6 +21,7 @@ var loadedModules = [
   require("./CyclingDisplayBox"),
   require("./SparkUI").default,
   require("./SparkStateProgress").default,
+  require("./SparkConfiguration").default,
   require("./HTMLPre").default,
   require("./BxHTML").default,
   require("./Foldout").default,

js/notebook/src/index.js

@@ -35,6 +35,7 @@ var loadedModules = [
   require("./CyclingDisplayBox"),
   require("./SparkUI").default,
   require("./SparkStateProgress").default,
+  require("./SparkConfiguration").default,
   require("./HTMLPre").default,
   require("./BxHTML").default,
   require("./Foldout").default,

kernel/base/src/main/java/com/twosigma/beakerx/widget/Box.java

@@ -32,12 +32,16 @@ public abstract class Box extends ValueWidget<String> {
   public static final String VIEW_NAME_VALUE = "BoxView";
   public static final String MODEL_NAME_VALUE = "BoxModel";
 
-  List<Widget> children;
+  private List<Widget> children;
 
   public Box(List<Widget> children) {
     this.children = children;
   }
 
+  public List<Widget> getChildren() {
+    return children;
+  }
+
   @Override
   protected HashMap<String, Serializable> content(HashMap<String, Serializable> content) {
     List<String> commIds = comIds();

kernel/base/src/main/java/com/twosigma/beakerx/widget/StringWidget.java

@@ -18,15 +18,16 @@
 import java.io.Serializable;
 import java.util.HashMap;
 
-import com.twosigma.beakerx.widget.ValueWidget;
-
 public abstract class StringWidget extends ValueWidget<String> {
 
+  public static final String PLACE_HOLDER = "placeholder";
+  private String placeholder = "";
+
   @Override
   protected HashMap<String, Serializable> content(HashMap<String, Serializable> content) {
     super.content(content);
     content.put(VALUE, this.value);
-    content.put("placeholder", "");
+    content.put("placeholder", this.placeholder);
     return content;
   }
 
@@ -35,4 +36,13 @@ public String getValueFromObject(Object input) {
     return getString(input);
   }
 
+  public String getPlaceholder() {
+    return placeholder;
+  }
+
+  public void setPlaceholder(String placeholder) {
+    this.placeholder = placeholder;
+    sendUpdate(PLACE_HOLDER, this.placeholder);
+  }
+
 }

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/Properties.java

@@ -0,0 +1,42 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx.widget;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class Properties {
+
+  private final VBox widget;
+
+  public Properties(List<PropertyItem> children) {
+    this.widget = new VBox(children.stream().map(x -> (Widget) x).collect(Collectors.toList()));
+  }
+
+  public List<PropertyItem> getItems() {
+    return this.widget.getChildren().stream()
+            .map(x -> (PropertyItem) x)
+            .collect(Collectors.toList());
+  }
+
+  public VBox getWidget() {
+    return widget;
+  }
+
+  public void add(PropertyItem propertyItem) {
+    this.widget.add(propertyItem);
+  }
+}

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java

@@ -0,0 +1,41 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx.widget;
+
+import static com.twosigma.beakerx.util.Preconditions.checkNotNull;
+import static java.util.Arrays.asList;
+
+public class PropertyItem extends HBox {
+
+  private Text name;
+  private Text value;
+  private Button remove;
+
+  public PropertyItem(Text name, Text value, Button remove) {
+    super(asList(checkNotNull(name), checkNotNull(value), checkNotNull(remove)));
+    this.name = name;
+    this.value = value;
+    this.remove = remove;
+  }
+
+  public String getNameAsString() {
+    return name.getValue();
+  }
+
+  public String getValueAsString() {
+    return value.getValue();
+  }
+}

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java

@@ -0,0 +1,110 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx.widget;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.EMPTY_LIST;
+import static java.util.Collections.singletonList;
+
+public class SparkConfiguration extends VBox {
+
+  public static final String VIEW_NAME_VALUE = "SparkConfigurationView";
+  public static final String MODEL_NAME_VALUE = "SparkConfigurationModel";
+
+  private Button add;
+  private HBox header;
+  private Properties properties;
+
+  public SparkConfiguration() {
+    super(new ArrayList<>());
+    this.add = createAddButton();
+    this.header = new HBox(singletonList(this.add));
+    add(this.header);
+    this.properties = new Properties(new ArrayList<>());
+    add(this.properties.getWidget());
+  }
+
+  private Button createAddButton() {
+    Button add = new Button();
+    add.setDescription("+");
+    add.registerOnClick((content, message) -> addProperty());
+    return add;
+  }
+
+  private void addProperty() {
+    Text name = new Text();
+    name.setPlaceholder("name");
+    Text value = new Text();
+    value.setPlaceholder("value");
+    Button remove = new Button();
+    remove.setDescription("-");
+    PropertyItem propertyItem = new PropertyItem(name, value, remove);
+    remove.registerOnClick((content, message) -> this.properties.getWidget().removeDOMWidget(propertyItem));
+    this.properties.add(propertyItem);
+  }
+
+  @Override
+  public String getModelNameValue() {
+    return MODEL_NAME_VALUE;
+  }
+
+  @Override
+  public String getViewNameValue() {
+    return VIEW_NAME_VALUE;
+  }
+
+  @Override
+  public String getModelModuleValue() {
+    return BeakerxWidget.MODEL_MODULE_VALUE;
+  }
+
+  @Override
+  public String getViewModuleValue() {
+    return BeakerxWidget.VIEW_MODULE_VALUE;
+  }
+
+  public List<Configuration> getConfiguration() {
+    if (this.properties == null) {
+      return EMPTY_LIST;
+    }
+    List<PropertyItem> children = this.properties.getItems();
+    return children.stream()
+            .map(x -> new Configuration(x.getNameAsString(), x.getValueAsString()))
+            .collect(Collectors.toList());
+  }
+
+  public static class Configuration {
+
+    private String name;
+    private String value;
+
+    public Configuration(String name, String value) {
+      this.name = name;
+      this.value = value;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public String getValue() {
+      return value;
+    }
+  }
+}

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java

@@ -21,13 +21,15 @@
 import org.apache.spark.SparkContext;
 import org.apache.spark.sql.SparkSession;
 
+import java.util.List;
+
 public interface SparkManager {
 
   TryResult configure(KernelFunctionality kernel, SparkUIManager sparkContextManager);
 
   SparkSession getOrCreate();
 
-  SparkConf getSparkConf();
+  SparkConf getSparkConf(List<SparkConfiguration.Configuration> configurations);
 
   SparkContext sparkContext();
 

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java

@@ -34,6 +34,7 @@
 import scala.collection.Iterator;
 
 import java.lang.reflect.Field;
+import java.util.List;
 import java.util.UUID;
 
 import static com.twosigma.beakerx.kernel.PlainCode.createSimpleEvaluationObject;
@@ -56,7 +57,7 @@ private SparkManagerImpl(SparkSession.Builder sparkSessionBuilder) {
   @Override
   public TryResult configure(KernelFunctionality kernel, SparkUIManager sparkContextManager) {
     this.sparkContextManager = sparkContextManager;
-    SparkConf sparkConf = configureSparkConf(getSparkConf());
+    SparkConf sparkConf = configureSparkConf(getSparkConf(sparkContextManager.getAdvancedOptions()));
     sparkSessionBuilder.config(sparkConf);
     SparkSession sparkSession = getOrCreate();
     addListener(getOrCreate().sparkContext());
@@ -97,7 +98,7 @@ private TryResult initSparkContextInShell(KernelFunctionality kernel) {
     return kernel.executeCode(addSc, seo);
   }
 
-  private SparkConf createSparkConf() {
+  private SparkConf createSparkConf(List<SparkConfiguration.Configuration> configurations) {
     SparkConf sparkConf = new SparkConf();
     try {
       Field options = this.sparkSessionBuilder.getClass().getDeclaredField("org$apache$spark$sql$SparkSession$Builder$$options");
@@ -107,14 +108,15 @@ private SparkConf createSparkConf() {
         Tuple2 x = (Tuple2) iterator.next();
         sparkConf.set((String) (x)._1, (String) (x)._2);
       }
+      configurations.forEach(x -> sparkConf.set(x.getName(), x.getValue()));
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
     return sparkConf;
   }
 
-  public SparkConf getSparkConf() {
-    return createSparkConf();
+  public SparkConf getSparkConf(List<SparkConfiguration.Configuration> configurations) {
+    return createSparkConf(configurations);
   }
 
   private SparkConf configureSparkConf(SparkConf sparkConf) {
@@ -189,7 +191,7 @@ private static boolean isLocalSpark(SparkConf sparkConf) {
   }
 
 
-  public static class SparkManagerFactoryImpl implements SparkManagerFactory{
+  public static class SparkManagerFactoryImpl implements SparkManagerFactory {
 
     @Override
     public SparkManager create(SparkSession.Builder sparkSessionBuilder) {