#7416 improve SparkUI widget - UI (#7439)

* #7416 improve SparkUI widget - UI

* #7416: add link to available-properties

* #7416 SparkUI config UI improvements

js/notebook/src/SparkUI.ts

@@ -91,10 +91,6 @@ class SparkUIView extends widgets.VBoxView {
   }
 
   private addSparUiWebUrl() {
-    if (this.sparkUiWebUrl) {
-      return;
-    }
-
     this.sparkUiWebUrl = this.model.get("sparkUiWebUrl");
 
     if (!this.sparkUiWebUrl) {
@@ -110,10 +106,6 @@ class SparkUIView extends widgets.VBoxView {
   }
 
   private addMasterUrl() {
-    if (this.sparkMasterUrl) {
-      return
-    }
-
     this.sparkMasterUrl = this.model.get("sparkMasterUrl");
 
     if (!this.sparkMasterUrl) {
@@ -162,7 +154,6 @@ class SparkUIView extends widgets.VBoxView {
           views.forEach((view) => {
             this.resolveChildren(view)
               .then((views) => {
-                this.setLabelsWidth(views);
                 this.handleLocalMasterUrl();
               })
               .catch(noop);
@@ -183,31 +174,6 @@ class SparkUIView extends widgets.VBoxView {
     });
   }
 
-  private setLabelsWidth(views): void {
-    let labels = [];
-    let lengths = [];
-
-    views.forEach((view) => {
-      const label = view.el.querySelector('.widget-label');
-
-      if (!label) {
-        return true;
-      }
-
-      lengths.push(this.getLabelWidth(label));
-      labels.push(label);
-    });
-
-    const maxWidth = Math.max.apply(null, lengths);
-    labels.forEach((label) => { label.style.width = `${maxWidth}px`; });
-  }
-
-  private getLabelWidth(labelEl): number {
-    labelEl.style.width = 'auto';
-
-    return labelEl.clientWidth;
-  }
-
   private createSparkMetricsWidget(): void {
     this.connectionStatusElement = this.el.querySelector('.bx-connection-status');
 

js/notebook/src/shared/style/spark.scss

@@ -147,3 +147,30 @@
     color: $bxBgColor1;
   }
 }
+
+.widget-spark-ui {
+  .bx-spark-config {
+    width: 606px;
+    max-width: 99%;
+
+    .widget-label {
+      margin-right: 10px;
+      width: 292px;
+    }
+
+    input {
+      width: 300px;
+      margin: 2px;
+    }
+
+    input,
+    .widget-label {
+      flex-grow: 1;
+      flex-shrink: 1;
+    }
+  }
+
+  .bx-config-name input {
+    text-align: right;
+  }
+}

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/Properties.java → kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java

@@ -18,11 +18,11 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
-public class Properties {
+public class PropertiesWidget {
 
   private final VBox widget;
 
-  public Properties(List<PropertyItem> children) {
+  public PropertiesWidget(List<PropertyItem> children) {
     this.widget = new VBox(children.stream().map(x -> (Widget) x).collect(Collectors.toList()));
   }
 

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java

@@ -16,12 +16,12 @@
 package com.twosigma.beakerx.widget;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
 import static java.util.Collections.EMPTY_LIST;
-import static java.util.Collections.singletonList;
 
 public class SparkConfiguration extends VBox {
 
@@ -30,15 +30,22 @@ public class SparkConfiguration extends VBox {
 
   private Button add;
   private HBox header;
-  private Properties properties;
+  private PropertiesWidget properties;
 
-  SparkConfiguration(Map<String, String> advancedSettings) {
+  SparkConfiguration(Map<String, String> advancedSettings, String sparkVersion) {
     super(new ArrayList<>());
     this.add = createAddButton();
-    this.header = new HBox(singletonList(this.add));
+    this.header = new HBox(asList(this.add, sparkVersionWidget(sparkVersion)));
     List<PropertyItem> propertyItems = createPropertyItems(advancedSettings);
-    this.properties = new Properties(propertyItems);
-    add(new VBox(Arrays.asList(this.header, this.properties.getWidget())));
+    this.properties = new PropertiesWidget(propertyItems);
+    add(new VBox(asList(this.header, this.properties.getWidget())));
+  }
+
+  private HTML sparkVersionWidget(String version) {
+    HTML html = new HTML();
+    String ap = String.format("https://spark.apache.org/docs/%s/configuration.html#available-properties", version);
+    html.setValue("<a target=\"_blank\" href=\"" + ap + "\">Available properties" + "</a>");
+    return html;
   }
 
   private List<PropertyItem> createPropertyItems(Map<String, String> advancedSettings) {
@@ -49,7 +56,8 @@ private List<PropertyItem> createPropertyItems(Map<String, String> advancedSetti
 
   private Button createAddButton() {
     Button add = new Button();
-    add.setDescription("+");
+    add.setTooltip("Add property");
+    add.setDomClasses(new ArrayList<>(asList("bx-button", "icon-add")));
     add.registerOnClick((content, message) -> addProperty());
     return add;
   }
@@ -69,9 +77,10 @@ private PropertyItem createPropertyItem(String name, String value) {
 
   private PropertyItem createPropertyItem(Text nameWidget, Text valueWidget) {
     nameWidget.setPlaceholder("name");
+    nameWidget.setDomClasses(new ArrayList<>(asList("bx-config-name")));
     valueWidget.setPlaceholder("value");
     Button remove = new Button();
-    remove.setDescription("-");
+    remove.setDomClasses(new ArrayList<>(asList("bx-button", "icon-close")));
     PropertyItem propertyItem = new PropertyItem(nameWidget, valueWidget, remove);
     remove.registerOnClick((content, message) -> this.properties.getWidget().removeDOMWidget(propertyItem));
     return propertyItem;

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngine.java

@@ -19,7 +19,6 @@
 import com.twosigma.beakerx.kernel.KernelFunctionality;
 import com.twosigma.beakerx.message.Message;
 import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
 import org.apache.spark.sql.SparkSession;
 
 import java.util.Map;
@@ -40,6 +39,8 @@ public interface SparkEngine {
 
   String getSparkMasterUrl();
 
+  String sparkVersion();
+
   interface SparkEngineFactory {
     SparkEngine create(SparkSession.Builder sparkSessionBuilder);
   }

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java

@@ -36,10 +36,12 @@
 import scala.Tuple2;
 import scala.collection.Iterator;
 
+import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
@@ -59,7 +61,7 @@ public class SparkEngineImpl implements SparkEngine {
 
   private SparkSession.Builder sparkSessionBuilder;
 
-  private SparkEngineImpl(SparkSession.Builder sparkSessionBuilder) {
+  SparkEngineImpl(SparkSession.Builder sparkSessionBuilder) {
     this.sparkSessionBuilder = sparkSessionBuilder;
     configureSparkSessionBuilder(this.sparkSessionBuilder);
   }
@@ -101,6 +103,19 @@ public String getSparkMasterUrl() {
     return conf.getAll().get("spark.master").get();
   }
 
+  @Override
+  public String sparkVersion() {
+    try {
+      InputStream sparkProps = Thread.currentThread().getContextClassLoader().
+              getResourceAsStream("spark-version-info.properties");
+      Properties props = new Properties();
+      props.load(sparkProps);
+      return props.getProperty("version");
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   private TryResult initSparkContextInShell(KernelFunctionality kernel, Message parent) {
     String addSc = String.format(("import com.twosigma.beakerx.widget.SparkVariable\n" +
                     "val %s = SparkVariable.getSparkSession()\n" +

kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java

@@ -107,15 +107,15 @@ private void createSparkView() {
     this.addMasterUrl(masterURL);
     this.addExecutorCores(executorCores);
     this.addExecutorMemory(executorMemory);
-    this.advancedOption = new SparkConfiguration(sparkEngine.getAdvanceSettings());
+    this.advancedOption = new SparkConfiguration(sparkEngine.getAdvanceSettings(),sparkEngine.sparkVersion());
     this.addAdvanceOptions(advancedOption);
     this.sendUpdate("sparkDefaultMasterUrl", SPARK_MASTER_DEFAULT);
   }
 
   private Text createExecutorCores() {
     Text cores = new Text();
     cores.setDescription("Executor cores");
-    cores.setDomClasses(new ArrayList<>(Arrays.asList("bx-spark-executor-cores")));
+    cores.setDomClasses(new ArrayList<>(Arrays.asList("bx-spark-config", "bx-spark-executor-cores")));
     if (getSparkConf().contains(SPARK_EXECUTOR_CORES)) {
       cores.setValue(getSparkConf().get(SPARK_EXECUTOR_CORES));
     } else {
@@ -131,7 +131,7 @@ private SparkConf getSparkConf() {
   private Text createExecutorMemory() {
     Text memory = new Text();
     memory.setDescription("Executor Memory");
-    memory.setDomClasses(new ArrayList<>(Arrays.asList("bx-spark-executor-memory")));
+    memory.setDomClasses(new ArrayList<>(Arrays.asList("bx-spark-config", "bx-spark-executor-memory")));
     if (getSparkConf().contains(SPARK_EXECUTOR_MEMORY)) {
       memory.setValue(getSparkConf().get(SPARK_EXECUTOR_MEMORY));
     } else {
@@ -143,7 +143,7 @@ private Text createExecutorMemory() {
   private Text createMasterURL() {
     Text masterURL = new Text();
     masterURL.setDescription("Master URL");
-    masterURL.setDomClasses(new ArrayList<>(Arrays.asList("bx-spark-master-url")));
+    masterURL.setDomClasses(new ArrayList<>(Arrays.asList("bx-spark-config", "bx-spark-master-url")));
     if (getSparkConf().contains(SPARK_MASTER)) {
       masterURL.setValue(getSparkConf().get(SPARK_MASTER));
     } else {

kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java

@@ -148,6 +148,11 @@ public String getSparkUiWebUrl() {
         public String getSparkMasterUrl() {
           return "";
         }
+
+        @Override
+        public String sparkVersion() {
+          return null;
+        }
       };
     }
   }

kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkEngineImplTest.java

@@ -0,0 +1,35 @@
+
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx.widget;
+
+import org.apache.spark.sql.SparkSession;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SparkEngineImplTest {
+
+  @Test
+  public void sparkVersion() {
+    //given
+    SparkEngineImpl sparkEngine = new SparkEngineImpl(SparkSession.builder());
+    //when
+    String version = sparkEngine.sparkVersion();
+    //then
+    assertThat(version).isEqualTo("2.2.1");
+  }
+}

kernel/sparkex/src/test/java/com/twosigma/beakerx/widget/SparkUITest.java

@@ -21,7 +21,6 @@
 import com.twosigma.beakerx.kernel.KernelManager;
 import com.twosigma.beakerx.message.Message;
 import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
 import org.apache.spark.sql.SparkSession;
 import org.junit.After;
 import org.junit.Before;
@@ -128,6 +127,11 @@ public String getSparkUiWebUrl() {
     public String getSparkMasterUrl() {
       return "";
     }
+
+    @Override
+    public String sparkVersion() {
+      return null;
+    }
   }
 
 }