#7426 improve spark connection status widget (#7432)

twosigma · May 25, 2018 · acb1858 · acb1858
1 parent a24a466
commit acb1858
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 19 deletions.
diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts
@@ -18,6 +18,7 @@ import {Widget} from "@phosphor/widgets";
 import BeakerXApi from "./tree/Utils/BeakerXApi";
 
 const widgets = require('./widgets');
+const bkUtils = require("./shared/bkUtils");
 
 class SparkUIModel extends widgets.VBoxModel {
   defaults() {
@@ -36,10 +37,20 @@ class SparkUIModel extends widgets.VBoxModel {
 class SparkUIView extends widgets.VBoxView {
   private sparkStats: Widget;
   private sparkAppId: string;
+  private sparkUiWebUrl: string;
+  private sparkMasterUrl: string;
   private apiCallIntervalId: number;
   private connectionLabelActive: HTMLElement;
   private connectionLabelMemory: HTMLElement;
   private connectionLabelDead: HTMLElement;
+  private connectionStatusElement: HTMLElement;
+
+  initialize(parameters) {
+    super.initialize(parameters);
+
+    this.openWebUi = this.openWebUi.bind(this);
+    this.openExecutors = this.openExecutors.bind(this);
+  }
 
   public render() {
     super.render();
@@ -53,10 +64,65 @@ class SparkUIView extends widgets.VBoxView {
     super.update();
 
     this.connectToApi();
+    this.addSparkUrls();
     this.addSparkMetricsWidget();
     this.updateLabels();
   }
 
+  private addSparkUrls() {
+    if (!this.connectionStatusElement) {
+      this.connectionStatusElement = this.el.querySelector('.bx-connection-status');
+    }
+
+    if (!this.connectionStatusElement) {
+      return;
+    }
+
+    this.addSparUiWebUrl();
+    this.addMasterUrl();
+  }
+
+  private addSparUiWebUrl() {
+    if (this.sparkUiWebUrl) {
+      return;
+    }
+
+    this.sparkUiWebUrl = this.model.get("sparkUiWebUrl");
+
+    if (!this.sparkUiWebUrl) {
+      return;
+    }
+
+    this.connectionStatusElement.removeEventListener('click', this.openWebUi);
+    this.connectionStatusElement.addEventListener('click', this.openWebUi);
+    this.sparkStats.node.removeEventListener('click', this.openExecutors);
+    this.sparkStats.node.addEventListener('click', this.openExecutors);
+    this.connectionStatusElement.style.cursor = 'pointer';
+    this.sparkStats.node.style.cursor = 'pointer';
+  }
+
+  private addMasterUrl() {
+    if (this.sparkMasterUrl) {
+      return
+    }
+
+    this.sparkMasterUrl = this.model.get("sparkMasterUrl");
+
+    if (!this.sparkMasterUrl) {
+      return;
+    }
+
+    this.connectionStatusElement.setAttribute('title', this.sparkMasterUrl);
+  }
+
+  private openWebUi() {
+    window.open(this.sparkUiWebUrl, '_blank');
+  }
+
+  private openExecutors() {
+    window.open(`${this.sparkUiWebUrl}/executors`, '_blank');
+  }
+
   private updateLabels() {
     const lengths = [];
     const labels = [];
@@ -117,9 +183,10 @@ class SparkUIView extends widgets.VBoxView {
   }
 
   private createSparkMetricsWidget(): void {
+    this.connectionStatusElement = this.el.querySelector('.bx-connection-status');
+
     if (this.sparkStats) {
-      this.el.querySelector('.bx-connection-status')
-        .insertAdjacentElement('afterend', this.sparkStats.node);
+      this.connectionStatusElement.insertAdjacentElement('afterend', this.sparkStats.node);
 
       return;
     }
@@ -129,14 +196,14 @@ class SparkUIView extends widgets.VBoxView {
     this.sparkStats.node.innerHTML = `
       <div class="active label label-info" title="Active Tasks">0</div> <div
       class="dead label label-danger" title="Dead Executors">0</div> <div
-      class="memory label label-default" title="Storage Memory">0</div>
+      class="memory label label-default" title="Storage Memory">0.0 B</div>
     `;
 
     this.connectionLabelActive = this.sparkStats.node.querySelector('.active');
     this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory');
     this.connectionLabelDead = this.sparkStats.node.querySelector('.dead');
 
-    this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node);
+    this.connectionStatusElement.insertAdjacentElement('afterend', this.sparkStats.node);
   }
 
   private connectToApi() {
@@ -202,7 +269,7 @@ class SparkUIView extends widgets.VBoxView {
     });
 
     this.connectionLabelActive.innerText = `${activeTasks}`;
-    this.connectionLabelMemory.innerText = `${storageMemory}`;
+    this.connectionLabelMemory.innerText = `${bkUtils.formatBytes(storageMemory)}`;
     this.connectionLabelDead.innerText = `${deadExecutors}`;
   }
 

diff --git a/js/notebook/src/shared/bkUtils.js b/js/notebook/src/shared/bkUtils.js
@@ -54,5 +54,15 @@ module.exports = {
   },
   newDeferred: function() {
     return jQuery.Deferred();
+  },
+  formatBytes: function(bytes) {
+    if (bytes == 0) return '0.0 B';
+
+    var k = 1000;
+    var dm = 1;
+    var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    var i = Math.floor(Math.log(bytes) / Math.log(k));
+
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
   }
 };
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java
@@ -38,6 +38,10 @@ public interface SparkManager {
 
   String getSparkAppId();
 
+  String getSparkUiWebUrl();
+
+  String getSparkMasterUrl();
+
   interface SparkManagerFactory {
     SparkManager create(SparkSession.Builder sparkSessionBuilder);
   }

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java
@@ -84,6 +84,17 @@ public String getSparkAppId() {
     return conf.getAll().get("spark.app.id").get();
   }
 
+  @Override
+  public String getSparkUiWebUrl() {
+    return getOrCreate().sparkContext().uiWebUrl().get();
+  }
+
+  @Override
+  public String getSparkMasterUrl() {
+    RuntimeConfig conf = getOrCreate().conf();
+    return conf.getAll().get("spark.master").get();
+  }
+
   @Override
   public SparkContext sparkContext() {
     return getOrCreate().sparkContext();

diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java
@@ -140,6 +140,8 @@ public void applicationStart() {
     sparkUI.clearView();
     sparkUI.addStatusPanel(createStatusPanel());
     sparkUI.sendUpdate("sparkAppId", sparkManager.getSparkAppId());
+    sparkUI.sendUpdate("sparkUiWebUrl", sparkManager.getSparkUiWebUrl());
+    sparkUI.sendUpdate("sparkMasterUrl", sparkManager.getSparkMasterUrl());
   }
 
   public void applicationEnd() {
@@ -151,14 +153,14 @@ public void applicationEnd() {
   private HBox createStatusPanel() {
     Label appStatus = createAppStatus();
     Button disconnect = createDisconnectButton();
-    HBox connectionPanel = new HBox(Arrays.asList(uiLink(), appStatus, disconnect));
+    HBox connectionPanel = new HBox(Arrays.asList(appStatus, disconnect));
     connectionPanel.setDomClasses(new ArrayList<>(Arrays.asList("bx-status-panel")));
     return connectionPanel;
   }
 
   private Label createAppStatus() {
     Label appStatus = new Label();
-    appStatus.setValue("Connected to " + getSparkConf().get("spark.master"));
+    appStatus.setValue("Connected");
     appStatus.setDomClasses(new ArrayList<>(Arrays.asList("bx-connection-status", "connected")));
     return appStatus;
   }
@@ -214,18 +216,6 @@ void taskEnd(int stageId, long taskId) {
     intProgress.addDone();
   }
 
-  private HTML uiLink() {
-    if (this.sparkManager.sparkContext().uiWebUrl().isDefined()) {
-      HTML html = new HTML();
-      html.setValue("<a target=\"_blank\" href=\"" + getSparkSession().sparkContext().uiWebUrl().get() + "\">Spark UI" + "</a>");
-      return html;
-    } else {
-      HTML html = new HTML();
-      html.setValue("<a target=\"_blank\" href=\"\">Spark UI " + "</a>");
-      return html;
-    }
-  }
-
   private String stageLink(int stageId) {
     if (getSparkSession().sparkContext().uiWebUrl().isDefined()) {
       return getSparkSession().sparkContext().uiWebUrl().get() + "/stages/stage/?id=" + stageId + "&attempt=0";

diff --git a/...sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/...sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java
@@ -132,6 +132,16 @@ public SparkSession.Builder getBuilder() {
       public String getSparkAppId() {
         return "sparkAppId1";
       }
+
+      @Override
+      public String getSparkUiWebUrl() {
+        return "sparkUiWebUrl";
+      }
+
+      @Override
+      public String getSparkMasterUrl() {
+        return "sparkMasterUrl";
+      }
     };
   }