diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts
index 4d2a5ee8a1..71589c45f3 100644
--- a/js/notebook/src/SparkUI.ts
+++ b/js/notebook/src/SparkUI.ts
@@ -18,6 +18,7 @@ import {Widget} from "@phosphor/widgets";
import BeakerXApi from "./tree/Utils/BeakerXApi";
const widgets = require('./widgets');
+const bkUtils = require("./shared/bkUtils");
class SparkUIModel extends widgets.VBoxModel {
defaults() {
@@ -36,10 +37,20 @@ class SparkUIModel extends widgets.VBoxModel {
class SparkUIView extends widgets.VBoxView {
private sparkStats: Widget;
private sparkAppId: string;
+ private sparkUiWebUrl: string;
+ private sparkMasterUrl: string;
private apiCallIntervalId: number;
private connectionLabelActive: HTMLElement;
private connectionLabelMemory: HTMLElement;
private connectionLabelDead: HTMLElement;
+ private connectionStatusElement: HTMLElement;
+
+ initialize(parameters) {
+ super.initialize(parameters);
+
+ this.openWebUi = this.openWebUi.bind(this);
+ this.openExecutors = this.openExecutors.bind(this);
+ }
public render() {
super.render();
@@ -53,10 +64,65 @@ class SparkUIView extends widgets.VBoxView {
super.update();
this.connectToApi();
+ this.addSparkUrls();
this.addSparkMetricsWidget();
this.updateLabels();
}
+ private addSparkUrls() {
+ if (!this.connectionStatusElement) {
+ this.connectionStatusElement = this.el.querySelector('.bx-connection-status');
+ }
+
+ if (!this.connectionStatusElement) {
+ return;
+ }
+
+ this.addSparUiWebUrl();
+ this.addMasterUrl();
+ }
+
+ private addSparUiWebUrl() {
+ if (this.sparkUiWebUrl) {
+ return;
+ }
+
+ this.sparkUiWebUrl = this.model.get("sparkUiWebUrl");
+
+ if (!this.sparkUiWebUrl) {
+ return;
+ }
+
+ this.connectionStatusElement.removeEventListener('click', this.openWebUi);
+ this.connectionStatusElement.addEventListener('click', this.openWebUi);
+ this.sparkStats.node.removeEventListener('click', this.openExecutors);
+ this.sparkStats.node.addEventListener('click', this.openExecutors);
+ this.connectionStatusElement.style.cursor = 'pointer';
+ this.sparkStats.node.style.cursor = 'pointer';
+ }
+
+ private addMasterUrl() {
+ if (this.sparkMasterUrl) {
+ return
+ }
+
+ this.sparkMasterUrl = this.model.get("sparkMasterUrl");
+
+ if (!this.sparkMasterUrl) {
+ return;
+ }
+
+ this.connectionStatusElement.setAttribute('title', this.sparkMasterUrl);
+ }
+
+ private openWebUi() {
+ window.open(this.sparkUiWebUrl, '_blank');
+ }
+
+ private openExecutors() {
+ window.open(`${this.sparkUiWebUrl}/executors`, '_blank');
+ }
+
private updateLabels() {
const lengths = [];
const labels = [];
@@ -117,9 +183,10 @@ class SparkUIView extends widgets.VBoxView {
}
private createSparkMetricsWidget(): void {
+ this.connectionStatusElement = this.el.querySelector('.bx-connection-status');
+
if (this.sparkStats) {
- this.el.querySelector('.bx-connection-status')
- .insertAdjacentElement('afterend', this.sparkStats.node);
+ this.connectionStatusElement.insertAdjacentElement('afterend', this.sparkStats.node);
return;
}
@@ -129,14 +196,14 @@ class SparkUIView extends widgets.VBoxView {
this.sparkStats.node.innerHTML = `
0
0
0
+ class="memory label label-default" title="Storage Memory">0.0 B
`;
this.connectionLabelActive = this.sparkStats.node.querySelector('.active');
this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory');
this.connectionLabelDead = this.sparkStats.node.querySelector('.dead');
- this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node);
+ this.connectionStatusElement.insertAdjacentElement('afterend', this.sparkStats.node);
}
private connectToApi() {
@@ -202,7 +269,7 @@ class SparkUIView extends widgets.VBoxView {
});
this.connectionLabelActive.innerText = `${activeTasks}`;
- this.connectionLabelMemory.innerText = `${storageMemory}`;
+ this.connectionLabelMemory.innerText = `${bkUtils.formatBytes(storageMemory)}`;
this.connectionLabelDead.innerText = `${deadExecutors}`;
}
diff --git a/js/notebook/src/shared/bkUtils.js b/js/notebook/src/shared/bkUtils.js
index 72dcdebfd9..4835aebc3a 100644
--- a/js/notebook/src/shared/bkUtils.js
+++ b/js/notebook/src/shared/bkUtils.js
@@ -54,5 +54,15 @@ module.exports = {
},
newDeferred: function() {
return jQuery.Deferred();
+ },
+ formatBytes: function(bytes) {
+ if (bytes == 0) return '0.0 B';
+
+ var k = 1000;
+ var dm = 1;
+ var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+ var i = Math.floor(Math.log(bytes) / Math.log(k));
+
+ return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
}
};
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java
index b0bd2eab67..cb4ce03c31 100644
--- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java
+++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java
@@ -38,6 +38,10 @@ public interface SparkManager {
String getSparkAppId();
+ String getSparkUiWebUrl();
+
+ String getSparkMasterUrl();
+
interface SparkManagerFactory {
SparkManager create(SparkSession.Builder sparkSessionBuilder);
}
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java
index 785a6b0e45..f573f2656f 100644
--- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java
+++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java
@@ -84,6 +84,17 @@ public String getSparkAppId() {
return conf.getAll().get("spark.app.id").get();
}
+ @Override
+ public String getSparkUiWebUrl() {
+ return getOrCreate().sparkContext().uiWebUrl().get();
+ }
+
+ @Override
+ public String getSparkMasterUrl() {
+ RuntimeConfig conf = getOrCreate().conf();
+ return conf.getAll().get("spark.master").get();
+ }
+
@Override
public SparkContext sparkContext() {
return getOrCreate().sparkContext();
diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java
index 369ed916a8..c879b7a1e6 100644
--- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java
+++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java
@@ -140,6 +140,8 @@ public void applicationStart() {
sparkUI.clearView();
sparkUI.addStatusPanel(createStatusPanel());
sparkUI.sendUpdate("sparkAppId", sparkManager.getSparkAppId());
+ sparkUI.sendUpdate("sparkUiWebUrl", sparkManager.getSparkUiWebUrl());
+ sparkUI.sendUpdate("sparkMasterUrl", sparkManager.getSparkMasterUrl());
}
public void applicationEnd() {
@@ -151,14 +153,14 @@ public void applicationEnd() {
private HBox createStatusPanel() {
Label appStatus = createAppStatus();
Button disconnect = createDisconnectButton();
- HBox connectionPanel = new HBox(Arrays.asList(uiLink(), appStatus, disconnect));
+ HBox connectionPanel = new HBox(Arrays.asList(appStatus, disconnect));
connectionPanel.setDomClasses(new ArrayList<>(Arrays.asList("bx-status-panel")));
return connectionPanel;
}
private Label createAppStatus() {
Label appStatus = new Label();
- appStatus.setValue("Connected to " + getSparkConf().get("spark.master"));
+ appStatus.setValue("Connected");
appStatus.setDomClasses(new ArrayList<>(Arrays.asList("bx-connection-status", "connected")));
return appStatus;
}
@@ -214,18 +216,6 @@ void taskEnd(int stageId, long taskId) {
intProgress.addDone();
}
- private HTML uiLink() {
- if (this.sparkManager.sparkContext().uiWebUrl().isDefined()) {
- HTML html = new HTML();
- html.setValue("Spark UI" + "");
- return html;
- } else {
- HTML html = new HTML();
- html.setValue("Spark UI " + "");
- return html;
- }
- }
-
private String stageLink(int stageId) {
if (getSparkSession().sparkContext().uiWebUrl().isDefined()) {
return getSparkSession().sparkContext().uiWebUrl().get() + "/stages/stage/?id=" + stageId + "&attempt=0";
diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java
index c86120368f..038c925a7d 100644
--- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java
+++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java
@@ -132,6 +132,16 @@ public SparkSession.Builder getBuilder() {
public String getSparkAppId() {
return "sparkAppId1";
}
+
+ @Override
+ public String getSparkUiWebUrl() {
+ return "sparkUiWebUrl";
+ }
+
+ @Override
+ public String getSparkMasterUrl() {
+ return "sparkMasterUrl";
+ }
};
}