From acb1858d680ab3f38d5f0f571ff3b25554ad2636 Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Sat, 26 May 2018 00:31:42 +0200 Subject: [PATCH] #7426 improve spark connection status widget (#7432) --- js/notebook/src/SparkUI.ts | 77 +++++++++++++++++-- js/notebook/src/shared/bkUtils.js | 10 +++ .../twosigma/beakerx/widget/SparkManager.java | 4 + .../beakerx/widget/SparkManagerImpl.java | 11 +++ .../beakerx/widget/SparkUIManager.java | 18 +---- .../magic/command/SparkMagicCommandTest.java | 10 +++ 6 files changed, 111 insertions(+), 19 deletions(-) diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index 4d2a5ee8a1..71589c45f3 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -18,6 +18,7 @@ import {Widget} from "@phosphor/widgets"; import BeakerXApi from "./tree/Utils/BeakerXApi"; const widgets = require('./widgets'); +const bkUtils = require("./shared/bkUtils"); class SparkUIModel extends widgets.VBoxModel { defaults() { @@ -36,10 +37,20 @@ class SparkUIModel extends widgets.VBoxModel { class SparkUIView extends widgets.VBoxView { private sparkStats: Widget; private sparkAppId: string; + private sparkUiWebUrl: string; + private sparkMasterUrl: string; private apiCallIntervalId: number; private connectionLabelActive: HTMLElement; private connectionLabelMemory: HTMLElement; private connectionLabelDead: HTMLElement; + private connectionStatusElement: HTMLElement; + + initialize(parameters) { + super.initialize(parameters); + + this.openWebUi = this.openWebUi.bind(this); + this.openExecutors = this.openExecutors.bind(this); + } public render() { super.render(); @@ -53,10 +64,65 @@ class SparkUIView extends widgets.VBoxView { super.update(); this.connectToApi(); + this.addSparkUrls(); this.addSparkMetricsWidget(); this.updateLabels(); } + private addSparkUrls() { + if (!this.connectionStatusElement) { + this.connectionStatusElement = this.el.querySelector('.bx-connection-status'); + } + + if (!this.connectionStatusElement) { + return; + } + + this.addSparUiWebUrl(); + this.addMasterUrl(); + } + + private addSparUiWebUrl() { + if (this.sparkUiWebUrl) { + return; + } + + this.sparkUiWebUrl = this.model.get("sparkUiWebUrl"); + + if (!this.sparkUiWebUrl) { + return; + } + + this.connectionStatusElement.removeEventListener('click', this.openWebUi); + this.connectionStatusElement.addEventListener('click', this.openWebUi); + this.sparkStats.node.removeEventListener('click', this.openExecutors); + this.sparkStats.node.addEventListener('click', this.openExecutors); + this.connectionStatusElement.style.cursor = 'pointer'; + this.sparkStats.node.style.cursor = 'pointer'; + } + + private addMasterUrl() { + if (this.sparkMasterUrl) { + return + } + + this.sparkMasterUrl = this.model.get("sparkMasterUrl"); + + if (!this.sparkMasterUrl) { + return; + } + + this.connectionStatusElement.setAttribute('title', this.sparkMasterUrl); + } + + private openWebUi() { + window.open(this.sparkUiWebUrl, '_blank'); + } + + private openExecutors() { + window.open(`${this.sparkUiWebUrl}/executors`, '_blank'); + } + private updateLabels() { const lengths = []; const labels = []; @@ -117,9 +183,10 @@ class SparkUIView extends widgets.VBoxView { } private createSparkMetricsWidget(): void { + this.connectionStatusElement = this.el.querySelector('.bx-connection-status'); + if (this.sparkStats) { - this.el.querySelector('.bx-connection-status') - .insertAdjacentElement('afterend', this.sparkStats.node); + this.connectionStatusElement.insertAdjacentElement('afterend', this.sparkStats.node); return; } @@ -129,14 +196,14 @@ class SparkUIView extends widgets.VBoxView { this.sparkStats.node.innerHTML = `
0
0
0
+ class="memory label label-default" title="Storage Memory">0.0 B `; this.connectionLabelActive = this.sparkStats.node.querySelector('.active'); this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory'); this.connectionLabelDead = this.sparkStats.node.querySelector('.dead'); - this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node); + this.connectionStatusElement.insertAdjacentElement('afterend', this.sparkStats.node); } private connectToApi() { @@ -202,7 +269,7 @@ class SparkUIView extends widgets.VBoxView { }); this.connectionLabelActive.innerText = `${activeTasks}`; - this.connectionLabelMemory.innerText = `${storageMemory}`; + this.connectionLabelMemory.innerText = `${bkUtils.formatBytes(storageMemory)}`; this.connectionLabelDead.innerText = `${deadExecutors}`; } diff --git a/js/notebook/src/shared/bkUtils.js b/js/notebook/src/shared/bkUtils.js index 72dcdebfd9..4835aebc3a 100644 --- a/js/notebook/src/shared/bkUtils.js +++ b/js/notebook/src/shared/bkUtils.js @@ -54,5 +54,15 @@ module.exports = { }, newDeferred: function() { return jQuery.Deferred(); + }, + formatBytes: function(bytes) { + if (bytes == 0) return '0.0 B'; + + var k = 1000; + var dm = 1; + var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } }; diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java index b0bd2eab67..cb4ce03c31 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java @@ -38,6 +38,10 @@ public interface SparkManager { String getSparkAppId(); + String getSparkUiWebUrl(); + + String getSparkMasterUrl(); + interface SparkManagerFactory { SparkManager create(SparkSession.Builder sparkSessionBuilder); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java index 785a6b0e45..f573f2656f 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java @@ -84,6 +84,17 @@ public String getSparkAppId() { return conf.getAll().get("spark.app.id").get(); } + @Override + public String getSparkUiWebUrl() { + return getOrCreate().sparkContext().uiWebUrl().get(); + } + + @Override + public String getSparkMasterUrl() { + RuntimeConfig conf = getOrCreate().conf(); + return conf.getAll().get("spark.master").get(); + } + @Override public SparkContext sparkContext() { return getOrCreate().sparkContext(); diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index 369ed916a8..c879b7a1e6 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -140,6 +140,8 @@ public void applicationStart() { sparkUI.clearView(); sparkUI.addStatusPanel(createStatusPanel()); sparkUI.sendUpdate("sparkAppId", sparkManager.getSparkAppId()); + sparkUI.sendUpdate("sparkUiWebUrl", sparkManager.getSparkUiWebUrl()); + sparkUI.sendUpdate("sparkMasterUrl", sparkManager.getSparkMasterUrl()); } public void applicationEnd() { @@ -151,14 +153,14 @@ public void applicationEnd() { private HBox createStatusPanel() { Label appStatus = createAppStatus(); Button disconnect = createDisconnectButton(); - HBox connectionPanel = new HBox(Arrays.asList(uiLink(), appStatus, disconnect)); + HBox connectionPanel = new HBox(Arrays.asList(appStatus, disconnect)); connectionPanel.setDomClasses(new ArrayList<>(Arrays.asList("bx-status-panel"))); return connectionPanel; } private Label createAppStatus() { Label appStatus = new Label(); - appStatus.setValue("Connected to " + getSparkConf().get("spark.master")); + appStatus.setValue("Connected"); appStatus.setDomClasses(new ArrayList<>(Arrays.asList("bx-connection-status", "connected"))); return appStatus; } @@ -214,18 +216,6 @@ void taskEnd(int stageId, long taskId) { intProgress.addDone(); } - private HTML uiLink() { - if (this.sparkManager.sparkContext().uiWebUrl().isDefined()) { - HTML html = new HTML(); - html.setValue("Spark UI" + ""); - return html; - } else { - HTML html = new HTML(); - html.setValue("Spark UI " + ""); - return html; - } - } - private String stageLink(int stageId) { if (getSparkSession().sparkContext().uiWebUrl().isDefined()) { return getSparkSession().sparkContext().uiWebUrl().get() + "/stages/stage/?id=" + stageId + "&attempt=0"; diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java index c86120368f..038c925a7d 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java @@ -132,6 +132,16 @@ public SparkSession.Builder getBuilder() { public String getSparkAppId() { return "sparkAppId1"; } + + @Override + public String getSparkUiWebUrl() { + return "sparkUiWebUrl"; + } + + @Override + public String getSparkMasterUrl() { + return "sparkMasterUrl"; + } }; }