diff --git a/beakerx/beakerx/handlers.py b/beakerx/beakerx/handlers.py
index fa667188e4..92b3e8d9e0 100644
--- a/beakerx/beakerx/handlers.py
+++ b/beakerx/beakerx/handlers.py
@@ -23,6 +23,39 @@
import os
+class SparkMetricsExecutorsHandler(APIHandler):
+ def data_received(self, chunk):
+ pass
+
+ @web.authenticated
+ @tornado.web.asynchronous
+ def get(self, id):
+
+ def handle_response(response):
+ self.finish(response.body)
+
+ url = "http://localhost:4040/api/v1/applications/" + id + "/allexecutors"
+ req = tornado.httpclient.HTTPRequest(
+ url=url,
+ method=self.request.method,
+ body=self.request.body,
+ headers=self.request.headers,
+ follow_redirects=False,
+ allow_nonstandard_methods=True
+ )
+
+ client = tornado.httpclient.AsyncHTTPClient()
+ try:
+ client.fetch(req, handle_response)
+ except tornado.httpclient.HTTPError as e:
+ if hasattr(e, 'response') and e.response:
+ handle_response(e.response)
+ else:
+ self.set_status(500)
+ self.write('Internal server error:\n' + str(e))
+ self.finish()
+
+
class SettingsHandler(APIHandler):
def data_received(self, chunk):
pass
@@ -69,10 +102,12 @@ def load_jupyter_server_extension(nbapp):
web_app = nbapp.web_app
host_pattern = '.*$'
settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings')
+ spark_metrics_executors_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/sparkmetrics/executors/(.*)')
version_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/version')
javadoc_route_pattern = url_path_join(web_app.settings['base_url'], '/static', '/javadoc/(.*)')
javadoc_lab_route_pattern = url_path_join(web_app.settings['base_url'], '/javadoc/(.*)')
web_app.add_handlers(host_pattern, [(settings_route_pattern, SettingsHandler)])
+ web_app.add_handlers(host_pattern, [(spark_metrics_executors_route_pattern, SparkMetricsExecutorsHandler)])
web_app.add_handlers(host_pattern, [(version_route_pattern, VersionHandler)])
web_app.add_handlers(host_pattern, [(javadoc_route_pattern, JavaDoc)])
web_app.add_handlers(host_pattern, [(javadoc_lab_route_pattern, JavaDoc)])
diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts
index b4fce478be..4d2a5ee8a1 100644
--- a/js/notebook/src/SparkUI.ts
+++ b/js/notebook/src/SparkUI.ts
@@ -14,6 +14,9 @@
* limitations under the License.
*/
+import {Widget} from "@phosphor/widgets";
+import BeakerXApi from "./tree/Utils/BeakerXApi";
+
const widgets = require('./widgets');
class SparkUIModel extends widgets.VBoxModel {
@@ -31,14 +34,26 @@ class SparkUIModel extends widgets.VBoxModel {
}
class SparkUIView extends widgets.VBoxView {
+ private sparkStats: Widget;
+ private sparkAppId: string;
+ private apiCallIntervalId: number;
+ private connectionLabelActive: HTMLElement;
+ private connectionLabelMemory: HTMLElement;
+ private connectionLabelDead: HTMLElement;
+
public render() {
super.render();
this.el.classList.add('widget-spark-ui');
+
+ this.addSparkMetricsWidget();
this.updateLabels();
}
public update() {
super.update();
+
+ this.connectToApi();
+ this.addSparkMetricsWidget();
this.updateLabels();
}
@@ -100,6 +115,115 @@ class SparkUIView extends widgets.VBoxView {
return labelEl.clientWidth;
}
+
+ private createSparkMetricsWidget(): void {
+ if (this.sparkStats) {
+ this.el.querySelector('.bx-connection-status')
+ .insertAdjacentElement('afterend', this.sparkStats.node);
+
+ return;
+ }
+
+ this.sparkStats = new Widget();
+ this.sparkStats.node.classList.add('bx-stats');
+ this.sparkStats.node.innerHTML = `
+
0
0
0
+ `;
+
+ this.connectionLabelActive = this.sparkStats.node.querySelector('.active');
+ this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory');
+ this.connectionLabelDead = this.sparkStats.node.querySelector('.dead');
+
+ this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node);
+ }
+
+ private connectToApi() {
+ let baseUrl;
+ let api;
+
+ this.sparkAppId = this.model.get('sparkAppId');
+
+ if (!this.sparkAppId) {
+ return;
+ }
+
+ try {
+ const coreutils = require('@jupyterlab/coreutils');
+ coreutils.PageConfig.getOption('pageUrl');
+ baseUrl = coreutils.PageConfig.getBaseUrl();
+ } catch(e) {
+ baseUrl = `${window.location.origin}/`;
+ }
+
+ api = new BeakerXApi(baseUrl);
+ this.setApiCallInterval(api);
+ }
+
+ private setApiCallInterval(api: BeakerXApi): void {
+ const sparkUrl = `${api.getApiUrl('sparkmetrics/executors')}/${this.sparkAppId}`;
+ const getMetrict = async () => {
+ try {
+ const response = await fetch(sparkUrl, { method: 'GET', credentials: 'include' });
+
+ if (!response.ok) {
+ return this.clearApiCallInterval();
+ }
+
+ const data = await response.json();
+ this.updateMetrics(data);
+ } catch(error) {
+ this.clearApiCallInterval();
+ }
+ };
+
+ this.clearApiCallInterval();
+ this.apiCallIntervalId = setInterval(getMetrict, 1000);
+ }
+
+ private clearApiCallInterval() {
+ clearInterval(this.apiCallIntervalId);
+ this.sparkAppId = null;
+ }
+
+ private updateMetrics(data: Array) {
+ let activeTasks: number = 0;
+ let deadExecutors: number = 0;
+ let storageMemory: number = 0;
+
+ data.forEach(execData => {
+ if (execData.isActive) {
+ activeTasks += execData.activeTasks;
+ storageMemory += execData.memoryUsed;
+ } else {
+ deadExecutors += 1;
+ }
+ });
+
+ this.connectionLabelActive.innerText = `${activeTasks}`;
+ this.connectionLabelMemory.innerText = `${storageMemory}`;
+ this.connectionLabelDead.innerText = `${deadExecutors}`;
+ }
+
+ private addSparkMetricsWidget() {
+ this.children_views.update(this.model.get('children')).then((views) => {
+ views.forEach((view) => {
+ view.children_views.update(view.model.get('children')).then((views) => {
+ views.forEach((view) => {
+ if (view instanceof widgets.LabelView && view.el.classList.contains('bx-connection-status')) {
+ this.createSparkMetricsWidget();
+ }
+ });
+ });
+ });
+ });
+ }
+
+ despose() {
+ super.dispose();
+ clearInterval(this.apiCallIntervalId);
+ }
}
export default {
diff --git a/js/notebook/src/shared/style/beakerx.scss b/js/notebook/src/shared/style/beakerx.scss
index 8c5020b97c..1e2ca5f180 100644
--- a/js/notebook/src/shared/style/beakerx.scss
+++ b/js/notebook/src/shared/style/beakerx.scss
@@ -14,14 +14,33 @@
* limitations under the License.
*/
-$focusColor: #66bb6a;
+@import "bxvariables";
.improveFonts .context-menu-root .context-menu-item span {
font-family: "Lato", Helvetica, sans-serif;
}
+.bx-button {
+ &[class*="icon"] {
+ font: normal normal normal 14px/1 FontAwesome;
+ border: 1px solid $bxBorderColor1;
+ width: auto;
+ height: 24px;
+ padding: 0 6px;
+ margin: 2px 4px;
+ }
+
+ &:before {
+ display: inline-block;
+ }
+
+ &.icon-close:before {
+ content: "\f00d" !important;
+ }
+}
+
.bko-focused {
- border: 1px solid $focusColor !important;
+ border: 1px solid $bxColorFocus !important;
&:before {
position: absolute;
@@ -31,7 +50,7 @@ $focusColor: #66bb6a;
width: 5px;
height: calc(100% + 2px);
content: '';
- background: $focusColor;
+ background: $bxColorFocus;
}
}
@@ -144,7 +163,7 @@ $focusColor: #66bb6a;
font-size: 14px;
line-height: 1.3em;
padding: 0 2.2em;
- border: 1px solid #ababab;
+ border: 1px solid $bxBorderColor1;
border-top: 0;
border-radius: 0 0 3px 3px;
text-overflow: ellipsis;
diff --git a/js/notebook/src/shared/style/bxvariables.scss b/js/notebook/src/shared/style/bxvariables.scss
index ce7623889b..e51c92ece2 100644
--- a/js/notebook/src/shared/style/bxvariables.scss
+++ b/js/notebook/src/shared/style/bxvariables.scss
@@ -18,6 +18,7 @@ $bxBorderColor1: #cccccc;
$bxBgColor1: #ffffff;
$bxBgColor2: #eeeeee;
+$bxColorFocus: #66bb6a;
$bxColorSuccess: #5cb85c;
$bxColorInfo: #5bc0de;
$bxColorWarning: #f0ad4e;
diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss
index 505f78bf08..9e68f99f0a 100644
--- a/js/notebook/src/shared/style/spark.scss
+++ b/js/notebook/src/shared/style/spark.scss
@@ -28,10 +28,6 @@
max-width: 600px;
}
-.bx-spark-stageProgressLabels {
- margin: 0;
-}
-
.bx-panel {
border: 1px solid $bxBorderColor1;
border-radius: 2px;
@@ -103,6 +99,11 @@
}
}
+.bx-spark-stageProgressLabels {
+ margin: 0;
+}
+
+.bx-stats,
.bx-spark-stageProgressLabels {
display: block;
line-height: 100%;
@@ -115,3 +116,32 @@
line-height: 150%;
}
}
+
+.bx-stats {
+ margin: 4px 0;
+ height: 20px;
+ line-height: 17px;
+}
+
+.bx-status-panel {
+ .bx-button {
+ margin-left: 20px;
+ }
+}
+
+.bx-connection-status {
+ background-color: transparent;
+ border: 1px solid $bxBorderColor1;
+ height: 20px !important;
+ line-height: 20px !important;
+ padding: 0 1em;
+ border-radius: 3px;
+ margin: 4px 6px;
+ min-width: 160px;
+
+ &.connected {
+ border: none;
+ background-color: $bxColorSuccess;
+ color: $bxBgColor1;
+ }
+}
diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java b/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java
index dbe3f8ce7f..213ba1d52f 100644
--- a/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java
+++ b/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java
@@ -21,7 +21,9 @@
import com.twosigma.beakerx.message.Message;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -31,9 +33,11 @@ public abstract class DOMWidget extends Widget {
public static final String SYNC_DATA = "state";
public static final String MODEL_MODULE_VALUE = "@jupyter-widgets/controls";
public static final String VIEW_MODULE_VALUE = "@jupyter-widgets/controls";
+ public static final String DOM_CLASSES = "_dom_classes";
private Layout layout;
protected Style style;
+ private List domClasses = new ArrayList<>();
private UpdateValueCallback updateValueCallback = () -> {
};
@@ -75,7 +79,7 @@ public Optional