diff --git a/beakerx/beakerx/handlers.py b/beakerx/beakerx/handlers.py index fa667188e4..92b3e8d9e0 100644 --- a/beakerx/beakerx/handlers.py +++ b/beakerx/beakerx/handlers.py @@ -23,6 +23,39 @@ import os +class SparkMetricsExecutorsHandler(APIHandler): + def data_received(self, chunk): + pass + + @web.authenticated + @tornado.web.asynchronous + def get(self, id): + + def handle_response(response): + self.finish(response.body) + + url = "http://localhost:4040/api/v1/applications/" + id + "/allexecutors" + req = tornado.httpclient.HTTPRequest( + url=url, + method=self.request.method, + body=self.request.body, + headers=self.request.headers, + follow_redirects=False, + allow_nonstandard_methods=True + ) + + client = tornado.httpclient.AsyncHTTPClient() + try: + client.fetch(req, handle_response) + except tornado.httpclient.HTTPError as e: + if hasattr(e, 'response') and e.response: + handle_response(e.response) + else: + self.set_status(500) + self.write('Internal server error:\n' + str(e)) + self.finish() + + class SettingsHandler(APIHandler): def data_received(self, chunk): pass @@ -69,10 +102,12 @@ def load_jupyter_server_extension(nbapp): web_app = nbapp.web_app host_pattern = '.*$' settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings') + spark_metrics_executors_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/sparkmetrics/executors/(.*)') version_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/version') javadoc_route_pattern = url_path_join(web_app.settings['base_url'], '/static', '/javadoc/(.*)') javadoc_lab_route_pattern = url_path_join(web_app.settings['base_url'], '/javadoc/(.*)') web_app.add_handlers(host_pattern, [(settings_route_pattern, SettingsHandler)]) + web_app.add_handlers(host_pattern, [(spark_metrics_executors_route_pattern, SparkMetricsExecutorsHandler)]) web_app.add_handlers(host_pattern, [(version_route_pattern, VersionHandler)]) web_app.add_handlers(host_pattern, [(javadoc_route_pattern, JavaDoc)]) web_app.add_handlers(host_pattern, [(javadoc_lab_route_pattern, JavaDoc)]) diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index b4fce478be..4d2a5ee8a1 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -14,6 +14,9 @@ * limitations under the License. */ +import {Widget} from "@phosphor/widgets"; +import BeakerXApi from "./tree/Utils/BeakerXApi"; + const widgets = require('./widgets'); class SparkUIModel extends widgets.VBoxModel { @@ -31,14 +34,26 @@ class SparkUIModel extends widgets.VBoxModel { } class SparkUIView extends widgets.VBoxView { + private sparkStats: Widget; + private sparkAppId: string; + private apiCallIntervalId: number; + private connectionLabelActive: HTMLElement; + private connectionLabelMemory: HTMLElement; + private connectionLabelDead: HTMLElement; + public render() { super.render(); this.el.classList.add('widget-spark-ui'); + + this.addSparkMetricsWidget(); this.updateLabels(); } public update() { super.update(); + + this.connectToApi(); + this.addSparkMetricsWidget(); this.updateLabels(); } @@ -100,6 +115,115 @@ class SparkUIView extends widgets.VBoxView { return labelEl.clientWidth; } + + private createSparkMetricsWidget(): void { + if (this.sparkStats) { + this.el.querySelector('.bx-connection-status') + .insertAdjacentElement('afterend', this.sparkStats.node); + + return; + } + + this.sparkStats = new Widget(); + this.sparkStats.node.classList.add('bx-stats'); + this.sparkStats.node.innerHTML = ` +
0
0
0
+ `; + + this.connectionLabelActive = this.sparkStats.node.querySelector('.active'); + this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory'); + this.connectionLabelDead = this.sparkStats.node.querySelector('.dead'); + + this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node); + } + + private connectToApi() { + let baseUrl; + let api; + + this.sparkAppId = this.model.get('sparkAppId'); + + if (!this.sparkAppId) { + return; + } + + try { + const coreutils = require('@jupyterlab/coreutils'); + coreutils.PageConfig.getOption('pageUrl'); + baseUrl = coreutils.PageConfig.getBaseUrl(); + } catch(e) { + baseUrl = `${window.location.origin}/`; + } + + api = new BeakerXApi(baseUrl); + this.setApiCallInterval(api); + } + + private setApiCallInterval(api: BeakerXApi): void { + const sparkUrl = `${api.getApiUrl('sparkmetrics/executors')}/${this.sparkAppId}`; + const getMetrict = async () => { + try { + const response = await fetch(sparkUrl, { method: 'GET', credentials: 'include' }); + + if (!response.ok) { + return this.clearApiCallInterval(); + } + + const data = await response.json(); + this.updateMetrics(data); + } catch(error) { + this.clearApiCallInterval(); + } + }; + + this.clearApiCallInterval(); + this.apiCallIntervalId = setInterval(getMetrict, 1000); + } + + private clearApiCallInterval() { + clearInterval(this.apiCallIntervalId); + this.sparkAppId = null; + } + + private updateMetrics(data: Array) { + let activeTasks: number = 0; + let deadExecutors: number = 0; + let storageMemory: number = 0; + + data.forEach(execData => { + if (execData.isActive) { + activeTasks += execData.activeTasks; + storageMemory += execData.memoryUsed; + } else { + deadExecutors += 1; + } + }); + + this.connectionLabelActive.innerText = `${activeTasks}`; + this.connectionLabelMemory.innerText = `${storageMemory}`; + this.connectionLabelDead.innerText = `${deadExecutors}`; + } + + private addSparkMetricsWidget() { + this.children_views.update(this.model.get('children')).then((views) => { + views.forEach((view) => { + view.children_views.update(view.model.get('children')).then((views) => { + views.forEach((view) => { + if (view instanceof widgets.LabelView && view.el.classList.contains('bx-connection-status')) { + this.createSparkMetricsWidget(); + } + }); + }); + }); + }); + } + + despose() { + super.dispose(); + clearInterval(this.apiCallIntervalId); + } } export default { diff --git a/js/notebook/src/shared/style/beakerx.scss b/js/notebook/src/shared/style/beakerx.scss index 8c5020b97c..1e2ca5f180 100644 --- a/js/notebook/src/shared/style/beakerx.scss +++ b/js/notebook/src/shared/style/beakerx.scss @@ -14,14 +14,33 @@ * limitations under the License. */ -$focusColor: #66bb6a; +@import "bxvariables"; .improveFonts .context-menu-root .context-menu-item span { font-family: "Lato", Helvetica, sans-serif; } +.bx-button { + &[class*="icon"] { + font: normal normal normal 14px/1 FontAwesome; + border: 1px solid $bxBorderColor1; + width: auto; + height: 24px; + padding: 0 6px; + margin: 2px 4px; + } + + &:before { + display: inline-block; + } + + &.icon-close:before { + content: "\f00d" !important; + } +} + .bko-focused { - border: 1px solid $focusColor !important; + border: 1px solid $bxColorFocus !important; &:before { position: absolute; @@ -31,7 +50,7 @@ $focusColor: #66bb6a; width: 5px; height: calc(100% + 2px); content: ''; - background: $focusColor; + background: $bxColorFocus; } } @@ -144,7 +163,7 @@ $focusColor: #66bb6a; font-size: 14px; line-height: 1.3em; padding: 0 2.2em; - border: 1px solid #ababab; + border: 1px solid $bxBorderColor1; border-top: 0; border-radius: 0 0 3px 3px; text-overflow: ellipsis; diff --git a/js/notebook/src/shared/style/bxvariables.scss b/js/notebook/src/shared/style/bxvariables.scss index ce7623889b..e51c92ece2 100644 --- a/js/notebook/src/shared/style/bxvariables.scss +++ b/js/notebook/src/shared/style/bxvariables.scss @@ -18,6 +18,7 @@ $bxBorderColor1: #cccccc; $bxBgColor1: #ffffff; $bxBgColor2: #eeeeee; +$bxColorFocus: #66bb6a; $bxColorSuccess: #5cb85c; $bxColorInfo: #5bc0de; $bxColorWarning: #f0ad4e; diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss index 505f78bf08..9e68f99f0a 100644 --- a/js/notebook/src/shared/style/spark.scss +++ b/js/notebook/src/shared/style/spark.scss @@ -28,10 +28,6 @@ max-width: 600px; } -.bx-spark-stageProgressLabels { - margin: 0; -} - .bx-panel { border: 1px solid $bxBorderColor1; border-radius: 2px; @@ -103,6 +99,11 @@ } } +.bx-spark-stageProgressLabels { + margin: 0; +} + +.bx-stats, .bx-spark-stageProgressLabels { display: block; line-height: 100%; @@ -115,3 +116,32 @@ line-height: 150%; } } + +.bx-stats { + margin: 4px 0; + height: 20px; + line-height: 17px; +} + +.bx-status-panel { + .bx-button { + margin-left: 20px; + } +} + +.bx-connection-status { + background-color: transparent; + border: 1px solid $bxBorderColor1; + height: 20px !important; + line-height: 20px !important; + padding: 0 1em; + border-radius: 3px; + margin: 4px 6px; + min-width: 160px; + + &.connected { + border: none; + background-color: $bxColorSuccess; + color: $bxBgColor1; + } +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java b/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java index dbe3f8ce7f..213ba1d52f 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java @@ -21,7 +21,9 @@ import com.twosigma.beakerx.message.Message; import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,9 +33,11 @@ public abstract class DOMWidget extends Widget { public static final String SYNC_DATA = "state"; public static final String MODEL_MODULE_VALUE = "@jupyter-widgets/controls"; public static final String VIEW_MODULE_VALUE = "@jupyter-widgets/controls"; + public static final String DOM_CLASSES = "_dom_classes"; private Layout layout; protected Style style; + private List domClasses = new ArrayList<>(); private UpdateValueCallback updateValueCallback = () -> { }; @@ -75,7 +79,7 @@ public Optional getSyncDataValue(Message msg) { ret = Optional.of(sync_data.get(VALUE)); } else if (sync_data.containsKey(INDEX)) { ret = Optional.of(sync_data.get(INDEX)); - } else if (sync_data.containsKey("outputs")){ + } else if (sync_data.containsKey("outputs")) { ret = Optional.of(sync_data.get("outputs")); } } @@ -104,12 +108,12 @@ public void doUpdateValueWithCallback(Object value) { } @Override - public String getModelModuleValue(){ + public String getModelModuleValue() { return MODEL_MODULE_VALUE; } @Override - public String getViewModuleValue(){ + public String getViewModuleValue() { return VIEW_MODULE_VALUE; } @@ -125,9 +129,19 @@ protected HashMap content(HashMap co content.put("font_weight", ""); content.put("background_color", null); content.put("color", null); + content.put(DOM_CLASSES, domClasses.toArray()); return content; } + public List getDomClasses() { + return domClasses; + } + + public void setDomClasses(List domClasses) { + this.domClasses = checkNotNull(domClasses); + sendUpdate(DOM_CLASSES, this.domClasses.toArray()); + } + public Layout getLayout() { if (layout == null) { layout = new Layout(); diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java b/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java index 3ed3cdfe7d..5cc6360a34 100644 --- a/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java +++ b/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java @@ -158,7 +158,7 @@ private static Optional getError(List messages) { } - private static Optional getUpdate(List messages) { + public static Optional getUpdate(List messages) { return messages.stream(). filter(x -> x.type().equals(JupyterMessages.COMM_MSG)). filter(x -> TestWidgetUtils.getData(x).get("method").equals("update")). diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java b/kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java new file mode 100644 index 0000000000..e57f82b411 --- /dev/null +++ b/kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.widget; + +import com.twosigma.beakerx.KernelTest; +import com.twosigma.beakerx.evaluator.EvaluatorResultTestWatcher; +import com.twosigma.beakerx.kernel.KernelManager; +import com.twosigma.beakerx.message.Message; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Map; + +import static com.twosigma.beakerx.widget.DOMWidget.DOM_CLASSES; +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; + +public class DOMWidgetTest { + + private KernelTest kernel; + private DOMWidget widget; + + @Before + public void setUp() throws Exception { + kernel = new KernelTest(); + KernelManager.register(kernel); + widget = new Button(); + } + + @After + public void tearDown() throws Exception { + KernelManager.register(null); + } + + @Test + public void shouldSendDomClassesUpdateMessage() { + //given + //when + widget.setDomClasses(asList("class1", "class2")); + //then + Message update = EvaluatorResultTestWatcher.getUpdate(kernel.getPublishedMessages()).get(); + Map state = TestWidgetUtils.getState(update); + assertThat(state.get(DOM_CLASSES)).isEqualTo(asList("class1", "class2").toArray()); + } +} \ No newline at end of file diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java index 37165bd83a..b0bd2eab67 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java @@ -36,6 +36,8 @@ public interface SparkManager { SparkSession.Builder getBuilder(); + String getSparkAppId(); + interface SparkManagerFactory { SparkManager create(SparkSession.Builder sparkSessionBuilder); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java index 5cb94e3f9f..785a6b0e45 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java @@ -31,6 +31,7 @@ import org.apache.spark.scheduler.SparkListenerStageSubmitted; import org.apache.spark.scheduler.SparkListenerTaskEnd; import org.apache.spark.scheduler.SparkListenerTaskStart; +import org.apache.spark.sql.RuntimeConfig; import org.apache.spark.sql.SparkSession; import scala.Tuple2; import scala.collection.Iterator; @@ -77,6 +78,12 @@ public SparkSession getOrCreate() { return sparkSessionBuilder.getOrCreate(); } + @Override + public String getSparkAppId() { + RuntimeConfig conf = getOrCreate().conf(); + return conf.getAll().get("spark.app.id").get(); + } + @Override public SparkContext sparkContext() { return getOrCreate().sparkContext(); diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index d5550b4941..369ed916a8 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -24,11 +24,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.twosigma.beakerx.kernel.PlainCode.createSimpleEvaluationObject; @@ -140,10 +136,10 @@ private void sendError(Message parentMessage, KernelFunctionality kernel, String seo.error(message); } - public void applicationStart() { sparkUI.clearView(); sparkUI.addStatusPanel(createStatusPanel()); + sparkUI.sendUpdate("sparkAppId", sparkManager.getSparkAppId()); } public void applicationEnd() { @@ -155,19 +151,22 @@ public void applicationEnd() { private HBox createStatusPanel() { Label appStatus = createAppStatus(); Button disconnect = createDisconnectButton(); - return new HBox(Arrays.asList(uiLink(), disconnect, appStatus)); + HBox connectionPanel = new HBox(Arrays.asList(uiLink(), appStatus, disconnect)); + connectionPanel.setDomClasses(new ArrayList<>(Arrays.asList("bx-status-panel"))); + return connectionPanel; } private Label createAppStatus() { Label appStatus = new Label(); appStatus.setValue("Connected to " + getSparkConf().get("spark.master")); + appStatus.setDomClasses(new ArrayList<>(Arrays.asList("bx-connection-status", "connected"))); return appStatus; } private Button createDisconnectButton() { Button disconnect = new Button(); disconnect.registerOnClick((content, message) -> getSparkSession().sparkContext().stop()); - disconnect.setDescription("Disconnect"); + disconnect.setDomClasses(new ArrayList<>(Arrays.asList("bx-button", "icon-close"))); return disconnect; } diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java index cf05cde1c6..c86120368f 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java @@ -127,6 +127,11 @@ public SparkContext sparkContext() { public SparkSession.Builder getBuilder() { return builder; } + + @Override + public String getSparkAppId() { + return "sparkAppId1"; + } }; } diff --git a/test/ipynb/groovy/DomClassesSupport.ipynb b/test/ipynb/groovy/DomClassesSupport.ipynb new file mode 100644 index 0000000000..cb453b0918 --- /dev/null +++ b/test/ipynb/groovy/DomClassesSupport.ipynb @@ -0,0 +1,40 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import com.twosigma.beakerx.widget.Button\n", + "bt = new Button()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bt.domClasses = [\"bx-button\", \"icon-close\"]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Groovy", + "language": "groovy", + "name": "groovy" + }, + "language_info": { + "codemirror_mode": "groovy", + "file_extension": ".groovy", + "mimetype": "", + "name": "Groovy", + "nbconverter_exporter": "", + "version": "2.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}