From 6c17bdef20cfdca75a5ef8cec4f6f1672972e8bf Mon Sep 17 00:00:00 2001 From: jarek Date: Wed, 23 May 2018 14:14:06 +0200 Subject: [PATCH 01/10] #7326: add endpoint for spark metrics executors --- beakerx/beakerx/handlers.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/beakerx/beakerx/handlers.py b/beakerx/beakerx/handlers.py index fa667188e4..92b3e8d9e0 100644 --- a/beakerx/beakerx/handlers.py +++ b/beakerx/beakerx/handlers.py @@ -23,6 +23,39 @@ import os +class SparkMetricsExecutorsHandler(APIHandler): + def data_received(self, chunk): + pass + + @web.authenticated + @tornado.web.asynchronous + def get(self, id): + + def handle_response(response): + self.finish(response.body) + + url = "http://localhost:4040/api/v1/applications/" + id + "/allexecutors" + req = tornado.httpclient.HTTPRequest( + url=url, + method=self.request.method, + body=self.request.body, + headers=self.request.headers, + follow_redirects=False, + allow_nonstandard_methods=True + ) + + client = tornado.httpclient.AsyncHTTPClient() + try: + client.fetch(req, handle_response) + except tornado.httpclient.HTTPError as e: + if hasattr(e, 'response') and e.response: + handle_response(e.response) + else: + self.set_status(500) + self.write('Internal server error:\n' + str(e)) + self.finish() + + class SettingsHandler(APIHandler): def data_received(self, chunk): pass @@ -69,10 +102,12 @@ def load_jupyter_server_extension(nbapp): web_app = nbapp.web_app host_pattern = '.*$' settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings') + spark_metrics_executors_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/sparkmetrics/executors/(.*)') version_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/version') javadoc_route_pattern = url_path_join(web_app.settings['base_url'], '/static', '/javadoc/(.*)') javadoc_lab_route_pattern = url_path_join(web_app.settings['base_url'], '/javadoc/(.*)') web_app.add_handlers(host_pattern, [(settings_route_pattern, SettingsHandler)]) + web_app.add_handlers(host_pattern, [(spark_metrics_executors_route_pattern, SparkMetricsExecutorsHandler)]) web_app.add_handlers(host_pattern, [(version_route_pattern, VersionHandler)]) web_app.add_handlers(host_pattern, [(javadoc_route_pattern, JavaDoc)]) web_app.add_handlers(host_pattern, [(javadoc_lab_route_pattern, JavaDoc)]) From 7c4bd57d1c7614e254d3911f9e99f0677aadaa3c Mon Sep 17 00:00:00 2001 From: jarek Date: Wed, 23 May 2018 14:44:12 +0200 Subject: [PATCH 02/10] #7326: send sparkAppId --- .../java/com/twosigma/beakerx/widget/SparkManager.java | 2 ++ .../com/twosigma/beakerx/widget/SparkManagerImpl.java | 9 ++++++++- .../java/com/twosigma/beakerx/widget/SparkUIManager.java | 4 +--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java index c41843e63b..a083a4b13e 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManager.java @@ -34,6 +34,8 @@ public interface SparkManager { SparkSession.Builder getBuilder(); + String getSparkAppId(); + interface SparkManagerFactory { SparkManager create(SparkSession.Builder sparkSessionBuilder); } diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java index 5499333c2f..284be6bfd6 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkManagerImpl.java @@ -31,6 +31,7 @@ import org.apache.spark.scheduler.SparkListenerStageSubmitted; import org.apache.spark.scheduler.SparkListenerTaskEnd; import org.apache.spark.scheduler.SparkListenerTaskStart; +import org.apache.spark.sql.RuntimeConfig; import org.apache.spark.sql.SparkSession; import scala.Tuple2; import scala.collection.Iterator; @@ -76,6 +77,12 @@ public SparkSession getOrCreate() { return sparkSessionBuilder.getOrCreate(); } + @Override + public String getSparkAppId() { + RuntimeConfig conf = getOrCreate().conf(); + return conf.getAll().get("spark.app.id").get(); + } + @Override public SparkContext sparkContext() { return getOrCreate().sparkContext(); @@ -191,7 +198,7 @@ private static boolean isLocalSpark(SparkConf sparkConf) { } - public static class SparkManagerFactoryImpl implements SparkManagerFactory{ + public static class SparkManagerFactoryImpl implements SparkManagerFactory { @Override public SparkManager create(SparkSession.Builder sparkSessionBuilder) { diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index 565bbe21d6..85c547cc0e 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -21,8 +21,6 @@ import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.message.Message; import org.apache.spark.sql.SparkSession; - -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -132,10 +130,10 @@ private void sendError(Message parentMessage, KernelFunctionality kernel, String seo.error(message); } - public void applicationStart() { sparkUI.clearView(); statusPanel = createStatusPanel(); + sparkUI.sendUpdate("sparkAppId", sparkManager.getSparkAppId()); } public void applicationEnd() { From ab3062c87aed3714f219f1de17d0c3911b3d7bd7 Mon Sep 17 00:00:00 2001 From: jarek Date: Wed, 23 May 2018 15:17:03 +0200 Subject: [PATCH 03/10] #7326: send sparkAppId --- .../beakerx/scala/magic/command/SparkMagicCommandTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java index 2044197207..e6068f0a71 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/scala/magic/command/SparkMagicCommandTest.java @@ -125,6 +125,11 @@ public SparkContext sparkContext() { public SparkSession.Builder getBuilder() { return builder; } + + @Override + public String getSparkAppId() { + return "sparkAppId1"; + } }; } From 8c54553ec02434ff2aa09f6eb208b71b116105f6 Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Tue, 22 May 2018 14:09:40 +0200 Subject: [PATCH 04/10] #7326 add button with icon css class support --- js/notebook/src/shared/style/beakerx.scss | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/js/notebook/src/shared/style/beakerx.scss b/js/notebook/src/shared/style/beakerx.scss index 9093147039..d8c83ab9ca 100644 --- a/js/notebook/src/shared/style/beakerx.scss +++ b/js/notebook/src/shared/style/beakerx.scss @@ -20,6 +20,20 @@ $focusColor: #66bb6a; font-family: "Lato", Helvetica, sans-serif; } +.BeakerX-button { + &[class*="icon"] { + font: normal normal normal 14px/1 FontAwesome; + } + + &:before { + display: inline-block; + } + + &.icon-close:before { + content: "\f00d" !important; + } +} + .bko-focused { border: 1px solid $focusColor !important; From 8b282b5085ae219544b0f1297660d0198b5fcb6b Mon Sep 17 00:00:00 2001 From: jarek Date: Tue, 22 May 2018 15:15:26 +0200 Subject: [PATCH 05/10] #7236: add _dom_classes support --- .../twosigma/beakerx/widget/DOMWidget.java | 20 ++++++- .../evaluator/EvaluatorResultTestWatcher.java | 2 +- .../beakerx/widget/DOMWidgetTest.java | 59 +++++++++++++++++++ test/ipynb/groovy/DomClassesSupport.ipynb | 40 +++++++++++++ 4 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java create mode 100644 test/ipynb/groovy/DomClassesSupport.ipynb diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java b/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java index dbe3f8ce7f..213ba1d52f 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java @@ -21,7 +21,9 @@ import com.twosigma.beakerx.message.Message; import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,9 +33,11 @@ public abstract class DOMWidget extends Widget { public static final String SYNC_DATA = "state"; public static final String MODEL_MODULE_VALUE = "@jupyter-widgets/controls"; public static final String VIEW_MODULE_VALUE = "@jupyter-widgets/controls"; + public static final String DOM_CLASSES = "_dom_classes"; private Layout layout; protected Style style; + private List domClasses = new ArrayList<>(); private UpdateValueCallback updateValueCallback = () -> { }; @@ -75,7 +79,7 @@ public Optional getSyncDataValue(Message msg) { ret = Optional.of(sync_data.get(VALUE)); } else if (sync_data.containsKey(INDEX)) { ret = Optional.of(sync_data.get(INDEX)); - } else if (sync_data.containsKey("outputs")){ + } else if (sync_data.containsKey("outputs")) { ret = Optional.of(sync_data.get("outputs")); } } @@ -104,12 +108,12 @@ public void doUpdateValueWithCallback(Object value) { } @Override - public String getModelModuleValue(){ + public String getModelModuleValue() { return MODEL_MODULE_VALUE; } @Override - public String getViewModuleValue(){ + public String getViewModuleValue() { return VIEW_MODULE_VALUE; } @@ -125,9 +129,19 @@ protected HashMap content(HashMap co content.put("font_weight", ""); content.put("background_color", null); content.put("color", null); + content.put(DOM_CLASSES, domClasses.toArray()); return content; } + public List getDomClasses() { + return domClasses; + } + + public void setDomClasses(List domClasses) { + this.domClasses = checkNotNull(domClasses); + sendUpdate(DOM_CLASSES, this.domClasses.toArray()); + } + public Layout getLayout() { if (layout == null) { layout = new Layout(); diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java b/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java index 3ed3cdfe7d..5cc6360a34 100644 --- a/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java +++ b/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java @@ -158,7 +158,7 @@ private static Optional getError(List messages) { } - private static Optional getUpdate(List messages) { + public static Optional getUpdate(List messages) { return messages.stream(). filter(x -> x.type().equals(JupyterMessages.COMM_MSG)). filter(x -> TestWidgetUtils.getData(x).get("method").equals("update")). diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java b/kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java new file mode 100644 index 0000000000..e57f82b411 --- /dev/null +++ b/kernel/base/src/test/java/com/twosigma/beakerx/widget/DOMWidgetTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.widget; + +import com.twosigma.beakerx.KernelTest; +import com.twosigma.beakerx.evaluator.EvaluatorResultTestWatcher; +import com.twosigma.beakerx.kernel.KernelManager; +import com.twosigma.beakerx.message.Message; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Map; + +import static com.twosigma.beakerx.widget.DOMWidget.DOM_CLASSES; +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; + +public class DOMWidgetTest { + + private KernelTest kernel; + private DOMWidget widget; + + @Before + public void setUp() throws Exception { + kernel = new KernelTest(); + KernelManager.register(kernel); + widget = new Button(); + } + + @After + public void tearDown() throws Exception { + KernelManager.register(null); + } + + @Test + public void shouldSendDomClassesUpdateMessage() { + //given + //when + widget.setDomClasses(asList("class1", "class2")); + //then + Message update = EvaluatorResultTestWatcher.getUpdate(kernel.getPublishedMessages()).get(); + Map state = TestWidgetUtils.getState(update); + assertThat(state.get(DOM_CLASSES)).isEqualTo(asList("class1", "class2").toArray()); + } +} \ No newline at end of file diff --git a/test/ipynb/groovy/DomClassesSupport.ipynb b/test/ipynb/groovy/DomClassesSupport.ipynb new file mode 100644 index 0000000000..8989cc2946 --- /dev/null +++ b/test/ipynb/groovy/DomClassesSupport.ipynb @@ -0,0 +1,40 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import com.twosigma.beakerx.widget.Button\n", + "bt = new Button()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bt.domClasses = [\"BeakerX-button\", \"icon-close\"]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Groovy", + "language": "groovy", + "name": "groovy" + }, + "language_info": { + "codemirror_mode": "groovy", + "file_extension": ".groovy", + "mimetype": "", + "name": "Groovy", + "nbconverter_exporter": "", + "version": "2.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 629af4883022942c8be8dd85b52b92891981cc54 Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Tue, 22 May 2018 15:02:36 +0200 Subject: [PATCH 06/10] #7326 improve spark connection status widget --- js/notebook/src/SparkUI.ts | 14 ++++++++++++++ .../twosigma/beakerx/widget/SparkUIManager.java | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index 993f5e2e78..2655b0fbaf 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -33,8 +33,14 @@ class SparkUIModel extends widgets.VBoxModel { } class SparkUIView extends widgets.VBoxView { + private connectionStatus: HTMLElement; + public render() { super.render(); + + this.el.classList.add('bx-spark-status-panel'); + + this.processConnectionWidget(); this.updateLabels(); } @@ -69,6 +75,14 @@ class SparkUIView extends widgets.VBoxView { width: 'auto', }).outerWidth(); } + + private processConnectionWidget() { + this.children_views.update(this.model.get('children')).then((views) => { + views.forEach((view) => { + debugger; + }); + }); + } } export default { diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index 85c547cc0e..c2db0ee17b 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -148,7 +148,7 @@ public void applicationEnd() { private HBox createStatusPanel() { Label appStatus = createAppStatus(); Button disconnect = createDisconnectButton(); - HBox statusPanel = new HBox(Arrays.asList(uiLink(), disconnect, appStatus)); + HBox statusPanel = new HBox(Arrays.asList(uiLink(), appStatus, disconnect)); sparkUI.add(statusPanel); return statusPanel; } From c390d88b5f5d218192ebb31705ad2a12b22fc00c Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Tue, 22 May 2018 15:48:22 +0200 Subject: [PATCH 07/10] #7326 connection progress basic styles --- js/notebook/src/SparkUI.ts | 10 +++++++--- js/notebook/src/shared/style/spark.scss | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index 2655b0fbaf..1156cf0704 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -38,14 +38,14 @@ class SparkUIView extends widgets.VBoxView { public render() { super.render(); - this.el.classList.add('bx-spark-status-panel'); - this.processConnectionWidget(); this.updateLabels(); } public update() { super.update(); + + this.processConnectionWidget(); this.updateLabels(); } @@ -79,7 +79,11 @@ class SparkUIView extends widgets.VBoxView { private processConnectionWidget() { this.children_views.update(this.model.get('children')).then((views) => { views.forEach((view) => { - debugger; + if (view instanceof widgets.LabelView) { + this.connectionStatus = view.pWidget; + this.connectionStatus.el.classList.add('bx-spark-connectionProgressBar'); + this.connectionStatus.el.classList.add('progress'); + } }); }); } diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss index a383d94dd1..95d91fa58e 100644 --- a/js/notebook/src/shared/style/spark.scss +++ b/js/notebook/src/shared/style/spark.scss @@ -14,6 +14,10 @@ * limitations under the License. */ +$colorSuccess: #4CAF50; +$colorWhite: #ffffff; +$colorBorder: #cfcfcf; + .foldout-preview { & > .bx-spark-stagePanel { padding: 0 15px; @@ -26,6 +30,7 @@ max-width: 600px; } +.bx-spark-connectionProgressBar, .bx-spark-stageProgressBar { margin: 0 10px; } @@ -42,3 +47,15 @@ line-height: 150%; } } + +.bx-spark-connectionProgressBar { + background-color: transparent; + border: 1px solid $colorBorder; + height: 20px; + margin: 6px; + line-height: 18px; + + &.connected { + background-color: $colorSuccess; + } +} From 38534fcd5cfe8d3247de6d816799b20ee6585b66 Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Wed, 23 May 2018 09:33:31 +0200 Subject: [PATCH 08/10] #7326 stylize connection status widget --- js/notebook/src/SparkUI.ts | 35 ++++++++++++++---- js/notebook/src/shared/style/beakerx.scss | 15 +++++--- js/notebook/src/shared/style/spark.scss | 36 +++++++++++++------ js/notebook/src/shared/style/variables.scss | 5 +++ .../beakerx/widget/SparkUIManager.java | 11 +++--- test/ipynb/groovy/DomClassesSupport.ipynb | 2 +- 6 files changed, 78 insertions(+), 26 deletions(-) create mode 100644 js/notebook/src/shared/style/variables.scss diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index 1156cf0704..ef0169f2e2 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -15,6 +15,8 @@ */ import * as $ from "jquery"; +import {Widget} from "@phosphor/widgets"; +import BeakerXApi from "./tree/Utils/BeakerXApi"; const widgets = require('./widgets'); @@ -33,7 +35,10 @@ class SparkUIModel extends widgets.VBoxModel { } class SparkUIView extends widgets.VBoxView { - private connectionStatus: HTMLElement; + private sparkStats: Widget; + private connectionLabelActive: HTMLElement; + private connectionLabelMemory: HTMLElement; + private connectionLabelDead: HTMLElement; public render() { super.render(); @@ -76,14 +81,32 @@ class SparkUIView extends widgets.VBoxView { }).outerWidth(); } + private appendSparkStats(): void { + this.sparkStats = new Widget(); + this.sparkStats.node.classList.add('bx-stats'); + this.sparkStats.node.innerHTML = ` +
0
0
0
+ `; + + this.connectionLabelActive = this.sparkStats.node.querySelector('.active'); + this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory'); + this.connectionLabelDead = this.sparkStats.node.querySelector('.dead'); + + this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node); + } + private processConnectionWidget() { this.children_views.update(this.model.get('children')).then((views) => { views.forEach((view) => { - if (view instanceof widgets.LabelView) { - this.connectionStatus = view.pWidget; - this.connectionStatus.el.classList.add('bx-spark-connectionProgressBar'); - this.connectionStatus.el.classList.add('progress'); - } + view.children_views.update(view.model.get('children')).then((views) => { + views.forEach((view) => { + if (view instanceof widgets.LabelView && view.el.classList.contains('bx-connection-status')) { + this.appendSparkStats(); + } + }); + }); }); }); } diff --git a/js/notebook/src/shared/style/beakerx.scss b/js/notebook/src/shared/style/beakerx.scss index d8c83ab9ca..6cadaa4bcd 100644 --- a/js/notebook/src/shared/style/beakerx.scss +++ b/js/notebook/src/shared/style/beakerx.scss @@ -14,15 +14,22 @@ * limitations under the License. */ +@import "./variables"; +$colorBorder: #cfcfcf; $focusColor: #66bb6a; .improveFonts .context-menu-root .context-menu-item span { font-family: "Lato", Helvetica, sans-serif; } -.BeakerX-button { +.bx-button { &[class*="icon"] { font: normal normal normal 14px/1 FontAwesome; + border: 1px solid $altoColor; + width: auto; + height: 24px; + padding: 0 6px; + margin: 2px 4px; } &:before { @@ -35,7 +42,7 @@ $focusColor: #66bb6a; } .bko-focused { - border: 1px solid $focusColor !important; + border: 1px solid $fernColor !important; &:before { position: absolute; @@ -45,7 +52,7 @@ $focusColor: #66bb6a; width: 5px; height: calc(100% + 2px); content: ''; - background: $focusColor; + background: $fernColor; } } @@ -158,7 +165,7 @@ $focusColor: #66bb6a; font-size: 14px; line-height: 1.3em; padding: 0 2.2em; - border: 1px solid #ababab; + border: 1px solid $silverChaliceColor; border-top: 0; border-radius: 0 0 3px 3px; text-overflow: ellipsis; diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss index 95d91fa58e..ad99c6fd40 100644 --- a/js/notebook/src/shared/style/spark.scss +++ b/js/notebook/src/shared/style/spark.scss @@ -14,9 +14,7 @@ * limitations under the License. */ -$colorSuccess: #4CAF50; -$colorWhite: #ffffff; -$colorBorder: #cfcfcf; +@import "./variables"; .foldout-preview { & > .bx-spark-stagePanel { @@ -30,11 +28,12 @@ $colorBorder: #cfcfcf; max-width: 600px; } -.bx-spark-connectionProgressBar, +.bx-connection-status, .bx-spark-stageProgressBar { margin: 0 10px; } +.bx-stats, .bx-spark-stageProgressLabels { display: block; line-height: 100%; @@ -48,14 +47,31 @@ $colorBorder: #cfcfcf; } } -.bx-spark-connectionProgressBar { - background-color: transparent; - border: 1px solid $colorBorder; +.bx-stats { + margin: 4px 0; height: 20px; - margin: 6px; - line-height: 18px; + line-height: 17px; +} + +.bx-status-panel { + .bx-button { + margin-left: 10px; + } +} + +.bx-connection-status { + background-color: transparent; + border: 1px solid $altoColor; + height: 20px !important; + line-height: 20px !important; + padding: 0 1em; + border-radius: 3px; + margin: 4px 6px; + min-width: 160px; &.connected { - background-color: $colorSuccess; + border: none; + background-color: $fruitSaladColor; + color: $whiteColor; } } diff --git a/js/notebook/src/shared/style/variables.scss b/js/notebook/src/shared/style/variables.scss new file mode 100644 index 0000000000..d8303a42ff --- /dev/null +++ b/js/notebook/src/shared/style/variables.scss @@ -0,0 +1,5 @@ +$altoColor: #cfcfcf; +$fernColor: #66bb6a; +$fruitSaladColor: #4CAF50; +$whiteColor: #ffffff; +$silverChaliceColor: #ababab; diff --git a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java index c2db0ee17b..168f512364 100644 --- a/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java +++ b/kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIManager.java @@ -21,9 +21,8 @@ import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.message.Message; import org.apache.spark.sql.SparkSession; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; + +import java.util.*; import static com.twosigma.beakerx.kernel.PlainCode.createSimpleEvaluationObject; @@ -149,20 +148,22 @@ private HBox createStatusPanel() { Label appStatus = createAppStatus(); Button disconnect = createDisconnectButton(); HBox statusPanel = new HBox(Arrays.asList(uiLink(), appStatus, disconnect)); + statusPanel.setDomClasses(new ArrayList<>(Arrays.asList("bx-status-panel"))); sparkUI.add(statusPanel); return statusPanel; } private Label createAppStatus() { Label appStatus = new Label(); - appStatus.setValue("Connected to " + this.sparkManager.getSparkConf().get("spark.master")); + appStatus.setDomClasses(new ArrayList<>(Arrays.asList("bx-connection-status", "connected"))); + appStatus.setValue(this.sparkManager.getSparkConf().get("spark.master")); return appStatus; } private Button createDisconnectButton() { Button disconnect = new Button(); disconnect.registerOnClick((content, message) -> getSparkSession().sparkContext().stop()); - disconnect.setDescription("Disconnect"); + disconnect.setDomClasses(new ArrayList<>(Arrays.asList("bx-button", "icon-close"))); return disconnect; } diff --git a/test/ipynb/groovy/DomClassesSupport.ipynb b/test/ipynb/groovy/DomClassesSupport.ipynb index 8989cc2946..cb453b0918 100644 --- a/test/ipynb/groovy/DomClassesSupport.ipynb +++ b/test/ipynb/groovy/DomClassesSupport.ipynb @@ -16,7 +16,7 @@ "metadata": {}, "outputs": [], "source": [ - "bt.domClasses = [\"BeakerX-button\", \"icon-close\"]" + "bt.domClasses = [\"bx-button\", \"icon-close\"]" ] } ], From 6baae7ed40e67738c6ebc6b9b6b2b6e9ba4e528c Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Thu, 24 May 2018 11:08:37 +0200 Subject: [PATCH 09/10] #7326 call spark REST Api for metrics --- js/notebook/src/SparkUI.ts | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index ef0169f2e2..972212f606 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -36,6 +36,8 @@ class SparkUIModel extends widgets.VBoxModel { class SparkUIView extends widgets.VBoxView { private sparkStats: Widget; + private sparkAppId: string; + private apiCallIntervalId: number; private connectionLabelActive: HTMLElement; private connectionLabelMemory: HTMLElement; private connectionLabelDead: HTMLElement; @@ -50,6 +52,7 @@ class SparkUIView extends widgets.VBoxView { public update() { super.update(); + this.connectToApi(); this.processConnectionWidget(); this.updateLabels(); } @@ -82,6 +85,10 @@ class SparkUIView extends widgets.VBoxView { } private appendSparkStats(): void { + if (this.sparkStats) { + return; + } + this.sparkStats = new Widget(); this.sparkStats.node.classList.add('bx-stats'); this.sparkStats.node.innerHTML = ` @@ -97,6 +104,54 @@ class SparkUIView extends widgets.VBoxView { this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node); } + private connectToApi() { + let baseUrl; + let api; + + this.sparkAppId = this.model.get('sparkAppId'); + + if (!this.sparkAppId) { + return; + } + + try { + const coreutils = require('@jupyterlab/coreutils'); + coreutils.PageConfig.getOption('pageUrl'); + baseUrl = coreutils.PageConfig.getBaseUrl(); + } catch(e) { + baseUrl = `${window.location.origin}/`; + } + + api = new BeakerXApi(baseUrl); + this.setApiCallInterval(api); + } + + private setApiCallInterval(api: BeakerXApi): void { + const sparkUrl = `${api.getApiUrl('sparkmetrics/executors')}/${this.sparkAppId}`; + const getMetrict = async () => { + try { + const response = await fetch(sparkUrl); + + if (!response.ok) { + console.log(response); + + return; + } + + const data = await response.json(); + this.updateMetrics(data); + } catch(e) { + clearInterval(this.apiCallIntervalId) + } + }; + + this.apiCallIntervalId = setInterval(getMetrict, 1000); + } + + private updateMetrics(data: object) { + console.log(data); + } + private processConnectionWidget() { this.children_views.update(this.model.get('children')).then((views) => { views.forEach((view) => { @@ -110,6 +165,11 @@ class SparkUIView extends widgets.VBoxView { }); }); } + + despose() { + super.dispose(); + clearInterval(this.apiCallIntervalId); + } } export default { From 8e23627cb997ce8e51ba5d0a8d86f5bcf9af9c02 Mon Sep 17 00:00:00 2001 From: Mariusz Jurowicz Date: Thu, 24 May 2018 13:30:36 +0200 Subject: [PATCH 10/10] #7326 update Spark metrics on Api call --- js/notebook/src/SparkUI.ts | 54 +++++++++++++++++-------- js/notebook/src/shared/style/spark.scss | 2 +- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/js/notebook/src/SparkUI.ts b/js/notebook/src/SparkUI.ts index 972212f606..b4f3ed2cd5 100644 --- a/js/notebook/src/SparkUI.ts +++ b/js/notebook/src/SparkUI.ts @@ -45,7 +45,7 @@ class SparkUIView extends widgets.VBoxView { public render() { super.render(); - this.processConnectionWidget(); + this.addSparkMetricsWidget(); this.updateLabels(); } @@ -53,7 +53,7 @@ class SparkUIView extends widgets.VBoxView { super.update(); this.connectToApi(); - this.processConnectionWidget(); + this.addSparkMetricsWidget(); this.updateLabels(); } @@ -84,17 +84,20 @@ class SparkUIView extends widgets.VBoxView { }).outerWidth(); } - private appendSparkStats(): void { + private createSparkMetricsWidget(): void { if (this.sparkStats) { + this.el.querySelector('.bx-connection-status') + .insertAdjacentElement('afterend', this.sparkStats.node); + return; } this.sparkStats = new Widget(); this.sparkStats.node.classList.add('bx-stats'); this.sparkStats.node.innerHTML = ` -
0
0
0
+
0
0
0
`; this.connectionLabelActive = this.sparkStats.node.querySelector('.active'); @@ -130,35 +133,54 @@ class SparkUIView extends widgets.VBoxView { const sparkUrl = `${api.getApiUrl('sparkmetrics/executors')}/${this.sparkAppId}`; const getMetrict = async () => { try { - const response = await fetch(sparkUrl); + const response = await fetch(sparkUrl, { method: 'GET', credentials: 'include' }); if (!response.ok) { - console.log(response); - - return; + return this.clearApiCallInterval(); } const data = await response.json(); this.updateMetrics(data); - } catch(e) { - clearInterval(this.apiCallIntervalId) + } catch(error) { + this.clearApiCallInterval(); } }; + this.clearApiCallInterval(); this.apiCallIntervalId = setInterval(getMetrict, 1000); } - private updateMetrics(data: object) { - console.log(data); + private clearApiCallInterval() { + clearInterval(this.apiCallIntervalId); + this.sparkAppId = null; + } + + private updateMetrics(data: Array) { + let activeTasks: number = 0; + let deadExecutors: number = 0; + let storageMemory: number = 0; + + data.forEach(execData => { + if (execData.isActive) { + activeTasks += execData.activeTasks; + storageMemory += execData.memoryUsed; + } else { + deadExecutors += 1; + } + }); + + this.connectionLabelActive.innerText = `${activeTasks}`; + this.connectionLabelMemory.innerText = `${storageMemory}`; + this.connectionLabelDead.innerText = `${deadExecutors}`; } - private processConnectionWidget() { + private addSparkMetricsWidget() { this.children_views.update(this.model.get('children')).then((views) => { views.forEach((view) => { view.children_views.update(view.model.get('children')).then((views) => { views.forEach((view) => { if (view instanceof widgets.LabelView && view.el.classList.contains('bx-connection-status')) { - this.appendSparkStats(); + this.createSparkMetricsWidget(); } }); }); diff --git a/js/notebook/src/shared/style/spark.scss b/js/notebook/src/shared/style/spark.scss index ad99c6fd40..27a228a6f2 100644 --- a/js/notebook/src/shared/style/spark.scss +++ b/js/notebook/src/shared/style/spark.scss @@ -55,7 +55,7 @@ .bx-status-panel { .bx-button { - margin-left: 10px; + margin-left: 20px; } }