#7326 improve spark status widget

beakerx/beakerx/handlers.py

@@ -23,6 +23,39 @@
 import os
 
 
+class SparkMetricsExecutorsHandler(APIHandler):
+    def data_received(self, chunk):
+        pass
+
+    @web.authenticated
+    @tornado.web.asynchronous
+    def get(self, id):
+
+        def handle_response(response):
+            self.finish(response.body)
+
+        url = "http://localhost:4040/api/v1/applications/" + id + "/allexecutors"
+        req = tornado.httpclient.HTTPRequest(
+            url=url,
+            method=self.request.method,
+            body=self.request.body,
+            headers=self.request.headers,
+            follow_redirects=False,
+            allow_nonstandard_methods=True
+        )
+
+        client = tornado.httpclient.AsyncHTTPClient()
+        try:
+            client.fetch(req, handle_response)
+        except tornado.httpclient.HTTPError as e:
+            if hasattr(e, 'response') and e.response:
+                handle_response(e.response)
+            else:
+                self.set_status(500)
+                self.write('Internal server error:\n' + str(e))
+                self.finish()
+
+
 class SettingsHandler(APIHandler):
     def data_received(self, chunk):
         pass
@@ -69,10 +102,12 @@ def load_jupyter_server_extension(nbapp):
     web_app = nbapp.web_app
     host_pattern = '.*$'
     settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings')
+    spark_metrics_executors_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/sparkmetrics/executors/(.*)')
     version_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/version')
     javadoc_route_pattern = url_path_join(web_app.settings['base_url'], '/static', '/javadoc/(.*)')
     javadoc_lab_route_pattern = url_path_join(web_app.settings['base_url'], '/javadoc/(.*)')
     web_app.add_handlers(host_pattern, [(settings_route_pattern, SettingsHandler)])
+    web_app.add_handlers(host_pattern, [(spark_metrics_executors_route_pattern, SparkMetricsExecutorsHandler)])
     web_app.add_handlers(host_pattern, [(version_route_pattern, VersionHandler)])
     web_app.add_handlers(host_pattern, [(javadoc_route_pattern, JavaDoc)])
     web_app.add_handlers(host_pattern, [(javadoc_lab_route_pattern, JavaDoc)])

js/notebook/src/SparkUI.ts

@@ -14,6 +14,9 @@
  *  limitations under the License.
  */
 
+import {Widget} from "@phosphor/widgets";
+import BeakerXApi from "./tree/Utils/BeakerXApi";
+
 const widgets = require('./widgets');
 
 class SparkUIModel extends widgets.VBoxModel {
@@ -31,14 +34,26 @@ class SparkUIModel extends widgets.VBoxModel {
 }
 
 class SparkUIView extends widgets.VBoxView {
+  private sparkStats: Widget;
+  private sparkAppId: string;
+  private apiCallIntervalId: number;
+  private connectionLabelActive: HTMLElement;
+  private connectionLabelMemory: HTMLElement;
+  private connectionLabelDead: HTMLElement;
+
   public render() {
     super.render();
     this.el.classList.add('widget-spark-ui');
+
+    this.addSparkMetricsWidget();
     this.updateLabels();
   }
 
   public update() {
     super.update();
+
+    this.connectToApi();
+    this.addSparkMetricsWidget();
     this.updateLabels();
   }
 
@@ -100,6 +115,115 @@ class SparkUIView extends widgets.VBoxView {
 
     return labelEl.clientWidth;
   }
+
+  private createSparkMetricsWidget(): void {
+    if (this.sparkStats) {
+      this.el.querySelector('.bx-connection-status')
+        .insertAdjacentElement('afterend', this.sparkStats.node);
+
+      return;
+    }
+
+    this.sparkStats = new Widget();
+    this.sparkStats.node.classList.add('bx-stats');
+    this.sparkStats.node.innerHTML = `
+      <div class="active label label-info" title="Active Tasks">0</div> <div
+      class="dead label label-danger" title="Dead Executors">0</div> <div
+      class="memory label label-default" title="Storage Memory">0</div>
+    `;
+
+    this.connectionLabelActive = this.sparkStats.node.querySelector('.active');
+    this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory');
+    this.connectionLabelDead = this.sparkStats.node.querySelector('.dead');
+
+    this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node);
+  }
+
+  private connectToApi() {
+    let baseUrl;
+    let api;
+
+    this.sparkAppId = this.model.get('sparkAppId');
+
+    if (!this.sparkAppId) {
+      return;
+    }
+
+    try {
+      const coreutils = require('@jupyterlab/coreutils');
+      coreutils.PageConfig.getOption('pageUrl');
+      baseUrl = coreutils.PageConfig.getBaseUrl();
+    } catch(e) {
+      baseUrl = `${window.location.origin}/`;
+    }
+
+    api = new BeakerXApi(baseUrl);
+    this.setApiCallInterval(api);
+  }
+
+  private setApiCallInterval(api: BeakerXApi): void {
+    const sparkUrl = `${api.getApiUrl('sparkmetrics/executors')}/${this.sparkAppId}`;
+    const getMetrict = async () => {
+      try {
+        const response = await fetch(sparkUrl, { method: 'GET', credentials: 'include' });
+
+        if (!response.ok) {
+          return this.clearApiCallInterval();
+        }
+
+        const data = await response.json();
+        this.updateMetrics(data);
+      } catch(error) {
+        this.clearApiCallInterval();
+      }
+    };
+
+    this.clearApiCallInterval();
+    this.apiCallIntervalId = setInterval(getMetrict, 1000);
+  }
+
+  private clearApiCallInterval() {
+    clearInterval(this.apiCallIntervalId);
+    this.sparkAppId = null;
+  }
+
+  private updateMetrics(data: Array<any>) {
+    let activeTasks: number = 0;
+    let deadExecutors: number = 0;
+    let storageMemory: number = 0;
+
+    data.forEach(execData => {
+      if (execData.isActive) {
+        activeTasks += execData.activeTasks;
+        storageMemory += execData.memoryUsed;
+      } else {
+        deadExecutors += 1;
+      }
+    });
+
+    this.connectionLabelActive.innerText = `${activeTasks}`;
+    this.connectionLabelMemory.innerText = `${storageMemory}`;
+    this.connectionLabelDead.innerText = `${deadExecutors}`;
+  }
+
+  private addSparkMetricsWidget() {
+    this.children_views.update(this.model.get('children')).then((views) => {
+      views.forEach((view) => {
+        view.children_views.update(view.model.get('children')).then((views) => {
+          views.forEach((view) => {
+            if (view instanceof widgets.LabelView && view.el.classList.contains('bx-connection-status')) {
+              this.createSparkMetricsWidget();
+            }
+          });
+        });
+      });
+    });
+  }
+
+  despose() {
+    super.dispose();
+    clearInterval(this.apiCallIntervalId);
+  }
 }
 
 export default {

js/notebook/src/shared/style/beakerx.scss

@@ -14,14 +14,33 @@
  *  limitations under the License.
  */
 
-$focusColor: #66bb6a;
+@import "bxvariables";
 
 .improveFonts .context-menu-root .context-menu-item span {
   font-family: "Lato", Helvetica, sans-serif;
 }
 
+.bx-button {
+  &[class*="icon"] {
+    font: normal normal normal 14px/1 FontAwesome;
+    border: 1px solid $bxBorderColor1;
+    width: auto;
+    height: 24px;
+    padding: 0 6px;
+    margin: 2px 4px;
+  }
+
+  &:before {
+    display: inline-block;
+  }
+
+  &.icon-close:before {
+    content: "\f00d" !important;
+  }
+}
+
 .bko-focused {
-  border: 1px solid $focusColor !important;
+  border: 1px solid $bxColorFocus !important;
 
   &:before {
     position: absolute;
@@ -31,7 +50,7 @@ $focusColor: #66bb6a;
     width: 5px;
     height: calc(100% + 2px);
     content: '';
-    background: $focusColor;
+    background: $bxColorFocus;
   }
 }
 
@@ -144,7 +163,7 @@ $focusColor: #66bb6a;
     font-size: 14px;
     line-height: 1.3em;
     padding: 0 2.2em;
-    border: 1px solid #ababab;
+    border: 1px solid $bxBorderColor1;
     border-top: 0;
     border-radius: 0 0 3px 3px;
     text-overflow: ellipsis;

js/notebook/src/shared/style/bxvariables.scss

@@ -18,6 +18,7 @@ $bxBorderColor1: #cccccc;
 $bxBgColor1: #ffffff;
 $bxBgColor2: #eeeeee;
 
+$bxColorFocus: #66bb6a;
 $bxColorSuccess: #5cb85c;
 $bxColorInfo: #5bc0de;
 $bxColorWarning: #f0ad4e;

js/notebook/src/shared/style/spark.scss

@@ -28,10 +28,6 @@
   max-width: 600px;
 }
 
-.bx-spark-stageProgressLabels {
-  margin: 0;
-}
-
 .bx-panel {
   border: 1px solid $bxBorderColor1;
   border-radius: 2px;
@@ -103,6 +99,11 @@
   }
 }
 
+.bx-spark-stageProgressLabels {
+  margin: 0;
+}
+
+.bx-stats,
 .bx-spark-stageProgressLabels {
   display: block;
   line-height: 100%;
@@ -115,3 +116,32 @@
     line-height: 150%;
   }
 }
+
+.bx-stats {
+  margin: 4px 0;
+  height: 20px;
+  line-height: 17px;
+}
+
+.bx-status-panel {
+  .bx-button {
+    margin-left: 20px;
+  }
+}
+
+.bx-connection-status {
+  background-color: transparent;
+  border: 1px solid $bxBorderColor1;
+  height: 20px !important;
+  line-height: 20px !important;
+  padding: 0 1em;
+  border-radius: 3px;
+  margin: 4px 6px;
+  min-width: 160px;
+
+  &.connected {
+    border: none;
+    background-color: $bxColorSuccess;
+    color: $bxBgColor1;
+  }
+}

kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java

@@ -21,7 +21,9 @@
 import com.twosigma.beakerx.message.Message;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -31,9 +33,11 @@ public abstract class DOMWidget extends Widget {
   public static final String SYNC_DATA = "state";
   public static final String MODEL_MODULE_VALUE = "@jupyter-widgets/controls";
   public static final String VIEW_MODULE_VALUE = "@jupyter-widgets/controls";
+  public static final String DOM_CLASSES = "_dom_classes";
 
   private Layout layout;
   protected Style style;
+  private List<String> domClasses = new ArrayList<>();
 
   private UpdateValueCallback updateValueCallback = () -> {
   };
@@ -75,7 +79,7 @@ public Optional<Object> getSyncDataValue(Message msg) {
             ret = Optional.of(sync_data.get(VALUE));
           } else if (sync_data.containsKey(INDEX)) {
             ret = Optional.of(sync_data.get(INDEX));
-          } else if (sync_data.containsKey("outputs")){
+          } else if (sync_data.containsKey("outputs")) {
             ret = Optional.of(sync_data.get("outputs"));
           }
         }
@@ -104,12 +108,12 @@ public void doUpdateValueWithCallback(Object value) {
   }
 
   @Override
-  public String getModelModuleValue(){
+  public String getModelModuleValue() {
     return MODEL_MODULE_VALUE;
   }
 
   @Override
-  public String getViewModuleValue(){
+  public String getViewModuleValue() {
     return VIEW_MODULE_VALUE;
   }
 
@@ -125,9 +129,19 @@ protected HashMap<String, Serializable> content(HashMap<String, Serializable> co
     content.put("font_weight", "");
     content.put("background_color", null);
     content.put("color", null);
+    content.put(DOM_CLASSES, domClasses.toArray());
     return content;
   }
 
+  public List<String> getDomClasses() {
+    return domClasses;
+  }
+
+  public void setDomClasses(List<String> domClasses) {
+    this.domClasses = checkNotNull(domClasses);
+    sendUpdate(DOM_CLASSES, this.domClasses.toArray());
+  }
+
   public Layout getLayout() {
     if (layout == null) {
       layout = new Layout();

kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java

@@ -158,7 +158,7 @@ private static Optional<Message> getError(List<Message> messages) {
   }
 
 
-  private static Optional<Message> getUpdate(List<Message> messages) {
+  public static Optional<Message> getUpdate(List<Message> messages) {
     return messages.stream().
             filter(x -> x.type().equals(JupyterMessages.COMM_MSG)).
             filter(x -> TestWidgetUtils.getData(x).get("method").equals("update")).