Skip to content

Commit

Permalink
#7326 improve spark status widget (#7423)
Browse files Browse the repository at this point in the history 
* #7326: add endpoint for spark metrics executors

* #7326: send sparkAppId

* #7326: send sparkAppId

* #7326 add button with icon css class support

* #7326 improve spark connection status widget

* #7236: add _dom_classes support

* #7326 connection progress basic styles

* #7326 stylize connection status widget

* #7326 call spark REST Api for metrics

* #7326 update Spark metrics on Api call
  • Loading branch information
@scottdraves
Mariusz Jurowicz authored and scottdraves committed May 24, 2018
1 parent 44d5cca commit a24a466
Show file tree
Hide file tree
Showing 13 changed files with 355 additions and 20 deletions.
35 changes: 35 additions & 0 deletions beakerx/beakerx/handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,39 @@
import os


class SparkMetricsExecutorsHandler(APIHandler):
def data_received(self, chunk):
pass

@web.authenticated
@tornado.web.asynchronous
def get(self, id):

def handle_response(response):
self.finish(response.body)

url = "http://localhost:4040/api/v1/applications/" + id + "/allexecutors"
req = tornado.httpclient.HTTPRequest(
url=url,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
allow_nonstandard_methods=True
)

client = tornado.httpclient.AsyncHTTPClient()
try:
client.fetch(req, handle_response)
except tornado.httpclient.HTTPError as e:
if hasattr(e, 'response') and e.response:
handle_response(e.response)
else:
self.set_status(500)
self.write('Internal server error:\n' + str(e))
self.finish()


class SettingsHandler(APIHandler):
def data_received(self, chunk):
pass
Expand Down Expand Up @@ -69,10 +102,12 @@ def load_jupyter_server_extension(nbapp):
web_app = nbapp.web_app
host_pattern = '.*$'
settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings')
spark_metrics_executors_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/sparkmetrics/executors/(.*)')
version_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/version')
javadoc_route_pattern = url_path_join(web_app.settings['base_url'], '/static', '/javadoc/(.*)')
javadoc_lab_route_pattern = url_path_join(web_app.settings['base_url'], '/javadoc/(.*)')
web_app.add_handlers(host_pattern, [(settings_route_pattern, SettingsHandler)])
web_app.add_handlers(host_pattern, [(spark_metrics_executors_route_pattern, SparkMetricsExecutorsHandler)])
web_app.add_handlers(host_pattern, [(version_route_pattern, VersionHandler)])
web_app.add_handlers(host_pattern, [(javadoc_route_pattern, JavaDoc)])
web_app.add_handlers(host_pattern, [(javadoc_lab_route_pattern, JavaDoc)])
Expand Down
124 changes: 124 additions & 0 deletions js/notebook/src/SparkUI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
* limitations under the License.
*/

import {Widget} from "@phosphor/widgets";
import BeakerXApi from "./tree/Utils/BeakerXApi";

const widgets = require('./widgets');

class SparkUIModel extends widgets.VBoxModel {
Expand All @@ -31,14 +34,26 @@ class SparkUIModel extends widgets.VBoxModel {
}

class SparkUIView extends widgets.VBoxView {
private sparkStats: Widget;
private sparkAppId: string;
private apiCallIntervalId: number;
private connectionLabelActive: HTMLElement;
private connectionLabelMemory: HTMLElement;
private connectionLabelDead: HTMLElement;

public render() {
super.render();
this.el.classList.add('widget-spark-ui');

this.addSparkMetricsWidget();
this.updateLabels();
}

public update() {
super.update();

this.connectToApi();
this.addSparkMetricsWidget();
this.updateLabels();
}

Expand Down Expand Up @@ -100,6 +115,115 @@ class SparkUIView extends widgets.VBoxView {

return labelEl.clientWidth;
}

private createSparkMetricsWidget(): void {
if (this.sparkStats) {
this.el.querySelector('.bx-connection-status')
.insertAdjacentElement('afterend', this.sparkStats.node);

return;
}

this.sparkStats = new Widget();
this.sparkStats.node.classList.add('bx-stats');
this.sparkStats.node.innerHTML = `
<div class="active label label-info" title="Active Tasks">0</div> <div
class="dead label label-danger" title="Dead Executors">0</div> <div
class="memory label label-default" title="Storage Memory">0</div>
`;

this.connectionLabelActive = this.sparkStats.node.querySelector('.active');
this.connectionLabelMemory = this.sparkStats.node.querySelector('.memory');
this.connectionLabelDead = this.sparkStats.node.querySelector('.dead');

this.el.querySelector('.bx-connection-status').insertAdjacentElement('afterend', this.sparkStats.node);
}

private connectToApi() {
let baseUrl;
let api;

this.sparkAppId = this.model.get('sparkAppId');

if (!this.sparkAppId) {
return;
}

try {
const coreutils = require('@jupyterlab/coreutils');
coreutils.PageConfig.getOption('pageUrl');
baseUrl = coreutils.PageConfig.getBaseUrl();
} catch(e) {
baseUrl = `${window.location.origin}/`;
}

api = new BeakerXApi(baseUrl);
this.setApiCallInterval(api);
}

private setApiCallInterval(api: BeakerXApi): void {
const sparkUrl = `${api.getApiUrl('sparkmetrics/executors')}/${this.sparkAppId}`;
const getMetrict = async () => {
try {
const response = await fetch(sparkUrl, { method: 'GET', credentials: 'include' });

if (!response.ok) {
return this.clearApiCallInterval();
}

const data = await response.json();
this.updateMetrics(data);
} catch(error) {
this.clearApiCallInterval();
}
};

this.clearApiCallInterval();
this.apiCallIntervalId = setInterval(getMetrict, 1000);
}

private clearApiCallInterval() {
clearInterval(this.apiCallIntervalId);
this.sparkAppId = null;
}

private updateMetrics(data: Array<any>) {
let activeTasks: number = 0;
let deadExecutors: number = 0;
let storageMemory: number = 0;

data.forEach(execData => {
if (execData.isActive) {
activeTasks += execData.activeTasks;
storageMemory += execData.memoryUsed;
} else {
deadExecutors += 1;
}
});

this.connectionLabelActive.innerText = `${activeTasks}`;
this.connectionLabelMemory.innerText = `${storageMemory}`;
this.connectionLabelDead.innerText = `${deadExecutors}`;
}

private addSparkMetricsWidget() {
this.children_views.update(this.model.get('children')).then((views) => {
views.forEach((view) => {
view.children_views.update(view.model.get('children')).then((views) => {
views.forEach((view) => {
if (view instanceof widgets.LabelView && view.el.classList.contains('bx-connection-status')) {
this.createSparkMetricsWidget();
}
});
});
});
});
}

despose() {
super.dispose();
clearInterval(this.apiCallIntervalId);
}
}

export default {
Expand Down
27 changes: 23 additions & 4 deletions js/notebook/src/shared/style/beakerx.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,33 @@
* limitations under the License.
*/

$focusColor: #66bb6a;
@import "bxvariables";

.improveFonts .context-menu-root .context-menu-item span {
font-family: "Lato", Helvetica, sans-serif;
}

.bx-button {
&[class*="icon"] {
font: normal normal normal 14px/1 FontAwesome;
border: 1px solid $bxBorderColor1;
width: auto;
height: 24px;
padding: 0 6px;
margin: 2px 4px;
}

&:before {
display: inline-block;
}

&.icon-close:before {
content: "\f00d" !important;
}
}

.bko-focused {
border: 1px solid $focusColor !important;
border: 1px solid $bxColorFocus !important;

&:before {
position: absolute;
Expand All @@ -31,7 +50,7 @@ $focusColor: #66bb6a;
width: 5px;
height: calc(100% + 2px);
content: '';
background: $focusColor;
background: $bxColorFocus;
}
}

Expand Down Expand Up @@ -144,7 +163,7 @@ $focusColor: #66bb6a;
font-size: 14px;
line-height: 1.3em;
padding: 0 2.2em;
border: 1px solid #ababab;
border: 1px solid $bxBorderColor1;
border-top: 0;
border-radius: 0 0 3px 3px;
text-overflow: ellipsis;
Expand Down
1 change: 1 addition & 0 deletions js/notebook/src/shared/style/bxvariables.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ $bxBorderColor1: #cccccc;
$bxBgColor1: #ffffff;
$bxBgColor2: #eeeeee;

$bxColorFocus: #66bb6a;
$bxColorSuccess: #5cb85c;
$bxColorInfo: #5bc0de;
$bxColorWarning: #f0ad4e;
Expand Down
38 changes: 34 additions & 4 deletions js/notebook/src/shared/style/spark.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,6 @@
max-width: 600px;
}

.bx-spark-stageProgressLabels {
margin: 0;
}

.bx-panel {
border: 1px solid $bxBorderColor1;
border-radius: 2px;
Expand Down Expand Up @@ -103,6 +99,11 @@
}
}

.bx-spark-stageProgressLabels {
margin: 0;
}

.bx-stats,
.bx-spark-stageProgressLabels {
display: block;
line-height: 100%;
Expand All @@ -115,3 +116,32 @@
line-height: 150%;
}
}

.bx-stats {
margin: 4px 0;
height: 20px;
line-height: 17px;
}

.bx-status-panel {
.bx-button {
margin-left: 20px;
}
}

.bx-connection-status {
background-color: transparent;
border: 1px solid $bxBorderColor1;
height: 20px !important;
line-height: 20px !important;
padding: 0 1em;
border-radius: 3px;
margin: 4px 6px;
min-width: 160px;

&.connected {
border: none;
background-color: $bxColorSuccess;
color: $bxBgColor1;
}
}
20 changes: 17 additions & 3 deletions kernel/base/src/main/java/com/twosigma/beakerx/widget/DOMWidget.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
import com.twosigma.beakerx.message.Message;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;

Expand All @@ -31,9 +33,11 @@ public abstract class DOMWidget extends Widget {
public static final String SYNC_DATA = "state";
public static final String MODEL_MODULE_VALUE = "@jupyter-widgets/controls";
public static final String VIEW_MODULE_VALUE = "@jupyter-widgets/controls";
public static final String DOM_CLASSES = "_dom_classes";

private Layout layout;
protected Style style;
private List<String> domClasses = new ArrayList<>();

private UpdateValueCallback updateValueCallback = () -> {
};
Expand Down Expand Up @@ -75,7 +79,7 @@ public Optional<Object> getSyncDataValue(Message msg) {
ret = Optional.of(sync_data.get(VALUE));
} else if (sync_data.containsKey(INDEX)) {
ret = Optional.of(sync_data.get(INDEX));
} else if (sync_data.containsKey("outputs")){
} else if (sync_data.containsKey("outputs")) {
ret = Optional.of(sync_data.get("outputs"));
}
}
Expand Down Expand Up @@ -104,12 +108,12 @@ public void doUpdateValueWithCallback(Object value) {
}

@Override
public String getModelModuleValue(){
public String getModelModuleValue() {
return MODEL_MODULE_VALUE;
}

@Override
public String getViewModuleValue(){
public String getViewModuleValue() {
return VIEW_MODULE_VALUE;
}

Expand All @@ -125,9 +129,19 @@ protected HashMap<String, Serializable> content(HashMap<String, Serializable> co
content.put("font_weight", "");
content.put("background_color", null);
content.put("color", null);
content.put(DOM_CLASSES, domClasses.toArray());
return content;
}

public List<String> getDomClasses() {
return domClasses;
}

public void setDomClasses(List<String> domClasses) {
this.domClasses = checkNotNull(domClasses);
sendUpdate(DOM_CLASSES, this.domClasses.toArray());
}

public Layout getLayout() {
if (layout == null) {
layout = new Layout();
Expand Down
2 changes: 1 addition & 1 deletion kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorResultTestWatcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ private static Optional<Message> getError(List<Message> messages) {
}


private static Optional<Message> getUpdate(List<Message> messages) {
public static Optional<Message> getUpdate(List<Message> messages) {
return messages.stream().
filter(x -> x.type().equals(JupyterMessages.COMM_MSG)).
filter(x -> TestWidgetUtils.getData(x).get("method").equals("update")).
Expand Down
Loading

0 comments on commit a24a466

Please sign in to comment.