#5742 add highlight support for jvm kernels magics (#7662)

* #5742 add highlight support for jvm kernels magics

* #5742 define line magic highlighter mode

* #5742 remove GrovyMode namespace, update types

* #5742 add line magic highlighting in Lab

* #5742 highlight Spark and classpath magics

* #5742 support line magics not only in first line of cell

js/lab/src/plugin/codeEditor.ts

@@ -18,10 +18,13 @@ import { CodeMirrorEditor } from "@jupyterlab/codemirror";
 import { Cell, CodeCell } from '@jupyterlab/cells';
 
 import 'codemirror/mode/groovy/groovy';
+import {NotebookPanel} from "@jupyterlab/notebook";
 
 const LINE_COMMENT_CHAR = '//';
+const LINE_MAGIC_MODE = 'line_magic';
+const CodeMirror = require("codemirror");
 
-export const registerCommentOutCmd = (panel) => {
+export const registerCommentOutCmd = (panel: NotebookPanel) => {
   const cells = panel.notebook.widgets || [];
 
   cells
@@ -50,3 +53,84 @@ const setCodeMirrorLineComment = (cell: Cell) => {
   mode.lineComment = LINE_COMMENT_CHAR;
   doc.mode = mode;
 };
+
+export function extendHighlightModes(panel: NotebookPanel) {
+  const cells = panel.notebook.widgets || [];
+
+  cells
+    .filter((cell) => (cell.editor instanceof CodeMirrorEditor))
+    .forEach(setLineMagicForCell);
+
+  CodeMirror.defineInitHook(addLineMagicsOverlay);
+}
+
+function setLineMagicForCell(cell: Cell) {
+  if (!(cell instanceof CodeCell)) {
+    return;
+  }
+
+  addLineMagicsOverlay((<CodeMirrorEditor>cell.editor).editor);
+}
+
+const lineMagicOverlay = {
+  startState() {
+    return { firstMatched: false, inMagicLine: false };
+  },
+
+  token(stream, state) {
+    if (stream.match(/^%(%classpath|%spark|\w+)/, false)) {
+      state.inMagicLine = true;
+    }
+
+    if (state.inMagicLine) {
+      stream.eat(() => true);
+
+      if (stream.eol()) {
+        state.inMagicLine = false;
+      }
+
+      return LINE_MAGIC_MODE;
+    }
+
+    stream.skipToEnd();
+
+    return null;
+  }
+};
+
+export function autoHighlightLineMagics(code_mirror) {
+  const current_mode = code_mirror.getOption('mode');
+
+  if (current_mode === LINE_MAGIC_MODE) {
+    return;
+  }
+
+  const re = /^%(%classpath|%spark|\w+)/;
+
+  code_mirror.eachLine((line) => {
+    if (line && line.text.match(re) !== null) {
+      // Add an overlay mode to recognize the first line as "line magic" instead
+      // of the mode used for the rest of the cell.
+      CodeMirror.defineMode(LINE_MAGIC_MODE, (config) => {
+        return CodeMirror.overlayMode(CodeMirror.getMode(config, current_mode), lineMagicOverlay);
+      });
+
+      code_mirror.setOption('mode', LINE_MAGIC_MODE);
+
+      return false;
+    }
+  });
+}
+
+export function addLineMagicsOverlay(editor: any) {
+  autoHighlightLineMagics(editor);
+
+  editor.off("focus", autoHighlightLineMagics);
+  editor.on("focus", autoHighlightLineMagics);
+  editor.off("change", autoHighlightLineMagics);
+  editor.on("change", autoHighlightLineMagics);
+  editor.off("blur", autoHighlightLineMagics);
+  editor.on("blur", autoHighlightLineMagics);
+
+  editor.refresh();
+}

js/lab/src/plugin/index.ts

@@ -21,7 +21,7 @@ import { INotebookModel, NotebookPanel } from '@jupyterlab/notebook';
 import { JupyterLab } from "@jupyterlab/application";
 import { ISettingRegistry } from "@jupyterlab/coreutils";
 import { registerCommTargets } from './comm';
-import { registerCommentOutCmd } from './codeEditor';
+import {extendHighlightModes, registerCommentOutCmd} from './codeEditor';
 import { enableInitializationCellsFeature } from './initializationCells';
 import UIOptionFeaturesHelper from "./UIOptionFeaturesHelper";
 
@@ -55,6 +55,7 @@ class BeakerxExtension implements DocumentRegistry.WidgetExtension {
     let settings = this.settings;
 
     Promise.all([panel.ready, panel.session.ready, context.ready]).then(function() {
+      extendHighlightModes(panel);
       enableInitializationCellsFeature(panel);
       registerCommentOutCmd(panel);
       registerCommTargets(panel, context);

js/notebook/src/extension/codeEditor.ts

@@ -0,0 +1,123 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/// <reference path='../types/index.d.ts'/>
+
+export const LINE_COMMENT_CHAR = '//';
+export const LINE_MAGIC_MODE = 'line_magic';
+
+export function extendWithLineComment(Jupyter: any, CodeMirror: any) {
+  CodeMirror.extendMode('groovy', { lineComment: LINE_COMMENT_CHAR });
+
+  Jupyter.notebook.get_cells().map(setCodeMirrorLineComment);
+}
+
+function setCodeMirrorLineComment(cell: any) {
+  if (cell.cell_type !== 'code') {
+    return;
+  }
+
+  const cm = cell.code_mirror;
+  const doc = cm.getDoc();
+  const mode = cm.getMode();
+
+  if (!mode.lineComment) {
+    mode.lineComment = LINE_COMMENT_CHAR;
+    doc.mode = mode;
+  }
+}
+
+export function extendHighlightModes(Jupyter: any) {
+  Jupyter.CodeCell.options_default.highlight_modes = {
+    ...Jupyter.CodeCell.options_default.highlight_modes,
+    magic_python: {reg: ['^%%python']},
+    magic_groovy: {reg: ['^%%groovy']},
+    magic_java: {reg: ['^%%java']},
+    magic_scala: {reg: ['^%%scala']},
+    magic_kotlin: {reg: ['^%%kotlin']},
+    magic_clojure: {reg: ['^%%clojure']},
+    magic_sql: {reg: ['^%%sql']},
+    magic_html: {reg: ['^%%html']}
+  };
+
+  Jupyter.notebook.get_cells().map(setLineMagicForCell);
+  CodeMirror.defineInitHook(addLineMagicsOverlay);
+}
+
+function setLineMagicForCell(cell) {
+  cell.auto_highlight();
+  addLineMagicsOverlay(cell.code_mirror);
+}
+
+const lineMagicOverlay = {
+  startState() {
+    return { firstMatched: false, inMagicLine: false };
+  },
+
+  token(stream, state) {
+    if (stream.match(/^%(%classpath|%spark|\w+)/)) {
+      state.inMagicLine = true;
+    }
+
+    if (state.inMagicLine) {
+      stream.eat(() => true);
+
+      if (stream.eol()) {
+        state.inMagicLine = false;
+      }
+
+      return LINE_MAGIC_MODE;
+    }
+
+    stream.skipToEnd();
+
+    return null;
+  }
+};
+
+export function autoHighlightLineMagics(code_mirror) {
+  const current_mode = code_mirror.getOption('mode');
+
+  if (current_mode === LINE_MAGIC_MODE) {
+    return;
+  }
+
+  const re = /^%(%classpath|%spark|\w+)/;
+
+  code_mirror.eachLine((line) => {
+    if (line && line.text.match(re) !== null) {
+      // Add an overlay mode to recognize the first line as "line magic" instead
+      // of the mode used for the rest of the cell.
+      CodeMirror.defineMode(LINE_MAGIC_MODE, (config) => {
+        return CodeMirror.overlayMode(CodeMirror.getMode(config, current_mode), lineMagicOverlay);
+      });
+
+      code_mirror.setOption('mode', LINE_MAGIC_MODE);
+
+      return false;
+    }
+  });
+}
+
+export function addLineMagicsOverlay(code_mirror) {
+  autoHighlightLineMagics(code_mirror);
+  code_mirror.off("focus", autoHighlightLineMagics);
+  code_mirror.on("focus", autoHighlightLineMagics);
+  code_mirror.off("change", autoHighlightLineMagics);
+  code_mirror.on("change", autoHighlightLineMagics);
+  code_mirror.off("blur", autoHighlightLineMagics);
+  code_mirror.on("blur", autoHighlightLineMagics);
+}

js/notebook/src/extension/index.ts

@@ -26,7 +26,6 @@ import {enableInitializationCellsFeature} from './initializationCells';
 import {Autotranslation} from './autotranslation';
 import {BeakerXKernel} from './kernel';
 import {displayHTML} from '../htmlOutput/htmlOutput';
-import {GroovyMode} from "./groovyModeExtension";
 import bkCoreManager from '../shared/bkCoreManager';
 
 import '../shared/style/beakerx.scss';
@@ -99,7 +98,8 @@ function setupNotebook() {
     })
     .catch((reason) => { console.error(log_prefix, 'unhandled error:', reason); });
 
-  GroovyMode.extendWithLineComment(Jupyter, CodeMirror);
+  extendWithLineComment(Jupyter, CodeMirror);
+  extendHighlightModes(Jupyter);
 }
 
 export const load_ipython_extension = () => {

js/notebook/src/types/BeakerXThemeHelper.d.ts

@@ -1,19 +1,3 @@
-/*
- *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 import { DataGrid } from "@phosphor/datagrid";
 export default class BeakerXThemeHelper {
     static readonly isDark: boolean;
@@ -28,6 +12,6 @@ export default class BeakerXThemeHelper {
     static readonly HIGHLIGHTED_CELL_BACKGROUND_ODD: string;
     static readonly MIN_LIGHTNESS_VALUE: number;
     static readonly MIN_SATURATION_VALUE: number;
-    private static getDarkStyle();
-    private static getLightStyle();
+    private static getDarkStyle;
+    private static getLightStyle;
 }

js/notebook/src/types/Spinner.d.ts

@@ -1,19 +1,3 @@
-/*
- *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 import widgets from './widgets';
 export declare class SpinnerModel extends widgets.DOMWidgetModel {
     defaults(): any;

js/notebook/src/types/global.env.d.ts

@@ -32,6 +32,7 @@ interface MapConstructor {
 
 declare var Map: MapConstructor;
 declare var CodeMirror: any;
+declare var Proxy: ProxyConstructor;
 
 declare interface NumberConstructor {
   isNaN: (number: number) => boolean,
@@ -48,7 +49,6 @@ interface ProxyConstructor {
   new <T extends object>(target: T, handler: ProxyHandler<T>): T;
 }
 
-declare var Proxy: ProxyConstructor;
 
 declare interface Array<T> {
   from: (arrayLike: any[]) => any[]

js/notebook/src/types/plot/plotSanitize.d.ts

@@ -1,4 +1,4 @@
-declare global  {
+declare global {
     interface Window {
         cssSchemaFixed: boolean;
         cssSchema: any;

js/notebook/src/types/shared/bkCoreManager.d.ts

@@ -0,0 +1,22 @@
+declare const bkCoreManager: {
+    _bkAppImpl: {
+        getBeakerObject: () => {
+            beakerObj: {
+                prefs: {
+                    outputColumnLimit: number;
+                    outputLineLimit: number;
+                    theme: {
+                        name: string;
+                        plotColors: string[];
+                    };
+                };
+            };
+        };
+    };
+    _prefs: {
+        getTheme: () => any;
+    };
+    getTheme: () => any;
+    getBkApp: () => any;
+};
+export default bkCoreManager;