jarek/5039: autotranslation [WIP] (#7542)

* #5039: add autotranslation service interface and NamespaceClient refactoring * #5039: (groovy) add sync from js to server * #5039: (scala) add support for autotranslation * #5039: add flask * #5039: change flask to tornado * #5039: changes after review * #5039: rename to BeakerClient * #5039: rename to BeakerXClient * #5039: rename to BeakerXClient * #5039: rename to BeakerClientManager * #5039: rename to BeakerXClientManager * #5039: rename to getBeaker * #5039: rename to getBeakerX * #5039: rename to beakerX * #5039: BeakerXClinetManager refactoring * Merge branch 'master' into jarek/5039_autotranslation_v2 # Conflicts: # kernel/base/src/main/java/com/twosigma/beakerx/evaluator/EvaluatorManager.java # kernel/base/src/main/java/com/twosigma/beakerx/kernel/Kernel.java # kernel/base/src/test/java/com/twosigma/beakerx/KernelTest.java # kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorTest.java # kernel/sql/src/main/java/com/twosigma/beakerx/sql/BeakerParser.java # kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLEvaluator.java # kernel/sql/src/main/java/com/twosigma/beakerx/sql/kernel/SQL.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLAutocompleteTest.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLBaseEvaluatorTest.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLEvaluatorTest.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLKernelTest.java * #7511: support structured data in scala kernel
twosigma · Jun 25, 2018 · 1ecc938 · 1ecc938
1 parent a33f7b2
commit 1ecc938
Show file tree

Hide file tree

Showing 85 changed files with 1,607 additions and 573 deletions.
diff --git a/beakerx/beakerx/__init__.py b/beakerx/beakerx/__init__.py
@@ -41,7 +41,6 @@ def _jupyter_server_extension_paths():
     return [dict(module="beakerx")]
 
 beakerx = BeakerX()
-
 def run():
     try:
         parse()

diff --git a/beakerx/beakerx/beakerx_autotranslation_server.py b/beakerx/beakerx/beakerx_autotranslation_server.py
@@ -0,0 +1,109 @@
+# Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tornado.ioloop
+import tornado.web
+import logging
+import os
+import random
+import string
+import socket
+
+import base64
+
+beakerx = {}
+
+logging.getLogger('tornado.access').disabled = True
+
+
+def basic_auth(f):
+    def auth(username, password):
+        return username == "beakerx" and password == os.environ["BEAKERX_AUTOTRANSLATION_PASSWORD"]
+
+    def _request_auth(handler):
+        handler.set_status(401)
+        return handler.finish()
+
+    def wrap(*args):
+        handler = args[0]
+        try:
+            auth_header = handler.request.headers.get('Authorization')
+            if (auth_header is None) or (not auth_header.startswith('Basic ')):
+                return _request_auth(handler)
+            auth_decoded = base64.b64decode(auth_header[6:])
+            username, password = auth_decoded.decode('UTF-8').split(':', 2)
+            if auth(username, password):
+                f(*args)
+            else:
+                _request_auth(handler)
+        except:
+            _request_auth(handler)
+
+    return wrap
+
+
+class MainSaveHandler(tornado.web.RequestHandler):
+
+    @basic_auth
+    def post(self):
+        input_json = tornado.escape.json_decode(self.request.body)
+        session_id = input_json["sessionId"]
+        name = input_json["name"]
+        json = input_json["json"]
+        if session_id not in beakerx:
+            beakerx[session_id] = {}
+
+        beakerx[session_id][name] = json
+        return self.finish("ok")
+
+
+class MainGetHandler(tornado.web.RequestHandler):
+
+    @basic_auth
+    def get(self, session_id, name):
+        if session_id in beakerx and name in beakerx[session_id]:
+            return self.finish(beakerx[session_id][name])
+        return self.finish("undefined")
+
+
+def make_app():
+    return tornado.web.Application([
+        (r"/autotransltion/(.*)/(.*)", MainGetHandler),
+        (r"/autotransltion/", MainSaveHandler),
+    ])
+
+
+def get_free_tcp_port():
+    tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    tcp.bind(('', 0))
+    addr, port = tcp.getsockname()
+    tcp.close()
+    return port
+
+
+def random_string_generator(size=128):
+    s = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in
+                range(size))
+    return s
+
+
+def init_env():
+    os.environ["BEAKERX_AUTOTRANSLATION_PASSWORD"] = random_string_generator()
+    os.environ["BEAKERX_AUTOTRANSLATION_PORT"] = str(get_free_tcp_port())
+
+
+def start_autotranslation_server():
+    init_env()
+    app = make_app()
+    app.listen(os.environ["BEAKERX_AUTOTRANSLATION_PORT"])
diff --git a/beakerx/beakerx/commands.py b/beakerx/beakerx/commands.py
@@ -58,18 +58,18 @@ def py4j_server_subparser(subparser):
     py4j_server_parser.add_argument("--port")
     py4j_server_parser.add_argument("--pyport")
     py4j_server_parser.add_argument("--kernel")
+    py4j_server_parser.add_argument("--context")
 
 
 def start_py4j_server(args):
-    Py4JServer(args.port, args.pyport, args.kernel)
+    Py4JServer(args.port, args.pyport, args.kernel, args.context)
 
 
 def run_jupyter(jupyter_commands):
     app.launch_new_instance(jupyter_commands)
 
 
 def init_parser():
-
     parser = argparse.ArgumentParser()
     parser.add_argument('--version', action='version', version=beakerx.__version__)
     parser.set_defaults(func=run_jupyter)
@@ -91,4 +91,3 @@ def parse():
         args.func(args)
     else:
         parser.parse_args(jupyter_commands)
-
diff --git a/beakerx/beakerx/handlers.py b/beakerx/beakerx/handlers.py
@@ -21,7 +21,7 @@
 import beakerx
 import tornado
 import os
-
+from .beakerx_autotranslation_server import start_autotranslation_server
 
 class SparkMetricsExecutorsHandler(APIHandler):
     def data_received(self, chunk):
@@ -99,6 +99,8 @@ def get(self, path):
 
 
 def load_jupyter_server_extension(nbapp):
+    start_autotranslation_server()
+
     web_app = nbapp.web_app
     host_pattern = '.*$'
     settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings')

diff --git a/beakerx/beakerx_magics/jvm_kernel_magic.py b/beakerx/beakerx_magics/jvm_kernel_magic.py
@@ -21,17 +21,18 @@
 
 class JVMKernelMagic:
 
-    def __init__(self, kernel_name):
+    def __init__(self, kernel_name, context):
         self.km = None
         self.kc = None
         self.comms = []
         self.kernel_name = kernel_name
+        self.context = context
         self.start()
 
     def start(self):
         self.km = KernelManager()
         self.km.kernel_name = self.kernel_name
-        self.km.start_kernel()
+        self.km.start_kernel(extra_arguments=[self.context])
         self.kc = self.km.client()
         self.kc.start_channels()
         self.kc.wait_for_ready()
@@ -66,8 +67,8 @@ def pass_msg(self, msg_raw):
 
 class PythonEntryPoint(object):
 
-    def __init__(self, kernel_name):
-        self.pm = JVMKernelMagic(kernel_name)
+    def __init__(self, kernel_name, context):
+        self.pm = JVMKernelMagic(kernel_name, context)
 
     def evaluate(self, code):
         print('code for evaluate {}'.format(code))
@@ -95,9 +96,9 @@ class Java:
 
 
 class Py4JServer:
-    def __init__(self, port, pyport, kernel_name):
+    def __init__(self, port, pyport, kernel_name, context):
         try:
-            pep = PythonEntryPoint(kernel_name)
+            pep = PythonEntryPoint(kernel_name, context)
         except NoSuchKernel:
             sys.exit(2)
         ClientServer(
@@ -108,4 +109,4 @@ def __init__(self, port, pyport, kernel_name):
 
 
 if __name__ == '__main__':
-    Py4JServer(sys.argv[1], sys.argv[2], sys.argv[3])
+    Py4JServer(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
diff --git a/js/notebook/src/extension.js b/js/notebook/src/extension.js
@@ -158,17 +158,35 @@ define([
 
     // assign Beaker methods to window
     if (window) {
-      if (!window.beakerx) {
-        window.beakerx = {};
-      }
-
       var plotApiList = plotApi.list();
       var bkApp = bkCoreManager.getBkApp();
       var bkObject = bkApp.getBeakerObject();
 
-      _.extend(window.beakerx, plotApiList);
-      _.extend(window.beakerx, htmlOutput);
-      window.beakerx.prefs = bkObject.beakerObj.prefs;
+      var beakerxInstance = {}
+      _.extend(beakerxInstance, plotApiList);
+      _.extend(beakerxInstance, htmlOutput);
+      beakerxInstance.prefs = bkObject.beakerObj.prefs;
+
+      if (!window.beakerx) {
+        var handler = {
+          get: function (obj, prop) {
+            return prop in obj ? obj[prop] : undefined;
+          },
+
+          set: function (obj, prop, value) {
+            obj[prop] = value;
+            var comm = Jupyter.notebook.kernel.comm_manager.new_comm('beaker.autotranslation', null, null, null, utils.uuid());
+            var data = {
+            };
+            data.name = prop;
+            data.value = value;
+            comm.send(data);
+            comm.close();
+            return true;
+          }
+        };
+        window.beakerx = new Proxy(beakerxInstance, handler);
+      }
     }
 
     if (inNotebook) {

diff --git a/kernel/base/build.gradle b/kernel/base/build.gradle
@@ -40,6 +40,7 @@ dependencies {
   compile group: 'org.apache.commons', name: 'commons-text', version: '1.1'
   compile group: 'org.apache.maven.shared', name: 'maven-invoker', version: '3.0.0'
   compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.25'
+  compile group: 'org.apache.httpcomponents', name: 'fluent-hc', version: '4.5.5'
 
   compile group: "org.zeromq", name: "jeromq", version: "0.3.5"
   compile group: 'org.apache.ivy', name: 'ivy', version: '2.4.0'

diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationService.java b/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationService.java
@@ -0,0 +1,27 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx;
+
+public interface AutotranslationService {
+
+  String update(String name, String json);
+
+  String get(String name);
+
+  String close();
+
+  String getContextAsString();
+}