From 1ecc9385dfe161ab8fe697dcc7996c1235cfa7a4 Mon Sep 17 00:00:00 2001 From: jaroslawmalekcodete Date: Tue, 26 Jun 2018 01:22:34 +0200 Subject: [PATCH] jarek/5039: autotranslation [WIP] (#7542) * #5039: add autotranslation service interface and NamespaceClient refactoring * #5039: (groovy) add sync from js to server * #5039: (scala) add support for autotranslation * #5039: add flask * #5039: change flask to tornado * #5039: changes after review * #5039: rename to BeakerClient * #5039: rename to BeakerXClient * #5039: rename to BeakerXClient * #5039: rename to BeakerClientManager * #5039: rename to BeakerXClientManager * #5039: rename to getBeaker * #5039: rename to getBeakerX * #5039: rename to beakerX * #5039: BeakerXClinetManager refactoring * Merge branch 'master' into jarek/5039_autotranslation_v2 # Conflicts: # kernel/base/src/main/java/com/twosigma/beakerx/evaluator/EvaluatorManager.java # kernel/base/src/main/java/com/twosigma/beakerx/kernel/Kernel.java # kernel/base/src/test/java/com/twosigma/beakerx/KernelTest.java # kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorTest.java # kernel/sql/src/main/java/com/twosigma/beakerx/sql/BeakerParser.java # kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLEvaluator.java # kernel/sql/src/main/java/com/twosigma/beakerx/sql/kernel/SQL.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLAutocompleteTest.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLBaseEvaluatorTest.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLEvaluatorTest.java # kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLKernelTest.java * #7511: support structured data in scala kernel --- beakerx/beakerx/__init__.py | 1 - .../beakerx/beakerx_autotranslation_server.py | 109 +++++++ beakerx/beakerx/commands.py | 5 +- beakerx/beakerx/handlers.py | 4 +- beakerx/beakerx_magics/jvm_kernel_magic.py | 15 +- js/notebook/src/extension.js | 32 +- kernel/base/build.gradle | 1 + .../beakerx/AutotranslationService.java | 27 ++ .../beakerx/AutotranslationServiceImpl.java | 144 +++++++++ .../com/twosigma/beakerx/BeakerXClient.java | 40 +++ .../beakerx/BeakerXClientManager.java | 35 +++ .../beakerx/BeakerXJsonSerializer.java | 22 ++ .../beakerx/DefaultBeakerXJsonSerializer.java | 54 ++++ .../com/twosigma/beakerx/NamespaceClient.java | 165 ++++++----- .../twosigma/beakerx/chart/ChartDetails.java | 6 +- .../beakerx/evaluator/BaseEvaluator.java | 19 +- .../twosigma/beakerx/evaluator/Evaluator.java | 3 + .../beakerx/evaluator/EvaluatorManager.java | 0 .../com/twosigma/beakerx/kernel/Kernel.java | 22 +- .../kernel/KernelConfigurationFile.java | 32 +- .../beakerx/kernel/KernelManager.java | 8 +- .../beakerx/kernel/MagicKernelManager.java | 275 +++++++++--------- .../kernel/comm/AutotranslationHandler.java | 49 ++++ .../kernel/comm/GetCodeCellsHandler.java | 4 +- .../beakerx/table/TableDisplayActions.java | 6 +- .../com/twosigma/beakerx/widget/Button.java | 16 +- .../java/com/twosigma/beakerx/KernelTest.java | 5 +- ...espaceClientShowProgressReportingTest.java | 3 +- .../twosigma/beakerx/NamespaceClientTest.java | 69 ++--- .../beakerx/evaluator/EvaluatorTest.java | 55 +++- .../clojure/evaluator/ClojureEvaluator.java | 19 +- .../beakerx/clojure/kernel/Clojure.java | 15 +- .../evaluator/ClojureEvaluatorTest.java | 7 +- .../kernel/ClojureAutocompleteTest.java | 3 +- .../clojure/kernel/ClojureKernelTest.java | 3 +- .../groovy/comm/GroovyCommOpenHandler.java | 28 +- .../groovy/evaluator/GroovyCodeRunner.java | 4 +- .../groovy/evaluator/GroovyEvaluator.java | 19 +- .../groovy/evaluator/GroovyWorkerThread.java | 11 - .../beakerx/groovy/kernel/Groovy.java | 21 +- .../beakerx/groovy/TestGroovyEvaluator.java | 53 +++- .../GroovyCompleteHandlerTest.java | 3 +- .../GroovyEvaluatorProgressReportingTest.java | 8 +- .../groovy/evaluator/GroovyEvaluatorTest.java | 7 +- .../javash/evaluator/JavaEvaluator.java | 18 +- .../javash/evaluator/JavaWorkerThread.java | 9 - .../twosigma/beakerx/javash/kernel/Java.java | 10 +- ...orAutocompleteClassNameExpressionTest.java | 3 +- .../JavaEvaluatorAutocompleteImportTest.java | 3 +- .../JavaEvaluatorAutocompleteTest.java | 3 +- .../evaluator/JavaBaseEvaluatorTest.java | 7 +- .../javash/evaluator/JavaEvaluatorTest.java | 3 +- .../beakerx/javash/kernel/JavaKernelTest.java | 3 +- .../kotlin/evaluator/KotlinEvaluator.java | 9 +- .../kotlin/evaluator/KotlinWorkerThread.java | 16 - .../beakerx/kotlin/kernel/Kotlin.java | 10 +- .../evaluator/KotlinBaseEvaluatorTest.java | 7 +- .../kotlin/evaluator/KotlinEvaluatorTest.java | 3 +- .../kotlin/kernel/KotlinKernelTest.java | 3 +- .../scala/evaluator/BeakerxObjectFactory.java | 2 +- .../evaluator/BeakerxObjectFactoryImpl.java | 19 +- .../scala/evaluator/ScalaEvaluator.java | 52 +--- .../scala/evaluator/ScalaWorkerThread.java | 15 - .../twosigma/beakerx/scala/kernel/Scala.java | 23 +- .../kernel/ScalaBeakerXJsonSerializer.java | 81 ++++++ .../beakerx/scala/TestScalaEvaluator.java | 5 +- .../evaluator/NoBeakerxObjectTestFactory.java | 2 +- .../evaluator/ScalaAutocompleteTest.java | 6 +- .../beakerx/scala/kernel/ScalaKernelTest.java | 13 +- .../beakerx/EnableSparkSupportTest.java | 16 +- .../twosigma/beakerx/sql/BeakerParser.java | 8 +- .../twosigma/beakerx/sql/QueryExecutor.java | 6 +- .../autocomplete/db/DbExplorerFactory.java | 5 +- .../beakerx/sql/evaluator/SQLCodeRunner.java | 7 +- .../beakerx/sql/evaluator/SQLEvaluator.java | 13 +- .../sql/evaluator/SQLWorkerThread.java | 10 +- .../com/twosigma/beakerx/sql/kernel/SQL.java | 11 +- .../beakerx/sql/SQLAutocompleteTest.java | 5 +- .../beakerx/sql/SQLBaseEvaluatorTest.java | 7 +- .../beakerx/sql/SQLEvaluatorTest.java | 3 +- .../twosigma/beakerx/sql/SQLKernelTest.java | 3 +- .../kernel/SQLKernelWIthDefaultEnvsTest.java | 3 +- .../SQLKernelWithoutDefaultJDBCUriTest.java | 3 +- .../AutotranslationBetweenKernels.ipynb | 99 +++++++ ...utotranslation_scala_structured_data.ipynb | 229 +++++++++++++++ 85 files changed, 1607 insertions(+), 573 deletions(-) create mode 100644 beakerx/beakerx/beakerx_autotranslation_server.py create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationService.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationServiceImpl.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClient.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClientManager.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/BeakerXJsonSerializer.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/DefaultBeakerXJsonSerializer.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/evaluator/EvaluatorManager.java create mode 100644 kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/AutotranslationHandler.java create mode 100644 kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/ScalaBeakerXJsonSerializer.java create mode 100644 test/ipynb/groovy/AutotranslationBetweenKernels.ipynb create mode 100644 test/ipynb/scala/autotranslation_scala_structured_data.ipynb diff --git a/beakerx/beakerx/__init__.py b/beakerx/beakerx/__init__.py index bc4fc71795..54a7eea2fa 100644 --- a/beakerx/beakerx/__init__.py +++ b/beakerx/beakerx/__init__.py @@ -41,7 +41,6 @@ def _jupyter_server_extension_paths(): return [dict(module="beakerx")] beakerx = BeakerX() - def run(): try: parse() diff --git a/beakerx/beakerx/beakerx_autotranslation_server.py b/beakerx/beakerx/beakerx_autotranslation_server.py new file mode 100644 index 0000000000..aab8a794b0 --- /dev/null +++ b/beakerx/beakerx/beakerx_autotranslation_server.py @@ -0,0 +1,109 @@ +# Copyright 2018 TWO SIGMA OPEN SOURCE, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tornado.ioloop +import tornado.web +import logging +import os +import random +import string +import socket + +import base64 + +beakerx = {} + +logging.getLogger('tornado.access').disabled = True + + +def basic_auth(f): + def auth(username, password): + return username == "beakerx" and password == os.environ["BEAKERX_AUTOTRANSLATION_PASSWORD"] + + def _request_auth(handler): + handler.set_status(401) + return handler.finish() + + def wrap(*args): + handler = args[0] + try: + auth_header = handler.request.headers.get('Authorization') + if (auth_header is None) or (not auth_header.startswith('Basic ')): + return _request_auth(handler) + auth_decoded = base64.b64decode(auth_header[6:]) + username, password = auth_decoded.decode('UTF-8').split(':', 2) + if auth(username, password): + f(*args) + else: + _request_auth(handler) + except: + _request_auth(handler) + + return wrap + + +class MainSaveHandler(tornado.web.RequestHandler): + + @basic_auth + def post(self): + input_json = tornado.escape.json_decode(self.request.body) + session_id = input_json["sessionId"] + name = input_json["name"] + json = input_json["json"] + if session_id not in beakerx: + beakerx[session_id] = {} + + beakerx[session_id][name] = json + return self.finish("ok") + + +class MainGetHandler(tornado.web.RequestHandler): + + @basic_auth + def get(self, session_id, name): + if session_id in beakerx and name in beakerx[session_id]: + return self.finish(beakerx[session_id][name]) + return self.finish("undefined") + + +def make_app(): + return tornado.web.Application([ + (r"/autotransltion/(.*)/(.*)", MainGetHandler), + (r"/autotransltion/", MainSaveHandler), + ]) + + +def get_free_tcp_port(): + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(('', 0)) + addr, port = tcp.getsockname() + tcp.close() + return port + + +def random_string_generator(size=128): + s = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in + range(size)) + return s + + +def init_env(): + os.environ["BEAKERX_AUTOTRANSLATION_PASSWORD"] = random_string_generator() + os.environ["BEAKERX_AUTOTRANSLATION_PORT"] = str(get_free_tcp_port()) + + +def start_autotranslation_server(): + init_env() + app = make_app() + app.listen(os.environ["BEAKERX_AUTOTRANSLATION_PORT"]) diff --git a/beakerx/beakerx/commands.py b/beakerx/beakerx/commands.py index e14dbf37ca..dc3f888af0 100644 --- a/beakerx/beakerx/commands.py +++ b/beakerx/beakerx/commands.py @@ -58,10 +58,11 @@ def py4j_server_subparser(subparser): py4j_server_parser.add_argument("--port") py4j_server_parser.add_argument("--pyport") py4j_server_parser.add_argument("--kernel") + py4j_server_parser.add_argument("--context") def start_py4j_server(args): - Py4JServer(args.port, args.pyport, args.kernel) + Py4JServer(args.port, args.pyport, args.kernel, args.context) def run_jupyter(jupyter_commands): @@ -69,7 +70,6 @@ def run_jupyter(jupyter_commands): def init_parser(): - parser = argparse.ArgumentParser() parser.add_argument('--version', action='version', version=beakerx.__version__) parser.set_defaults(func=run_jupyter) @@ -91,4 +91,3 @@ def parse(): args.func(args) else: parser.parse_args(jupyter_commands) - diff --git a/beakerx/beakerx/handlers.py b/beakerx/beakerx/handlers.py index 92b3e8d9e0..19e43c80e9 100644 --- a/beakerx/beakerx/handlers.py +++ b/beakerx/beakerx/handlers.py @@ -21,7 +21,7 @@ import beakerx import tornado import os - +from .beakerx_autotranslation_server import start_autotranslation_server class SparkMetricsExecutorsHandler(APIHandler): def data_received(self, chunk): @@ -99,6 +99,8 @@ def get(self, path): def load_jupyter_server_extension(nbapp): + start_autotranslation_server() + web_app = nbapp.web_app host_pattern = '.*$' settings_route_pattern = url_path_join(web_app.settings['base_url'], '/beakerx', '/settings') diff --git a/beakerx/beakerx_magics/jvm_kernel_magic.py b/beakerx/beakerx_magics/jvm_kernel_magic.py index c08d4cf101..6a01f6f238 100644 --- a/beakerx/beakerx_magics/jvm_kernel_magic.py +++ b/beakerx/beakerx_magics/jvm_kernel_magic.py @@ -21,17 +21,18 @@ class JVMKernelMagic: - def __init__(self, kernel_name): + def __init__(self, kernel_name, context): self.km = None self.kc = None self.comms = [] self.kernel_name = kernel_name + self.context = context self.start() def start(self): self.km = KernelManager() self.km.kernel_name = self.kernel_name - self.km.start_kernel() + self.km.start_kernel(extra_arguments=[self.context]) self.kc = self.km.client() self.kc.start_channels() self.kc.wait_for_ready() @@ -66,8 +67,8 @@ def pass_msg(self, msg_raw): class PythonEntryPoint(object): - def __init__(self, kernel_name): - self.pm = JVMKernelMagic(kernel_name) + def __init__(self, kernel_name, context): + self.pm = JVMKernelMagic(kernel_name, context) def evaluate(self, code): print('code for evaluate {}'.format(code)) @@ -95,9 +96,9 @@ class Java: class Py4JServer: - def __init__(self, port, pyport, kernel_name): + def __init__(self, port, pyport, kernel_name, context): try: - pep = PythonEntryPoint(kernel_name) + pep = PythonEntryPoint(kernel_name, context) except NoSuchKernel: sys.exit(2) ClientServer( @@ -108,4 +109,4 @@ def __init__(self, port, pyport, kernel_name): if __name__ == '__main__': - Py4JServer(sys.argv[1], sys.argv[2], sys.argv[3]) + Py4JServer(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) diff --git a/js/notebook/src/extension.js b/js/notebook/src/extension.js index ac24641308..9d7992d20e 100644 --- a/js/notebook/src/extension.js +++ b/js/notebook/src/extension.js @@ -158,17 +158,35 @@ define([ // assign Beaker methods to window if (window) { - if (!window.beakerx) { - window.beakerx = {}; - } - var plotApiList = plotApi.list(); var bkApp = bkCoreManager.getBkApp(); var bkObject = bkApp.getBeakerObject(); - _.extend(window.beakerx, plotApiList); - _.extend(window.beakerx, htmlOutput); - window.beakerx.prefs = bkObject.beakerObj.prefs; + var beakerxInstance = {} + _.extend(beakerxInstance, plotApiList); + _.extend(beakerxInstance, htmlOutput); + beakerxInstance.prefs = bkObject.beakerObj.prefs; + + if (!window.beakerx) { + var handler = { + get: function (obj, prop) { + return prop in obj ? obj[prop] : undefined; + }, + + set: function (obj, prop, value) { + obj[prop] = value; + var comm = Jupyter.notebook.kernel.comm_manager.new_comm('beaker.autotranslation', null, null, null, utils.uuid()); + var data = { + }; + data.name = prop; + data.value = value; + comm.send(data); + comm.close(); + return true; + } + }; + window.beakerx = new Proxy(beakerxInstance, handler); + } } if (inNotebook) { diff --git a/kernel/base/build.gradle b/kernel/base/build.gradle index c6bb2b703e..9b7c64c2c5 100644 --- a/kernel/base/build.gradle +++ b/kernel/base/build.gradle @@ -40,6 +40,7 @@ dependencies { compile group: 'org.apache.commons', name: 'commons-text', version: '1.1' compile group: 'org.apache.maven.shared', name: 'maven-invoker', version: '3.0.0' compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.25' + compile group: 'org.apache.httpcomponents', name: 'fluent-hc', version: '4.5.5' compile group: "org.zeromq", name: "jeromq", version: "0.3.5" compile group: 'org.apache.ivy', name: 'ivy', version: '2.4.0' diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationService.java b/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationService.java new file mode 100644 index 0000000000..162319de98 --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationService.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx; + +public interface AutotranslationService { + + String update(String name, String json); + + String get(String name); + + String close(); + + String getContextAsString(); +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationServiceImpl.java b/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationServiceImpl.java new file mode 100644 index 0000000000..31a94bdee6 --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/AutotranslationServiceImpl.java @@ -0,0 +1,144 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import com.google.gson.Gson; +import org.apache.http.client.fluent.Request; +import org.apache.commons.codec.binary.Base64; +import org.apache.http.entity.ContentType; + +public class AutotranslationServiceImpl implements AutotranslationService { + + public static final String AUTHORIZATION = "Authorization"; + public static final String LOCALHOST = "http://localhost:"; + public static final String AUTOTRANSLTION = "/autotransltion/"; + public static final String BEAKERX = "beakerx"; + private final String contextAsString; + private final AutotranslationContext context; + + public static AutotranslationService createAsSubkernel(String configuration) { + return new AutotranslationServiceImpl(configuration); + } + + public static AutotranslationService createAsMainKernel(String id) { + Map context = new HashMap<>(); + context.put("contextId", id); + context.put("port", autotranslationPort()); + Gson gson = new Gson(); + String contextAsString = gson.toJson(context); + return new AutotranslationServiceImpl(contextAsString); + } + + private AutotranslationServiceImpl(String configuration) { + Gson gson = new Gson(); + Map map = gson.fromJson(configuration, Map.class); + String c = (String) map.get("contextId"); + String port = (String) map.get("port"); + this.context = new AutotranslationContext(c, port); + this.contextAsString = configuration; + } + + public String getContextAsString() { + return contextAsString; + } + + @Override + public String update(String name, String json) { + try { + String reply = Request.Post(LOCALHOST + this.context.getPort() + AUTOTRANSLTION) + .addHeader(AUTHORIZATION, auth()) + .bodyString(createBody(name, json), ContentType.APPLICATION_JSON) + .execute().returnContent().asString(); + if (!reply.equals("ok")) { + throw new RuntimeException(reply); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return json; + } + + @Override + public String get(String name) { + String valueString = ""; + try { + valueString = Request + .Get(LOCALHOST + this.context.getPort() + AUTOTRANSLTION + this.context.getContextId() + "/" + name) + .addHeader(AUTHORIZATION, auth()) + .execute() + .returnContent() + .asString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return valueString; + } + + private String createBody(String name, String json) { + Map context = new HashMap<>(); + context.put("name", name); + context.put("json", json); + context.put("sessionId", this.context.getContextId()); + Gson gson = new Gson(); + return gson.toJson(context); + } + + private String auth() { + String authString = getBasic_auth_username() + ":" + getBasic_auth_password(); + return "Basic " + Base64.encodeBase64String(authString.getBytes(StandardCharsets.UTF_8)); + } + + private static String getBasic_auth_username() { + return BEAKERX; + } + + private static String getBasic_auth_password() { + return System.getenv("BEAKERX_AUTOTRANSLATION_PASSWORD"); + } + + private static String autotranslationPort() { + return System.getenv("BEAKERX_AUTOTRANSLATION_PORT"); + } + + @Override + public String close() { + return "ok"; + } + + private static class AutotranslationContext { + private String contextId; + private String port; + + AutotranslationContext(String contextId, String port) { + this.contextId = contextId; + this.port = port; + } + + public String getContextId() { + return contextId; + } + + public String getPort() { + return port; + } + } + +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClient.java b/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClient.java new file mode 100644 index 0000000000..9a94dbda6b --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClient.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx; + +import java.util.List; +import java.util.concurrent.SynchronousQueue; + +public interface BeakerXClient { + + void showProgressUpdate(String message, int progress); + + void delBeaker(); + + String update(String name, Object value); + + Object set(String name, Object value); + + Object get(final String name); + + SynchronousQueue getMessageQueue(String channel); + + List getCodeCells(String tagFilter); + + void runByTag(String tag); + + String getContext(); +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClientManager.java b/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClientManager.java new file mode 100644 index 0000000000..5aa196fc6a --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXClientManager.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx; + + +public class BeakerXClientManager { + + public static String BEAKER_X_CLIENT_MANAGER_PATH = BeakerXClientManager.class.getName(); + public static String BEAKER_X_CLIENT_MANAGER_GET = BeakerXClientManager.class.getSimpleName() + ".get()"; + + private static BeakerXClient beakerXClientInst; + + public static BeakerXClient register(BeakerXClient beakerXClient) { + beakerXClientInst = beakerXClient; + return beakerXClientInst = beakerXClient; + } + + public static BeakerXClient get() { + return beakerXClientInst; + } + +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXJsonSerializer.java b/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXJsonSerializer.java new file mode 100644 index 0000000000..b445303ac4 --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/BeakerXJsonSerializer.java @@ -0,0 +1,22 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx; + +public interface BeakerXJsonSerializer { + + String toJson(Object value); + +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/DefaultBeakerXJsonSerializer.java b/kernel/base/src/main/java/com/twosigma/beakerx/DefaultBeakerXJsonSerializer.java new file mode 100644 index 0000000000..f92b46cd19 --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/DefaultBeakerXJsonSerializer.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.twosigma.beakerx.jvm.serialization.BasicObjectSerializer; +import com.twosigma.beakerx.table.TableDisplayToJson; + +import java.io.IOException; +import java.io.StringWriter; + +import static com.fasterxml.jackson.databind.SerializationFeature.WRITE_ENUMS_USING_TO_STRING; + +public class DefaultBeakerXJsonSerializer implements BeakerXJsonSerializer { + + private final BasicObjectSerializer objectSerializer; + private ObjectMapper objectMapper; + + public DefaultBeakerXJsonSerializer() { + this.objectMapper = new ObjectMapper(); + objectMapper.enable(WRITE_ENUMS_USING_TO_STRING); + objectMapper.registerModule(TableDisplayToJson.tableDisplayModule()); + objectSerializer = new BasicObjectSerializer(); + } + + @Override + public String toJson(Object value) { + try { + StringWriter sw = new StringWriter(); + JsonGenerator jgen = objectMapper.getFactory().createGenerator(sw); + objectSerializer.writeObject(value, jgen, true); + jgen.flush(); + sw.flush(); + return sw.toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/NamespaceClient.java b/kernel/base/src/main/java/com/twosigma/beakerx/NamespaceClient.java index b1cb3ec399..1c51c2342b 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/NamespaceClient.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/NamespaceClient.java @@ -15,102 +15,82 @@ */ package com.twosigma.beakerx; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.twosigma.beakerx.evaluator.InternalVariable; +import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; +import com.twosigma.beakerx.kernel.KernelConfigurationFile; import com.twosigma.beakerx.kernel.comm.Comm; import com.twosigma.beakerx.kernel.comm.TargetNamesEnum; -import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; -import com.twosigma.beakerx.jvm.serialization.BasicObjectSerializer; -import com.twosigma.beakerx.jvm.serialization.BeakerObjectConverter; -import com.twosigma.beakerx.table.TableDisplayToJson; -import java.io.IOException; import java.io.Serializable; -import java.io.StringWriter; import java.util.HashMap; -import java.util.Map; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Map; import java.util.concurrent.SynchronousQueue; -import static com.fasterxml.jackson.databind.SerializationFeature.WRITE_ENUMS_USING_TO_STRING; import static com.twosigma.beakerx.kernel.msg.JupyterMessages.COMM_MSG; -public class NamespaceClient { +public class NamespaceClient implements BeakerXClient { - private static Map nsClients = new ConcurrentHashMap<>(); - private static String currentSession; private static Map> messagePool = new HashMap<>(); - private ObjectMapper objectMapper; - private BeakerObjectConverter objectSerializer; - private SimpleEvaluationObject currentCeo = null; private Comm autotranslationComm = null; private Comm codeCellsComm = null; private Comm tagRunComm = null; + private AutotranslationService autotranslationService; + private BeakerXJsonSerializer beakerXJsonSerializer; - public NamespaceClient() { - SimpleModule module = TableDisplayToJson.tableDisplayModule(); - objectMapper = new ObjectMapper(); - objectMapper.enable(WRITE_ENUMS_USING_TO_STRING); - objectMapper.registerModule(module); - objectSerializer = new BasicObjectSerializer(); + public NamespaceClient(AutotranslationService autotranslationService, BeakerXJsonSerializer beakerXJsonSerializer) { + this.autotranslationService = autotranslationService; + this.beakerXJsonSerializer = beakerXJsonSerializer; } + @Override public synchronized void showProgressUpdate(String message, int progress) { SimpleEvaluationObject seo = InternalVariable.getSimpleEvaluationObject(); seo.structuredUpdate(message, progress); } - public SimpleEvaluationObject getOutputObj() { - return currentCeo; + @Override + public synchronized void delBeaker() { + autotranslationService.close(); } - public synchronized void setOutputObj(SimpleEvaluationObject input) { - currentCeo = input; + @Override + public synchronized Object get(final String name) { + return autotranslationService.get(name); } - public synchronized static NamespaceClient getBeaker() { - if (currentSession != null) { - return nsClients.get(currentSession); + @Override + public synchronized String update(String name, Object value) { + try { + String json = getJson(value); + autotranslationService.update(name, json); + return json; + } catch (Exception e) { + throw new RuntimeException(e); } - return null; } - public synchronized static NamespaceClient getBeaker(String session) { - currentSession = session; - if (!nsClients.containsKey(session)) { - nsClients.put(session, new NamespaceClient()); + @Override + public synchronized Object set(String name, Object value) { + String json = update(name, value); + try { + Comm c = getAutotranslationComm(); + HashMap data = new HashMap<>(); + HashMap state = new HashMap<>(); + state.put("name", name); + state.put("value", json); + state.put("sync", true); + data.put("state", state); + data.put("buffer_paths", new HashMap<>()); + c.send(COMM_MSG, Comm.Buffer.EMPTY, new Comm.Data(data)); + return value; + } catch (Exception e) { + throw new RuntimeException(e); } - return nsClients.get(currentSession); - } - - public synchronized static void delBeaker(String sessionId) { - nsClients.remove(sessionId); - currentSession = null; - } - - public synchronized Object set(String name, Object value) throws IOException { - Comm c = getAutotranslationComm(); - HashMap data = new HashMap<>(); - HashMap state = new HashMap<>(); - state.put("name", name); - state.put("value", getJson(value)); - state.put("sync", true); - data.put("state", state); - data.put("buffer_paths", new HashMap<>()); - c.send(COMM_MSG, Comm.Buffer.EMPTY, new Comm.Data(data)); - return value; } - protected String getJson(Object value) throws IOException { - StringWriter sw = new StringWriter(); - JsonGenerator jgen = objectMapper.getFactory().createGenerator(sw); - objectSerializer.writeObject(value, jgen, true); - jgen.flush(); - sw.flush(); - return sw.toString(); + private String getJson(Object value) { + return beakerXJsonSerializer.toJson(value); } //TODO : Not Implemented @@ -123,12 +103,8 @@ public Object unset(String name) { throw new RuntimeException("This option is not implemented now"); } - //TODO : Not Implemented - public synchronized Object get(final String name) { - throw new RuntimeException("This option is not implemented now"); - } - - public static SynchronousQueue getMessageQueue(String channel) { + @Override + public SynchronousQueue getMessageQueue(String channel) { SynchronousQueue result = messagePool.get(channel); if (result == null) { result = new SynchronousQueue(); @@ -137,7 +113,7 @@ public static SynchronousQueue getMessageQueue(String channel) { return result; } - protected Comm getAutotranslationComm() { + private Comm getAutotranslationComm() { if (autotranslationComm == null) { autotranslationComm = new Comm(TargetNamesEnum.BEAKER_AUTOTRANSLATION); autotranslationComm.open(); @@ -145,7 +121,7 @@ protected Comm getAutotranslationComm() { return autotranslationComm; } - protected Comm getCodeCellsComm() { + private Comm getCodeCellsComm() { if (codeCellsComm == null) { codeCellsComm = new Comm(TargetNamesEnum.BEAKER_GETCODECELLS); codeCellsComm.open(); @@ -153,7 +129,7 @@ protected Comm getCodeCellsComm() { return codeCellsComm; } - protected Comm getTagRunComm() { + private Comm getTagRunComm() { if (tagRunComm == null) { tagRunComm = new Comm(TargetNamesEnum.BEAKER_TAG_RUN); tagRunComm.open(); @@ -161,22 +137,27 @@ protected Comm getTagRunComm() { return tagRunComm; } - - public List getCodeCells(String tagFilter) throws IOException, InterruptedException { + @Override + public List getCodeCells(String tagFilter) { // first send message to get cells - Comm c = getCodeCellsComm(); - HashMap data = new HashMap<>(); - HashMap state = new HashMap<>(); - state.put("name", "CodeCells"); - state.put("value", getJson(tagFilter)); - data.put("state", state); - data.put("buffer_paths", new HashMap<>()); - c.send(COMM_MSG, Comm.Buffer.EMPTY, new Comm.Data(data)); - // block - Object cells = getMessageQueue("CodeCells").take(); - return (List) cells; + try { + Comm c = getCodeCellsComm(); + HashMap data = new HashMap<>(); + HashMap state = new HashMap<>(); + state.put("name", "CodeCells"); + state.put("value", getJson(tagFilter)); + data.put("state", state); + data.put("buffer_paths", new HashMap<>()); + c.send(COMM_MSG, Comm.Buffer.EMPTY, new Comm.Data(data)); + // block + Object cells = getMessageQueue("CodeCells").take(); + return (List) cells; + } catch (Exception e) { + throw new RuntimeException(e); + } } + @Override public synchronized void runByTag(String tag) { Comm c = getTagRunComm(); HashMap data = new HashMap<>(); @@ -187,4 +168,20 @@ public synchronized void runByTag(String tag) { c.send(COMM_MSG, Comm.Buffer.EMPTY, new Comm.Data(data)); } + @Override + public String getContext() { + return this.autotranslationService.getContextAsString(); + } + + public static NamespaceClient create(String id, KernelConfigurationFile configurationFile) { + return create(id, configurationFile, new DefaultBeakerXJsonSerializer()); + } + + public static NamespaceClient create(String id, KernelConfigurationFile configurationFile, BeakerXJsonSerializer serializer) { + if (configurationFile.getContext().isPresent()) { + return new NamespaceClient(AutotranslationServiceImpl.createAsSubkernel(configurationFile.getContext().get()), serializer); + } else { + return new NamespaceClient(AutotranslationServiceImpl.createAsMainKernel(id), serializer); + } + } } \ No newline at end of file diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/chart/ChartDetails.java b/kernel/base/src/main/java/com/twosigma/beakerx/chart/ChartDetails.java index 0362dbed1c..92a88dcf89 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/chart/ChartDetails.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/chart/ChartDetails.java @@ -16,10 +16,10 @@ package com.twosigma.beakerx.chart; +import com.twosigma.beakerx.BeakerXClientManager; import com.twosigma.beakerx.widget.CommActions; import org.apache.commons.lang3.StringUtils; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.chart.actions.CategoryGraphicsActionObject; import com.twosigma.beakerx.chart.actions.CombinedPlotActionObject; import com.twosigma.beakerx.chart.actions.GraphicsActionObject; @@ -90,9 +90,9 @@ protected void onActionDetails(HashMap content, Message message) { info.setGraphics(g); updateDetails(info); if (CommActions.ONCLICK.equals(info.getActionType())) { - NamespaceClient.getBeaker().runByTag(info.getTag()); + BeakerXClientManager.get().runByTag(info.getTag()); } else if (CommActions.ONKEY.equals(info.getActionType())) { - NamespaceClient.getBeaker().runByTag(info.getTag()); + BeakerXClientManager.get().runByTag(info.getTag()); } } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/BaseEvaluator.java b/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/BaseEvaluator.java index 4fef0b8d43..17f90d2f2e 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/BaseEvaluator.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/BaseEvaluator.java @@ -15,10 +15,11 @@ */ package com.twosigma.beakerx.evaluator; +import com.twosigma.beakerx.BeakerXClient; +import com.twosigma.beakerx.BeakerXClientManager; import com.twosigma.beakerx.DefaultJVMVariables; import com.twosigma.beakerx.inspect.Inspect; import com.twosigma.beakerx.inspect.InspectResult; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; import com.twosigma.beakerx.jvm.threads.CellExecutor; @@ -54,16 +55,23 @@ public abstract class BaseEvaluator implements Evaluator { protected Imports imports; private final CellExecutor executor; private Path tempFolder; + private BeakerXClient beakerXClient; protected EvaluatorParameters evaluatorParameters; private EvaluatorHooks cancelHooks = new EvaluatorHooks(); protected ExecutorService executorService; - public BaseEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters) { + public BaseEvaluator(String id, + String sId, + CellExecutor cellExecutor, + TempFolderFactory tempFolderFactory, + EvaluatorParameters evaluatorParameters, + BeakerXClient beakerXClient) { shellId = id; sessionId = sId; executor = cellExecutor; tempFolder = tempFolderFactory.createTempFolder(); + this.beakerXClient = BeakerXClientManager.register(beakerXClient); outDir = getOrCreateFile(tempFolder.toString() + File.separator + "outDir").getPath(); classPath = new Classpath(); classPath.add(new PathToJar(outDir)); @@ -100,6 +108,11 @@ public ClassLoader getClassLoaderForImport() { return getClassLoader(); } + @Override + public BeakerXClient getBeakerX() { + return beakerXClient; + } + @Override public List addJarsToClasspath(List paths) { LinkedList addedPaths = new LinkedList<>(); @@ -227,7 +240,7 @@ public Path getTempFolder() { @Override public void exit() { - NamespaceClient.delBeaker(getSessionId()); + beakerXClient.delBeaker(); removeTempFolder(); } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/Evaluator.java b/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/Evaluator.java index 6b36ab1d12..95787bffba 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/Evaluator.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/Evaluator.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.evaluator; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.inspect.InspectResult; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.kernel.AddImportStatus; @@ -74,4 +75,6 @@ public interface Evaluator { String getOutDir(); void registerCancelHook(Hook hook); + + BeakerXClient getBeakerX(); } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/EvaluatorManager.java b/kernel/base/src/main/java/com/twosigma/beakerx/evaluator/EvaluatorManager.java new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/Kernel.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/Kernel.java index 8e69ff36a8..5f82c92c5d 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/Kernel.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/Kernel.java @@ -73,12 +73,18 @@ public Kernel(final String sessionId, final Evaluator evaluator, this( sessionId, evaluator, - kernelSocketsFactory, () -> System.exit(0), - new EnvCacheFolderFactory(), customMagicCommands); - } - - protected Kernel(final String sessionId, final Evaluator evaluator, final KernelSocketsFactory kernelSocketsFactory, - CloseKernelAction closeKernelAction, CacheFolderFactory cacheFolderFactory, CustomMagicCommandsFactory customMagicCommands) { + kernelSocketsFactory, + () -> System.exit(0), + new EnvCacheFolderFactory(), + customMagicCommands); + } + + protected Kernel(final String sessionId, + final Evaluator evaluator, + final KernelSocketsFactory kernelSocketsFactory, + CloseKernelAction closeKernelAction, + CacheFolderFactory cacheFolderFactory, + CustomMagicCommandsFactory customMagicCommands) { this.sessionId = sessionId; this.cacheFolderFactory = cacheFolderFactory; this.kernelSocketsFactory = kernelSocketsFactory; @@ -303,7 +309,7 @@ public void registerCancelHook(Hook hook) { public PythonEntryPoint getPythonEntryPoint(String kernelName) throws NoSuchKernelException { MagicKernelManager manager = magicKernels.get(kernelName); if (manager == null) { - manager = new MagicKernelManager(kernelName); + manager = new MagicKernelManager(kernelName, evaluator.getBeakerX().getContext()); magicKernels.put(kernelName, manager); } return manager.getPythonEntryPoint(); @@ -316,7 +322,7 @@ public MagicKernelManager getManagerByCommId(String commId) { } @Override - public void addCommIdManagerMapping(String commId, String kernel) { + public void addCommIdManagerMapping(String commId, String kernel){ commKernelMapping.put(commId, kernel); } } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelConfigurationFile.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelConfigurationFile.java index d8609b7a1a..8fee3103fb 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelConfigurationFile.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelConfigurationFile.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.kernel; +import com.google.gson.Gson; import com.twosigma.beakerx.message.MessageSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,15 +24,29 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.Map; +import java.util.Optional; public class KernelConfigurationFile implements ConfigurationFile { private static final Logger logger = LoggerFactory.getLogger(KernelConfigurationFile.class); private File config; private Config configuration; + private Optional context = Optional.empty(); public KernelConfigurationFile(final String[] args) { - this.config = getConfig(args); + if (args.length > 2) { + logger.error("Invalid parameters passed to the Kernel. Expected one or two parameter, found " + args.length); + for (String string : args) { + logger.error(string); + } + System.exit(1); + } + this.config = getConfig(args[0]); + if (args.length == 2) { + String contextAsJson = args[1]; + this.context = Optional.of(contextAsJson); + } } @Override @@ -54,16 +69,8 @@ private byte[] readConfig() { return bytes; } - private File getConfig(final String[] args) { - if (args.length != 1) { - logger.error("Invalid parameters passed to the Kernel. Expected one parameter, found " + args.length); - for (String string : args) { - logger.error(string); - } - System.exit(1); - } - - File config = new File(args[0]); + private File getConfig(final String path) { + File config = new File(path); if (!config.exists()) { logger.error("Kernel configuration not found."); System.exit(1); @@ -71,4 +78,7 @@ private File getConfig(final String[] args) { return config; } + public Optional getContext() { + return context; + } } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelManager.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelManager.java index c289381d51..bc06214516 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelManager.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/KernelManager.java @@ -18,14 +18,14 @@ public class KernelManager { - private static KernelFunctionality groovyKernelInst; + private static KernelFunctionality kernelInst; - public static void register(KernelFunctionality groovyKernel) { - groovyKernelInst = groovyKernel; + public static void register(KernelFunctionality kernelFunctionality) { + kernelInst = kernelFunctionality; } public static KernelFunctionality get() { - return groovyKernelInst; + return kernelInst; } } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/MagicKernelManager.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/MagicKernelManager.java index 668d984d76..812615a496 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/MagicKernelManager.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/MagicKernelManager.java @@ -34,158 +34,161 @@ public class MagicKernelManager { - ClientServer clientServer = null; - private PythonEntryPoint pep = null; - private Process pythonProcess = null; - - private static String DEFAULT_PORT = "25333"; - private static String DEFAULT_PYTHON_PORT = "25334"; - private static int NO_SUCH_KERNEL_CODE = 2; - private static String PY4J_INIT_MESSAGE = "Py4j server is running"; - - private Integer port = null; - private Integer pythonPort = null; - - private final String kernelName; - - public MagicKernelManager(String kernelName) { - this.kernelName = kernelName; + ClientServer clientServer = null; + private PythonEntryPoint pep = null; + private Process pythonProcess = null; + + private static String DEFAULT_PORT = "25333"; + private static String DEFAULT_PYTHON_PORT = "25334"; + private static int NO_SUCH_KERNEL_CODE = 2; + private static String PY4J_INIT_MESSAGE = "Py4j server is running"; + + private Integer port = null; + private Integer pythonPort = null; + + private final String kernelName; + private String context; + + public MagicKernelManager(String kernelName, String context) { + this.kernelName = kernelName; + this.context = context; + } + + private void initPythonProcess() throws NoSuchKernelException { + //cleanup communication resources if already in use + exit(); + + port = findFreePort(); + pythonPort = findFreePort(); + + try { + ProcessBuilder pb = new ProcessBuilder(getPy4jCommand()); + pb.redirectError(ProcessBuilder.Redirect.INHERIT); + pythonProcess = pb.start(); + BufferedReader br = new BufferedReader(new InputStreamReader(pythonProcess.getInputStream())); + while (!PY4J_INIT_MESSAGE.equals(br.readLine()) && pythonProcess.isAlive()) { + //wait for python process to initialize properly + } + if (!pythonProcess.isAlive() && pythonProcess.exitValue() == NO_SUCH_KERNEL_CODE) { + throw new NoSuchKernelException(kernelName); + } + } catch (IOException e) { + e.printStackTrace(); } - - private void initPythonProcess() throws NoSuchKernelException { - //cleanup communication resources if already in use - exit(); - - port = findFreePort(); - pythonPort = findFreePort(); - - try { - ProcessBuilder pb = new ProcessBuilder(getPy4jCommand()); - pb.redirectError(ProcessBuilder.Redirect.INHERIT); - pythonProcess = pb.start(); - BufferedReader br = new BufferedReader(new InputStreamReader(pythonProcess.getInputStream())); - while (!PY4J_INIT_MESSAGE.equals(br.readLine()) && pythonProcess.isAlive()){ - //wait for python process to initialize properly - } - if (!pythonProcess.isAlive() && pythonProcess.exitValue() == NO_SUCH_KERNEL_CODE) { - throw new NoSuchKernelException(kernelName); - } - } catch (IOException e) { - e.printStackTrace(); - } + } + + protected String[] getPy4jCommand() { + return new String[]{ + "beakerx", + "py4j_server", + "--port", port == null ? DEFAULT_PORT : String.valueOf(port), + "--pyport", pythonPort == null ? DEFAULT_PYTHON_PORT : String.valueOf(pythonPort), + "--kernel", kernelName, + "--context", this.context + }; + } + + public void exit() { + if (pep != null) { + pep.shutdownKernel(); + pep = null; } - - protected String[] getPy4jCommand() { - return new String[] { - "beakerx", - "py4j_server", - "--port", port == null ? DEFAULT_PORT : String.valueOf(port), - "--pyport", pythonPort == null ? DEFAULT_PYTHON_PORT : String.valueOf(pythonPort), - "--kernel", kernelName - }; + if (clientServer != null) { + clientServer.shutdown(); + clientServer = null; } - - public void exit() { - if (pep != null) { - pep.shutdownKernel(); - pep = null; - } - if (clientServer != null) { - clientServer.shutdown(); - clientServer = null; - } - if (pythonProcess != null) { - pythonProcess.destroy(); - pythonProcess = null; - } + if (pythonProcess != null) { + pythonProcess.destroy(); + pythonProcess = null; } + } - public PythonEntryPoint getPythonEntryPoint() throws NoSuchKernelException { - if (pythonProcess == null || !pythonProcess.isAlive() || clientServer == null) { - initPythonProcess(); - } - if (pep == null) { - this.pep = initPythonEntryPoint(); - } - return pep; + public PythonEntryPoint getPythonEntryPoint() throws NoSuchKernelException { + if (pythonProcess == null || !pythonProcess.isAlive() || clientServer == null) { + initPythonProcess(); } - - private PythonEntryPoint initPythonEntryPoint() { - if (this.clientServer == null) { - initClientServer(); - } - return (PythonEntryPoint) clientServer.getPythonServerEntryPoint(new Class[] {PythonEntryPoint.class}); + if (pep == null) { + this.pep = initPythonEntryPoint(); } + return pep; + } - private void initClientServer() { - this.clientServer = new ClientServer(port, GatewayServer.defaultAddress(), pythonPort, - GatewayServer.defaultAddress(), GatewayServer.DEFAULT_CONNECT_TIMEOUT, - GatewayServer.DEFAULT_READ_TIMEOUT, ServerSocketFactory.getDefault(), SocketFactory.getDefault(), null); + private PythonEntryPoint initPythonEntryPoint() { + if (this.clientServer == null) { + initClientServer(); } - - private List getIopubMessages() throws IOException { - List messages = new ArrayList<>(); - while (true) { - String iopubMsg = pep.getIopubMsg(); - if (iopubMsg.equals("null")) break; - messages.add(parseMessage(iopubMsg)); - } - return messages; + return (PythonEntryPoint) clientServer.getPythonServerEntryPoint(new Class[]{PythonEntryPoint.class}); + } + + private void initClientServer() { + this.clientServer = new ClientServer(port, GatewayServer.defaultAddress(), pythonPort, + GatewayServer.defaultAddress(), GatewayServer.DEFAULT_CONNECT_TIMEOUT, + GatewayServer.DEFAULT_READ_TIMEOUT, ServerSocketFactory.getDefault(), SocketFactory.getDefault(), null); + } + + private List getIopubMessages() throws IOException { + List messages = new ArrayList<>(); + while (true) { + String iopubMsg = pep.getIopubMsg(); + if (iopubMsg.equals("null")) break; + messages.add(parseMessage(iopubMsg)); } - - private Message parseMessage(String stringJson) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - JsonNode json = mapper.readTree(stringJson); - Message msg = new Message(mapper.convertValue(json.get("header"), Header.class)); - msg.setContent(mapper.convertValue(json.get("content"), Map.class)); - msg.setMetadata(mapper.convertValue(json.get("metadata"), Map.class)); - msg.setBuffers(mapper.convertValue(json.get("buffers"), List.class)); - List identities = mapper.convertValue(json.get("comm_id"), List.class); - msg.setIdentities(identities == null ? new ArrayList<>() : identities); - return msg; + return messages; + } + + private Message parseMessage(String stringJson) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode json = mapper.readTree(stringJson); + Message msg = new Message(mapper.convertValue(json.get("header"), Header.class)); + msg.setContent(mapper.convertValue(json.get("content"), Map.class)); + msg.setMetadata(mapper.convertValue(json.get("metadata"), Map.class)); + msg.setBuffers(mapper.convertValue(json.get("buffers"), List.class)); + List identities = mapper.convertValue(json.get("comm_id"), List.class); + msg.setIdentities(identities == null ? new ArrayList<>() : identities); + return msg; + } + + public List handleMsg(Message message) { + List messages = new ArrayList<>(); + try { + getPythonEntryPoint(); + } catch (NoSuchKernelException e) { + return messages; } + pep.sendMessage(new ObjectMapper().valueToTree(message).toString()); - public List handleMsg(Message message) { - List messages = new ArrayList<>(); - try { - getPythonEntryPoint(); - } catch (NoSuchKernelException e) { - return messages; - } - pep.sendMessage(new ObjectMapper().valueToTree(message).toString()); - - try { - messages = getIopubMessages(); - } catch (IOException e) { - e.printStackTrace(); - } - - for (Message msg : messages) { - msg.setParentHeader(message.getHeader()); - } - return messages; + try { + messages = getIopubMessages(); + } catch (IOException e) { + e.printStackTrace(); } - private Integer findFreePort() { - ServerSocket socket = null; + for (Message msg : messages) { + msg.setParentHeader(message.getHeader()); + } + return messages; + } + + public static Integer findFreePort() { + ServerSocket socket = null; + try { + socket = new ServerSocket(0); + socket.setReuseAddress(true); + int port = socket.getLocalPort(); + try { + socket.close(); + } catch (IOException e) { + } + return port; + } catch (IOException e) { + } finally { + if (socket != null) { try { - socket = new ServerSocket(0); - socket.setReuseAddress(true); - int port = socket.getLocalPort(); - try { - socket.close(); - } catch (IOException e) { - } - return port; + socket.close(); } catch (IOException e) { - } finally { - if (socket != null) { - try { - socket.close(); - } catch (IOException e) { - } - } } - return null; + } } + return null; + } } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/AutotranslationHandler.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/AutotranslationHandler.java new file mode 100644 index 0000000000..83abfa277d --- /dev/null +++ b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/AutotranslationHandler.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.kernel.comm; + +import com.twosigma.beakerx.BeakerXClientManager; +import com.twosigma.beakerx.kernel.KernelFunctionality; +import com.twosigma.beakerx.message.Message; + +import java.util.Map; + +import static com.twosigma.beakerx.handler.KernelHandlerWrapper.wrapBusyIdle; + +public class AutotranslationHandler extends BaseHandler { + + public AutotranslationHandler(KernelFunctionality kernel) { + super(kernel); + } + + @Override + public void handle(Message message) { + wrapBusyIdle(kernel, message, () -> { + handleMsg(message, kernel); + }); + } + + private void handleMsg(Message message, KernelFunctionality kernel) { + Map data = (Map) message.getContent().get("data"); + BeakerXClientManager.get().update((String) data.get("name"), data.get("value")); + } + + @Override + public String getHandlerCommand() { + return ""; + } + +} \ No newline at end of file diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/GetCodeCellsHandler.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/GetCodeCellsHandler.java index b7face9d11..32d3ae3fdc 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/GetCodeCellsHandler.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/comm/GetCodeCellsHandler.java @@ -17,8 +17,8 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; +import com.twosigma.beakerx.BeakerXClientManager; import com.twosigma.beakerx.CodeCell; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.jvm.serialization.BasicObjectSerializer; import com.twosigma.beakerx.jvm.serialization.BeakerObjectConverter; import com.twosigma.beakerx.kernel.KernelFunctionality; @@ -54,7 +54,7 @@ public void handle(Message message) { private void handleMsg(Message message) { try { List cells = getBeakerCodeCells(getValueFromData(message, getHandlerCommand())); - NamespaceClient.getMessageQueue("CodeCells").put(cells); + BeakerXClientManager.get().getMessageQueue("CodeCells").put(cells); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/table/TableDisplayActions.java b/kernel/base/src/main/java/com/twosigma/beakerx/table/TableDisplayActions.java index f271263a03..bdffa97e28 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/table/TableDisplayActions.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/table/TableDisplayActions.java @@ -15,7 +15,7 @@ */ package com.twosigma.beakerx.table; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClientManager; import com.twosigma.beakerx.message.Message; import com.twosigma.beakerx.table.action.TableActionDetails; import com.twosigma.beakerx.widget.CommActions; @@ -87,11 +87,11 @@ private void onActionDetails(HashMap content, Message message) { tableDisplay.setDetails(details); if (CommActions.CONTEXT_MENU_CLICK.equals(details.getActionType())) { if (tableDisplay.getContextMenuTags() != null && !tableDisplay.getContextMenuTags().isEmpty() && details.getContextMenuItem() != null && !details.getContextMenuItem().isEmpty()) { - NamespaceClient.getBeaker().runByTag(tableDisplay.getContextMenuTags().get(details.getContextMenuItem())); + BeakerXClientManager.get().runByTag(tableDisplay.getContextMenuTags().get(details.getContextMenuItem())); } } else if (CommActions.DOUBLE_CLICK.equals(details.getActionType())) { if (tableDisplay.getDoubleClickTag() != null && !tableDisplay.getDoubleClickTag().isEmpty()) { - NamespaceClient.getBeaker().runByTag(tableDisplay.getDoubleClickTag()); + BeakerXClientManager.get().runByTag(tableDisplay.getDoubleClickTag()); } } } diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/widget/Button.java b/kernel/base/src/main/java/com/twosigma/beakerx/widget/Button.java index 6e2f0018b7..d574617bdc 100644 --- a/kernel/base/src/main/java/com/twosigma/beakerx/widget/Button.java +++ b/kernel/base/src/main/java/com/twosigma/beakerx/widget/Button.java @@ -15,7 +15,7 @@ */ package com.twosigma.beakerx.widget; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClientManager; import com.twosigma.beakerx.handler.Handler; import com.twosigma.beakerx.message.Message; @@ -34,7 +34,7 @@ public class Button extends ValueWidget { public static final String ICON = "icon"; public static final String BUTTON_STYLE = "button_style"; - private String tooltip= EMPTY_STRING; + private String tooltip = EMPTY_STRING; private String tag; private String icon = EMPTY_STRING; private String button_style = EMPTY_STRING; @@ -59,7 +59,7 @@ protected HashMap content(HashMap co content.put(BUTTON_STYLE, button_style); return content; } - + public String getTooltip() { return tooltip; @@ -88,12 +88,12 @@ private void handleOnClick(Message message) { handleCommEventSync(message, CommActions.CLICK, this::onClick); } - public void onClick(HashMap content, Message message){ - if(actionPerformed != null){ + public void onClick(HashMap content, Message message) { + if (actionPerformed != null) { actionPerformed.executeAction(content, message); } - if(getTag() != null && !getTag().isEmpty()){ - NamespaceClient.getBeaker().runByTag(getTag()); + if (getTag() != null && !getTag().isEmpty()) { + BeakerXClientManager.get().runByTag(getTag()); } } @@ -121,5 +121,5 @@ public void setButton_style(String button_style) { this.button_style = button_style; sendUpdate(BUTTON_STYLE, button_style); } - + } \ No newline at end of file diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/KernelTest.java b/kernel/base/src/test/java/com/twosigma/beakerx/KernelTest.java index 367ef3f3cf..7c87b4217c 100644 --- a/kernel/base/src/test/java/com/twosigma/beakerx/KernelTest.java +++ b/kernel/base/src/test/java/com/twosigma/beakerx/KernelTest.java @@ -413,7 +413,7 @@ public void registerCancelHook(Hook hook) { public PythonEntryPoint getPythonEntryPoint(String kernelName) throws NoSuchKernelException { MagicKernelManager manager = magicKernels.get(kernelName); if (manager == null) { - manager = new MagicKernelManager(kernelName); + manager = new MagicKernelManager(kernelName,"kernelTestContext"); magicKernels.put(kernelName, manager); } return manager.getPythonEntryPoint(); @@ -425,6 +425,5 @@ public MagicKernelManager getManagerByCommId(String commId) { } @Override - public void addCommIdManagerMapping(String commId, String kernel) { - } + public void addCommIdManagerMapping(String commId, String kernel) {} } diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientShowProgressReportingTest.java b/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientShowProgressReportingTest.java index 14d8d93d8a..84dec44439 100644 --- a/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientShowProgressReportingTest.java +++ b/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientShowProgressReportingTest.java @@ -28,13 +28,12 @@ public class NamespaceClientShowProgressReportingTest { - private static String SESSION_ID = "sessionId"; private NamespaceClient namespaceClient; private KernelTest kernel; @Before public void setUp() { - namespaceClient = NamespaceClient.getBeaker(SESSION_ID); + namespaceClient = new NamespaceClient(new NamespaceClientTest.AutotranslationServiceTestImpl(), new DefaultBeakerXJsonSerializer()); kernel = new KernelTest(); KernelManager.register(kernel); } diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientTest.java b/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientTest.java index 661247b554..97d5848185 100644 --- a/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientTest.java +++ b/kernel/base/src/test/java/com/twosigma/beakerx/NamespaceClientTest.java @@ -33,13 +33,12 @@ public class NamespaceClientTest { - private static String SESSION_ID = "sessionId"; private NamespaceClient namespaceClient; private KernelTest kernel; @Before public void setUp() { - namespaceClient = NamespaceClient.getBeaker(SESSION_ID); + namespaceClient = new NamespaceClient(new AutotranslationServiceTestImpl(), new DefaultBeakerXJsonSerializer()); kernel = new KernelTest(); KernelManager.register(kernel); } @@ -50,39 +49,11 @@ public void tearDown() throws Exception { KernelManager.register(null); } - @Test - public void getNamespaceClientBySessionId_returnNamespaceClient() { - //when - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker(SESSION_ID); - //then - assertThat(namespaceClient).isNotNull(); - assertThat(curNamespaceClient).isEqualTo(namespaceClient); - } - - @Test - public void getNamespaceClientByCurrentSessionId_returnNamespaceClient() { - //when - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker(); - //then - assertThat(curNamespaceClient).isNotNull(); - assertThat(curNamespaceClient).isEqualTo(namespaceClient); - } - - @Test - public void deleteNamespaceClientBySessionId_deleteNamespaceClient() { - //when - NamespaceClient.delBeaker(SESSION_ID); - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker(); - //then - assertThat(curNamespaceClient).isNull(); - } - @Test public void setData_returnValue() throws Exception { //given - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker("returnValue"); //when - Object value = curNamespaceClient.set("x", new Integer(10)); + Object value = namespaceClient.set("x", new Integer(10)); //then assertThat(value).isNotNull(); assertThat(value).isEqualTo(new Integer(10)); @@ -91,9 +62,8 @@ public void setData_returnValue() throws Exception { @Test public void setData_setAutotranslationData() throws Exception { //given - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker("setAutotranslationData"); //when - curNamespaceClient.set("x", new Integer(10)); + namespaceClient.set("x", new Integer(10)); //then assertThat(kernel.getPublishedMessages()).isNotEmpty(); Map data = (Map) kernel.getPublishedMessages().get(1).getContent().get("data"); @@ -127,9 +97,8 @@ public void setBigInt() throws Exception { }} ); - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker("setBigInt"); //when - curNamespaceClient.set("table_with_longs", table); + namespaceClient.set("table_with_longs", table); //then assertThat(kernel.getPublishedMessages()).isNotEmpty(); Map data = (Map) kernel.getPublishedMessages().get(2).getContent().get("data"); @@ -141,20 +110,42 @@ public void setBigInt() throws Exception { @Test public void setData_sendCommMessage() throws Exception { //given - NamespaceClient curNamespaceClient = NamespaceClient.getBeaker("sendCommMessage"); //when - curNamespaceClient.set("x", new Integer(10)); + namespaceClient.set("x", new Integer(10)); //then assertThat(kernel.getPublishedMessages()).isNotEmpty(); } - public static boolean isJSONValid(Object jsonInString ) { + public static boolean isJSONValid(Object jsonInString) { try { final ObjectMapper mapper = new ObjectMapper(); - mapper.readTree((String)jsonInString); + mapper.readTree((String) jsonInString); return true; } catch (IOException e) { return false; } } + + public static class AutotranslationServiceTestImpl implements AutotranslationService { + + @Override + public String update(String name, String json) { + return null; + } + + @Override + public String get(String name) { + return null; + } + + @Override + public String close() { + return null; + } + + @Override + public String getContextAsString() { + return null; + } + } } diff --git a/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorTest.java b/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorTest.java index ef4c3b5a54..2ef4506e09 100644 --- a/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorTest.java +++ b/kernel/base/src/test/java/com/twosigma/beakerx/evaluator/EvaluatorTest.java @@ -16,6 +16,8 @@ package com.twosigma.beakerx.evaluator; +import com.twosigma.beakerx.BeakerXClient; +import com.twosigma.beakerx.CodeCell; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.inspect.InspectResult; @@ -35,6 +37,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.SynchronousQueue; public class EvaluatorTest extends BaseEvaluator { @@ -55,7 +59,7 @@ public EvaluatorTest() { } public EvaluatorTest(String id, String sId, CellExecutor cellExecutor, EvaluatorParameters kernelParameters) { - super(id, sId, cellExecutor, getTestTempFolderFactory(), kernelParameters); + super(id, sId, cellExecutor, getTestTempFolderFactory(), kernelParameters, new BeakexClientTestImpl()); } @Override @@ -190,4 +194,53 @@ public int getResetEnvironmentCounter() { return resetEnvironmentCounter; } + + public static class BeakexClientTestImpl implements BeakerXClient { + + @Override + public void showProgressUpdate(String message, int progress) { + + } + + @Override + public void delBeaker() { + + } + + @Override + public String update(String name, Object value) { + return null; + } + + @Override + public Object set(String name, Object value) { + return null; + } + + @Override + public Object get(String name) { + return null; + } + + @Override + public SynchronousQueue getMessageQueue(String channel) { + return null; + } + + @Override + public List getCodeCells(String tagFilter) { + return null; + } + + @Override + public void runByTag(String tag) { + + } + + @Override + public String getContext() { + return null; + } + } + } diff --git a/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluator.java b/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluator.java index 306b04d2df..300760bfcf 100644 --- a/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluator.java +++ b/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluator.java @@ -21,6 +21,7 @@ import clojure.lang.RT; import clojure.lang.Symbol; import clojure.lang.Var; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.clojure.autocomplete.ClojureAutocomplete; @@ -54,14 +55,24 @@ public class ClojureEvaluator extends BaseEvaluator { private DynamicClassLoader loader; private Var clojureLoadString = null; - public ClojureEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters) { - super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters); + public ClojureEvaluator(String id, + String sId, + CellExecutor cellExecutor, + TempFolderFactory tempFolderFactory, + EvaluatorParameters evaluatorParameters, + BeakerXClient beakerxClient) { + super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters, beakerxClient); requirements = new ArrayList<>(); init(); } - public ClojureEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters) { - this(id, sId, new BeakerCellExecutor("clojure"), new TempFolderFactoryImpl(), evaluatorParameters); + public ClojureEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + this(id, + sId, + new BeakerCellExecutor("clojure"), + new TempFolderFactoryImpl(), + evaluatorParameters, + beakerxClient); } @Override diff --git a/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/kernel/Clojure.java b/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/kernel/Clojure.java index 889cc3b447..f466f46892 100644 --- a/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/kernel/Clojure.java +++ b/kernel/clojure/src/main/java/com/twosigma/beakerx/clojure/kernel/Clojure.java @@ -21,6 +21,7 @@ import clojure.lang.LazySeq; import com.fasterxml.jackson.databind.ObjectMapper; +import com.twosigma.beakerx.AutotranslationServiceImpl; import com.twosigma.beakerx.DisplayerDataMapper; import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.clojure.evaluator.ClojureEvaluator; @@ -76,18 +77,18 @@ public KernelHandler getKernelInfoHandler(Kernel kernel) { public static void main(final String[] args) throws InterruptedException, IOException { KernelRunner.run(() -> { String id = uuid(); + KernelConfigurationFile configurationFile = new KernelConfigurationFile(args); KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl( - new KernelConfigurationFile(args)); - ClojureEvaluator evaluator = new ClojureEvaluator(id, id, getKernelParameters()); - initNamespaceClient(id); + configurationFile); + ClojureEvaluator evaluator = new ClojureEvaluator( + id, + id, + getKernelParameters(), + NamespaceClient.create(id, configurationFile)); return new Clojure(id, evaluator, kernelSocketsFactory); }); } - private static void initNamespaceClient(String id) { - NamespaceClient.getBeaker(id); - } - @Override protected void configureMagicCommands() { super.configureMagicCommands(); diff --git a/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluatorTest.java b/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluatorTest.java index a517459c54..810ecb4853 100644 --- a/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluatorTest.java +++ b/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/evaluator/ClojureEvaluatorTest.java @@ -2,6 +2,7 @@ import com.twosigma.beakerx.evaluator.BaseEvaluator; import com.twosigma.beakerx.evaluator.EvaluatorBaseTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.evaluator.TempFolderFactory; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -16,7 +17,7 @@ public class ClojureEvaluatorTest extends EvaluatorBaseTest { @BeforeClass public static void setUpClass() throws Exception { - evaluator = new ClojureEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + evaluator = new ClojureEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass @@ -26,12 +27,12 @@ public static void tearDown() throws Exception { @Override protected BaseEvaluator createNewEvaluator() { - return new ClojureEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + return new ClojureEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Override protected BaseEvaluator createNewEvaluator(TempFolderFactory tempFolderFactory) { - return new ClojureEvaluator("id", "sid", cellExecutor(), tempFolderFactory, KERNEL_PARAMETERS); + return new ClojureEvaluator("id", "sid", cellExecutor(), tempFolderFactory, KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Override diff --git a/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureAutocompleteTest.java b/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureAutocompleteTest.java index 680b78d968..9150f2a4f3 100644 --- a/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureAutocompleteTest.java +++ b/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureAutocompleteTest.java @@ -39,7 +39,8 @@ public static void setUpClass() throws Exception { "sid", cellExecutor(), EvaluatorTest.getTestTempFolderFactory(), - KERNEL_PARAMETERS + KERNEL_PARAMETERS, + new EvaluatorTest.BeakexClientTestImpl() ); } diff --git a/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureKernelTest.java b/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureKernelTest.java index 3cb34d7fe4..21c8a68037 100644 --- a/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureKernelTest.java +++ b/kernel/clojure/src/test/java/com/twosigma/beakerx/clojure/kernel/ClojureKernelTest.java @@ -18,6 +18,7 @@ import com.twosigma.beakerx.KernelExecutionTest; import com.twosigma.beakerx.KernelSocketsServiceTest; import com.twosigma.beakerx.clojure.evaluator.ClojureEvaluator; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.Kernel; import com.twosigma.beakerx.kernel.KernelSocketsFactory; @@ -48,7 +49,7 @@ public class ClojureKernelTest extends KernelExecutionTest { @Override protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { - ClojureEvaluator evaluator = new ClojureEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + ClojureEvaluator evaluator = new ClojureEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); return new Clojure(sessionId, evaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory()); } diff --git a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/comm/GroovyCommOpenHandler.java b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/comm/GroovyCommOpenHandler.java index 0bea322fe6..d073eb2dcf 100644 --- a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/comm/GroovyCommOpenHandler.java +++ b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/comm/GroovyCommOpenHandler.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.kernel.KernelFunctionality; import com.twosigma.beakerx.handler.Handler; +import com.twosigma.beakerx.kernel.comm.AutotranslationHandler; import com.twosigma.beakerx.kernel.comm.GetCodeCellsHandler; import com.twosigma.beakerx.kernel.comm.KernelControlCommandListHandler; import com.twosigma.beakerx.kernel.comm.KernelControlInterrupt; @@ -24,26 +25,31 @@ import com.twosigma.beakerx.message.Message; import com.twosigma.beakerx.kernel.handler.CommOpenHandler; -public class GroovyCommOpenHandler extends CommOpenHandler{ +public class GroovyCommOpenHandler extends CommOpenHandler { - private Handler[] KERNEL_CONTROL_CHANNEL_HANDLERS = { - new KernelControlInterrupt(kernel), - new KernelControlCommandListHandler(kernel)}; + private Handler[] KERNEL_CONTROL_CHANNEL_HANDLERS = { + new KernelControlInterrupt(kernel), + new KernelControlCommandListHandler(kernel)}; + + private Handler[] KERNEL_GET_CODECELLS_CHANNEL_HANDLER = { + new GetCodeCellsHandler(kernel)}; + + private Handler[] AUTOTRANSLATION_HANDLER = { + new AutotranslationHandler(kernel)}; - private Handler[] KERNEL_GET_CODECELLS_CHANNEL_HANDLER = { - new GetCodeCellsHandler(kernel)}; public GroovyCommOpenHandler(KernelFunctionality kernel) { super(kernel); } - public Handler[] getKernelControlChanelHandlers(String targetName){ - if(TargetNamesEnum.KERNEL_CONTROL_CHANNEL.getTargetName().equalsIgnoreCase(targetName)){ + public Handler[] getKernelControlChanelHandlers(String targetName) { + if (TargetNamesEnum.KERNEL_CONTROL_CHANNEL.getTargetName().equalsIgnoreCase(targetName)) { return (Handler[]) KERNEL_CONTROL_CHANNEL_HANDLERS; - }else if(TargetNamesEnum.BEAKER_GETCODECELLS.getTargetName().equalsIgnoreCase(targetName)){ + } else if (TargetNamesEnum.BEAKER_GETCODECELLS.getTargetName().equalsIgnoreCase(targetName)) { return (Handler[]) KERNEL_GET_CODECELLS_CHANNEL_HANDLER; - } - else{ + } else if (TargetNamesEnum.BEAKER_AUTOTRANSLATION.getTargetName().equalsIgnoreCase(targetName)) { + return (Handler[]) AUTOTRANSLATION_HANDLER; + } else { return (Handler[]) new Handler[0]; } } diff --git a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyCodeRunner.java b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyCodeRunner.java index 89c93b4044..4ec14baef2 100644 --- a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyCodeRunner.java +++ b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyCodeRunner.java @@ -15,10 +15,8 @@ */ package com.twosigma.beakerx.groovy.evaluator; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.evaluator.Evaluator; -import com.twosigma.beakerx.evaluator.InternalVariable; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; import groovy.lang.Script; import org.codehaus.groovy.runtime.StackTraceUtils; @@ -91,7 +89,7 @@ private TryResult handleError(String scriptName, Throwable e) { } private Object runScript(Script script) { - groovyEvaluator.getScriptBinding().setVariable(Evaluator.BEAKER_VARIABLE_NAME, NamespaceClient.getBeaker(groovyEvaluator.getSessionId())); + groovyEvaluator.getScriptBinding().setVariable(Evaluator.BEAKER_VARIABLE_NAME, groovyEvaluator.getBeakerX()); script.setBinding(groovyEvaluator.getScriptBinding()); return script.run(); } diff --git a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluator.java b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluator.java index e691778401..39a882134a 100644 --- a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluator.java +++ b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluator.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.groovy.evaluator; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.evaluator.BaseEvaluator; @@ -53,12 +54,22 @@ public class GroovyEvaluator extends BaseEvaluator { private ImportCustomizer icz; private BeakerXUrlClassLoader beakerxUrlClassLoader; - public GroovyEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters) { - this(id, sId, new BeakerCellExecutor("groovy"), new TempFolderFactoryImpl(), evaluatorParameters); + public GroovyEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + this(id, + sId, + new BeakerCellExecutor("groovy"), + new TempFolderFactoryImpl(), + evaluatorParameters, + beakerxClient); } - public GroovyEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters) { - super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters); + public GroovyEvaluator(String id, + String sId, + CellExecutor cellExecutor, + TempFolderFactory tempFolderFactory, + EvaluatorParameters evaluatorParameters, + BeakerXClient beakerxClient) { + super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters, beakerxClient); cps = new GroovyClasspathScanner(); gac = createGroovyAutocomplete(cps); outDir = envVariablesFilter(outDir, System.getenv()); diff --git a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyWorkerThread.java b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyWorkerThread.java index 0aa9b82938..acca4d5e9b 100644 --- a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyWorkerThread.java +++ b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/evaluator/GroovyWorkerThread.java @@ -39,15 +39,9 @@ public TryResult call() { NamespaceClient nc = null; TryResult r; try { - nc = NamespaceClient.getBeaker(groovyEvaluator.getSessionId()); - nc.setOutputObj(j.outputObject); j.outputObject.started(); String code = j.codeToBeExecuted; r = groovyEvaluator.executeTask(new GroovyCodeRunner(groovyEvaluator, code, j.outputObject)); - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } } catch (Throwable e) { if (e instanceof GroovyNotFoundException) { logger.warn(e.getLocalizedMessage()); @@ -56,11 +50,6 @@ public TryResult call() { e.printStackTrace(); r = TryResult.createError(e.getLocalizedMessage()); } - } finally { - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } } return r; } diff --git a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/kernel/Groovy.java b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/kernel/Groovy.java index d55e1bb789..3d28715df3 100644 --- a/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/kernel/Groovy.java +++ b/kernel/groovy/src/main/java/com/twosigma/beakerx/groovy/kernel/Groovy.java @@ -18,6 +18,8 @@ import static com.twosigma.beakerx.DefaultJVMVariables.IMPORTS; import static com.twosigma.beakerx.kernel.Utils.uuid; +import com.twosigma.beakerx.AutotranslationServiceImpl; +import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.evaluator.Evaluator; import com.twosigma.beakerx.groovy.comm.GroovyCommOpenHandler; import com.twosigma.beakerx.groovy.evaluator.GroovyEvaluator; @@ -40,11 +42,17 @@ public class Groovy extends Kernel { - private Groovy(final String id, final Evaluator evaluator, KernelSocketsFactory kernelSocketsFactory) { + private Groovy(final String id, + final Evaluator evaluator, + KernelSocketsFactory kernelSocketsFactory) { super(id, evaluator, kernelSocketsFactory, new CustomMagicCommandsEmptyImpl()); } - public Groovy(final String id, final Evaluator evaluator, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction, CacheFolderFactory cacheFolderFactory) { + public Groovy(final String id, + final Evaluator evaluator, + KernelSocketsFactory kernelSocketsFactory, + CloseKernelAction closeKernelAction, + CacheFolderFactory cacheFolderFactory) { super(id, evaluator, kernelSocketsFactory, closeKernelAction, cacheFolderFactory, new CustomMagicCommandsEmptyImpl()); } @@ -61,9 +69,14 @@ public KernelHandler getKernelInfoHandler(Kernel kernel) { public static void main(final String[] args) throws InterruptedException, IOException { KernelRunner.run(() -> { String id = uuid(); + KernelConfigurationFile configurationFile = new KernelConfigurationFile(args); KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl( - new KernelConfigurationFile(args)); - GroovyEvaluator evaluator = new GroovyEvaluator(id, id, getEvaluatorParameters()); + configurationFile); + + GroovyEvaluator evaluator = new GroovyEvaluator(id, + id, + getEvaluatorParameters(), + NamespaceClient.create(id, configurationFile)); return new Groovy(id, evaluator, kernelSocketsFactory); }); } diff --git a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/TestGroovyEvaluator.java b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/TestGroovyEvaluator.java index f542507bb5..8f299c99bf 100644 --- a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/TestGroovyEvaluator.java +++ b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/TestGroovyEvaluator.java @@ -15,7 +15,10 @@ */ package com.twosigma.beakerx.groovy; +import com.twosigma.beakerx.AutotranslationService; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.evaluator.BaseEvaluator; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.evaluator.TempFolderFactory; import com.twosigma.beakerx.groovy.evaluator.GroovyEvaluator; import com.twosigma.beakerx.groovy.kernel.GroovyDefaultVariables; @@ -29,23 +32,61 @@ public class TestGroovyEvaluator { + public static AutotranslationService autotranslationService() { + return new AutotranslationService() { + @Override + public String update(String name, String json) { + return null; + } + + @Override + public String get(String name) { + return null; + } + + @Override + public String close() { + return null; + } + + @Override + public String getContextAsString() { + return null; + } + }; + } + + public static BaseEvaluator groovyEvaluator(BeakerXClient client) { + GroovyEvaluator evaluator = new GroovyEvaluator( + "id", + "sid", + cellExecutor(), + getTestTempFolderFactory(), + getKernelParameters(), + client); + return evaluator; + } + + public static BaseEvaluator groovyEvaluator() { GroovyEvaluator evaluator = new GroovyEvaluator( "id", "sid", cellExecutor(), getTestTempFolderFactory(), - getKernelParameters()); + getKernelParameters(), + new EvaluatorTest.BeakexClientTestImpl()); return evaluator; } public static BaseEvaluator groovyEvaluator(TempFolderFactory tempFolderFactory) { GroovyEvaluator evaluator = new GroovyEvaluator( - "id", - "sid", - cellExecutor(), - tempFolderFactory, - getKernelParameters()); + "id", + "sid", + cellExecutor(), + tempFolderFactory, + getKernelParameters(), + new EvaluatorTest.BeakexClientTestImpl()); return evaluator; } diff --git a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/autocomplete/GroovyCompleteHandlerTest.java b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/autocomplete/GroovyCompleteHandlerTest.java index 471e937e36..0eb5b6ff34 100644 --- a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/autocomplete/GroovyCompleteHandlerTest.java +++ b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/autocomplete/GroovyCompleteHandlerTest.java @@ -49,7 +49,8 @@ public static void setUpClass() { "sid", cellExecutor(), EvaluatorTest.getTestTempFolderFactory(), - new EvaluatorParameters(new HashedMap()) + new EvaluatorParameters(new HashedMap()), + new EvaluatorTest.BeakexClientTestImpl() ); groovyKernel = new GroovyKernelMock("sid", groovyEvaluator); diff --git a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorProgressReportingTest.java b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorProgressReportingTest.java index 15caaef140..9fdd2ca8c1 100644 --- a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorProgressReportingTest.java +++ b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorProgressReportingTest.java @@ -15,9 +15,10 @@ */ package com.twosigma.beakerx.groovy.evaluator; +import com.twosigma.beakerx.DefaultBeakerXJsonSerializer; +import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.evaluator.BaseEvaluator; -import com.twosigma.beakerx.evaluator.InternalVariable; import com.twosigma.beakerx.groovy.TestGroovyEvaluator; import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; @@ -42,13 +43,14 @@ public class GroovyEvaluatorProgressReportingTest { + public static final String GROOVY_EVALUATOR_PROGRESS_REPORTING_TEST = "groovyEvaluatorProgressReportingTest"; private BaseEvaluator groovyEvaluator; private KernelTest groovyKernel; @Before public void setUp() throws Exception { - groovyEvaluator = TestGroovyEvaluator.groovyEvaluator(); - groovyKernel = new KernelTest("groovyEvaluatorProgressReportingTest", groovyEvaluator); + groovyEvaluator = TestGroovyEvaluator.groovyEvaluator(new NamespaceClient(TestGroovyEvaluator.autotranslationService(), new DefaultBeakerXJsonSerializer())); + groovyKernel = new KernelTest(GROOVY_EVALUATOR_PROGRESS_REPORTING_TEST, groovyEvaluator); KernelManager.register(groovyKernel); } diff --git a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorTest.java b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorTest.java index e40a93f62e..fd443b7316 100644 --- a/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorTest.java +++ b/kernel/groovy/src/test/java/com/twosigma/beakerx/groovy/evaluator/GroovyEvaluatorTest.java @@ -16,7 +16,7 @@ package com.twosigma.beakerx.groovy.evaluator; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.groovy.kernel.GroovyDefaultVariables; import com.twosigma.beakerx.groovy.kernel.GroovyKernelMock; import com.twosigma.beakerx.kernel.KernelManager; @@ -43,7 +43,6 @@ public class GroovyEvaluatorTest { static GroovyClassLoader groovyClassLoader; static GroovyKernelMock groovyKernel; static Binding scriptBinding; - static GroovyEvaluator groovyEvaluator; static ImportCustomizer icz = new ImportCustomizer(); @BeforeClass @@ -54,11 +53,11 @@ public static void initClassStubData() throws IOException { params.put(CLASSPATH, var.getClassPath()); EvaluatorParameters kernelParameters = new EvaluatorParameters(params); - GroovyEvaluator groovyEvaluator = new GroovyEvaluator("123", "345", cellExecutor(), getTestTempFolderFactory(), kernelParameters); + GroovyEvaluator groovyEvaluator = new GroovyEvaluator("123", "345", cellExecutor(), getTestTempFolderFactory(), kernelParameters, new EvaluatorTest.BeakexClientTestImpl()); groovyEvaluator.updateEvaluatorParameters(kernelParameters); groovyClassLoader = newEvaluator(groovyEvaluator.getImports(), groovyEvaluator.getClasspath(), groovyEvaluator.getOutDir(), icz, newParentClassLoader(groovyEvaluator.getClasspath())); scriptBinding = new Binding(); - scriptBinding.setVariable("beaker", NamespaceClient.getBeaker("345")); + scriptBinding.setVariable("beaker", groovyEvaluator.getBeakerX()); groovyKernel = new GroovyKernelMock("groovyEvaluatorTest", groovyEvaluator); KernelManager.register(groovyKernel); } diff --git a/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluator.java b/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluator.java index a998c81f8a..aac31e67b3 100644 --- a/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluator.java +++ b/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluator.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.javash.evaluator; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.evaluator.BaseEvaluator; @@ -44,12 +45,17 @@ public class JavaEvaluator extends BaseEvaluator { private JavaAutocomplete jac; private JavaBeakerXUrlClassLoader loader = null; - public JavaEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters) { - this(id, sId, new BeakerCellExecutor("javash"), new TempFolderFactoryImpl(), evaluatorParameters); + public JavaEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + this(id, sId, new BeakerCellExecutor("javash"), new TempFolderFactoryImpl(), evaluatorParameters, beakerxClient); } - public JavaEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters) { - super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters); + public JavaEvaluator(String id, + String sId, + CellExecutor cellExecutor, + TempFolderFactory tempFolderFactory, + EvaluatorParameters evaluatorParameters, + BeakerXClient beakerxClient) { + super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters, beakerxClient); packageId = "com.twosigma.beaker.javash.bkr" + shellId.split("-")[0]; cps = new JavaClasspathScanner(); jac = createJavaAutocomplete(cps); @@ -58,7 +64,7 @@ public JavaEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolde @Override public ClassLoader getClassLoaderForImport() { - return getJavaClassLoader().getJavaURLClassLoader(); + return getJavaClassLoader().getJavaURLClassLoader(); } @Override @@ -131,7 +137,7 @@ public String getPackageId() { } private JavaBeakerXUrlClassLoader newClassLoader() { - JavaBeakerXUrlClassLoader loader = new JavaBeakerXUrlClassLoader(ClassLoader.getSystemClassLoader(),new PathToJar(outDir)); + JavaBeakerXUrlClassLoader loader = new JavaBeakerXUrlClassLoader(ClassLoader.getSystemClassLoader(), new PathToJar(outDir)); loader.addInitPathToJars(getClasspath().getPaths()); return loader; } diff --git a/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaWorkerThread.java b/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaWorkerThread.java index 41d15a0192..2ae6921d86 100644 --- a/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaWorkerThread.java +++ b/kernel/java/src/main/java/com/twosigma/beakerx/javash/evaluator/JavaWorkerThread.java @@ -15,7 +15,6 @@ */ package com.twosigma.beakerx.javash.evaluator; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.evaluator.JobDescriptor; @@ -33,20 +32,12 @@ public JavaWorkerThread(JavaEvaluator javaEvaluator, JobDescriptor j) { @Override public TryResult call() throws Exception { - NamespaceClient nc = null; TryResult r; try { - nc = NamespaceClient.getBeaker(javaEvaluator.getSessionId()); - nc.setOutputObj(j.outputObject); r = javaEvaluator.executeTask(new JavaCodeRunner(javaEvaluator, j.outputObject, j)); } catch (Throwable e) { e.printStackTrace(); r = TryResult.createError(e.getLocalizedMessage()); - } finally { - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } } return r; } diff --git a/kernel/java/src/main/java/com/twosigma/beakerx/javash/kernel/Java.java b/kernel/java/src/main/java/com/twosigma/beakerx/javash/kernel/Java.java index 31bbcf573a..d2981512c9 100644 --- a/kernel/java/src/main/java/com/twosigma/beakerx/javash/kernel/Java.java +++ b/kernel/java/src/main/java/com/twosigma/beakerx/javash/kernel/Java.java @@ -18,6 +18,8 @@ import static com.twosigma.beakerx.DefaultJVMVariables.IMPORTS; import static com.twosigma.beakerx.kernel.Utils.uuid; +import com.twosigma.beakerx.AutotranslationServiceImpl; +import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.evaluator.Evaluator; import com.twosigma.beakerx.handler.KernelHandler; import com.twosigma.beakerx.javash.comm.JavaCommOpenHandler; @@ -62,9 +64,13 @@ public KernelHandler getKernelInfoHandler(Kernel kernel) { public static void main(final String[] args) throws InterruptedException, IOException { KernelRunner.run(() -> { String id = uuid(); - JavaEvaluator e = new JavaEvaluator(id, id, getKernelParameters()); + KernelConfigurationFile configurationFile = new KernelConfigurationFile(args); KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl( - new KernelConfigurationFile(args)); + configurationFile); + JavaEvaluator e = new JavaEvaluator(id, + id, + getKernelParameters(), + NamespaceClient.create(id, configurationFile)); return new Java(id, e, kernelSocketsFactory); }); } diff --git a/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteClassNameExpressionTest.java b/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteClassNameExpressionTest.java index 9ce4dd1031..5bba818ce9 100644 --- a/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteClassNameExpressionTest.java +++ b/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteClassNameExpressionTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.evaluator.BaseEvaluator; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.javash.evaluator.JavaEvaluator; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -33,7 +34,7 @@ public class JavaEvaluatorAutocompleteClassNameExpressionTest { @BeforeClass public static void setUpClass() throws Exception { - evaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + evaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass diff --git a/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteImportTest.java b/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteImportTest.java index 6889476d9a..6c03119601 100644 --- a/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteImportTest.java +++ b/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteImportTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.autocomplete.JVMEvaluatorAutocompleteImportTest; import com.twosigma.beakerx.evaluator.BaseEvaluator; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.javash.evaluator.JavaEvaluator; import com.twosigma.beakerx.javash.kernel.JavaDefaultVariables; import com.twosigma.beakerx.kernel.EvaluatorParameters; @@ -38,7 +39,7 @@ public static void setUpClass() throws Exception { HashMap map = new HashMap<>(); map.put(IMPORTS, new JavaDefaultVariables().getImports()); EvaluatorParameters kernelParameters = new EvaluatorParameters(map); - groovyEvaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), kernelParameters); + groovyEvaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), kernelParameters, new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass diff --git a/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteTest.java b/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteTest.java index 59fb008177..0964b6d6d6 100644 --- a/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteTest.java +++ b/kernel/java/src/test/java/com/twosigma/beakerx/javash/autocomplete/JavaEvaluatorAutocompleteTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.evaluator.BaseEvaluator; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.javash.evaluator.JavaEvaluator; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,7 +37,7 @@ public class JavaEvaluatorAutocompleteTest { @BeforeClass public static void setUpClass() throws Exception { - evaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + evaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS,new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass diff --git a/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaBaseEvaluatorTest.java b/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaBaseEvaluatorTest.java index 7a161e0a28..dbbfd8d318 100644 --- a/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaBaseEvaluatorTest.java +++ b/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaBaseEvaluatorTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.evaluator.BaseEvaluator; import com.twosigma.beakerx.evaluator.EvaluatorBaseTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.evaluator.TempFolderFactory; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -31,7 +32,7 @@ public class JavaBaseEvaluatorTest extends EvaluatorBaseTest { @BeforeClass public static void setUpClass() throws Exception { - evaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + evaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass @@ -41,12 +42,12 @@ public static void tearDown() throws Exception { @Override protected BaseEvaluator createNewEvaluator() { - return new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + return new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Override protected BaseEvaluator createNewEvaluator(TempFolderFactory tempFolderFactory) { - return new JavaEvaluator("id", "sid", cellExecutor(), tempFolderFactory, KERNEL_PARAMETERS); + return new JavaEvaluator("id", "sid", cellExecutor(), tempFolderFactory, KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Override diff --git a/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluatorTest.java b/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluatorTest.java index a6a758c517..f7706d7417 100644 --- a/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluatorTest.java +++ b/kernel/java/src/test/java/com/twosigma/beakerx/javash/evaluator/JavaEvaluatorTest.java @@ -18,6 +18,7 @@ import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.chart.xychart.Plot; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.javash.kernel.JavaKernelMock; import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; @@ -37,7 +38,7 @@ public class JavaEvaluatorTest { @BeforeClass public static void setUp() throws Exception { - javaEvaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + javaEvaluator = new JavaEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); JavaKernelMock kernel = new JavaKernelMock("id", javaEvaluator); KernelManager.register(kernel); } diff --git a/kernel/java/src/test/java/com/twosigma/beakerx/javash/kernel/JavaKernelTest.java b/kernel/java/src/test/java/com/twosigma/beakerx/javash/kernel/JavaKernelTest.java index d2dd31f918..c25b85a979 100644 --- a/kernel/java/src/test/java/com/twosigma/beakerx/javash/kernel/JavaKernelTest.java +++ b/kernel/java/src/test/java/com/twosigma/beakerx/javash/kernel/JavaKernelTest.java @@ -16,6 +16,7 @@ package com.twosigma.beakerx.javash.kernel; import com.twosigma.beakerx.KernelExecutionTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.javash.evaluator.JavaEvaluator; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.Kernel; @@ -30,7 +31,7 @@ public class JavaKernelTest extends KernelExecutionTest { @Override protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { - JavaEvaluator evaluator = new JavaEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + JavaEvaluator evaluator = new JavaEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); return new Java(sessionId, evaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory()); } diff --git a/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluator.java b/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluator.java index f4ce856dda..63c844575d 100644 --- a/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluator.java +++ b/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluator.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.kotlin.evaluator; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.autocomplete.ClasspathScanner; @@ -51,12 +52,12 @@ public class KotlinEvaluator extends BaseEvaluator { private ReplClassLoader loader = null; private BeakerXUrlClassLoader kotlinClassLoader; - public KotlinEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters) { - this(id, sId, new BeakerCellExecutor("kotlin"), new TempFolderFactoryImpl(), evaluatorParameters); + public KotlinEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + this(id, sId, new BeakerCellExecutor("kotlin"), new TempFolderFactoryImpl(), evaluatorParameters, beakerxClient); } - public KotlinEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters) { - super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters); + public KotlinEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters, beakerxClient); cps = new ClasspathScanner(); createRepl(); } diff --git a/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinWorkerThread.java b/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinWorkerThread.java index b13dfea64d..f1251b7a28 100644 --- a/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinWorkerThread.java +++ b/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/evaluator/KotlinWorkerThread.java @@ -15,7 +15,6 @@ */ package com.twosigma.beakerx.kotlin.evaluator; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.evaluator.JobDescriptor; @@ -37,34 +36,19 @@ public KotlinWorkerThread(KotlinEvaluator kotlinEvaluator, JobDescriptor j) { @Override public TryResult call() throws Exception { - NamespaceClient nc = null; TryResult either; try { - - nc = NamespaceClient.getBeaker(kotlinEvaluator.getSessionId()); - nc.setOutputObj(j.outputObject); - j.outputObject.started(); try { KotlinCodeRunner kotlinCodeRunner = new KotlinCodeRunner(j.outputObject, kotlinEvaluator.getClassLoader(), kotlinEvaluator.getRepl(), j.codeToBeExecuted); either = kotlinEvaluator.executeTask(kotlinCodeRunner); - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } - } catch (Exception e) { either = TryResult.createError(e.getMessage()); } } catch (Throwable e) { e.printStackTrace(); either = TryResult.createError(e.getMessage()); - } finally { - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } } return either; } diff --git a/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/kernel/Kotlin.java b/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/kernel/Kotlin.java index 5a45016fff..41bd6bc5a8 100644 --- a/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/kernel/Kotlin.java +++ b/kernel/kotlin/src/main/java/com/twosigma/beakerx/kotlin/kernel/Kotlin.java @@ -18,6 +18,8 @@ import static com.twosigma.beakerx.DefaultJVMVariables.IMPORTS; import static com.twosigma.beakerx.kernel.Utils.uuid; +import com.twosigma.beakerx.AutotranslationServiceImpl; +import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.evaluator.Evaluator; import com.twosigma.beakerx.handler.KernelHandler; import com.twosigma.beakerx.kernel.CacheFolderFactory; @@ -68,9 +70,13 @@ public static EvaluatorParameters getKernelParameters() { public static void main(final String[] args) throws InterruptedException, IOException { KernelRunner.run(() -> { String id = uuid(); - KotlinEvaluator e = new KotlinEvaluator(id, id, getKernelParameters()); + KernelConfigurationFile configurationFile = new KernelConfigurationFile(args); KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl( - new KernelConfigurationFile(args)); + configurationFile); + KotlinEvaluator e = new KotlinEvaluator(id, + id, + getKernelParameters(), + NamespaceClient.create(id, configurationFile)); return new Kotlin(id, e, kernelSocketsFactory); }); } diff --git a/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinBaseEvaluatorTest.java b/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinBaseEvaluatorTest.java index 1f46047bc1..5e6fb5bb0f 100644 --- a/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinBaseEvaluatorTest.java +++ b/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinBaseEvaluatorTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.evaluator.BaseEvaluator; import com.twosigma.beakerx.evaluator.EvaluatorBaseTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.evaluator.TempFolderFactory; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -31,7 +32,7 @@ public class KotlinBaseEvaluatorTest extends EvaluatorBaseTest { @BeforeClass public static void setUpClass() throws Exception { - evaluator = new KotlinEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + evaluator = new KotlinEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass @@ -41,12 +42,12 @@ public static void tearDown() throws Exception { @Override protected BaseEvaluator createNewEvaluator() { - return new KotlinEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + return new KotlinEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Override protected BaseEvaluator createNewEvaluator(TempFolderFactory tempFolderFactory) { - return new KotlinEvaluator("id", "sid", cellExecutor(), tempFolderFactory, KERNEL_PARAMETERS); + return new KotlinEvaluator("id", "sid", cellExecutor(), tempFolderFactory, KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Override diff --git a/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluatorTest.java b/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluatorTest.java index 9401aaa903..e40a67f6fa 100644 --- a/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluatorTest.java +++ b/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/evaluator/KotlinEvaluatorTest.java @@ -16,6 +16,7 @@ package com.twosigma.beakerx.kotlin.evaluator; import com.twosigma.beakerx.TryResult; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; import com.twosigma.beakerx.kernel.EvaluatorParameters; import com.twosigma.beakerx.kernel.KernelManager; @@ -42,7 +43,7 @@ public class KotlinEvaluatorTest { @BeforeClass public static void setUp() throws Exception { - evaluator = new KotlinEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + evaluator = new KotlinEvaluator("id", "sid", cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); KotlinKernelMock kernel = new KotlinKernelMock("id", evaluator); KernelManager.register(kernel); } diff --git a/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/kernel/KotlinKernelTest.java b/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/kernel/KotlinKernelTest.java index 4367fd76a1..5a31a1c906 100644 --- a/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/kernel/KotlinKernelTest.java +++ b/kernel/kotlin/src/test/java/com/twosigma/beakerx/kotlin/kernel/KotlinKernelTest.java @@ -16,6 +16,7 @@ package com.twosigma.beakerx.kotlin.kernel; import com.twosigma.beakerx.KernelExecutionTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.Kernel; import com.twosigma.beakerx.kernel.KernelSocketsFactory; @@ -30,7 +31,7 @@ public class KotlinKernelTest extends KernelExecutionTest { @Override protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { - KotlinEvaluator evaluator = new KotlinEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS); + KotlinEvaluator evaluator = new KotlinEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); return new Kotlin(sessionId, evaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory()); } diff --git a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactory.java b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactory.java index dc7e507b8e..24bdad4a72 100644 --- a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactory.java +++ b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactory.java @@ -17,5 +17,5 @@ public interface BeakerxObjectFactory { - String create(String sessionId); + String create(); } diff --git a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactoryImpl.java b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactoryImpl.java index e7995d825f..ecd7db0cb9 100644 --- a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactoryImpl.java +++ b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/BeakerxObjectFactoryImpl.java @@ -15,23 +15,26 @@ */ package com.twosigma.beakerx.scala.evaluator; +import static com.twosigma.beakerx.BeakerXClientManager.BEAKER_X_CLIENT_MANAGER_GET; +import static com.twosigma.beakerx.BeakerXClientManager.BEAKER_X_CLIENT_MANAGER_PATH; + public class BeakerxObjectFactoryImpl implements BeakerxObjectFactory { @Override - public String create(String sessionId) { - return "import com.twosigma.beakerx.NamespaceClient\n" + + public String create() { + return "import " + BEAKER_X_CLIENT_MANAGER_PATH + "\n" + "import language.dynamics\n" + - "var _beaker = NamespaceClient.getBeaker(\"" + sessionId + "\")\n" + - "object beaker extends Dynamic {\n" + - " def selectDynamic( field : String ) = _beaker.get(field)\n" + + "var _beakerx = " + BEAKER_X_CLIENT_MANAGER_GET + "\n" + + "object beakerx extends Dynamic {\n" + + " def selectDynamic( field : String ) = _beakerx.get(field)\n" + " def updateDynamic (field : String)(value : Any) : Any = {\n" + - " _beaker.set(field,value)\n" + + " _beakerx.set(field,value)\n" + " return value\n" + " }\n" + " def applyDynamic(methodName: String)(args: AnyRef*) = {\n" + " def argtypes = args.map(_.getClass)\n" + - " def method = _beaker.getClass.getMethod(methodName, argtypes: _*)\n" + - " method.invoke(_beaker,args: _*)\n" + + " def method = _beakerx.getClass.getMethod(methodName, argtypes: _*)\n" + + " method.invoke(_beakerx,args: _*)\n" + " }\n" + "}\n"; } diff --git a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaEvaluator.java b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaEvaluator.java index 881f12998d..3d4540e425 100644 --- a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaEvaluator.java +++ b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaEvaluator.java @@ -16,30 +16,19 @@ package com.twosigma.beakerx.scala.evaluator; -import com.google.inject.Provider; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.evaluator.BaseEvaluator; import com.twosigma.beakerx.evaluator.JobDescriptor; import com.twosigma.beakerx.evaluator.TempFolderFactory; -import com.twosigma.beakerx.evaluator.TempFolderFactoryImpl; import com.twosigma.beakerx.jvm.classloader.BeakerXUrlClassLoader; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; -import com.twosigma.beakerx.jvm.serialization.BeakerObjectConverter; -import com.twosigma.beakerx.jvm.threads.BeakerCellExecutor; import com.twosigma.beakerx.jvm.threads.CellExecutor; import com.twosigma.beakerx.kernel.EvaluatorParameters; import com.twosigma.beakerx.kernel.ImportPath; import com.twosigma.beakerx.kernel.PathToJar; -import com.twosigma.beakerx.scala.serializers.ScalaCollectionDeserializer; -import com.twosigma.beakerx.scala.serializers.ScalaCollectionSerializer; -import com.twosigma.beakerx.scala.serializers.ScalaListOfPrimitiveTypeMapsSerializer; -import com.twosigma.beakerx.scala.serializers.ScalaMapDeserializer; -import com.twosigma.beakerx.scala.serializers.ScalaMapSerializer; -import com.twosigma.beakerx.scala.serializers.ScalaPrimitiveTypeListOfListSerializer; -import com.twosigma.beakerx.scala.serializers.ScalaPrimitiveTypeMapSerializer; -import com.twosigma.beakerx.scala.serializers.ScalaTableDeSerializer; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,18 +40,11 @@ public class ScalaEvaluator extends BaseEvaluator { private final static Logger logger = LoggerFactory.getLogger(ScalaEvaluator.class.getName()); private BeakerxObjectFactory beakerxObjectFactory; - private final Provider objectSerializerProvider; - private static boolean autoTranslationSetup = false; private BeakerXUrlClassLoader classLoader; private ScalaEvaluatorGlue shell; - public ScalaEvaluator(String id, String sId, Provider osp, EvaluatorParameters evaluatorParameters) { - this(id, sId, osp, new BeakerCellExecutor("scala"), new BeakerxObjectFactoryImpl(), new TempFolderFactoryImpl(), evaluatorParameters); - } - - public ScalaEvaluator(String id, String sId, Provider osp, CellExecutor cellExecutor, BeakerxObjectFactory beakerxObjectFactory, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters) { - super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters); - this.objectSerializerProvider = osp; + public ScalaEvaluator(String id, String sId, CellExecutor cellExecutor, BeakerxObjectFactory beakerxObjectFactory, TempFolderFactory tempFolderFactory, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters, beakerxClient); this.beakerxObjectFactory = beakerxObjectFactory; this.classLoader = newClassLoader(); this.shell = createNewEvaluator(); @@ -135,11 +117,11 @@ private String adjustImport(String imp) { private ScalaEvaluatorGlue createNewEvaluator(ScalaEvaluatorGlue shell) { ScalaEvaluatorGlue newEvaluator = createNewEvaluator(); - setLineId(newEvaluator,shell.interpreter().lastRequest().lineRep().lineId()); + setLineId(newEvaluator, shell.interpreter().lastRequest().lineRep().lineId()); return newEvaluator; } - private void setLineId(ScalaEvaluatorGlue newEvaluator,int lines) { + private void setLineId(ScalaEvaluatorGlue newEvaluator, int lines) { for (int i = newEvaluator.interpreter().lastRequest().lineRep().lineId(); i < lines; i++) { newEvaluator.evaluate2("\"\""); } @@ -153,9 +135,7 @@ private ScalaEvaluatorGlue createNewEvaluator() { addImportsToShell(shell, getImports().getImportPaths()); } logger.debug("creating beaker object"); - // ensure object is created - NamespaceClient.getBeaker(getSessionId()); - String r = shell.evaluate2(this.beakerxObjectFactory.create(getSessionId())); + String r = shell.evaluate2(this.beakerxObjectFactory.create()); if (r != null && !r.isEmpty()) { logger.warn("ERROR creating beaker object: {}", r); } @@ -201,22 +181,4 @@ private String createLoaderCp() { } return loader_cp + File.pathSeparatorChar + System.getProperty("java.class.path"); } - - public void setupAutoTranslation() { - if (autoTranslationSetup) - return; - - objectSerializerProvider.get().addfTypeSerializer(new ScalaCollectionSerializer(objectSerializerProvider.get())); - objectSerializerProvider.get().addfTypeSerializer(new ScalaMapSerializer(objectSerializerProvider.get())); - objectSerializerProvider.get().addfTypeSerializer(new ScalaPrimitiveTypeListOfListSerializer(objectSerializerProvider.get())); - objectSerializerProvider.get().addfTypeSerializer(new ScalaListOfPrimitiveTypeMapsSerializer(objectSerializerProvider.get())); - objectSerializerProvider.get().addfTypeSerializer(new ScalaPrimitiveTypeMapSerializer(objectSerializerProvider.get())); - - objectSerializerProvider.get().addfTypeDeserializer(new ScalaCollectionDeserializer(objectSerializerProvider.get())); - objectSerializerProvider.get().addfTypeDeserializer(new ScalaMapDeserializer(objectSerializerProvider.get())); - objectSerializerProvider.get().addfTypeDeserializer(new ScalaTableDeSerializer(objectSerializerProvider.get())); - - autoTranslationSetup = true; - } - } \ No newline at end of file diff --git a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaWorkerThread.java b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaWorkerThread.java index b25934c166..ea7bf9832b 100644 --- a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaWorkerThread.java +++ b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/evaluator/ScalaWorkerThread.java @@ -15,7 +15,6 @@ */ package com.twosigma.beakerx.scala.evaluator; -import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.evaluator.JobDescriptor; import org.slf4j.Logger; @@ -37,27 +36,13 @@ public ScalaWorkerThread(ScalaEvaluator scalaEvaluator, JobDescriptor j) { @Override public TryResult call() throws Exception { - NamespaceClient nc = null; TryResult either; - try { j.outputObject.started(); - - nc = NamespaceClient.getBeaker(scalaEvaluator.getSessionId()); - nc.setOutputObj(j.outputObject); either = scalaEvaluator.executeTask(new ScalaCodeRunner(scalaEvaluator, j.codeToBeExecuted, j.outputObject)); - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } } catch (Throwable e) { e.printStackTrace(); either = TryResult.createError(e.getMessage()); - } finally { - if (nc != null) { - nc.setOutputObj(null); - nc = null; - } } return either; } diff --git a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/Scala.java b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/Scala.java index 380599554d..9f0e16233b 100644 --- a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/Scala.java +++ b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/Scala.java @@ -16,8 +16,11 @@ package com.twosigma.beakerx.scala.kernel; import com.twosigma.beakerx.DisplayerDataMapper; +import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.evaluator.Evaluator; +import com.twosigma.beakerx.evaluator.TempFolderFactoryImpl; import com.twosigma.beakerx.handler.KernelHandler; +import com.twosigma.beakerx.jvm.threads.BeakerCellExecutor; import com.twosigma.beakerx.kernel.CacheFolderFactory; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.CustomMagicCommandsFactory; @@ -30,6 +33,7 @@ import com.twosigma.beakerx.kernel.handler.CommOpenHandler; import com.twosigma.beakerx.message.Message; import com.twosigma.beakerx.scala.comm.ScalaCommOpenHandler; +import com.twosigma.beakerx.scala.evaluator.BeakerxObjectFactoryImpl; import com.twosigma.beakerx.scala.evaluator.ScalaEvaluator; import com.twosigma.beakerx.scala.handler.ScalaKernelInfoHandler; import com.twosigma.beakerx.scala.magic.command.CustomMagicCommandsImpl; @@ -73,12 +77,17 @@ public KernelHandler getKernelInfoHandler(Kernel kernel) { public static void main(final String[] args) { KernelRunner.run(() -> { String id = uuid(); - ScalaEvaluator se = new ScalaEvaluator(id, id, - null, - getKernelParameters());//TODO check what to put, need for autotranslation - - //js.setupAutoTranslation(); -- uncomment - KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl(new KernelConfigurationFile(args)); + KernelConfigurationFile configurationFile = new KernelConfigurationFile(args); + KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl(configurationFile); + NamespaceClient namespaceClient = NamespaceClient.create(id, configurationFile, new ScalaBeakerXJsonSerializer()); + ScalaEvaluator se = new ScalaEvaluator( + id, + id, + new BeakerCellExecutor("scala"), + new BeakerxObjectFactoryImpl(), + new TempFolderFactoryImpl(), + getKernelParameters(), + namespaceClient); return new Scala(id, se, kernelSocketsFactory); }); } @@ -103,7 +112,7 @@ private static Object asJava(Object scalaObject) { return scalaObject; } - public static EvaluatorParameters getKernelParameters() { + private static EvaluatorParameters getKernelParameters() { HashMap kernelParameters = new HashMap<>(); kernelParameters.put(IMPORTS, new ScalaDefaultVariables().getImports()); return new EvaluatorParameters(kernelParameters); diff --git a/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/ScalaBeakerXJsonSerializer.java b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/ScalaBeakerXJsonSerializer.java new file mode 100644 index 0000000000..3046f61407 --- /dev/null +++ b/kernel/scala/src/main/java/com/twosigma/beakerx/scala/kernel/ScalaBeakerXJsonSerializer.java @@ -0,0 +1,81 @@ +/* + * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twosigma.beakerx.scala.kernel; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.twosigma.beakerx.BeakerXJsonSerializer; +import com.twosigma.beakerx.jvm.serialization.BasicObjectSerializer; +import com.twosigma.beakerx.jvm.serialization.BeakerObjectConverter; +import com.twosigma.beakerx.scala.serializers.ScalaCollectionDeserializer; +import com.twosigma.beakerx.scala.serializers.ScalaCollectionSerializer; +import com.twosigma.beakerx.scala.serializers.ScalaListOfPrimitiveTypeMapsSerializer; +import com.twosigma.beakerx.scala.serializers.ScalaMapDeserializer; +import com.twosigma.beakerx.scala.serializers.ScalaMapSerializer; +import com.twosigma.beakerx.scala.serializers.ScalaPrimitiveTypeListOfListSerializer; +import com.twosigma.beakerx.scala.serializers.ScalaPrimitiveTypeMapSerializer; +import com.twosigma.beakerx.scala.serializers.ScalaTableDeSerializer; +import com.twosigma.beakerx.table.TableDisplayToJson; + +import java.io.IOException; +import java.io.StringWriter; + +import static com.fasterxml.jackson.databind.SerializationFeature.WRITE_ENUMS_USING_TO_STRING; + +public class ScalaBeakerXJsonSerializer implements BeakerXJsonSerializer { + + private final ObjectMapper objectMapper; + private final BeakerObjectConverter serializer; + + public ScalaBeakerXJsonSerializer() { + this.objectMapper = new ObjectMapper(); + this.objectMapper.enable(WRITE_ENUMS_USING_TO_STRING); + this.objectMapper.registerModule(TableDisplayToJson.tableDisplayModule()); + this.serializer = createSerializer(); + } + + @Override + public String toJson(Object value) { + StringWriter sw = new StringWriter(); + try { + JsonGenerator jgen = objectMapper.getFactory().createGenerator(sw); + boolean success = serializer.writeObject(value, jgen, true); + jgen.flush(); + sw.flush(); + if (success) { + return sw.toString(); + } else { + return value.toString(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static BeakerObjectConverter createSerializer() { + BasicObjectSerializer serializer = new BasicObjectSerializer(); + serializer.addfTypeSerializer(new ScalaCollectionSerializer(serializer)); + serializer.addfTypeSerializer(new ScalaMapSerializer(serializer)); + serializer.addfTypeSerializer(new ScalaPrimitiveTypeListOfListSerializer(serializer)); + serializer.addfTypeSerializer(new ScalaListOfPrimitiveTypeMapsSerializer(serializer)); + serializer.addfTypeSerializer(new ScalaPrimitiveTypeMapSerializer(serializer)); + serializer.addfTypeDeserializer(new ScalaCollectionDeserializer(serializer)); + serializer.addfTypeDeserializer(new ScalaMapDeserializer(serializer)); + serializer.addfTypeDeserializer(new ScalaTableDeSerializer(serializer)); + return serializer; + } + +} diff --git a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/TestScalaEvaluator.java b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/TestScalaEvaluator.java index 374b60c372..562f8d64e0 100644 --- a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/TestScalaEvaluator.java +++ b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/TestScalaEvaluator.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.scala; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.evaluator.TempFolderFactory; import com.twosigma.beakerx.kernel.EvaluatorParameters; import com.twosigma.beakerx.scala.evaluator.NoBeakerxObjectTestFactory; @@ -30,12 +31,12 @@ public class TestScalaEvaluator { public static ScalaEvaluator evaluator() { - ScalaEvaluator evaluator = new ScalaEvaluator("id", "sid", null, cellExecutor(), new NoBeakerxObjectTestFactory(), getTestTempFolderFactory(),getKernelParameters()); + ScalaEvaluator evaluator = new ScalaEvaluator("id", "sid", cellExecutor(), new NoBeakerxObjectTestFactory(), getTestTempFolderFactory(), getKernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); return evaluator; } public static ScalaEvaluator evaluator(TempFolderFactory tempFolderFactory) { - ScalaEvaluator evaluator = new ScalaEvaluator("id", "sid", null, cellExecutor(), new NoBeakerxObjectTestFactory(), tempFolderFactory, getKernelParameters()); + ScalaEvaluator evaluator = new ScalaEvaluator("id", "sid", cellExecutor(), new NoBeakerxObjectTestFactory(), tempFolderFactory, getKernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); return evaluator; } diff --git a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/NoBeakerxObjectTestFactory.java b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/NoBeakerxObjectTestFactory.java index ed4978d263..a11bf5e4f5 100644 --- a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/NoBeakerxObjectTestFactory.java +++ b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/NoBeakerxObjectTestFactory.java @@ -18,7 +18,7 @@ public class NoBeakerxObjectTestFactory implements BeakerxObjectFactory { @Override - public String create(String sessionId) { + public String create() { return "()"; } } diff --git a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/ScalaAutocompleteTest.java b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/ScalaAutocompleteTest.java index 7a39d31ea4..8973f3be96 100644 --- a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/ScalaAutocompleteTest.java +++ b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/evaluator/ScalaAutocompleteTest.java @@ -38,7 +38,7 @@ public class ScalaAutocompleteTest { @BeforeClass public static void setUpClass() throws Exception { - scalaEvaluator = new ScalaEvaluator("id", "sid", null, cellExecutor(), new NoBeakerxObjectTestFactory(), getTestTempFolderFactory(), EvaluatorTest.KERNEL_PARAMETERS); + scalaEvaluator = new ScalaEvaluator("id", "sid", cellExecutor(), new NoBeakerxObjectTestFactory(), getTestTempFolderFactory(), EvaluatorTest.KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); } @Before @@ -98,8 +98,8 @@ public void autocomplete_internalDeclarationsAreVisible() { public void autocomplete_interpretedResultsVisible() { // This test needs a fresh ScalaEvaluator to modify without disturbing other tests final ScalaEvaluator localEvaluator = - new ScalaEvaluator("id", "sid", null, cellExecutor(), new NoBeakerxObjectTestFactory(), - getTestTempFolderFactory(), EvaluatorTest.KERNEL_PARAMETERS); + new ScalaEvaluator("id", "sid", cellExecutor(), new NoBeakerxObjectTestFactory(), + getTestTempFolderFactory(), EvaluatorTest.KERNEL_PARAMETERS, new EvaluatorTest.BeakexClientTestImpl()); try { //when diff --git a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/kernel/ScalaKernelTest.java b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/kernel/ScalaKernelTest.java index 41e25aaf2f..5226e5a7f9 100644 --- a/kernel/scala/src/test/java/com/twosigma/beakerx/scala/kernel/ScalaKernelTest.java +++ b/kernel/scala/src/test/java/com/twosigma/beakerx/scala/kernel/ScalaKernelTest.java @@ -43,8 +43,15 @@ public class ScalaKernelTest extends KernelExecutionTest { @Override protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { - ScalaEvaluator evaluator = new ScalaEvaluator(sessionId, sessionId, null, cellExecutor(), new NoBeakerxObjectTestFactory(), getTestTempFolderFactory(), EvaluatorTest.KERNEL_PARAMETERS); - return new Scala(sessionId, evaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory(),new CustomMagicCommandsEmptyImpl()); + ScalaEvaluator evaluator = new ScalaEvaluator( + sessionId, + sessionId, + cellExecutor(), + new NoBeakerxObjectTestFactory(), + getTestTempFolderFactory(), + EvaluatorTest.KERNEL_PARAMETERS, + new EvaluatorTest.BeakexClientTestImpl()); + return new Scala(sessionId, evaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory(), new CustomMagicCommandsEmptyImpl()); } @Override @@ -91,7 +98,7 @@ public void noKeepVariablesWhenAddJar() throws Exception { //then Optional result = waitForErrorMessage(getKernelSocketsService().getKernelSockets()); assertThat(result).isPresent(); - assertThat((String)result.get().getContent().get("text")).contains("not found"); + assertThat((String) result.get().getContent().get("text")).contains("not found"); } private void runStatement(String code) throws InterruptedException { diff --git a/kernel/sparkex/src/test/java/com/twosigma/beakerx/EnableSparkSupportTest.java b/kernel/sparkex/src/test/java/com/twosigma/beakerx/EnableSparkSupportTest.java index c142579ff9..e8def3d4cf 100644 --- a/kernel/sparkex/src/test/java/com/twosigma/beakerx/EnableSparkSupportTest.java +++ b/kernel/sparkex/src/test/java/com/twosigma/beakerx/EnableSparkSupportTest.java @@ -53,11 +53,11 @@ public class EnableSparkSupportTest extends KernelSetUpFixtureTest { protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { ScalaEvaluator evaluator = new ScalaEvaluator(sessionId, sessionId, - null, TestBeakerCellExecutor.cellExecutor(), new NoBeakerxObjectTestFactory(), EvaluatorTest.getTestTempFolderFactory(), - getKernelParameters()); + getKernelParameters() + , new EvaluatorTest.BeakexClientTestImpl()); return new Scala(sessionId, evaluator, kernelSocketsFactory, @@ -95,12 +95,12 @@ private void runSparkDataset(String returnStatement) throws InterruptedException String peoplePath = EnableSparkSupportTest.class.getClassLoader().getResource("people.json").getPath(); String code = "val spark = SparkSession\n" + - " .builder\n" + - " .appName(\"jupyter\")\n" + - " .master(\"local[*]\")\n" + - " .getOrCreate()\n" + - "val ds = spark.read.json(\"file://" + peoplePath + "\")\n" - + returnStatement; + " .builder\n" + + " .appName(\"jupyter\")\n" + + " .master(\"local[*]\")\n" + + " .getOrCreate()\n" + + "val ds = spark.read.json(\"file://" + peoplePath + "\")\n" + + returnStatement; Message messageWithCode = MessageFactoryTest.getExecuteRequestMessage(code); getKernelSocketsService().handleMsg(messageWithCode); diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/BeakerParser.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/BeakerParser.java index 42b28655ee..b4de07d456 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/BeakerParser.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/BeakerParser.java @@ -15,7 +15,7 @@ */ package com.twosigma.beakerx.sql; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; import java.io.IOException; import java.sql.SQLException; @@ -41,8 +41,8 @@ public class BeakerParser { public static final String NO_DATASOURCES_ERROR = "No datasource"; protected final JDBCClient jdbcClient; - - private NamespaceClient client; + + private BeakerXClient client; private ConnectionStringHolder dbURI; private Map inputs = new HashMap<>(); private Set outputs = new HashSet<>(); @@ -51,7 +51,7 @@ public class BeakerParser { private List results = new ArrayList<>(); - public BeakerParser(String script, NamespaceClient client, ConnectionStringHolder defaultConnectionString, Map namedConnectionString, JDBCClient jdbcClient) throws IOException, DBConnectionException { + public BeakerParser(String script, BeakerXClient client, ConnectionStringHolder defaultConnectionString, Map namedConnectionString, JDBCClient jdbcClient) throws IOException, DBConnectionException { this.client = client; this.jdbcClient = jdbcClient; this.defaultConnectionString = defaultConnectionString; diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/QueryExecutor.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/QueryExecutor.java index d9bd582f65..2c4d6175df 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/QueryExecutor.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/QueryExecutor.java @@ -15,7 +15,7 @@ */ package com.twosigma.beakerx.sql; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.jvm.object.OutputCell; import com.twosigma.beakerx.jvm.object.OutputContainer; import com.twosigma.beakerx.table.TableDisplay; @@ -47,7 +47,7 @@ public QueryExecutor(JDBCClient jdbcClient) { this.jdbcClient = jdbcClient; } - public synchronized Object executeQuery(String script, NamespaceClient namespaceClient, ConnectionStringHolder defaultConnectionString, Map namedConnectionString) + public synchronized Object executeQuery(String script, BeakerXClient namespaceClient, ConnectionStringHolder defaultConnectionString, Map namedConnectionString) throws SQLException, IOException, ReadVariableException { BeakerParser beakerParser = new BeakerParser(script, namespaceClient, defaultConnectionString, namedConnectionString, jdbcClient); @@ -183,7 +183,7 @@ private void adoptResult(BeakerParseResult queryLine, QueryResult queryResult, L } } - private QueryResult executeQuery(int currentIterationIndex, BeakerParseResult queryLine, Connection conn, NamespaceClient namespaceClient) throws SQLException, ReadVariableException { + private QueryResult executeQuery(int currentIterationIndex, BeakerParseResult queryLine, Connection conn, BeakerXClient namespaceClient) throws SQLException, ReadVariableException { QueryResult queryResult = new QueryResult(); diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/autocomplete/db/DbExplorerFactory.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/autocomplete/db/DbExplorerFactory.java index 80dcbf050c..8d1f3088e8 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/autocomplete/db/DbExplorerFactory.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/autocomplete/db/DbExplorerFactory.java @@ -16,7 +16,8 @@ package com.twosigma.beakerx.sql.autocomplete.db; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; +import com.twosigma.beakerx.BeakerXClientManager; import com.twosigma.beakerx.sql.BeakerParser; import com.twosigma.beakerx.sql.ConnectionStringHolder; import com.twosigma.beakerx.sql.DBConnectionException; @@ -39,7 +40,7 @@ public class DbExplorerFactory { public static DbInfo getDbInfo(String txt, JDBCClient jdbcClient, String sessionId, ConnectionStringHolder defaultConnectionString, Map namedConnectionString) { - final NamespaceClient namespaceClient = NamespaceClient.getBeaker(sessionId); + final BeakerXClient namespaceClient = BeakerXClientManager.get(); final BeakerParser beakerParser; try { beakerParser = new BeakerParser(txt, namespaceClient, diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLCodeRunner.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLCodeRunner.java index f0ceb20d50..40656c3d00 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLCodeRunner.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLCodeRunner.java @@ -15,9 +15,8 @@ */ package com.twosigma.beakerx.sql.evaluator; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; -import com.twosigma.beakerx.evaluator.InternalVariable; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; import com.twosigma.beakerx.sql.ReadVariableException; import org.slf4j.Logger; @@ -34,9 +33,9 @@ class SQLCodeRunner implements Callable { private SQLEvaluator sqlEvaluator; private final SimpleEvaluationObject simpleEvaluationObject; - private final NamespaceClient namespaceClient; + private final BeakerXClient namespaceClient; - SQLCodeRunner(SQLEvaluator sqlEvaluator, SimpleEvaluationObject seo, NamespaceClient namespaceClient) { + SQLCodeRunner(SQLEvaluator sqlEvaluator, SimpleEvaluationObject seo, BeakerXClient namespaceClient) { this.sqlEvaluator = sqlEvaluator; this.simpleEvaluationObject = seo; this.namespaceClient = namespaceClient; diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLEvaluator.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLEvaluator.java index 6f9d441e77..f1dd02737a 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLEvaluator.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLEvaluator.java @@ -16,7 +16,7 @@ package com.twosigma.beakerx.sql.evaluator; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.autocomplete.AutocompleteResult; import com.twosigma.beakerx.autocomplete.ClasspathScanner; @@ -67,16 +67,17 @@ public class SQLEvaluator extends BaseEvaluator { private JDBCClient jdbcClient; private DynamicClassLoaderSimple loader; - public SQLEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters) { - this(id, sId, new BeakerCellExecutor("sql"), new TempFolderFactoryImpl(), evaluatorParameters); + public SQLEvaluator(String id, String sId, EvaluatorParameters evaluatorParameters, BeakerXClient beakerxClient) { + this(id, sId, new BeakerCellExecutor("sql"), new TempFolderFactoryImpl(), evaluatorParameters, beakerxClient); } public SQLEvaluator(String id, String sId, CellExecutor cellExecutor, TempFolderFactory tempFolderFactory, - EvaluatorParameters evaluatorParameters) { - super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters); + EvaluatorParameters evaluatorParameters, + BeakerXClient beakerxClient) { + super(id, sId, cellExecutor, tempFolderFactory, evaluatorParameters,beakerxClient); cps = new ClasspathScanner(); sac = createSqlAutocomplete(cps); loader = reloadClassLoader(); @@ -238,7 +239,7 @@ public List getListOfConnectiononWhoNeedDialog() { return ret; } - public Object executeQuery(String expression, NamespaceClient namespaceClient, ConnectionStringHolder defaultConnectionString, Map namedConnectionString) throws SQLException, IOException, ReadVariableException { + public Object executeQuery(String expression, BeakerXClient namespaceClient, ConnectionStringHolder defaultConnectionString, Map namedConnectionString) throws SQLException, IOException, ReadVariableException { return queryExecutor.executeQuery(expression, namespaceClient, defaultConnectionString, namedConnectionString); } diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLWorkerThread.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLWorkerThread.java index 47571fe97f..380e8b36b5 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLWorkerThread.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/evaluator/SQLWorkerThread.java @@ -15,7 +15,7 @@ */ package com.twosigma.beakerx.sql.evaluator; -import com.twosigma.beakerx.NamespaceClient; +import com.twosigma.beakerx.BeakerXClient; import com.twosigma.beakerx.TryResult; import com.twosigma.beakerx.evaluator.JobDescriptor; import org.slf4j.Logger; @@ -36,20 +36,16 @@ class SQLWorkerThread implements Callable { @Override public TryResult call() throws Exception { - NamespaceClient namespaceClient = null; + BeakerXClient namespaceClient = null; TryResult r; job.getSimpleEvaluationObject().started(); try { job.getSimpleEvaluationObject().setOutputHandler(); - namespaceClient = NamespaceClient.getBeaker(sqlEvaluator.getSessionId()); - namespaceClient.setOutputObj(job.getSimpleEvaluationObject()); + namespaceClient = sqlEvaluator.getBeakerX(); r = sqlEvaluator.executeTask(new SQLCodeRunner(sqlEvaluator, job.getSimpleEvaluationObject(), namespaceClient)); } finally { job.getSimpleEvaluationObject().clrOutputHandler(); - if (namespaceClient != null) { - namespaceClient.setOutputObj(null); - } } return r; } diff --git a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/kernel/SQL.java b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/kernel/SQL.java index 5ef5d8971d..993894a7dd 100644 --- a/kernel/sql/src/main/java/com/twosigma/beakerx/sql/kernel/SQL.java +++ b/kernel/sql/src/main/java/com/twosigma/beakerx/sql/kernel/SQL.java @@ -21,6 +21,7 @@ import static com.twosigma.beakerx.sql.magic.command.DefaultDataSourcesMagicCommand.DEFAULT_DATASOURCE; import com.twosigma.beakerx.DefaultJVMVariables; +import com.twosigma.beakerx.NamespaceClient; import com.twosigma.beakerx.evaluator.Evaluator; import com.twosigma.beakerx.handler.KernelHandler; import com.twosigma.beakerx.kernel.CacheFolderFactory; @@ -44,6 +45,7 @@ import com.twosigma.beakerx.util.BeakerXSystem; import com.twosigma.beakerx.util.BeakerXSystemImpl; +import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -52,6 +54,7 @@ public class SQL extends Kernel { + private final static Logger logger = Logger.getLogger(SQL.class.getName()); public static final String BEAKERX_SQL_DEFAULT_JDBC = "BEAKERX_SQL_DEFAULT_JDBC"; @@ -73,18 +76,19 @@ public KernelHandler getKernelInfoHandler(Kernel kernel) { return new SQLKernelInfoHandler(kernel); } - public static void main(final String[] args) { KernelRunner.run(() -> { String id = uuid(); + KernelConfigurationFile configurationFile = new KernelConfigurationFile(args); KernelSocketsFactoryImpl kernelSocketsFactory = new KernelSocketsFactoryImpl( - new KernelConfigurationFile(args)); + configurationFile); EvaluatorParameters params = getKernelParameters(new BeakerXSystemImpl()); - SQLEvaluator evaluator = new SQLEvaluator(id, id, params); + SQLEvaluator evaluator = new SQLEvaluator(id, id, params,NamespaceClient.create(id, configurationFile)); return new SQL(id, evaluator, kernelSocketsFactory); }); } + static EvaluatorParameters getKernelParameters(BeakerXSystem beakerXSystem) { Map kernelParameters = new HashMap<>(); kernelParameters.put(IMPORTS, new DefaultJVMVariables().getImports()); @@ -103,6 +107,7 @@ private static Map getDefaultConnectionString(BeakerXSystem beak return kernelParameters; } + static class SQLCustomMagicCommandsImpl implements CustomMagicCommandsFactory { @Override public List customMagicCommands(KernelFunctionality kernel) { diff --git a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLAutocompleteTest.java b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLAutocompleteTest.java index 4e8efdbe02..4e2e43bf52 100644 --- a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLAutocompleteTest.java +++ b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLAutocompleteTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.KernelTest; import com.twosigma.beakerx.autocomplete.AutocompleteResult; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; import com.twosigma.beakerx.kernel.EvaluatorParameters; @@ -41,7 +42,7 @@ public class SQLAutocompleteTest { @Before public void setUp() throws Exception { - sqlEvaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters()); + sqlEvaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); kernelTest = new KernelTest("id1", sqlEvaluator); } @@ -97,7 +98,7 @@ public void shouldAutocompleteAfterSpace() throws Exception { assertThat(autocomplete.getStartIndex()).isEqualTo(code.length()); } - private void givenColorTable() { + private void givenColorTable() { SimpleEvaluationObject seo = new SimpleEvaluationObject(SQLForColorTable.CREATE_AND_SELECT_ALL); sqlEvaluator.evaluate(seo, SQLForColorTable.CREATE_AND_SELECT_ALL); } diff --git a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLBaseEvaluatorTest.java b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLBaseEvaluatorTest.java index 68d9c33ec8..5ad77721de 100644 --- a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLBaseEvaluatorTest.java +++ b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLBaseEvaluatorTest.java @@ -24,7 +24,6 @@ import org.junit.AfterClass; import org.junit.BeforeClass; -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -39,7 +38,7 @@ public class SQLBaseEvaluatorTest extends EvaluatorBaseTest { @BeforeClass public static void setUpClass() throws Exception { - evaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters()); + evaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); } @AfterClass @@ -49,13 +48,13 @@ public static void tearDown() throws Exception { @Override protected BaseEvaluator createNewEvaluator() { - SQLEvaluator evaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(),kernelParameters()); + SQLEvaluator evaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); return evaluator; } @Override protected BaseEvaluator createNewEvaluator(TempFolderFactory tempFolderFactory) { - SQLEvaluator evaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), tempFolderFactory, kernelParameters()); + SQLEvaluator evaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), tempFolderFactory, kernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); return evaluator; } diff --git a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLEvaluatorTest.java b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLEvaluatorTest.java index 3837757501..7ed0148f34 100644 --- a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLEvaluatorTest.java +++ b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLEvaluatorTest.java @@ -17,6 +17,7 @@ import com.twosigma.beakerx.KernelTest; import com.twosigma.beakerx.TryResult; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.KernelManager; import com.twosigma.beakerx.jvm.object.OutputCell; import com.twosigma.beakerx.jvm.object.SimpleEvaluationObject; @@ -44,7 +45,7 @@ public class SQLEvaluatorTest { @Before public void setUp() throws Exception { - sqlEvaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters()); + sqlEvaluator = new SQLEvaluator("shellId1", "sessionId1", cellExecutor(), getTestTempFolderFactory(), kernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); sqlEvaluator.updateEvaluatorParameters(kernelParameters()); kernelTest = new KernelTest("1", sqlEvaluator); KernelManager.register(kernelTest); diff --git a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLKernelTest.java b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLKernelTest.java index 88bc66f0ca..e358c6de92 100644 --- a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLKernelTest.java +++ b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/SQLKernelTest.java @@ -18,6 +18,7 @@ import com.twosigma.beakerx.KernelSetUpFixtureTest; import com.twosigma.beakerx.KernelSocketsServiceTest; import com.twosigma.beakerx.KernelSocketsTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.Kernel; import com.twosigma.beakerx.kernel.EvaluatorParameters; @@ -48,7 +49,7 @@ public class SQLKernelTest extends KernelSetUpFixtureTest { @Override protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { - SQLEvaluator sqlEvaluator = new SQLEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), kernelParameters()); + SQLEvaluator sqlEvaluator = new SQLEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), kernelParameters(), new EvaluatorTest.BeakexClientTestImpl()); Kernel sqlKernel = new SQL(sessionId, sqlEvaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory()); return sqlKernel; } diff --git a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWIthDefaultEnvsTest.java b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWIthDefaultEnvsTest.java index bb041aa9b0..1c2a76db3f 100644 --- a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWIthDefaultEnvsTest.java +++ b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWIthDefaultEnvsTest.java @@ -15,6 +15,7 @@ */ package com.twosigma.beakerx.sql.kernel; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.EvaluatorParameters; import com.twosigma.beakerx.kernel.Kernel; @@ -39,7 +40,7 @@ protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocke } return null; }); - SQLEvaluator sqlEvaluator = new SQLEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), kernelParameters); + SQLEvaluator sqlEvaluator = new SQLEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), kernelParameters, new EvaluatorTest.BeakexClientTestImpl()); Kernel sqlKernel = new SQL(sessionId, sqlEvaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory()); return sqlKernel; } diff --git a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWithoutDefaultJDBCUriTest.java b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWithoutDefaultJDBCUriTest.java index bc9e161aca..2f951fdedb 100644 --- a/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWithoutDefaultJDBCUriTest.java +++ b/kernel/sql/src/test/java/com/twosigma/beakerx/sql/kernel/SQLKernelWithoutDefaultJDBCUriTest.java @@ -16,6 +16,7 @@ package com.twosigma.beakerx.sql.kernel; import com.twosigma.beakerx.KernelSetUpFixtureTest; +import com.twosigma.beakerx.evaluator.EvaluatorTest; import com.twosigma.beakerx.kernel.CloseKernelAction; import com.twosigma.beakerx.kernel.EvaluatorParameters; import com.twosigma.beakerx.kernel.Kernel; @@ -41,7 +42,7 @@ public class SQLKernelWithoutDefaultJDBCUriTest extends KernelSetUpFixtureTest { @Override protected Kernel createKernel(String sessionId, KernelSocketsFactory kernelSocketsFactory, CloseKernelAction closeKernelAction) { EvaluatorParameters kernelParameters = getKernelParameters(name -> null); - SQLEvaluator sqlEvaluator = new SQLEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), kernelParameters); + SQLEvaluator sqlEvaluator = new SQLEvaluator(sessionId, sessionId, cellExecutor(), getTestTempFolderFactory(), kernelParameters,new EvaluatorTest.BeakexClientTestImpl()); return new SQL(sessionId, sqlEvaluator, kernelSocketsFactory, closeKernelAction, getCacheFolderFactory()); } diff --git a/test/ipynb/groovy/AutotranslationBetweenKernels.ipynb b/test/ipynb/groovy/AutotranslationBetweenKernels.ipynb new file mode 100644 index 0000000000..e77359ea7a --- /dev/null +++ b/test/ipynb/groovy/AutotranslationBetweenKernels.ipynb @@ -0,0 +1,99 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "beakerx.x =1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%javascript\n", + "console.log(beakerx.x)\n", + "beakerx.x =2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "beakerx.x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%scala\n", + "beakerx.x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%scala\n", + "beakerx.x = 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "beakerx.x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Groovy", + "language": "groovy", + "name": "groovy" + }, + "language_info": { + "codemirror_mode": "groovy", + "file_extension": ".groovy", + "mimetype": "", + "name": "Groovy", + "nbconverter_exporter": "", + "version": "2.4.3" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": false, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": false, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test/ipynb/scala/autotranslation_scala_structured_data.ipynb b/test/ipynb/scala/autotranslation_scala_structured_data.ipynb new file mode 100644 index 0000000000..48dfaaa57a --- /dev/null +++ b/test/ipynb/scala/autotranslation_scala_structured_data.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Structured data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_beakerx.set(\"seq\", Seq(1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_beakerx.set(\"map\", Map(\"1\" -> 2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%javascript\n", + "alert(JSON.stringify(beakerx.seq))\n", + "alert(JSON.stringify(beakerx.map))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_beakerx.get(\"map\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_beakerx.set(\"int\", 11)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%javascript\n", + "alert(JSON.stringify(beakerx.int))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import collection.JavaConverters._\n", + "\n", + "_beakerx.set(\"map\", Map(\"1\" -> 2).asJava)\n", + "_beakerx.set(\"seq\", Seq(1).asJava)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%javascript\n", + "\n", + "alert(JSON.stringify(beakerx.seq))\n", + "alert(JSON.stringify(beakerx.map))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Scala", + "language": "scala", + "name": "scala" + }, + "language_info": { + "codemirror_mode": "text/x-scala", + "file_extension": ".scala", + "mimetype": "", + "name": "Scala", + "nbconverter_exporter": "", + "version": "2.11.12" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": false, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": false, + "toc_window_display": false + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "47aec6d4-e0fa-44ae-bd17-cf171beac794": { + "model_module": "beakerx", + "model_module_version": "*", + "model_name": "TableDisplayModel", + "state": { + "model": { + "alignmentForColumn": {}, + "alignmentForType": {}, + "cellHighlighters": [], + "columnNames": [ + "Key", + "Value" + ], + "columnOrder": [], + "columnsFrozen": {}, + "columnsFrozenRight": {}, + "columnsVisible": {}, + "contextMenuItems": [], + "contextMenuTags": {}, + "dataFontSize": null, + "doubleClickTag": null, + "fontColor": [], + "hasDoubleClickAction": false, + "hasIndex": null, + "headerFontSize": null, + "headersVertical": false, + "rendererForColumn": {}, + "rendererForType": {}, + "stringFormatForColumn": {}, + "stringFormatForTimes": null, + "stringFormatForType": {}, + "subtype": "Dictionary", + "timeZone": null, + "tooManyRows": false, + "tooltips": [], + "type": "TableDisplay", + "types": [], + "values": [ + [ + "1", + 2 + ] + ] + } + } + }, + "aed45398-26c4-48cd-992f-584f7b358e6c": { + "model_module": "beakerx", + "model_module_version": "*", + "model_name": "TableDisplayModel", + "state": { + "model": { + "alignmentForColumn": {}, + "alignmentForType": {}, + "cellHighlighters": [], + "columnNames": [ + "Key", + "Value" + ], + "columnOrder": [], + "columnsFrozen": {}, + "columnsFrozenRight": {}, + "columnsVisible": {}, + "contextMenuItems": [], + "contextMenuTags": {}, + "dataFontSize": null, + "doubleClickTag": null, + "fontColor": [], + "hasDoubleClickAction": false, + "hasIndex": null, + "headerFontSize": null, + "headersVertical": false, + "rendererForColumn": {}, + "rendererForType": {}, + "stringFormatForColumn": {}, + "stringFormatForTimes": null, + "stringFormatForType": {}, + "subtype": "Dictionary", + "timeZone": null, + "tooManyRows": false, + "tooltips": [], + "type": "TableDisplay", + "types": [], + "values": [ + [ + "1", + 2 + ] + ] + } + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}