Merge branch 'master' of https://github.com/twosigma/beakerx into luk…

…asz/7625

README.md

@@ -24,14 +24,14 @@
 [![NPM version](https://badge.fury.io/js/beakerx.svg)](http://badge.fury.io/js/beakerx)
 [![PyPI Version](https://badge.fury.io/py/beakerx.svg)](http://badge.fury.io/py/beakerx)
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/beakerx/badges/version.svg)](https://anaconda.org/conda-forge/beakerx)
-[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/twosigma/beakerx/0.21.0?filepath=StartHere.ipynb)
+[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/twosigma/beakerx/1.0.0?filepath=StartHere.ipynb)
 
 BeakerX is a collection of JVM kernels and interactive widgets for
 plotting, tables, autotranslation, and other extensions to Jupyter
 Notebook.  BeakerX is in beta and under active development.
 
 The [documentation](https://github.com/twosigma/beakerx/blob/master/StartHere.ipynb) consists of tutorial notebooks on GitHub.
-You can try it in the cloud for free with [Binder](https://mybinder.org/v2/gh/twosigma/beakerx/0.21.0?filepath=StartHere.ipynb).
+You can try it in the cloud for free with [Binder](https://mybinder.org/v2/gh/twosigma/beakerx/1.0.0?filepath=StartHere.ipynb).
 
 BeakerX is the successor to the [Beaker Notebook (source code
 archive)](https://github.com/twosigma/beaker-notebook-archive).  It

StartHere.ipynb

@@ -6,9 +6,9 @@
    "source": [
     "# Tutorials, Examples, and Demos\n",
     "\n",
-    "[Jupyter](http://jupyter.org/) and [BeakerX](http://beakerx.com/) are based on the idea of the lab notebook, brought to life in your web browser. Each notebook is a place for recording the written ideas, data, images, spreadsheets, diagrams, equations, and especially code, that one produces in the course of research. You can analyze, visualize, and document data and science, using multiple programming languages.  BeakerX is an extension of Jupyter, including kernels for the JVM langauges, autotranslation between languages ([prototype](https://github.com/twosigma/beakerx/issues/5039)), interactive plots, tables, and more.\n",
+    "[Jupyter](http://jupyter.org/) and [BeakerX](http://beakerx.com/) are based on the idea of the lab notebook, brought to life in your web browser. Each notebook is a place for recording the written ideas, data, images, spreadsheets, diagrams, equations, and especially code, that one produces in the course of research. You can analyze, visualize, and document data and science, using multiple programming languages.  BeakerX is an extension of Jupyter, including kernels for the JVM langauges, autotranslation between languages ([prototype](https://github.com/twosigma/beakerx/issues/5039)), interactive plots, tables, Spark, and more.\n",
     "\n",
-    "This is a beta release. There are known issues, please see our [issue tracker](https://github.com/twosigma/beakerx/issues). Your [email feedback](mailto:beakerx-feedback@twosigma.com) is most welcome.\n",
+    "Please see our [issue tracker](https://github.com/twosigma/beakerx/issues) for documentation of the known bugs and plans for future development. Your [email feedback](mailto:beakerx-feedback@twosigma.com) is most welcome.\n",
     "\n",
     "[The Jupyter documentation](http://jupyter-notebook.readthedocs.io/en/latest/examples/Notebook/Running%20Code.html) covers interacting with code cells, markdown, and notebooks. The tutorials below show the features of the BeakerX extension.\n",
     "\n",

beakerx/beakerx/_version.py

@@ -1,2 +1,2 @@
-version_info = (0, 21, 0)
+version_info = (1, 0, 0)
 __version__ = '.'.join(map(str, version_info))

beakerx/beakerx/runtime.py

@@ -423,13 +423,13 @@ def __get__(self, model_instance, model_class):
         def f():
             display_html(TableDisplay(model_instance))
         return f
-
+
+
 class BeakerX:
-    """Runtime support for Python code in BeakerX."""
-    _comm = Comm(target_name='beakerx.autotranslation')
 
     def __init__(self):
         BeakerX.pandas_display_table()
+        self._comm = None
 
     @staticmethod
     def pandas_display_default():
@@ -445,12 +445,18 @@ def set4(self, var, val, unset, sync):
             val = transform(val)
             args['value'] = json.dumps(val, cls=DataFrameEncoder)
             state = {'state': args}
+        if self._comm is None:
+            self.init_autotranslation_comm()
         self._comm.send(data=state)
 
+    def init_autotranslation_comm(self):
+        self._comm = Comm(target_name='beakerx.autotranslation')
+        self._comm.open()
+
     def get(self, var):
         result = autotranslation_get(var)
         if result == 'undefined':
-            raise NameError('name \'' + var + '\' is not defined in notebook namespace')
+            raise NameError('name \'' + var + '\' is not defined on the beakerx object')
         return transformBack(json.loads(result))
 
     def set_session(self, id):
@@ -578,9 +584,14 @@ def __setattr__(self, name, value):
         if 'session_id' == name:
             self.__dict__['session_id'] = value
             return
+        if '_comm' == name:
+            self.__dict__['_comm'] = value
+            return
         return self.set(name, value)
 
     def __getattr__(self, name):
+        if '_comm' == name:
+            return self.__dict__['_comm']
         return self.get(name)
 
     def __contains__(self, name):

beakerx/beakerx/tabledisplay/tableitems.py

@@ -32,7 +32,6 @@ class TimeUnit(Enum):
     HOURS = 7
 
 
-
 class ColumnType(Enum):
     String = "string"
     Double = "double"
@@ -65,9 +64,15 @@ def __init__(self, min=4, max=4):
         self.minDecimals = min
         self.maxDecimals = max
 
+
+class ImageFormat:
+    type = "image"
+
+
 class HTMLFormat:
     type = "html"
 
+
 class HighlightStyle(Enum):
     FULL_ROW = 1
     SINGLE_COLUMN = 2
@@ -91,12 +96,19 @@ def getDataBarsRenderer(include_text=True):
 
 
 class TableDisplayStringFormat:
+
     @staticmethod
     def getDecimalFormat(min, max):
         return DecimalStringFormat(min, max)
+
+    @staticmethod
     def getHTMLFormat():
         return HTMLFormat()
 
+    @staticmethod
+    def getImageFormat():
+        return ImageFormat()
+
 class TableDisplayCellHighlighter:
     FULL_ROW = HighlightStyle.FULL_ROW
     SINGLE_COLUMN = HighlightStyle.SINGLE_COLUMN

doc/groovy/GeneralAutotranslation.ipynb

@@ -6,7 +6,7 @@
    "source": [
     "# General Autotranslation\n",
     "\n",
-    "The `beakerx` object synchronizes values between languages, including JavaScript, Python, Groovy, and the other JVM languages.  BeakerX's current implementation works with metaprogramming and serializing the objects to JSON, so it's not recommended for more than a few MB of data.  Using [Arrow](https://arrow.apache.org/) and shared memory to remove that limitation is on the [agenda](https://github.com/twosigma/beakerx/issues/7577).\n",
+    "The `beakerx` object synchronizes values between languages, including JavaScript, Python, Groovy, Scala, and SQL.  BeakerX's current implementation works with metaprogramming and serializing the objects to JSON, so it's not recommended for more than a few MB of data.  Using [Arrow](https://arrow.apache.org/) and shared memory to remove that limitation is on the [agenda](https://github.com/twosigma/beakerx/issues/7577), as is support in [Java](https://github.com/twosigma/beakerx/issues/7652), [Clojure](https://github.com/twosigma/beakerx/issues/7653), and [Kotlin](https://github.com/twosigma/beakerx/issues/7654).\n",
     "\n",
     "Start in Groovy:"
    ]
@@ -88,6 +88,35 @@
     "%%scala\n",
     "beakerx.bar"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Pandas DataFrames\n",
+    "\n",
+    "Pandas DataFrames are supported directly in Python.  The resulting data is available in Groovy, however it's wrapped in a Map instead of a TableDisplay, see [#7610](https://github.com/twosigma/beakerx/issues/7610)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%python\n",
+    "import pandas as pd\n",
+    "beakerx.rates = pd.read_csv('../resources/data/interest-rates.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "beakerx.rates.values[0]"
+   ]
   }
  ],
  "metadata": {

doc/groovy/TableAPI.ipynb

@@ -424,7 +424,7 @@
    "source": [
     "## HTML format\n",
     "\n",
-    "HTML format allows markup and styling of the cell's content.  Interactive JavaScript is not supported however."
+    "HTML format allows markup and styling of the cell's content.  Interactive JavaScript and images are not supported however.  See below for image support."
    ]
   },
   {
@@ -440,6 +440,52 @@
     "tbl"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Image format\n",
+    "\n",
+    "Image format allows loading any image supported by the web browser, including PNG, JPG, and SVG formats.  It can be a relative URL served from the same directories as the notebook itself, an absolute URL to anywhere on the web, or a data URL with encoded data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tbl = new TableDisplay(['Peacock': '../resources/img/butterfly1.jpg',\n",
+    "                        'Morpho': '../resources/img/butterfly2.jpg',\n",
+    "                        'Swallowtail': '../resources/img/butterfly3.jpg']);\n",
+    "tbl.setStringFormatForColumn(\"Value\", TableDisplayStringFormat.getImageFormat())\n",
+    "tbl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tbl = new TableDisplay(['https://en.wikipedia.org/wiki/IPython': 'http://jupyter.org/assets/nav_logo.svg',\n",
+    "                        'https://en.wikipedia.org/wiki/Two_Sigma': 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/54/2_sigma_logo.png/150px-2_sigma_logo.png']);\n",
+    "tbl.setStringFormatForColumn(\"Value\", TableDisplayStringFormat.getImageFormat())\n",
+    "tbl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tbl = new TableDisplay(['up': 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+DQo8c3ZnIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgaGVpZ2h0PSI2NCIgd2lkdGg9IjY0IiB2aWV3Qm94PSIwIDAgNjQgNjQiPg0KICA8cG9seWdvbiBmaWxsPSIjNmJhMmM3IiBwb2ludHM9IjMwLDQgNCw2MCA2MCw2MCI+PC9wb2x5Z29uPg0KPC9zdmc+DQo=',\n",
+    "                        'down': 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCFET0NUWVBFIHN2ZyBQVUJMSUMgIi0vL1czQy8vRFREIFNWRyAxLjEvL0VOIiAiaHR0cDovL3d3dy53My5vcmcvR3JhcGhpY3MvU1ZHLzEuMS9EVEQvc3ZnMTEuZHRkIj4KPHN2ZyB2ZXJzaW9uPSIxLjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIGhlaWdodD0iNjQiIHdpZHRoPSI2NCIgdmlld0JveD0iMCAwIDY0IDY0Ij4KICA8cG9seWdvbiBmaWxsPSIjNmJhMmM3IiBwb2ludHM9IjMwLDYwIDYwLDQgNCw0Ij48L3BvbHlnb24+Cjwvc3ZnPgo=']);\n",
+    "tbl.setStringFormatForColumn(\"Value\", TableDisplayStringFormat.getImageFormat())\n",
+    "tbl"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

doc/scala/Flint.ipynb

@@ -23,9 +23,9 @@
    "outputs": [],
    "source": [
     "%%classpath add mvn\n",
-    "com.github.twosigma flint master-SNAPSHOT\n",
     "org.apache.spark spark-sql_2.11 2.2.1\n",
-    "org.apache.spark spark-mllib_2.11 2.2.1"
+    "org.apache.spark spark-mllib_2.11 2.2.1\n",
+    "com.github.twosigma flint 6055a7a231"
    ]
   },
   {
@@ -178,7 +178,7 @@
    "outputs": [],
    "source": [
     "// Calculate logarithm of a column\n",
-    "val logVolumeRdd = pricesRdd.addColumns(\"logVolume\" -> DoubleType -> { row => Math.log(row.getAs[Double](\"volume\")) })\n",
+    "val logVolumeRdd = pricesRdd.addColumns(\"logVolume\" -> DoubleType -> { row => scala.math.log(row.getAs[Double](\"volume\")) })\n",
     "preview(pricesRdd)"
    ]
   },
@@ -189,7 +189,7 @@
    "outputs": [],
    "source": [
     "// Raise a column to an exponent\n",
-    "val squaredVolumeRdd = pricesRdd.addColumns(\"squaredVolume\" -> DoubleType -> { row => Math.pow(row.getAs[Double](\"volume\"), 2) })\n",
+    "val squaredVolumeRdd = pricesRdd.addColumns(\"squaredVolume\" -> DoubleType -> { row => scala.math.pow(row.getAs[Double](\"volume\"), 2) })\n",
     "preview(squaredVolumeRdd)"
    ]
   },
@@ -339,8 +339,8 @@
     "// Compute the Z score across an interval\n",
     "val zScoreRdd = pricesRdd.addColumnsForCycle(\"volumeZScore\" -> DoubleType -> { rows: Seq[Row] =>\n",
     "    val mean = rows.map(_.getAs[Double](\"volume\")).sum / rows.size\n",
-    "    val stddev = Math.sqrt(rows.map { row =>\n",
-    "        Math.pow(row.getAs[Double](\"close\") - mean, 2)\n",
+    "    val stddev = scala.math.sqrt(rows.map { row =>\n",
+    "        scala.math.pow(row.getAs[Double](\"close\") - mean, 2)\n",
     "    }.sum ) / (rows.size - 1)\n",
     "    rows.map { row =>\n",
     "        row -> (row.getAs[Double](\"close\") - mean) / stddev\n",

doc/scala/SparkUI.ipynb

@@ -20,7 +20,7 @@
    "outputs": [],
    "source": [
     "%%classpath add mvn\n",
-    "org.apache.spark spark-sql_2.11 2.3.1"
+    "org.apache.spark spark-sql_2.11 2.2.1"
    ]
   },
   {
@@ -62,7 +62,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You can also provide a `--connect` (or `-c`) option to automatically connect with the cluster."
+    "You can also provide a `--start` (or `-s`) option to automatically start a session with a cluster (or a local instance)."
    ]
   },
   {
@@ -71,7 +71,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%spark --connect\n",
+    "%%spark --start\n",
     "SparkSession.builder().master(\"local[100]\")"
    ]
   },

doc/sql/Sql.ipynb

@@ -98,6 +98,27 @@
     "                          tableFinish)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%groovy\n",
+    "beakerx.citiesTable.values[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%python\n",
+    "from beakerx import beakerx\n",
+    "beakerx.citiesTable.loc[0]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {

js/lab/package.json

@@ -1,6 +1,6 @@
 {
   "name": "beakerx-jupyterlab",
-  "version": "0.21.0",
+  "version": "1.0.0",
   "description": "BeakerX: Beaker Extensions for JupyterLab",
   "author": "Two Sigma Open Source, LLC",
   "main": "dist/index.js",

js/lab/src/theme/static/fonts.css

@@ -68,10 +68,11 @@ div.output_text {
   font-family: "Lato", Helvetica, sans-serif;
 }
 
-improveFonts .jp-RenderedText pre {
+.improveFonts .jp-RenderedText pre {
   font-family: "Roboto Mono", monospace, sans-serif;
 }
 
-.improveFonts .jp-RenderedMarkdown {
+.improveFonts .jp-RenderedMarkdown,
+.improveFonts .jp-RenderedHTML {
   font-family: "Lato", Helvetica, sans-serif;
 }

js/notebook/package.json

@@ -1,6 +1,6 @@
 {
   "name": "beakerx",
-  "version": "0.21.0",
+  "version": "1.0.0",
   "description": "BeakerX: Beaker Extensions for Jupyter",
   "author": "Two Sigma Open Source, LLC",
   "main": "src/index.ts",

js/notebook/src/SparkStateProgress.ts

@@ -91,8 +91,6 @@ export class SparkStateProgressView extends widgets.VBoxView {
     this.progressLabelWaiting.innerText = `${valueWaiting}`;
     this.progressLabelAll.innerText = max;
 
-    this.updateLabelWidths();
-
     return super.update();
   }
 
@@ -205,6 +203,8 @@ export class SparkStateProgressView extends widgets.VBoxView {
     this.progressLabelWaiting = this.progressLabels.querySelector('.waiting');
     this.progressLabelAll = this.progressLabels.querySelector('.all');
 
+    this.updateLabelWidths();
+
     return $(this.progressLabels);
   }