From 28eb3549c8a1f39b81ad5250dcbdf817a82ed23c Mon Sep 17 00:00:00 2001 From: Doris Lee Date: Fri, 21 May 2021 20:52:57 -0700 Subject: [PATCH] New functionality and docs for export to data reports (#378) * added return HTML option to save_as_html; new docs for export * black * update github action * update python workflow * update workflow --- .github/workflows/python-app.yml | 13 +- doc/source/guide/export.rst | 264 ++++++++++++++++++------------- lux/core/frame.py | 11 +- 3 files changed, 175 insertions(+), 113 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index cc9e1bae..ee2ff91c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -44,9 +44,16 @@ jobs: pip install wheel pip install -r requirements.txt pip install -r requirements-dev.txt - pip uninstall -y lux-widget - pip install git+https://github.com/lux-org/lux-widget pip install sqlalchemy + + pip uninstall -y lux-widget + pip install git+git://github.com/lux-org/lux-widget.git + # Temporary Fix (#372) + cd /opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/luxwidget/ + mkdir labextension + cd labextension + wget https://raw.githubusercontent.com/lux-org/lux-widget/master/luxwidget/nbextension/package.json + - name: Upload data to Postgres run: | python lux/data/upload_car_data.py @@ -57,4 +64,4 @@ jobs: - name: Test with Pytest and Code Coverage Report run: | pytest --cov-report term --cov=lux tests/ tests_sql/ - bash <(curl -s https://codecov.io/bash) + bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/doc/source/guide/export.rst b/doc/source/guide/export.rst index c0571fd8..64020836 100644 --- a/doc/source/guide/export.rst +++ b/doc/source/guide/export.rst @@ -4,7 +4,7 @@ Exporting Vis From Widget .. note:: You can follow along this tutorial in a Jupyter notebook. [`Github `_] [`Binder `_] -In this tutorial, we look at the `Happy Planet Index `_ dataset, which contains metrics related to well-being for 140 countries around the world. We demonstrate how you can select visualizations of interest and export them for further analysis. +In this tutorial, we will first look at how you can access and export a single visualizations in the widget as code to further work with in the notebook. Then, we walk through how you can export more than one visualizations in the widget as an interactive report for sharing or presentation. .. code-block:: python @@ -13,51 +13,17 @@ In this tutorial, we look at the `Happy Planet Index `_ dataset, which contains metrics related to well-being for 140 countries around the world. Note that for the convienience of this tutorial, we set Lux as the default display so we don't have to Toggle from the Pandas table display everytime we print the dataframe. -Exporting widget visualizations as static HTML ----------------------------------------------------------------- +Working with a Single Visualizations from Widget +-------------------------------------------------- -Let's say that you are interested in sharing the visualizations displayed in Lux with others, you can export the visualizations into a static HTML using the following command: - -.. code-block:: python - - df.save_as_html() - -By default, the file is saved as `export.html`, you can optionally specify the HTML filename in the input parameter. - -.. code-block:: python - - df.save_as_html('hpi.html') - -Selecting visualizations from recommendation widget ----------------------------------------------------------------- - -You can also click on visualizations of interest and export them into a separate widget for further processing. - -.. code-block:: python - - df - -.. image:: https://github.com/lux-org/lux-resources/blob/master/doc_img/export-1.gif?raw=true - :width: 700 - :align: center - :alt: 1) scroll through Correlation, then 2) click on any 3 visualization (let's say 2nd, 5th and something towards the end), then 3) click on the export button and make sure the blue message box show up - -.. code-block:: python - - bookmarked_charts = df.exported - bookmarked_charts - -.. image:: ../img/export-2.png - :width: 700 - :align: center - :alt: add screenshot of exported VisList (include the Out[] __repr__ string) in screenshot - -From the dataframe recommendations, the visualization showing the relationship between `GDPPerCapita` and `Footprint` is very interesting. In particular, there is an outlier with extremely high ecological footprint as well as high GDP per capita. So we click on this visualization and click on the export button. +Selecting a Single Visualization as :code:`Vis` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +From the dataframe recommendations, the visualization showing the relationship between :code:`GDPPerCapita` and :code:`Footprint` is very interesting. In particular, there is an outlier with extremely high ecological footprint as well as high GDP per capita. So we click on this visualization and click on the export button. .. code-block:: python @@ -79,7 +45,7 @@ From the dataframe recommendations, the visualization showing the relationship b :alt: add screenshot of exported vis Setting Vis as the Updated Intent ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Often, we might be interested in other visualizations that is related to a visualization of interest and want to learn more. With the exported Vis, we can update the intent associated with dataframe to be based on the selected Vis to get more recommendations related to this visualization. @@ -91,64 +57,21 @@ Often, we might be interested in other visualizations that is related to a visua .. image:: ../img/export-5.png :width: 700 :align: center - :alt: add screenshot - -Accessing Widget State ------------------------- - -We can access the set of recommendations generated for the dataframes via the properties `recommendation`. - -.. code-block:: python - - df.recommendation - -.. image:: ../img/export-6.png - :width: 700 - :align: center - :alt: add screenshot - -The resulting output is a dictionary, keyed by the name of the recommendation category. - -.. code-block:: python - - df.recommendation["Enhance"] - -.. image:: ../img/export-7.png - :width: 700 - :align: center - :alt: add screenshot - -You can also access the vis represented by the current intent via the property `current_vis`. - -.. code-block:: python - df.current_vis - -.. image:: ../img/export-8.png - :width: 700 - :align: center - :alt: add screenshot Exporting Visualizations as Code --------------------------------- - -Let's revist our earlier recommendations by clearing the specified intent. - -.. code-block:: python - - df.clear_intent() - df +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: https://github.com/lux-org/lux-resources/blob/master/doc_img/export-9.gif?raw=true :width: 700 :align: center :alt: 1) click on `Occurrence` tab, then 2) hover around the SubRegion v.s. Number of Records chart -Looking at the Occurrence tab, we are interested in the bar chart distribution of country `SubRegion`. +Looking at the Occurrence tab, we can select and export the bar chart distribution of :code:`SubRegion`. .. code-block:: python - vis = df.recommendation["Occurrence"][0] + vis = df.exported[0] vis .. image:: ../img/export-10.png @@ -164,8 +87,8 @@ To allow further edits of visualizations, visualizations can be exported to code print (vis.to_code("altair")) print (vis.to_code("vegalite")) -Exporting Visualizations to Matplotlib -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Exporting to Matplotlib +========================== We can also export the visualization as code in `Matplotlib `_. .. code-block:: python @@ -211,21 +134,14 @@ This code can be copy-and-pasted back into a new notebook cell for further editi :align: center :alt: add screenshot -Exporting Visualizations to Altair -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Exporting to Altair +========================== .. code-block:: python print (vis.to_altair()) -.. .. image:: ../img/export-11.png -.. :width: 700 -.. :align: center -.. :alt: add screenshot - -.. This can be copy-and-pasted back into a new notebook cell for further editing. - .. code-block:: python import altair as alt @@ -249,8 +165,8 @@ Exporting Visualizations to Altair :align: center :alt: add screenshot -Exporting Visualizations to Vega-Lite -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Exporting to Vega-Lite +========================== You can also export this as Vega-Lite specification and view/edit the specification on `Vega Editor `_. @@ -263,10 +179,10 @@ You can also export this as Vega-Lite specification and view/edit the specificat :align: center :alt: add screenshot of what this looks like in Vega Editor -Exporting Standalone Visualizations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Exporting Standalone Visualization Code +==================================================== -Let's say now we are interested in the scatter plot of the `HPIRank` and `HappyPlanetIndex`. +Let's say now we are interested in the scatter plot of the :code:`HPIRank` and :code:`HappyPlanetIndex`. .. code-block:: python @@ -292,4 +208,140 @@ If we wanted to include the actual data in the returned codeblock, we would use .. image:: ../img/export-15.png :width: 700 :align: center - :alt: screenshot of code with embedded data \ No newline at end of file + :alt: screenshot of code with embedded data + + + +Working with Multiple Visualizations from Widget +-------------------------------------------------- + +We have seen how you can select and export a single visualization of interest. Now let's say that we found several visualizations that displays interesting trends to filter to and investigate further. + +Selecting Multiple Visualization as :code:`VisList` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To do this, we can select one or more visualizations of interest and click on the export button to extract the selected visualizations into a separate widget for further processing. + + +.. image:: https://github.com/lux-org/lux-resources/blob/master/doc_img/export-1.gif?raw=true + :width: 700 + :align: center + :alt: 1) scroll through Correlation, then 2) click on any 3 visualization (let's say 2nd, 5th and something towards the end), then 3) click on the export button and make sure the blue message box show up + +After clicking on the export button, the selected visualizations are stored inside the :code:`exported` property of the dataframe as a :code:`VisList`, which can be accessed programmatically. + +.. code-block:: python + + bookmarked_charts = df.exported + bookmarked_charts + +.. image:: ../img/export-2.png + :width: 700 + :align: center + :alt: add screenshot of exported VisList (include the Out[] __repr__ string) in screenshot + + +Accessing Widget Recommendations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We can access the set of recommendations generated for the dataframes via the dataframe property :code:`recommendation`. + +.. code-block:: python + + df.recommendation + +.. image:: ../img/export-6.png + :width: 700 + :align: center + +The resulting output is a Python dictionary, with the key as the name of the recommendation category and values as a :code:`VisList` of recommendations. + +.. code-block:: python + + df.recommendation["Enhance"] + +.. image:: ../img/export-7.png + :width: 700 + :align: center + + +Exporting Entire Dataframe Widgets +----------------------------------- + +Let's say that you are interested in export all the visualizations recommended by Lux, you can export the dataframe widget to an interactive HTML report or data application to share with others. By integrating Lux with `Streamlit `_ or `DataPane `_, you can intersperse Markdown text, images, and other rich graphics and widgets, alongside the output widget generated by Lux to compose and author your own data science narrative. + +Exporting to HTML Report +~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can export the visualizations into a static HTML using the following command: + +.. code-block:: python + + df.save_as_html() + +By default, the file is saved as `export.html`, you can optionally specify the HTML filename in the input parameter. + +.. code-block:: python + + df.save_as_html('hpi.html') + +If you would like to output HTML be returned directly via :code:`save_as_html`, you can input the parameter :code:`output=True`. + +.. code-block:: python + + html_content = df.save_as_html(output=True) + +The HTML export functionality is the basis for exporting to interactive data apps, described next. + +Exporting to Streamlit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`Streamlit `_ is a Python library that simplifies the process of creating interactive data apps that can be shared and deployed anywhere. +To integrate Lux with Streamlit, you can wrap the HTML widget output with a `Streamlit Component `_. As shown below, you can create a file called :code:`app.py`: + +.. code-block:: python + + import streamlit as st + import streamlit.components.v1 as components + from pathlib import Path + import pandas as pd + import lux + + def app(): + st.title('Analysis of Happy Planet Index Dataset') + st.write('Check out these cool visualizations!') + df = pd.read_csv("https://raw.githubusercontent.com/lux-org/lux-datasets/master/data/hpi.csv") + export_file = 'visualizations.html' + html_content = df.save_as_html(output=True) + components.html(html_content, width=800, height=350) + + app() + +After running :code:`streamlit run app.py` in the command line, you should find the Streamlit app hosted in your browser (e.g., :code:`localhost:8501`): + +.. image:: https://github.com/lux-org/lux-resources/blob/master/doc_img/export-streamlit.png?raw=true + :width: 700 + :align: center + +Exporting to DataPane +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`DataPane `_ is a Python library used for creating interactive data science reports. To integrate Lux with a DataPane report, you can wrap the HTML widget output with a `dp.HTML `_ block as shown below: + +.. code-block:: python + + import pandas as pd + import datapane as dp + import lux + df = pd.read_csv("https://raw.githubusercontent.com/lux-org/lux-datasets/master/data/hpi.csv") + html_content = df.save_as_html(output=True) + dp.Report("## Analysis of Happy Planet Index Dataset\n Check out these cool visualizations!", + dp.HTML(html_content) + ).save(path='report.html', open=True) + +You should find that a webpage :code:`report.html` that pops up automatically. + + +.. image:: https://github.com/lux-org/lux-resources/blob/master/doc_img/export-datapane.png?raw=true + :width: 700 + :align: center \ No newline at end of file diff --git a/lux/core/frame.py b/lux/core/frame.py index e4aaad06..4880deb2 100644 --- a/lux/core/frame.py +++ b/lux/core/frame.py @@ -749,7 +749,7 @@ def rec_to_JSON(recs): del rec_lst[idx]["collection"] return rec_lst - def save_as_html(self, filename: str = "export.html") -> None: + def save_as_html(self, filename: str = "export.html", output=False): """ Save dataframe widget as static HTML file @@ -831,9 +831,12 @@ def save_as_html(self, filename: str = "export.html") -> None: rendered_template = html_template.format( header=header, manager_state=manager_state, widget_view=widget_view ) - with open(filename, "w") as fp: - fp.write(rendered_template) - print(f"Saved HTML to {filename}") + if output: + return rendered_template + else: + with open(filename, "w") as fp: + fp.write(rendered_template) + print(f"Saved HTML to {filename}") # Overridden Pandas Functions def head(self, n: int = 5):