Merge pull request #1696 from CartoDB/chore/ch110997/automate-noteboo…

…ks-guides-examples-execution

Added tests for notebook execution

tests/README.md

@@ -41,6 +41,51 @@ Execute a single test
 pytest tests/unit/io/test_carto.py::test_read_carto
 ```
 
+## Executing Jupyter Notebooks tests
+
+These tests execute all the Jupyter Notebooks contained in the `docs/examples` and `docs/guides` directories and overwrite the notebook with the executed versions (if the `OVERWRITE` parameter is set to `true`).
+
+Create a virtual environment
+
+```
+virtualenv -p python3 venv
+source venv/bin/activate
+```
+
+Install the required dependencies
+
+```
+pip install -r requirements.txt
+pip install -r tests/notebooks/requirements.txt
+```
+
+Set your credentials
+
+Create the file tests/notebooks/creds.json with the following structure:
+
+```
+{
+  "username": "your_username",
+  "api_key": "your_api_key"
+}
+```
+
+Execute the tests
+
+```
+pytest [-s] tests/notebooks/test_notebooks.py
+```
+
+Environment variables:
+ - `SCOPE`: (default `all`): Scope of the tests: all, guides, examples (Example: `SCOPE=guides`)
+ - `OVERWRITE` (default `true`): Overwrites the notebooks with the result of the execution (Example: `OVERWRITE=false`)
+ - `TIMEOUT` (default `600`): Notebook timeout for each cell (Example: `TIMEOUT=100`)
+ - `KERNEL` (default `python3`): Kernel used to execute the notebooks (Example: `KERNEL=python3`)
+
+Notes:
+ - You can select the notebooks to be executed by overriding the value of the `EXECUTE_NOTEBOOKS` variable in `test_notebooks.py` (if not set, all the notebooks in the `docs/examples` and `docs/guides` directories will be executed)
+ - You can also select the notebooks to be avoided by overriding the value of the `AVOID_NOTEBOOKS` variable in `test_notebooks.py`
+
 ## File structure
 
 ```

tests/notebooks/creds.sample.json

@@ -0,0 +1,4 @@
+{
+  "username": "your_username",
+  "api_key": "your_api_key"
+}

tests/notebooks/requirements.txt

@@ -0,0 +1,8 @@
+notebook==6.1.4
+pytest==6.1.1
+pytest-mock==3.3.1
+nbconvert==6.0.7
+ipykernel==5.3.4
+ipywidgets==7.5.1
+matplotlib==3.3.2
+xlrd==1.2.0

tests/notebooks/test_notebooks.py

@@ -0,0 +1,90 @@
+import os
+import time
+import glob
+import pytest
+import logging
+import nbformat
+import subprocess
+from nbconvert.preprocessors import ExecutePreprocessor
+
+
+logging.basicConfig(level=logging.INFO)
+
+EXECUTE_NOTEBOOKS = []
+AVOID_NOTEBOOKS = [
+    'docs/examples/data_observatory/download_dataset.ipynb',
+    'docs/examples/data_management/change_carto_table_privacy.ipynb',
+    'docs/examples/publish_and_share/publish_visualization_layout.ipynb',
+    'docs/examples/publish_and_share/publish_visualization_private_table.ipynb',
+    'docs/examples/publish_and_share/publish_visualization_public_table.ipynb',
+    'docs/examples/_debug/testing_polygons_features.ipynb',
+    'docs/examples/_debug/enrichment_big_polygons.ipynb',
+]
+
+OVERWRITE = os.environ.get('OVERWRITE', 'true').lower() == 'true'
+TIMEOUT = int(os.environ.get('TIMEOUT', 600))
+KERNEL = os.environ.get('KERNEL', 'python3').lower()
+SCOPE = os.environ.get('SCOPE', 'all').lower()
+
+with open('tests/notebooks/creds.json', 'r') as creds_file:
+    CREDS_FILE = creds_file.read()
+
+
+def find_notebooks():
+    notebooks = []
+
+    if EXECUTE_NOTEBOOKS:
+        notebooks = list(set(EXECUTE_NOTEBOOKS) - set(AVOID_NOTEBOOKS))
+    else:
+        if SCOPE in ['all', 'guides']:
+            notebooks += glob.glob('docs/guides/**/*.ipynb', recursive=True)
+        if SCOPE in ['all', 'examples']:
+            notebooks += glob.glob('docs/examples/**/*.ipynb', recursive=True)
+        notebooks = list(set(notebooks) - set(AVOID_NOTEBOOKS))
+
+    notebooks.sort()
+    return notebooks
+
+
+class TestNotebooks:
+    def teardown(self):
+        time.sleep(0.1)
+
+    def custom_setup(self, path):
+        with open('{}/creds.json'.format(path), 'w') as creds_file:
+            creds_file.write(CREDS_FILE)
+
+    def custom_teardown(self, path):
+        os.remove('{}/creds.json'.format(path))
+
+    @pytest.mark.parametrize('notebook_filename', find_notebooks())
+    def test_docs(self, notebook_filename):
+        try:
+            path = os.path.dirname(notebook_filename)
+
+            self.custom_setup(path)
+            self.execute_notebook(notebook_filename, path)
+        finally:
+            self.custom_teardown(path)
+
+    def execute_notebook(self, notebook_filename, path):
+        with open(notebook_filename) as f:
+            logging.info('\nExecuting notebook: %s', notebook_filename)
+
+            nb = nbformat.read(f, as_version=4)
+            ep = ExecutePreprocessor(timeout=TIMEOUT, kernel_name=KERNEL, allow_errors=OVERWRITE,
+                                     store_widget_state=OVERWRITE)
+            ep.preprocess(nb, {'metadata': {'path': path}})
+
+            if OVERWRITE:
+                logging.info('Overwriting notebook: %s', notebook_filename)
+                with open(notebook_filename, 'w') as fwrite:
+                    nbformat.write(nb, fwrite)
+
+                logging.info('Trusting notebook: %s', notebook_filename)
+                p_jupyter = subprocess.Popen('jupyter trust {}'.format(notebook_filename), shell=True,
+                                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                _, stderr_jupyter = p_jupyter.communicate()
+
+                if len(stderr_jupyter) > 0:
+                    raise RuntimeError('Error trusting the notebook ({}): {}'.format(notebook_filename, stderr_jupyter))