chore: convert examples to notebook

Signed-off-by: ThibaultFy <50656860+ThibaultFy@users.noreply.github.com>

.binder/postBuild

@@ -1,18 +1,3 @@
 #!/bin/bash
 # binder post build script
 set -ex
-
-(cd docs && make html)
-
-# uninstall docs requirements for a lighter docker image
-pip uninstall -y -r docs/requirements.txt
-
-# move examples to the notebooks folder
-mv docs/build/html/notebooks .
-mv examples notebooks/examples
-
-# delete everything but the notebooks folder and the substra dependencies
-shopt -s extglob
-rm -rf .[!.]*
-rm -rf !(notebooks|docs)
-(cd docs && rm -rf !(src))

.binder/runtime.txt

@@ -1 +1 @@
-python-3.8
+python-3.10

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
 
       - name: Clone substra-tools
         uses: actions/checkout@v2
@@ -40,7 +40,7 @@ jobs:
 
       - name: Install substra, substra-tools and substrafl
         run: |
-          pip install -e './substrafl[dev]'
+          pip install -e ./substrafl
           pip install -e ./substra
           pip install -e ./substra-tools
 
@@ -49,8 +49,14 @@ jobs:
           cp -r substra/references docs/source/documentation/references
           cp -r substrafl/docs/api docs/source/substrafl_doc/
 
+      - name: Install Pandoc
+        run: |
+          sudo wget https://github.com/jgm/pandoc/releases/download/3.1.8/pandoc-3.1.8-1-amd64.deb
+          sudo dpkg -i pandoc-3.1.8-1-amd64.deb
+
       - name: Install requirements
-        run: pip install -r requirements.txt
+        run: |
+          pip install -r requirements.txt
 
       - name: Sphinx make
         working-directory: ./docs

.gitignore

@@ -135,8 +135,6 @@ _build/
 # Misc build artefacts
 tmp/**
 
-docs/source/examples/**
-doc/source/substrafl_doc/examples/**
 docs/source/documentation/references/**
 docs/source/substrafl_doc/api
 

.readthedocs.yaml

@@ -8,17 +8,12 @@ version: 2
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.8"
+    python: "mambaforge-22.9"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/source/conf.py
   fail_on_warning: True
 
-# Optionally build your docs in additional formats such as PDF
-formats: []
-
-# Optionally set the version of Python and requirements required to build your docs
-python:
-  install:
-    - requirements: requirements.txt
+conda:
+  environment: environment.yml

Makefile

@@ -1,27 +1,22 @@
 install-examples-dependencies:
-	pip3 install -r examples/substra_core/diabetes_example/assets/requirements.txt \
-    -r examples/substra_core/titanic_example/assets/requirements.txt \
-    -r examples/substrafl/get_started/torch_fedavg_assets/requirements.txt \
-    -r examples/substrafl/go_further/sklearn_fedavg_assets/requirements.txt \
-    -r examples/substrafl/go_further/torch_cyclic_assets/requirements.txt \
-    -r examples/substrafl/go_further/diabetes_substrafl_assets/requirements.txt \
+	pip3 install -r examples_requirements.txt
 
 examples: example-substra example-substrafl
 
 example-substra: example-core-diabetes example-core-titanic
 
 example-core-diabetes:
-	cd examples/substra_core/diabetes_example/ && python run_diabetes.py
+	cd docs/source/examples/substra_core/diabetes_example/ && ipython -c "%run run_diabetes.ipynb"
 example-core-titanic:
-	cd examples/substra_core/titanic_example/ && python run_titanic.py
+	cd docs/source/examples/substra_core/titanic_example/ && ipython -c "%run run_titanic.ipynb"
 
 example-substrafl: example-fl-mnist example-fl-iris example-fl-cyclic example-fl-diabetes
 
 example-fl-mnist:
-	cd examples/substrafl/get_started/ && python run_mnist_torch.py
+	cd docs/source/examples/substrafl/get_started/ && ipython -c "%run run_mnist_torch.ipynb"
 example-fl-iris:
-	cd examples/substrafl/go_further/ && python run_iris_sklearn.py
+	cd docs/source/examples/substrafl/go_further/ && ipython -c "%run run_iris_sklearn.ipynb"
 example-fl-cyclic:
-	cd examples/substrafl/go_further/ && python run_mnist_cyclic.py
+	cd docs/source/examples/substrafl/go_further/ && ipython -c "%run run_mnist_cyclic.ipynb"
 example-fl-diabetes:
-	cd examples/substrafl/go_further/ && python run_diabetes_substrafl.py
+	cd docs/source/examples/substrafl/go_further/ && ipython -c "%run run_diabetes_substrafl.ipynb"

docker/substra-documentation-examples/Dockerfile

@@ -21,7 +21,8 @@ RUN cd substra && python -m pip install --no-cache-dir -e .
 RUN cd substra-tools && python -m pip install --no-cache-dir -e .
 
 COPY substra-documentation/Makefile substra-documentation/
-COPY substra-documentation/examples substra-documentation/examples/
+COPY substra-documentation/examples_requirements.txt substra-documentation/
+COPY substra-documentation/docs/source/examples substra-documentation/docs/source/examples/
 
 RUN cd substra-documentation && make install-examples-dependencies
 

docs/Makefile

@@ -7,9 +7,7 @@ SPHINXOPTS    ?= -W --keep-going -n
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build
-SUBSTRAEXAMPLEDIR 		= source/examples/substra_core
 SUBSTRADOCDIR 		= source/documentation/references
-SUBSTRAFLEXAMPLEDIR	= source/substrafl_doc/examples
 SUBSTRAFLDOCDIR	= source/substrafl_doc/api
 
 # Put it first so that "make" without argument is like "make help".
@@ -33,8 +31,6 @@ clean:
 	rm -rf $(BUILDDIR)
 	rm -rf $(SUBSTRADOCDIR)
 	rm -rf $(SUBSTRAFLDOCDIR)
-	rm -rf $(SUBSTRAEXAMPLEDIR)
-	rm -rf $(SUBSTRAFLEXAMPLEDIR)
 # Delete the local worker folders in substra-documentation
 	find .. -type d -name local-worker -prune -exec rm -rf {} \;
 # Delete the tmp folders in substra-documentation

docs/requirements.txt

@@ -11,7 +11,9 @@ myst-parser==0.16.1
 # but docutils 0.17 changed the output html markup, breaking the RTD theme
 # original issue: https://github.com/sphinx-doc/sphinx/issues/9051
 docutils==0.16
-sphinx-gallery==0.7.0
 sphinx-fontawesome==0.0.6
 sphinx-copybutton==0.5.0
 pyyaml==6.0
+nbsphinx==0.9.3
+pandoc==2.3
+git-python==1.0.3

docs/source/conf.py

@@ -24,9 +24,7 @@
 import git
 import yaml
 
-from sphinx_gallery.sorting import ExplicitOrder
-
-TMP_FOLDER = Path(__file__).parents[2] / "tmp"
+TMP_FOLDER = Path(__file__).parent / "tmp"
 TMP_FOLDER.mkdir(exist_ok=True)
 
 # Generate a JSON compatibility table
@@ -99,6 +97,31 @@ def __call__(self, directory):
         return directory
 
 
+# Nbsphinx config
+
+nbsphinx_thumbnails = {
+    "examples/substra_core/diabetes_example/run_diabetes": "_static/example_thumbnail/diabetes.png",
+    "examples/substra_core/titanic_example/run_titanic": "_static/example_thumbnail/titanic.jpg",
+    "examples/substrafl/get_started/run_mnist_torch": "_static/example_thumbnail/mnist.png",
+    "examples/substrafl/go_further/run_diabetes_substrafl": "_static/example_thumbnail/diabetes.png",
+    "examples/substrafl/go_further/run_iris_sklearn": "_static/example_thumbnail/iris.jpg",
+    "examples/substrafl/go_further/run_mnist_cyclic": "_static/example_thumbnail/cyclic-mnist.png",
+}
+
+nbsphinx_prolog = r"""
+{% set docname = 'doc/' + env.doc2path(env.docname, base=None) %}
+
+.. raw:: html
+
+    <div class="notebook note">
+    Launch notebook online <span style="white-space: nowrap;"><a href="https://mybinder.org/v2/gh/Substra/substra-documentation/{{ env.config.release|e }}?filepath={{ docname|e }}"><img alt="Binder badge" src="https://mybinder.org/badge_logo.svg" style="vertical-align:text-bottom"> </a></span>
+    or download it <span style="white-space: nowrap;"><a href="{{ env.docname.split('/')|last|e + '.ipynb' }}"><img alt="Download badge" src="https://img.shields.io/badge/download_-notebook-orange?logo=jupyter" style="vertical-align:text-bottom"></a></span>
+    </div>
+"""
+
+nbsphinx_epilog = nbsphinx_prolog
+
+
 # zip the assets directory found in the examples directory and place it in the current dir
 def zip_dir(source_dir, zip_file_name):
     # Create archive with compressed files
@@ -111,29 +134,29 @@ def zip_dir(source_dir, zip_file_name):
                 )
 
 
-assets_dir_titanic = Path(__file__).parents[2] / "examples" / "substra_core" / "titanic_example" / "assets"
+assets_dir_titanic = Path(__file__).parent / "examples" / "substra_core" / "titanic_example" / "assets"
 zip_dir(assets_dir_titanic, "titanic_assets.zip")
 
-assets_dir_diabetes = Path(__file__).parents[2] / "examples" / "substra_core" / "diabetes_example" / "assets"
+assets_dir_diabetes = Path(__file__).parent / "examples" / "substra_core" / "diabetes_example" / "assets"
 zip_dir(assets_dir_diabetes, "diabetes_assets.zip")
 
 assets_dir_substrafl_torch_fedavg = (
-    Path(__file__).parents[2] / "examples" / "substrafl" / "get_started" / "torch_fedavg_assets"
+    Path(__file__).parent / "examples" / "substrafl" / "get_started" / "torch_fedavg_assets"
 )
 zip_dir(assets_dir_substrafl_torch_fedavg, "torch_fedavg_assets.zip")
 
 assets_dir_substrafl_diabetes = (
-    Path(__file__).parents[2] / "examples" / "substrafl" / "go_further" / "diabetes_substrafl_assets"
+    Path(__file__).parent / "examples" / "substrafl" / "go_further" / "diabetes_substrafl_assets"
 )
 zip_dir(assets_dir_substrafl_diabetes, "diabetes_substrafl_assets.zip")
 
 assets_dir_substrafl_sklearn_fedavg = (
-    Path(__file__).parents[2] / "examples" / "substrafl" / "go_further" / "sklearn_fedavg_assets"
+    Path(__file__).parent / "examples" / "substrafl" / "go_further" / "sklearn_fedavg_assets"
 )
 zip_dir(assets_dir_substrafl_sklearn_fedavg, "sklearn_fedavg_assets.zip")
 
 assets_dir_substrafl_sklearn_fedavg = (
-    Path(__file__).parents[2] / "examples" / "substrafl" / "go_further" / "torch_cyclic_assets"
+    Path(__file__).parent / "examples" / "substrafl" / "go_further" / "torch_cyclic_assets"
 )
 zip_dir(assets_dir_substrafl_sklearn_fedavg, "torch_cyclic_assets.zip")
 
@@ -248,7 +271,6 @@ def reformat_md_section_links(file_path: Path):
 
 for file_path in Path(".").rglob("*.md"):
     reformat_md_section_links(file_path)
-
 # -- Project information -----------------------------------------------------
 
 project = "Substra"
@@ -267,7 +289,9 @@ def reformat_md_section_links(file_path: Path):
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx_gallery.gen_gallery"]
+extensions = [
+    "nbsphinx",
+]
 
 extensions.extend(
     [
@@ -297,7 +321,6 @@ def reformat_md_section_links(file_path: Path):
     "torch": ("https://pytorch.org/docs/stable/", None),
 }
 
-
 ################
 # Substrafl API
 ################
@@ -379,7 +402,7 @@ def reformat_md_section_links(file_path: Path):
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**/description.md"]
 
 rst_epilog = f"""
 .. |substra_version| replace:: {importlib.import_module('substra').__version__}
@@ -414,21 +437,3 @@ def reformat_md_section_links(file_path: Path):
 html_context = {
     "display_github": False,
 }
-
-sphinx_gallery_conf = {
-    "remove_config_comments": True,
-    "doc_module": "substra",
-    "reference_url": {"Substra": None},
-    "examples_dirs": ["../../examples/substra_core", "../../examples/substrafl"],
-    "gallery_dirs": ["examples/substra_core", "examples/substrafl"],
-    "subsection_order": ExplicitOrder(
-        [
-            "../../examples/substra_core/titanic_example",
-            "../../examples/substra_core/diabetes_example",
-            "../../examples/substrafl/get_started",
-            "../../examples/substrafl/go_further",
-        ]
-    ),
-    "download_all_examples": False,
-    "filename_pattern": "/run_",
-}