From 16cc26f6cb64b4061b4b4399f896875eb128170b Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:05:06 +0100 Subject: [PATCH 1/9] refactor: rename fix-first-nbcell to set-nb-cells --- .pre-commit-hooks.yaml | 27 +++++++++---------- README.md | 2 +- setup.cfg | 2 +- .../{fix_first_nbcell.py => set_nb_cells.py} | 9 ++++--- 4 files changed, 20 insertions(+), 20 deletions(-) rename src/repoma/{fix_first_nbcell.py => set_nb_cells.py} (95%) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 8e80dae8..0f619523 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -5,20 +5,6 @@ always_run: true pass_filenames: false -- id: fix-first-nbcell - name: Check and fix first cells in notebooks - description: > - Add or replace the first cell in a Jupyter notebook by adding install - statements and IPython configurations. - entry: fix-first-nbcell - exclude: > - (?x)^( - docs/adr/.* - )$ - language: python - types: - - jupyter - - id: fix-nbformat-version name: Set nbformat minor version to 4 and remove cell IDs entry: fix-nbformat-version @@ -41,3 +27,16 @@ language: python types: - jupyter + +- id: set-nb-cells + name: Add or update default cells in a Jupyter notebook + description: > + Add or replace certain default cells in a Jupyter notebook. + entry: set-nb-cells + exclude: > + (?x)^( + docs/adr/.* + )$ + language: python + types: + - jupyter diff --git a/README.md b/README.md index f4ab20f3..d0186e2c 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ repos: rev: "" hooks: - id: check-dev-files - - id: fix-first-nbcell - id: fix-nbformat-version - id: format-setup-cfg - id: pin-nb-requirements + - id: set-nb-cells ``` then run `pre-commit autoupdate`. This example lists all available hooks diff --git a/setup.cfg b/setup.cfg index 4f8cae4b..7c9b3476 100644 --- a/setup.cfg +++ b/setup.cfg @@ -83,10 +83,10 @@ dev = [options.entry_points] console_scripts = check-dev-files = repoma.check_dev_files:main - fix-first-nbcell = repoma.fix_first_nbcell:main fix-nbformat-version = repoma.fix_nbformat_version:main format-setup-cfg = repoma.format_setup_cfg:main pin-nb-requirements = repoma.pin_nb_requirements:main + set-nb-cells = repoma.set_nb_cells:main [options.packages.find] where = src diff --git a/src/repoma/fix_first_nbcell.py b/src/repoma/set_nb_cells.py similarity index 95% rename from src/repoma/fix_first_nbcell.py rename to src/repoma/set_nb_cells.py index 9203dbf2..2e0efb94 100644 --- a/src/repoma/fix_first_nbcell.py +++ b/src/repoma/set_nb_cells.py @@ -1,4 +1,4 @@ -"""Add install statements to first cell in a Jupyter notebook. +"""Add or update standard cells in a Jupyter notebook. Notebook servers like Google Colaboratory and Deepnote do not install a package automatically, so this has to be done through a code cell. At the same time, @@ -10,11 +10,12 @@ Notebooks can be ignored by making the first cell a `Markdown cell `_ -and starting its content with: +and writing the following `Markdown comment +`_: .. code-block:: markdown - + """ import argparse @@ -166,7 +167,7 @@ def _insert_autolink_concat(filename: str) -> None: def _skip_notebook( - filename: str, ignore_statement: str = "" + filename: str, ignore_statement: str = "" ) -> bool: notebook = nbformat.read(filename, as_version=nbformat.NO_CONVERT) for cell in notebook["cells"]: From e0d8dbd92c566db48a2d7a7dc8c10515c9446654 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:08:47 +0100 Subject: [PATCH 2/9] chore: put install cell before config cell --- src/repoma/set_nb_cells.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 2e0efb94..95e152b6 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -92,12 +92,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: args = parser.parse_args(argv) for filename in args.filenames: - _update_cell( - filename, - new_content=__CONFIG_CELL_CONTENT.strip("\n"), - new_metadata=__CONFIG_CELL_METADATA, - cell_id=0, - ) + cell_id = 0 if args.add_install_cell: cell_content = __INSTALL_CELL_CONTENT.strip("\n") if args.extras_require: @@ -112,8 +107,15 @@ def main(argv: Optional[Sequence[str]] = None) -> int: filename, new_content=cell_content, new_metadata=__INSTALL_CELL_METADATA, - cell_id=1, + cell_id=cell_id, ) + cell_id += 1 + _update_cell( + filename, + new_content=__CONFIG_CELL_CONTENT.strip("\n"), + new_metadata=__CONFIG_CELL_METADATA, + cell_id=cell_id, + ) _insert_autolink_concat(filename) return 0 From 9467a2da3e5e58bd55f1e1bdd59f532887afc102 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:10:14 +0100 Subject: [PATCH 3/9] feat: add IPython.display import statement --- src/repoma/set_nb_cells.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 95e152b6..166aa789 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -34,6 +34,8 @@ %config InlineBackend.figure_formats = ['svg'] import os +from IPython.display import display + STATIC_WEB_PAGE = {"EXECUTE_NB", "READTHEDOCS"}.intersection(os.environ) """ __CONFIG_CELL_METADATA: dict = { From 7351575a693fad643aa8ce86cb009a3db4959f7b Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:25:39 +0100 Subject: [PATCH 4/9] fix: only add IPython import if IPython is additional package --- src/repoma/set_nb_cells.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 166aa789..0ff91295 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -34,8 +34,6 @@ %config InlineBackend.figure_formats = ['svg'] import os -from IPython.display import display - STATIC_WEB_PAGE = {"EXECUTE_NB", "READTHEDOCS"}.intersection(os.environ) """ __CONFIG_CELL_METADATA: dict = { @@ -91,6 +89,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ), type=str, ) + parser.add_argument( + "--no-config-cell", + action="store_true", + help="Do not add configuration cell.", + ) args = parser.parse_args(argv) for filename in args.filenames: @@ -112,12 +115,19 @@ def main(argv: Optional[Sequence[str]] = None) -> int: cell_id=cell_id, ) cell_id += 1 - _update_cell( - filename, - new_content=__CONFIG_CELL_CONTENT.strip("\n"), - new_metadata=__CONFIG_CELL_METADATA, - cell_id=cell_id, - ) + if not args.no_config_cell: + config_cell_content = __CONFIG_CELL_CONTENT + if "ipython" in args.additional_packages.lower(): + config_cell_content = config_cell_content.replace( + "import os", + "import os\n\nfrom IPython.display import display", + ) + _update_cell( + filename, + new_content=config_cell_content.strip("\n"), + new_metadata=__CONFIG_CELL_METADATA, + cell_id=cell_id, + ) _insert_autolink_concat(filename) return 0 From b6478ec15c03b6c6c4114c555db2ab021585d8ca Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:35:41 +0100 Subject: [PATCH 5/9] fix: ignore F401 error --- src/repoma/set_nb_cells.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 0ff91295..2098f6a3 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -120,7 +120,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if "ipython" in args.additional_packages.lower(): config_cell_content = config_cell_content.replace( "import os", - "import os\n\nfrom IPython.display import display", + "import os\n\nfrom IPython.display import display # noqa:" + " F401", ) _update_cell( filename, From bc49e1136806204fbc423fc5e1bb0da5f66a4927 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:45:44 +0100 Subject: [PATCH 6/9] fix: put IPython import in separate cell sphinx-codeautolink does not detect display function if there is cell magic in the cell --- src/repoma/set_nb_cells.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 2098f6a3..450a39ce 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -116,19 +116,26 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) cell_id += 1 if not args.no_config_cell: - config_cell_content = __CONFIG_CELL_CONTENT - if "ipython" in args.additional_packages.lower(): - config_cell_content = config_cell_content.replace( - "import os", - "import os\n\nfrom IPython.display import display # noqa:" - " F401", - ) _update_cell( filename, - new_content=config_cell_content.strip("\n"), + new_content=__CONFIG_CELL_CONTENT.strip("\n"), new_metadata=__CONFIG_CELL_METADATA, cell_id=cell_id, ) + cell_id += 1 + if "ipython" in args.additional_packages.lower(): + _update_cell( + filename, + new_content=( + "from IPython.display import display # noqa: F401" + ), + new_metadata={ + **__CONFIG_CELL_METADATA, + "tags": ["hide-cell"], + }, + cell_id=cell_id, + ) + cell_id += 1 _insert_autolink_concat(filename) return 0 From 778fc57d131b822d5027f4f69f0014d004ddeaf8 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 13:11:54 +0100 Subject: [PATCH 7/9] docs: add IPython.display import as autolink-preface --- src/repoma/set_nb_cells.py | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 450a39ce..9eac6219 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -21,7 +21,7 @@ import argparse import sys from textwrap import dedent -from typing import Optional, Sequence +from typing import List, Optional, Sequence import nbformat @@ -116,27 +116,23 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) cell_id += 1 if not args.no_config_cell: + config_cell_content = __CONFIG_CELL_CONTENT + if "ipython" in args.additional_packages.lower(): + config_cell_content = config_cell_content.replace( + "import os", + "import os\n\nfrom IPython.display import display # noqa:" + " F401", + ) _update_cell( filename, - new_content=__CONFIG_CELL_CONTENT.strip("\n"), + new_content=config_cell_content.strip("\n"), new_metadata=__CONFIG_CELL_METADATA, cell_id=cell_id, ) - cell_id += 1 - if "ipython" in args.additional_packages.lower(): - _update_cell( - filename, - new_content=( - "from IPython.display import display # noqa: F401" - ), - new_metadata={ - **__CONFIG_CELL_METADATA, - "tags": ["hide-cell"], - }, - cell_id=cell_id, - ) - cell_id += 1 - _insert_autolink_concat(filename) + imports = [] + if "ipython" in args.additional_packages.lower(): + imports.append("from IPython.display import display") + _insert_autolink_concat(filename, imports) return 0 @@ -163,7 +159,9 @@ def _update_cell( nbformat.write(notebook, filename) -def _insert_autolink_concat(filename: str) -> None: +def _insert_autolink_concat( + filename: str, imports: Optional[List[str]] = None +) -> None: if _skip_notebook( filename, ignore_statement="" ): @@ -173,7 +171,11 @@ def _insert_autolink_concat(filename: str) -> None: ```{autolink-concat} ``` """ - expected_cell_content = dedent(expected_cell_content).strip() + expected_cell_content = dedent(expected_cell_content) + if imports is not None: + preface = "```{autolink-preface}\n" + "\n".join(imports) + "\n```" + expected_cell_content += preface + expected_cell_content = expected_cell_content.strip() for cell_id, cell in enumerate(notebook["cells"]): if cell["cell_type"] != "markdown": continue From 426635c3f1b1e40119640540010328f14e601a5f Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 13:15:59 +0100 Subject: [PATCH 8/9] fix: substitute autolink-concat cell --- src/repoma/set_nb_cells.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index 9eac6219..eced275d 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -184,7 +184,10 @@ def _insert_autolink_concat( return new_cell = nbformat.v4.new_markdown_cell(expected_cell_content) del new_cell["id"] # following nbformat_minor = 4 - notebook["cells"].insert(cell_id, new_cell) + if cell_content.startswith("```{autolink-concat}"): + notebook["cells"][cell_id] = new_cell + else: + notebook["cells"].insert(cell_id, new_cell) nbformat.validate(notebook) nbformat.write(notebook, filename) return From 7bd5740453b6747aeba471e4b36ffd71344f3620 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sun, 6 Feb 2022 13:21:52 +0100 Subject: [PATCH 9/9] revert: revert commits 426635c, 778fc57, bc49e11 - Revert "fix: substitute autolink-concat cell" This reverts commit 426635c3f1b1e40119640540010328f14e601a5f. - Revert "docs: add IPython.display import as autolink-preface" This reverts commit 778fc57d131b822d5027f4f69f0014d004ddeaf8. - Revert "fix: put IPython import in separate cell" This reverts commit bc49e1136806204fbc423fc5e1bb0da5f66a4927. --- src/repoma/set_nb_cells.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/repoma/set_nb_cells.py b/src/repoma/set_nb_cells.py index eced275d..2098f6a3 100644 --- a/src/repoma/set_nb_cells.py +++ b/src/repoma/set_nb_cells.py @@ -21,7 +21,7 @@ import argparse import sys from textwrap import dedent -from typing import List, Optional, Sequence +from typing import Optional, Sequence import nbformat @@ -129,10 +129,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: new_metadata=__CONFIG_CELL_METADATA, cell_id=cell_id, ) - imports = [] - if "ipython" in args.additional_packages.lower(): - imports.append("from IPython.display import display") - _insert_autolink_concat(filename, imports) + _insert_autolink_concat(filename) return 0 @@ -159,9 +156,7 @@ def _update_cell( nbformat.write(notebook, filename) -def _insert_autolink_concat( - filename: str, imports: Optional[List[str]] = None -) -> None: +def _insert_autolink_concat(filename: str) -> None: if _skip_notebook( filename, ignore_statement="" ): @@ -171,11 +166,7 @@ def _insert_autolink_concat( ```{autolink-concat} ``` """ - expected_cell_content = dedent(expected_cell_content) - if imports is not None: - preface = "```{autolink-preface}\n" + "\n".join(imports) + "\n```" - expected_cell_content += preface - expected_cell_content = expected_cell_content.strip() + expected_cell_content = dedent(expected_cell_content).strip() for cell_id, cell in enumerate(notebook["cells"]): if cell["cell_type"] != "markdown": continue @@ -184,10 +175,7 @@ def _insert_autolink_concat( return new_cell = nbformat.v4.new_markdown_cell(expected_cell_content) del new_cell["id"] # following nbformat_minor = 4 - if cell_content.startswith("```{autolink-concat}"): - notebook["cells"][cell_id] = new_cell - else: - notebook["cells"].insert(cell_id, new_cell) + notebook["cells"].insert(cell_id, new_cell) nbformat.validate(notebook) nbformat.write(notebook, filename) return