Merge pull request #2875 from djhoese/doc-mas-rapido

Make documentation generation faster

.github/workflows/ci.yaml

@@ -80,7 +80,7 @@ jobs:
           numpy \
           pandas \
           scipy
-          conda remove --force-remove -y pykdtree pyresample python-geotiepoints pyhdf netcdf4 h5py cftime astropy pyerfa
+          conda remove --force-remove -y pykdtree pyresample python-geotiepoints pyhdf netcdf4 h5py cftime astropy pyerfa || true
           python -m pip install --upgrade --no-deps --pre --no-build-isolation \
           pyerfa \
           git+https://github.com/storpipfugl/pykdtree \

.readthedocs.yml

@@ -18,5 +18,7 @@ build:
       - git fetch --tags
     pre_install:
       - git update-index --assume-unchanged doc/rtd_environment.yml doc/source/conf.py
+    pre_build:
+      - cd doc/source && if [ "$READTHEDOCS_VERSION" = "latest" ] || [ "$READTHEDOCS_VERSION" = "stable" ]; then python generate_area_def_list.py; else touch area_def_list.rst; fi
 conda:
   environment: doc/rtd_environment.yml

doc/rtd_environment.yml

@@ -31,7 +31,7 @@ dependencies:
   - xarray
   - zarr
   - xarray-datatree
-  - cartopy
+  - geoviews
   - pip:
     - graphviz
     - pytest-lazy-fixtures

doc/source/conf.py

@@ -24,15 +24,9 @@
 sys.path.append(os.path.abspath("../../"))
 sys.path.append(os.path.abspath(os.path.dirname(__file__)))
 
-from pyresample.area_config import (  # noqa: E402
-    _create_area_def_from_dict,
-    _read_yaml_area_file_content,
-    generate_area_def_rst_list,
-)
-from reader_table import generate_reader_table, rst_table_header, rst_table_row  # noqa: E402
+from reader_table import generate_reader_table  # noqa: E402
 
 import satpy  # noqa: E402
-from satpy.resample import get_area_file  # noqa: E402
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -86,26 +80,6 @@ def __getattr__(cls, name):
 with open("reader_table.rst", mode="w") as f:
     f.write(generate_reader_table())
 
-# create table from area definition yaml file
-area_file = get_area_file()[0]
-
-area_dict = _read_yaml_area_file_content(area_file)
-area_table = [rst_table_header("Area Definitions", header=["Name", "Description", "Projection"],
-                               widths="auto", class_name="area-table")]
-
-for aname, params in area_dict.items():
-    area = _create_area_def_from_dict(aname, params)
-    if not hasattr(area, "_repr_html_"):
-        continue
-
-    area_table.append(rst_table_row([f"`{aname}`_", area.description,
-                                     area.proj_dict.get("proj")]))
-
-with open("area_def_list.rst", mode="w") as f:
-    f.write("".join(area_table))
-    f.write("\n\n")
-    f.write(generate_area_def_rst_list(area_file))
-
 # -- General configuration -----------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions

doc/source/dev_guide/index.rst

@@ -128,6 +128,15 @@ additional packages are needed. These can be installed with ::
 
     pip install -e ".[doc]"
 
+Generating the documentation requires a one-time script to generate a list
+of previews of all of the AreaDefinition objects used by the documentation.
+This script can take 2+ minutes to execute so it is run separately from the
+normal documentation build process. To run it::
+
+    cd doc/source/
+    python generate_area_def_list.py
+    cd ../../
+
 After editing the source files there the documentation can be generated locally::
 
     cd doc

doc/source/docutils.conf

@@ -0,0 +1,2 @@
+[parsers]
+line_length_limit=30000000

doc/source/generate_area_def_list.py

@@ -0,0 +1,152 @@
+"""Generate the area definition list restructuredtext document.
+
+This should be run once before generating the sphinx documentation to
+produce the ``area_def_list.rst`` file referenced by ``satpy/resample.py``.
+
+"""
+import logging
+import pathlib
+import sys
+from datetime import datetime
+
+import bokeh
+import geoviews as gv
+import geoviews.feature as gf
+from bokeh.embed import components
+from jinja2 import Template
+from pyresample._formatting_html import _load_static_files
+from pyresample.area_config import area_repr, load_area
+from pyresample.utils.proj4 import ignore_pyproj_proj_warnings
+from reader_table import rst_table_header, rst_table_row
+
+from satpy.resample import get_area_file
+
+logger = logging.getLogger(__name__)
+
+gv.extension("bokeh")
+
+
+TEMPLATE = '''
+
+{{ table_header }}
+{% for area_name, area_def in areas.items() if area_def._repr_html_ is defined %}
+{{ create_table_row(area_name, area_def) }}
+{% endfor %}
+
+
+.. raw:: html
+
+    {{ resources }}
+    {{ pyr_icons_svg | indent(5) }}
+    <style>
+    {{ pyr_css_style | indent(5) }}
+    </style>
+    {{ script | indent(5)}}
+
+{% for area_name, area_div in area_divs_dict.items() %}
+
+{{ area_name }}
+{{ rst_underline('^', area_name|length) }}
+
+.. raw:: html
+
+    {{ area_repr(areas[area_name], map_content=area_div, include_header=False, include_static_files=False) |
+       indent(5) }}
+    <br>
+
+{% endfor %}
+'''  # noqa: Q001
+
+
+def main():
+    """Parse CLI arguments and generate area definition list file."""
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser(description="Generate restructuredtext area definition list for sphinx documentation")
+    parser.add_argument("--area-file",
+                        help="Input area YAML file to read")
+    parser.add_argument("-o", "--output-file",
+                        type=pathlib.Path,
+                        help="HTML or restructuretext filename to create. "
+                             "Defaults to 'area_def_list.rst' in the "
+                             "documentation source directory.")
+    args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO)
+
+    if args.output_file is None:
+        args.output_file = str(pathlib.Path(__file__).resolve().parent / "area_def_list.rst")
+    area_file = args.area_file
+    if area_file is None:
+        area_file = get_area_file()[0]
+
+    area_list = load_area(area_file)
+    areas_dict = {_area_name(area): area for area in area_list}
+    logger.info(f"Generating bokeh plots ({datetime.now()})...")
+    script, divs_dict = _generate_html_map_divs(areas_dict)
+    logger.info(f"Done generating bokeh plots ({datetime.now()})")
+
+    def rst_underline(ch, num_chars):
+        return ch * num_chars
+
+    template = Template(TEMPLATE)
+    icons_svg, css_style = _load_static_files()
+    logger.info(f"Rendering document ({datetime.now()})...")
+    res = template.render(
+        resources=bokeh.resources.CDN.render(),
+        script=script,
+        area_divs_dict=divs_dict,
+        areas=areas_dict,
+        rst_underline=rst_underline,
+        area_repr=area_repr,
+        pyr_icons_svg=icons_svg,
+        pyr_css_style=css_style,
+        table_header=rst_table_header("Area Definitions", header=["Name", "Description", "Projection"],
+                                      widths="auto", class_name="area-table"),
+        create_table_row=_area_table_row,
+    )
+    logger.info(f"Done rendering document ({datetime.now()})")
+
+    with open(args.output_file, mode="w") as f:
+        f.write(res)
+
+
+def _area_name(area_def) -> str:
+    if hasattr(area_def, "attrs"):
+        # pyresample 2
+        return area_def.attrs["name"]
+    # pyresample 1
+    return area_def.area_id
+
+
+def _area_table_row(area_name, area_def):
+    with ignore_pyproj_proj_warnings():
+        area_proj = area_def.proj_dict.get("proj")
+    return rst_table_row([f"`{area_name}`_", area_def.description, area_proj])
+
+
+def _generate_html_map_divs(areas_dict: dict) -> tuple[str, dict]:
+    areas_bokeh_models = {}
+    for area_name, area_def in areas_dict.items():
+        if not hasattr(area_def, "to_cartopy_crs"):
+            logger.info(f"Skipping {area_name} because it can't be converted to cartopy CRS")
+            continue
+        crs = area_def.to_cartopy_crs()
+
+        features = gv.Overlay([gf.ocean, gf.land, gf.borders, gf.coastline])
+        f = gv.render(
+            features.opts(
+                toolbar=None,
+                default_tools=[],
+                projection=crs,
+                xlim=crs.bounds[:2],
+                ylim=crs.bounds[2:],
+            ),
+            backend="bokeh")
+        areas_bokeh_models[area_name] = f
+
+    script, divs_dict = components(areas_bokeh_models)
+    return script, divs_dict
+
+
+if __name__ == "__main__":
+    sys.exit(main())