From 10c63025ffb9538973e61f15fb19436eda28938e Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 6 Aug 2024 10:35:24 -0500 Subject: [PATCH 1/9] Remove PROJ/pyproj warnings when generating areas previews in docs --- doc/source/conf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 62156e9760..248c9f7a07 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -29,6 +29,7 @@ _read_yaml_area_file_content, generate_area_def_rst_list, ) +from pyresample.utils.proj4 import ignore_pyproj_proj_warnings from reader_table import generate_reader_table, rst_table_header, rst_table_row # noqa: E402 import satpy # noqa: E402 @@ -98,8 +99,9 @@ def __getattr__(cls, name): if not hasattr(area, "_repr_html_"): continue - area_table.append(rst_table_row([f"`{aname}`_", area.description, - area.proj_dict.get("proj")])) + with ignore_pyproj_proj_warnings(): + area_proj = area.proj_dict.get("proj") + area_table.append(rst_table_row([f"`{aname}`_", area.description, area_proj])) with open("area_def_list.rst", mode="w") as f: f.write("".join(area_table)) From c514fe8ca48ecf0017dd462844abe02e82b5988a Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 27 Aug 2024 14:10:39 -0500 Subject: [PATCH 2/9] Move area list generation to separate script And to use bokeh --- .readthedocs.yml | 3 + doc/rtd_environment.yml | 5 +- doc/source/conf.py | 30 +----- doc/source/generate_area_def_list.py | 152 +++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 31 deletions(-) create mode 100644 doc/source/generate_area_def_list.py diff --git a/.readthedocs.yml b/.readthedocs.yml index 9f3d7bd1b5..1eeb6b2332 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -18,5 +18,8 @@ build: - git fetch --tags pre_install: - git update-index --assume-unchanged doc/rtd_environment.yml doc/source/conf.py + pre_build: + - cd doc/source + - python generate_area_def_list.py conda: environment: doc/rtd_environment.yml diff --git a/doc/rtd_environment.yml b/doc/rtd_environment.yml index abd8add616..2ffcceb8d4 100644 --- a/doc/rtd_environment.yml +++ b/doc/rtd_environment.yml @@ -16,7 +16,7 @@ dependencies: - numpy - pillow - pooch - - pyresample +# - pyresample - pytest - python-eccodes - python-geotiepoints @@ -31,8 +31,9 @@ dependencies: - xarray - zarr - xarray-datatree - - cartopy + - geoviews - pip: - graphviz - pytest-lazy-fixtures + - git+https://github.com/djhoese/pyresample.git@feat-html-bokeh - .. # relative path to the satpy project diff --git a/doc/source/conf.py b/doc/source/conf.py index 248c9f7a07..2e41793c15 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -24,16 +24,9 @@ sys.path.append(os.path.abspath("../../")) sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from pyresample.area_config import ( # noqa: E402 - _create_area_def_from_dict, - _read_yaml_area_file_content, - generate_area_def_rst_list, -) -from pyresample.utils.proj4 import ignore_pyproj_proj_warnings -from reader_table import generate_reader_table, rst_table_header, rst_table_row # noqa: E402 +from reader_table import generate_reader_table # noqa: E402 import satpy # noqa: E402 -from satpy.resample import get_area_file # noqa: E402 # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -87,27 +80,6 @@ def __getattr__(cls, name): with open("reader_table.rst", mode="w") as f: f.write(generate_reader_table()) -# create table from area definition yaml file -area_file = get_area_file()[0] - -area_dict = _read_yaml_area_file_content(area_file) -area_table = [rst_table_header("Area Definitions", header=["Name", "Description", "Projection"], - widths="auto", class_name="area-table")] - -for aname, params in area_dict.items(): - area = _create_area_def_from_dict(aname, params) - if not hasattr(area, "_repr_html_"): - continue - - with ignore_pyproj_proj_warnings(): - area_proj = area.proj_dict.get("proj") - area_table.append(rst_table_row([f"`{aname}`_", area.description, area_proj])) - -with open("area_def_list.rst", mode="w") as f: - f.write("".join(area_table)) - f.write("\n\n") - f.write(generate_area_def_rst_list(area_file)) - # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions diff --git a/doc/source/generate_area_def_list.py b/doc/source/generate_area_def_list.py new file mode 100644 index 0000000000..2c5d8d17b7 --- /dev/null +++ b/doc/source/generate_area_def_list.py @@ -0,0 +1,152 @@ +"""Generate the area definition list restructuredtext document. + +This should be run once before generating the sphinx documentation to +produce the ``area_def_list.rst`` file referenced by ``satpy/resample.py``. + +""" +import logging +import pathlib +import sys +from datetime import datetime + +import bokeh +import geoviews as gv +import geoviews.feature as gf +from bokeh.embed import components +from jinja2 import Template +from pyresample._formatting_html import _load_static_files +from pyresample.area_config import area_repr, load_area +from pyresample.utils.proj4 import ignore_pyproj_proj_warnings +from reader_table import rst_table_header, rst_table_row + +from satpy.resample import get_area_file + +logger = logging.getLogger(__name__) + +gv.extension("bokeh") + + +TEMPLATE = ''' + +{{ table_header }} +{% for area_name, area_def in areas.items() if area_def._repr_html_ is defined %} +{{ create_table_row(area_name, area_def) }} +{% endfor %} + + +.. raw:: html + + {{ resources }} + {{ pyr_icons_svg | indent(5) }} + + {{ script | indent(5)}} + +{% for area_name, area_div in area_divs_dict.items() %} + +{{ area_name }} +{{ rst_underline('^', area_name|length) }} + +.. raw:: html + + {{ area_repr(areas[area_name], map_content=area_div, include_header=False, include_static_files=False) | + indent(5) }} +
+ +{% endfor %} +''' # noqa: Q001 + + +def main(): + """Parse CLI arguments and generate area definition list file.""" + from argparse import ArgumentParser + + parser = ArgumentParser(description="Generate restructuredtext area definition list for sphinx documentation") + parser.add_argument("--area-file", + help="Input area YAML file to read") + parser.add_argument("-o", "--output-file", + type=pathlib.Path, + help="HTML or restructuretext filename to create. " + "Defaults to 'area_def_list.rst' in the " + "documentation source directory.") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + + if args.output_file is None: + args.output_file = str(pathlib.Path(__file__).resolve().parent / "area_def_list.rst") + area_file = args.area_file + if area_file is None: + area_file = get_area_file()[0] + + area_list = load_area(area_file) + areas_dict = {_area_name(area): area for area in area_list} + logger.info(f"Generating bokeh plots ({datetime.now()})...") + script, divs_dict = _generate_html_map_divs(areas_dict) + logger.info(f"Done generating bokeh plots ({datetime.now()})") + + def rst_underline(ch, num_chars): + return ch * num_chars + + template = Template(TEMPLATE) + icons_svg, css_style = _load_static_files() + logger.info(f"Rendering document ({datetime.now()})...") + res = template.render( + resources=bokeh.resources.CDN.render(), + script=script, + area_divs_dict=divs_dict, + areas=areas_dict, + rst_underline=rst_underline, + area_repr=area_repr, + pyr_icons_svg=icons_svg, + pyr_css_style=css_style, + table_header=rst_table_header("Area Definitions", header=["Name", "Description", "Projection"], + widths="auto", class_name="area-table"), + create_table_row=_area_table_row, + ) + logger.info(f"Done rendering document ({datetime.now()})") + + with open(args.output_file, mode="w") as f: + f.write(res) + + +def _area_name(area_def) -> str: + if hasattr(area_def, "attrs"): + # pyresample 2 + return area_def.attrs["name"] + # pyresample 1 + return area_def.area_id + + +def _area_table_row(area_name, area_def): + with ignore_pyproj_proj_warnings(): + area_proj = area_def.proj_dict.get("proj") + return rst_table_row([f"`{area_name}`_", area_def.description, area_proj]) + + +def _generate_html_map_divs(areas_dict: dict) -> tuple[str, dict]: + areas_bokeh_models = {} + for area_name, area_def in areas_dict.items(): + if not hasattr(area_def, "to_cartopy_crs"): + logger.info(f"Skipping {area_name} because it can't be converted to cartopy CRS") + continue + crs = area_def.to_cartopy_crs() + + features = gv.Overlay([gf.ocean, gf.land, gf.borders, gf.coastline]) + f = gv.render( + features.opts( + toolbar=None, + default_tools=[], + projection=crs, + xlim=crs.bounds[:2], + ylim=crs.bounds[2:], + ), + backend="bokeh") + areas_bokeh_models[area_name] = f + + script, divs_dict = components(areas_bokeh_models) + return script, divs_dict + + +if __name__ == "__main__": + sys.exit(main()) From ff2fa34eb1c2e8965e5f21b89287be5d1ea06da2 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 27 Aug 2024 14:21:55 -0500 Subject: [PATCH 3/9] Remove non-working pyresample dev install in RTD --- doc/rtd_environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/rtd_environment.yml b/doc/rtd_environment.yml index 2ffcceb8d4..5bc7dabe95 100644 --- a/doc/rtd_environment.yml +++ b/doc/rtd_environment.yml @@ -16,7 +16,7 @@ dependencies: - numpy - pillow - pooch -# - pyresample + - pyresample - pytest - python-eccodes - python-geotiepoints @@ -35,5 +35,4 @@ dependencies: - pip: - graphviz - pytest-lazy-fixtures - - git+https://github.com/djhoese/pyresample.git@feat-html-bokeh - .. # relative path to the satpy project From 56e4d5b313867f24812594e98512c81f51d0c157 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 29 Aug 2024 09:07:34 -0500 Subject: [PATCH 4/9] Fix readthedocs running area def script --- .readthedocs.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 1eeb6b2332..b786e6ebce 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,7 +19,6 @@ build: pre_install: - git update-index --assume-unchanged doc/rtd_environment.yml doc/source/conf.py pre_build: - - cd doc/source - - python generate_area_def_list.py + - cd doc/source && python generate_area_def_list.py conda: environment: doc/rtd_environment.yml From fe930206e3ffa48ebb88117e4a8db473c4d9ed60 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 29 Aug 2024 09:20:08 -0500 Subject: [PATCH 5/9] Add docutils config to workaround long bokeh lines --- doc/source/docutils.conf | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/source/docutils.conf diff --git a/doc/source/docutils.conf b/doc/source/docutils.conf new file mode 100644 index 0000000000..70bcbbb2e7 --- /dev/null +++ b/doc/source/docutils.conf @@ -0,0 +1,2 @@ +[parsers] +line_length_limit=30000000 From e804b7068fd94ecd0e18385c889ac0d808ef2f76 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 29 Aug 2024 10:03:01 -0500 Subject: [PATCH 6/9] Allow conda unstable remove to always succeed --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9d9e552da5..f7b5f447c4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -80,7 +80,7 @@ jobs: numpy \ pandas \ scipy - conda remove --force-remove -y pykdtree pyresample python-geotiepoints pyhdf netcdf4 h5py cftime astropy pyerfa + conda remove --force-remove -y pykdtree pyresample python-geotiepoints pyhdf netcdf4 h5py cftime astropy pyerfa || true python -m pip install --upgrade --no-deps --pre --no-build-isolation \ pyerfa \ git+https://github.com/storpipfugl/pykdtree \ From 084b1144b2019a86d47c0dc9e5f2f62752a8c565 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 29 Aug 2024 11:19:16 -0500 Subject: [PATCH 7/9] Add documentation about generating area list docs --- doc/source/dev_guide/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/dev_guide/index.rst b/doc/source/dev_guide/index.rst index bbd9bedaf4..0df47a866c 100644 --- a/doc/source/dev_guide/index.rst +++ b/doc/source/dev_guide/index.rst @@ -128,6 +128,15 @@ additional packages are needed. These can be installed with :: pip install -e ".[doc]" +Generating the documentation requires a one-time script to generate a list +of previews of all of the AreaDefinition objects used by the documentation. +This script can take 2+ minutes to execute so it is run separately from the +normal documentation build process. To run it:: + + cd doc/source/ + python generate_area_def_list.py + cd ../../ + After editing the source files there the documentation can be generated locally:: cd doc From d25ec20ace9f72bb38cdd515c9ea56e5ce90d9bd Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 1 Sep 2024 09:55:47 -0500 Subject: [PATCH 8/9] Turn off area def list generation if not latest/stable RTD --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index b786e6ebce..f6f9662c1c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,6 +19,6 @@ build: pre_install: - git update-index --assume-unchanged doc/rtd_environment.yml doc/source/conf.py pre_build: - - cd doc/source && python generate_area_def_list.py + - cd doc/source && if [[ $READTHEDOCS_VERSION =~ latest|stable ]]; then python generate_area_def_list.py; else touch area_def_list.rst; fi conda: environment: doc/rtd_environment.yml From 5b2ac6855afd2aaaefc46754f8c15d66b51d88ab Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 1 Sep 2024 10:05:27 -0500 Subject: [PATCH 9/9] Switch to 'sh' compatible syntax in RTD config --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index f6f9662c1c..8404e7da46 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,6 +19,6 @@ build: pre_install: - git update-index --assume-unchanged doc/rtd_environment.yml doc/source/conf.py pre_build: - - cd doc/source && if [[ $READTHEDOCS_VERSION =~ latest|stable ]]; then python generate_area_def_list.py; else touch area_def_list.rst; fi + - cd doc/source && if [ "$READTHEDOCS_VERSION" = "latest" ] || [ "$READTHEDOCS_VERSION" = "stable" ]; then python generate_area_def_list.py; else touch area_def_list.rst; fi conda: environment: doc/rtd_environment.yml