Add esmvalcore.dataset module (#1877)

Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com>

doc/api/esmvalcore.dataset.rst

@@ -0,0 +1,5 @@
+Dataset
+=======
+
+.. automodule:: esmvalcore.dataset
+    :no-show-inheritance:

doc/api/esmvalcore.rst

@@ -11,6 +11,7 @@ library. This section documents the public API of ESMValCore.
 
    esmvalcore.cmor
    esmvalcore.config
+   esmvalcore.dataset
    esmvalcore.esgf
    esmvalcore.exceptions
    esmvalcore.iris_helpers

doc/conf.py

@@ -45,6 +45,8 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'autodocsumm',
+    'nbsphinx',
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',
@@ -54,7 +56,6 @@
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'sphinx.ext.napoleon',
-    'autodocsumm',
 ]
 
 autodoc_default_options = {

doc/example-notebooks.rst

@@ -0,0 +1,13 @@
+Example notebooks
+*****************
+
+Example notebooks are available in the
+`notebooks <https://github.com/ESMValGroup/ESMValCore/tree/main/notebooks>`__
+folder and can also be viewed here.
+These notebooks demonstrate the use of the :ref:`Python API <api>`.
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    notebooks/*

doc/gensidebar.py

@@ -72,6 +72,7 @@ def _header(project, text):
 
     _header("esmvalcore", "ESMValCore")
     _write("esmvalcore", "Getting started", "quickstart/index")
+    _write("esmvalcore", "Example notebooks", "example-notebooks")
     _write("esmvalcore", "The recipe format", "recipe/index")
     _write("esmvalcore", "Diagnostic script interfaces", "interfaces")
     _write("esmvalcore", "Development", "develop/index")

doc/notebooks

@@ -0,0 +1 @@
+../notebooks

doc/quickstart/configure.rst

@@ -62,6 +62,11 @@ omitted in the file.
   # on a computer without an internet connection.
   offline: true
 
+  # Search ESGF for files even when files are available locally --- true/[false]
+  # This option is useful to make sure you have the latest version of all files.
+  # Remember to set ``offline: false`` if this option is set to ``true``.
+  always_search_esgf: false
+
   # Auxiliary data directory
   # Used by some recipes to look for additional datasets.
   auxiliary_data_dir: ~/auxiliary_data

environment.yml

@@ -41,6 +41,8 @@ dependencies:
   - yamale
   # Python packages needed for building docs
   - autodocsumm>=0.2.2
+  - ipython
+  - nbsphinx
   - sphinx>=5
   - sphinx_rtd_theme
   # Python packages needed for testing

esmvalcore/_main.py

@@ -74,7 +74,7 @@ def process_recipe(recipe_file: Path, session):
     import shutil
     import warnings
 
-    from ._recipe import read_recipe_file
+    from esmvalcore._recipe.recipe import read_recipe_file
     if not recipe_file.is_file():
         import errno
         raise OSError(errno.ENOENT, "Specified recipe file does not exist",

esmvalcore/_recipe/__init__.py

@@ -0,0 +1 @@
+"""Load and run recipes."""

esmvalcore/_recipe_checks.py → esmvalcore/_recipe/check.py

@@ -9,10 +9,10 @@
 import isodate
 import yamale
 
-from .exceptions import InputFilesNotFound, RecipeError
-from .local import _get_start_end_year
-from .preprocessor import TIME_PREPROCESSORS, PreprocessingTask
-from .preprocessor._multimodel import STATISTIC_MAPPING
+from esmvalcore.exceptions import InputFilesNotFound, RecipeError
+from esmvalcore.local import _get_start_end_year, _parse_period
+from esmvalcore.preprocessor import TIME_PREPROCESSORS, PreprocessingTask
+from esmvalcore.preprocessor._multimodel import STATISTIC_MAPPING
 
 logger = logging.getLogger(__name__)
 
@@ -141,18 +141,20 @@ def data_availability(input_files, var, patterns, log=True):
 
     if not input_files:
         raise InputFilesNotFound(
-            f"Missing data for {var['alias']}: {var['short_name']}")
+            f"Missing data for {var.get('alias', 'dataset')}: "
+            f"{var['short_name']}")
 
-    if var['frequency'] == 'fx':
-        # check time availability only for non-fx variables
+    if 'timerange' not in var:
         return
-    start_year = var['start_year']
-    end_year = var['end_year']
+
+    start_date, end_date = _parse_period(var['timerange'])
+    start_year = int(start_date[0:4])
+    end_year = int(end_date[0:4])
     required_years = set(range(start_year, end_year + 1, 1))
     available_years = set()
 
-    for filename in input_files:
-        start, end = _get_start_end_year(filename)
+    for file in input_files:
+        start, end = _get_start_end_year(file.name)
         available_years.update(range(start, end + 1))
 
     missing_years = required_years - available_years
@@ -246,8 +248,7 @@ def _verify_keep_input_datasets(keep_input_datasets):
     if not isinstance(keep_input_datasets, bool):
         raise RecipeError(
             "Invalid value encountered for `keep_input_datasets`."
-            f"Must be defined as a boolean. Got {keep_input_datasets}."
-        )
+            f"Must be defined as a boolean. Got {keep_input_datasets}.")
 
 
 def _verify_arguments(given, expected):