Provenance hints to user_note and user_error and error handling for missing coupling direction

src/esm_master/task.py

@@ -1,18 +1,16 @@
 import os
-import sys
-import subprocess
-import shlex  # contains shlex.split that respects quoted strings
-
 # deniz: it is better to use more pathlib in the future so that dir/path
 # operations will be more portable (supported since Python 3.4, 2014)
 import pathlib
-
-from .software_package import software_package
-from esm_parser import user_error
+import shlex  # contains shlex.split that respects quoted strings
+import subprocess
+import sys
 
 import esm_environment
 import esm_plugin_manager
+from esm_tools import user_error
 
+from .software_package import software_package
 
 ######################################################################################
 ################################# class "task" #######################################

src/esm_motd/esm_motd.py

@@ -3,11 +3,12 @@
 import urllib.request
 from time import sleep
 
-import esm_parser
-import esm_utilities
 import yaml
 
+import esm_parser
 import esm_tools
+import esm_utilities
+from esm_tools import user_error
 
 
 class MessageOfTheDayError(Exception):
@@ -83,7 +84,7 @@ def action_handler(self, action, time, package, version):
             if action == "sleep":
                 sleep(time)
             elif action == "error":
-                esm_parser.user_error(
+                user_error(
                     "Version",
                     (
                         f"Version {version} of '{package}' package has been tagged as "

src/esm_parser/esm_parser.py

@@ -53,19 +53,15 @@
 Specific documentation for classes and functions are given below:
 """
 # Python 2 and 3 version agnostic compatiability:
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-
-import pdb
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
 
 # Python Standard Library imports
 import collections
 import copy
 import logging
 import os
-import re
+import pdb
 import shutil
 import socket
 import subprocess
@@ -79,21 +75,20 @@
 
 # Always import externals before any non standard library imports
 
+import coloredlogs
 # Third-Party Imports
 import numpy
-import coloredlogs
-import colorama
 import yaml
 
-# functions reading in dict from file
-from .yaml_to_dict import *
-from .provenance import *
-
+# Loader for package yamls
+import esm_tools
 # Date class
 from esm_calendar import Date
+from esm_tools import user_error, user_note
 
-# Loader for package yamls
-import esm_tools
+from .provenance import *
+# functions reading in dict from file
+from .yaml_to_dict import *
 
 # Logger and related constants
 logger = logging.getLogger("root")
@@ -2789,51 +2784,6 @@ def find_key(d_search, k_search, exc_strings="", level="", paths2finds=[], sep="
     return paths2finds
 
 
-def user_note(note_heading, note_text, color=colorama.Fore.YELLOW, dsymbols=["``"]):
-    """
-    Notify the user about something. In the future this should also write in the log.
-
-    Parameters
-    ----------
-    note_heading : str
-        Note type used for the heading.
-    text : str
-        Text clarifying the note.
-    """
-    reset_s = colorama.Style.RESET_ALL
-
-    if isinstance(note_text, list):
-        new_note_text = ""
-        for item in note_text:
-            new_note_text = f"{new_note_text}- {item}\n"
-        note_text = new_note_text
-
-    for dsymbol in dsymbols:
-        note_text = re.sub(
-            f"{dsymbol}([^{dsymbol}]*){dsymbol}", f"{color}\\1{reset_s}", str(note_text)
-        )
-    print(f"\n{color}{note_heading}\n{'-' * len(note_heading)}{reset_s}")
-    print(f"{note_text}\n")
-
-
-def user_error(error_type, error_text, exit_code=1, dsymbols=["``"]):
-    """
-    User-friendly error using ``sys.exit()`` instead of an ``Exception``.
-
-    Parameters
-    ----------
-    error_type : str
-        Error type used for the error heading.
-    text : str
-        Text clarifying the error.
-    exit_code : int
-        The exit code to send back to the parent process (default to 1)
-    """
-    error_title = "ERROR: " + error_type
-    user_note(error_title, error_text, color=colorama.Fore.RED, dsymbols=dsymbols)
-    sys.exit(exit_code)
-
-
 class GeneralConfig(dict):  # pragma: no cover
     """All configs do this!"""
 

src/esm_parser/provenance.py

@@ -523,6 +523,23 @@ def get_provenance(self, index=-1):
 
         return provenance_dict
 
+    def extract_first_nested_values_provenance(self):
+        """
+        Recursively loops through the dictionary keys and returns the first provenance
+        found in the nested values.
+
+        Returns
+        -------
+        first_provenance : esm_parser.provenance.Provenance
+            The first provenance found in the nested values
+        """
+        first_provenance = None
+        for key, val in self.items():
+            if isinstance(val, PROVENANCE_MAPPINGS):
+                return val.extract_first_nested_values_provenance()
+            elif hasattr(val, "provenance"):
+                return val.provenance[-1]
+
     def __setitem__(self, key, val):
         """
         Any time an item in a DictWithProvenance is set, extend the old provenance of
@@ -773,6 +790,23 @@ def get_provenance(self, index=-1):
 
         return provenance_list
 
+    def extract_first_nested_values_provenance(self):
+        """
+        Recursively loops through the list elements and returns the first provenance
+        found in the nested values.
+
+        Returns
+        -------
+        first_provenance : esm_parser.provenance.Provenance
+            The first provenance found in the nested values
+        """
+        first_provenance = None
+        for elem in self:
+            if isinstance(elem, PROVENANCE_MAPPINGS):
+                return elem.extract_first_nested_values_provenance()
+            elif hasattr(elem, "provenance"):
+                return elem.provenance[-1]
+
     def __setitem__(self, indx, val):
         """
         Any time an item in a ListWithProvenance is set, extend the old provenance of

src/esm_parser/yaml_to_dict.py

@@ -11,6 +11,7 @@
 
 import esm_parser
 import esm_tools
+from esm_tools import user_error
 
 from .provenance import *
 
@@ -135,7 +136,7 @@ def constructor_env_variables(loader, node):
                 for env_var in envvar_matches:
                     # first check if the variable exists in the shell environment
                     if not os.getenv(env_var):
-                        esm_parser.user_error(
+                        user_error(
                             f"{env_var} is not defined",
                             f"{env_var} is not an environment variable. Exiting",
                         )
@@ -227,10 +228,10 @@ def yaml_file_to_dict(filepath):
         except yaml.scanner.ScannerError as yaml_error:
             logger.debug(f"Your file {filepath + extension} has syntax issues!")
             error = EsmConfigFileError(filepath + extension, yaml_error)
-            esm_parser.user_error("Yaml syntax", f"{error}")
+            user_error("Yaml syntax", f"{error}")
         except Exception as error:
             logger.exception(error)
-            esm_parser.user_error(
+            user_error(
                 "Yaml syntax",
                 f"Syntax error in ``{filepath}``\n\n``Details:\n``{error}",
             )
@@ -331,7 +332,7 @@ def check_changes_duplicates(yamldict_all, fpath):
             # If more than one ``_changes`` without ``choose_`` return error
             if len(changes_no_choose) > 1:
                 changes_no_choose = [x.replace(",", ".") for x in changes_no_choose]
-                esm_parser.user_error(
+                user_error(
                     "YAML syntax",
                     "More than one ``_changes`` out of a ``choose_`` in "
                     + fpath
@@ -347,7 +348,7 @@ def check_changes_duplicates(yamldict_all, fpath):
                 changes_group.remove(changes_no_choose[0])
                 if len(changes_group) > 0:
                     changes_group = [x.replace(",", ".") for x in changes_group]
-                    esm_parser.user_error(
+                    user_error(
                         "YAML syntax",
                         "The general ``"
                         + changes_no_choose[0]
@@ -394,7 +395,7 @@ def check_changes_duplicates(yamldict_all, fpath):
                                 ","
                             )[0]
                         if case == sub_case:
-                            esm_parser.user_error(
+                            user_error(
                                 "YAML syntax",
                                 "The following ``_changes`` can be accessed "
                                 + "simultaneously in "
@@ -413,7 +414,7 @@ def check_changes_duplicates(yamldict_all, fpath):
                     else:
                         # If these ``choose_`` are different they can be accessed
                         # simultaneously, then it returns an error
-                        esm_parser.user_error(
+                        user_error(
                             "YAML syntax",
                             "The following ``_changes`` can be accessed "
                             + "simultaneously in "
@@ -452,7 +453,7 @@ def check_changes_duplicates(yamldict_all, fpath):
                 add_group.remove(add_no_choose[0])
                 if len(add_group) > 0:
                     add_group = [x.replace(",", ".") for x in add_group]
-                    esm_parser.user_error(
+                    user_error(
                         "YAML syntax",
                         "The general ``"
                         + add_no_choose[0]
@@ -469,7 +470,7 @@ def check_changes_duplicates(yamldict_all, fpath):
 def check_for_empty_components(yaml_load, fpath):
     for key, value in yaml_load.items():
         if not value:
-            esm_parser.user_error(
+            user_error(
                 "YAML syntax",
                 f"The component ``{key}`` is empty in the file ``{fpath}``. ESM-Tools does"
                 + " not support empty components, either add some variables to the "
@@ -540,7 +541,7 @@ def map_constructor(loader, node, deep=False):
             value = loader.construct_object(value_node, deep=deep)
 
             if key in mapping:
-                esm_parser.user_error(
+                user_error(
                     "Duplicated variables",
                     "Key ``{0}`` is duplicated {1}\n\n".format(
                         key, str(key_node.start_mark).replace("  ", "").split(",")[0]
@@ -623,7 +624,7 @@ def construct_scalar(self, node):
                         self.env_variables.append((env_var, rval))
                         return rval
                     else:
-                        esm_parser.user_error(
+                        user_error(
                             f"{env_var} is not defined",
                             f"{env_var} is not an environment variable. Exiting",
                         )
@@ -868,4 +869,4 @@ def dump(self, data, stream=None, **kw):
         if not self.add_comments:
             return super().dump(data, stream, **kw)
         self._add_origin_comments(data)
-        return super().dump(data, stream, **kw)
+        return super().dump(data, stream, **kw)

src/esm_runscripts/batch_system.py

@@ -4,10 +4,12 @@
 import sys
 import textwrap
 
-import esm_environment
-from esm_parser import find_variable, user_error, user_note
 from loguru import logger
 
+import esm_environment
+from esm_parser import find_variable
+from esm_tools import user_error, user_note
+
 from . import dataprocess, helpers, prepare
 from .pbs import Pbs
 from .slurm import Slurm
@@ -814,7 +816,7 @@ def calc_launcher_flags(config, model, cluster):
             cpus_per_proc = config[model].get("cpus_per_proc", omp_num_threads)
             # Check for CPUs and OpenMP threads
             if omp_num_threads > cpus_per_proc:
-                esm_parser.user_error(
+                user_error(
                     "OpenMP configuration",
                     (
                         "The number of OpenMP threads cannot be larger than the number"
@@ -826,7 +828,7 @@ def calc_launcher_flags(config, model, cluster):
         elif "nproca" in config[model] and "nprocb" in config[model]:
             # ``nproca``/``nprocb`` not compatible with ``omp_num_threads``
             if omp_num_threads > 1:
-                esm_parser.user_note(
+                user_note(
                     "nproc",
                     "``nproca``/``nprocb`` not compatible with ``omp_num_threads``",
                 )

src/esm_runscripts/cli.py

@@ -15,7 +15,7 @@
 from loguru import logger
 
 from esm_motd import check_all_esm_packages
-from esm_parser import user_error
+from esm_tools import user_error
 
 from .helpers import SmartSink
 from .sim_objects import *

src/esm_runscripts/compute.py

@@ -9,12 +9,13 @@
 import questionary
 import yaml
 from colorama import Back, Fore, Style, init
+from loguru import logger
 
 import esm_calendar
 import esm_parser
 import esm_runscripts
 import esm_tools
-from loguru import logger
+from esm_tools import user_error, user_note
 
 from .batch_system import batch_system
 from .filelists import copy_files, log_used_files
@@ -444,7 +445,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type):
             # If the --update flag is used, notify that the target script will
             # be updated and do update it
             if gconfig["update"]:
-                esm_parser.user_note(
+                user_note(
                     f"Original {file_type} different from target",
                     differences + "\n" + f"{scriptsdir + '/' + tfile} will be updated!",
                 )
@@ -454,7 +455,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type):
             # If the --update flag is not called, exit with an error showing the
             # user how to proceed
             else:
-                esm_parser.user_note(
+                user_note(
                     f"Original {file_type} different from target",
                     differences
                     + "\n"
@@ -546,7 +547,7 @@ def copy_tools_to_thisrun(config):
         # exit right away to prevent further recursion. There might still be
         # running instances of esmr_runscripts and something like
         # `killall esm_runscripts` might be required
-        esm_parser.user_error(error_type, error_text)
+        user_error(error_type, error_text)
 
     # If ``fromdir`` and ``scriptsdir`` are the same, this is already a computing
     # simulation which means we want to use the script in the experiment folder,