From 78650a292b738556c626c5d5a29f78d45c49a552 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 28 Mar 2022 21:43:30 +0200 Subject: [PATCH 01/15] Add CompassConfigParser This is a "meta" config parser that keeps a dictionary of config parsers and their sources to combine when needed. The custom config parser allows provenance of the source of different config options and allows the "user" config options to always take precedence over other config options (even if they are added later). --- compass/config.py | 355 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 263 insertions(+), 92 deletions(-) diff --git a/compass/config.py b/compass/config.py index dd9db6ceab..e65438d601 100644 --- a/compass/config.py +++ b/compass/config.py @@ -1,119 +1,290 @@ +from configparser import RawConfigParser, ConfigParser, ExtendedInterpolation import os -import configparser -from io import StringIO from importlib import resources +import inspect -def duplicate_config(config): +class CompassConfigParser: """ - Make a deep copy of config to changes can be made without affecting the - original - - Parameters - ---------- - config : configparser.ConfigParser - Configuration options - - Returns - ------- - new_config : configparser.ConfigParser - Deep copy of configuration options + A "meta" config parser that keeps a dictionary of config parsers and their + sources to combine when needed. The custom config parser allows provenance + of the source of different config options and allows the "user" config + options to always take precedence over other config options (even if they + are added later). """ + def __init__(self): + """ + Make a new (empty) config parser + """ - config_string = StringIO() - config.write(config_string) - # We must reset the buffer to make it ready for reading. - config_string.seek(0) - new_config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - new_config.read_file(config_string) - return new_config + self._configs = dict() + self._user_config = dict() + self._combined = None + def add_user_config(self, filename): + """ + Add a the contents of a user config file to the parser. These options + take precedence over all other options. -def add_config(config, package, config_file, exception=True): - """ - Add the contents of a config file within a package to the current config - parser + Parameters + ---------- + filename : str + The relative or absolute path to the config file + """ + self._add(filename, user=True) - Parameters - ---------- - config : configparser.ConfigParser - Configuration options + def add_from_file(self, filename): + """ + Add the contents of a config file to the parser. - package : str or Package - The package where ``config_file`` is found + Parameters + ---------- + filename : str + The relative or absolute path to the config file + """ + self._add(filename, user=False) - config_file : str - The name of the config file to add + def add_from_package(self, package, config_filename, exception=True): + """ + Add the contents of a config file to the parser. - exception : bool - Whether to raise an exception if the config file isn't found - """ - try: - with resources.path(package, config_file) as path: - config.read(path) - except (ModuleNotFoundError, FileNotFoundError, TypeError): - if exception: - raise + Parameters + ---------- + package : str or Package + The package where ``config_filename`` is found + config_filename : str + The name of the config file to add -def merge_other_config(config, other_config): - """ - Add config options from the other config parser to this one + exception : bool, optional + Whether to raise an exception if the config file isn't found + """ + try: + with resources.path(package, config_filename) as path: + self._add(path, user=False) + except (ModuleNotFoundError, FileNotFoundError, TypeError): + if exception: + raise - Parameters - ---------- - config : configparser.ConfigParser - Configuration options + def get(self, section, option): + """ + Get an option value for a given section. - other_config : configparser.ConfigParser - Configuration options to add - """ - for section in other_config.sections(): - if not config.has_section(section): - config.add_section(section) - for key, value in other_config.items(section): - config.set(section, key, value) + Parameters + ---------- + section : str + The name of the config section + option : str + The name of the config option -def ensure_absolute_paths(config): - """ - make sure all paths in the paths, namelists and streams sections are - absolute paths + Returns + ------- + value : str + The value of the config option + """ + if self._combined is None: + self._combine() + return self._combined.get(section, option) - Parameters - ---------- - config : configparser.ConfigParser - Configuration options - """ - for section in ['paths', 'namelists', 'streams', 'executables']: - for option, value in config.items(section): - value = os.path.abspath(value) - config.set(section, option, value) + def getint(self, section, option): + """ + Get an option integer value for a given section. + Parameters + ---------- + section : str + The name of the config section -def get_source_file(source_path, source, config): - """ - Get an absolute path given a tag name for that path + option : str + The name of the config option - Parameters - ---------- - source_path : str - The keyword path for a path as defined in :ref:`dev_config`, - a config option from a relative or absolute directory for the source + Returns + ------- + value : int + The value of the config option + """ + if self._combined is None: + self._combine() + return self._combined.getint(section, option) - source : str - The basename or relative path of the source within the ``source_path`` - directory + def getfloat(self, section, option): + """ + Get an option float value for a given section. - config : configparser.ConfigParser - Configuration options used to determine the the absolute paths for the - given ``source_path`` - """ + Parameters + ---------- + section : str + The name of the config section + + option : str + The name of the config option + + Returns + ------- + value : float + The value of the config option + """ + if self._combined is None: + self._combine() + return self._combined.getfloat(section, option) + + def getboolean(self, section, option): + """ + Get an option boolean value for a given section. + + Parameters + ---------- + section : str + The name of the config section + + option : str + The name of the config option + + Returns + ------- + value : bool + The value of the config option + """ + if self._combined is None: + self._combine() + return self._combined.getboolean(section, option) + + def getlist(self, section, option, dtype=str): + """ + Get an option value as a list for a given section. + + Parameters + ---------- + section : str + The name of the config section + + option : str + The name of the config option + + dtype : {Type[str], Type[int], Type[float]} + The type of the elements in the list + + Returns + ------- + value : list + The value of the config option parsed into a list + """ + values = self.get(section, option) + values = [dtype(value) for value in values.replace(',', ' ').split()] + return values + + def has_option(self, section, option): + """ + Whether the given section has the given option + + Parameters + ---------- + section : str + The name of the config section + + option : str + The name of the config option + + Returns + ------- + found : bool + Whether the option was found in the section + """ + if self._combined is None: + self._combine() + return self._combined.has_option(section, option) + + def set(self, section, option, value=None): + """ + Set the value of the given option in the given section. The file from + which this function was called is also retained for provenance. + + Parameters + ---------- + section : str + The name of the config section + + option : str + The name of the config option + + value : str, optional + The value to set the option to + """ + calling_frame = inspect.stack(context=2)[1] + filename = os.path.abspath(calling_frame.filename) + if filename not in self._configs: + self._configs[filename] = RawConfigParser() + config = self._configs[filename] + if not config.has_section(section): + config.add_section(section) + config.set(section, option, value) + self._combined = None + + def write(self, fp): + """ + Write the config options to the given file pointer. + + Parameters + ---------- + fp : typing.TestIO + The file pointer to write to. + """ + if self._combined is None: + self._combine() + self._combined.write(fp) + + def __getitem__(self, section): + """ + Get get the config options for a given section. + + Parameters + ---------- + section : str + The name of the section to retrieve. + + Returns + ------- + section_proxy : configparser.SectionProxy + The config options for the given section. + """ + if self._combined is None: + self._combine() + return self._combined[section] + + def _add(self, filename, user): + filename = os.path.abspath(filename) + config = RawConfigParser() + if not os.path.exists(filename): + raise FileNotFoundError(f'Config file does not exist: {filename}') + config.read(filenames=filename) + if user: + self._user_config = {filename: config} + else: + self._configs[filename] = config + self._combined = None - if config.has_option('paths', source_path): - source_path = config.get('paths', source_path) + def _combine(self): + self._combined = ConfigParser(interpolation=ExtendedInterpolation()) + configs = dict(self._configs) + configs.update(self._user_config) + for source, config in configs.items(): + for section in config.sections(): + if not self._combined.has_section(section): + self._combined.add_section(section) + for key, value in config.items(section): + self._combined.set(section, key, value) + self._ensure_absolute_paths() - source_file = '{}/{}'.format(source_path, source) - source_file = os.path.abspath(source_file) - return source_file + def _ensure_absolute_paths(self): + """ + make sure all paths in the paths, namelists, streams, and executables + sections are absolute paths + """ + config = self._combined + for section in ['paths', 'namelists', 'streams', 'executables']: + if not config.has_section(section): + continue + for option, value in config.items(section): + value = os.path.abspath(value) + config.set(section, option, value) From 7fcbbce44a7470edf1d3000d164334197dc9cc62 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 28 Mar 2022 21:48:29 +0200 Subject: [PATCH 02/15] Update config parsing in framework --- compass/cache.py | 8 +++--- compass/io.py | 2 +- compass/model.py | 2 +- compass/parallel.py | 6 ++--- compass/provenance.py | 11 +++----- compass/run.py | 22 +++++++-------- compass/setup.py | 62 ++++++++++++++----------------------------- compass/step.py | 3 +-- compass/testcase.py | 3 +-- 9 files changed, 43 insertions(+), 76 deletions(-) diff --git a/compass/cache.py b/compass/cache.py index ac2d4269ae..ffcb40bb4d 100644 --- a/compass/cache.py +++ b/compass/cache.py @@ -4,11 +4,10 @@ from datetime import datetime import os from importlib import resources -import configparser import shutil import pickle -from compass.config import add_config +from compass.config import CompassConfigParser def update_cache(step_paths, date_string=None, dry_run=False): @@ -39,9 +38,8 @@ def update_cache(step_paths, date_string=None, dry_run=False): if invalid: raise ValueError('You must cache files from either Anvil or Chrysalis') - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - add_config(config, 'compass.machines', '{}.cfg'.format(machine)) + config = CompassConfigParser() + config.add_from_package('compass.machines', '{}.cfg'.format(machine)) if date_string is None: date_string = datetime.now().strftime("%y%m%d") diff --git a/compass/io.py b/compass/io.py index 2b3580585a..8b6a720d43 100644 --- a/compass/io.py +++ b/compass/io.py @@ -18,7 +18,7 @@ def download(url, dest_path, config, exceptions=True): The path (including file name) where the downloaded file should be saved - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options used to find custom paths if ``dest_path`` is a config option diff --git a/compass/model.py b/compass/model.py index f734626e81..2cdde29b96 100644 --- a/compass/model.py +++ b/compass/model.py @@ -76,7 +76,7 @@ def partition(cores, config, logger, graph_file='graph.info'): cores : int The number of cores that the model should be run on - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for the test case, used to get the partitioning executable diff --git a/compass/parallel.py b/compass/parallel.py index 85b2a32bdb..aa701d8a06 100644 --- a/compass/parallel.py +++ b/compass/parallel.py @@ -10,7 +10,7 @@ def get_available_cores_and_nodes(config): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for the test case Returns @@ -49,7 +49,7 @@ def check_parallel_system(config): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options Raises @@ -77,7 +77,7 @@ def set_cores_per_node(config): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options """ parallel_system = config.get('parallel', 'system') diff --git a/compass/provenance.py b/compass/provenance.py index aa6f83f318..55d1e23059 100644 --- a/compass/provenance.py +++ b/compass/provenance.py @@ -1,9 +1,8 @@ import os import sys import subprocess -import configparser -from compass.config import add_config +from compass.config import CompassConfigParser def write(work_dir, test_cases, mpas_core=None, config_filename=None, @@ -114,13 +113,11 @@ def write(work_dir, test_cases, mpas_core=None, config_filename=None, def _get_mpas_git_version(mpas_core, config_filename, mpas_model_path): if mpas_model_path is None: - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) + config = CompassConfigParser() # add the config options for the MPAS core - add_config(config, 'compass.{}'.format(mpas_core), - '{}.cfg'.format(mpas_core)) + config.add_from_package(f'compass.{mpas_core}', f'{mpas_core}.cfg') if config_filename is not None: - config.read(config_filename) + config.add_user_config(config_filename) if not config.has_option('paths', 'mpas_model'): raise ValueError('Couldn\'t find MPAS model. Not in user config ' 'file or passed with -p flag.') diff --git a/compass/run.py b/compass/run.py index 90cb10b7e4..0b7d417501 100644 --- a/compass/run.py +++ b/compass/run.py @@ -2,7 +2,6 @@ import sys import os import pickle -import configparser import time import glob @@ -10,6 +9,7 @@ import mpas_tools.io from compass.parallel import check_parallel_system, set_cores_per_node from compass.logging import log_method_call +from compass.config import CompassConfigParser def run_suite(suite_name, quiet=False): @@ -50,9 +50,8 @@ def run_suite(suite_name, quiet=False): test_case = next(iter(test_suite['test_cases'].values())) config_filename = os.path.join(test_case.work_dir, test_case.config_filename) - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - config.read(config_filename) + config = CompassConfigParser() + config.add_from_file(config_filename) check_parallel_system(config) # start logging to stdout/stderr @@ -92,9 +91,8 @@ def run_suite(suite_name, quiet=False): os.chdir(test_case.work_dir) - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - config.read(test_case.config_filename) + config = CompassConfigParser() + config.add_from_file(test_case.config_filename) test_case.config = config set_cores_per_node(test_case.config) @@ -216,9 +214,8 @@ def run_test_case(steps_to_run=None, steps_not_to_run=None): with open('test_case.pickle', 'rb') as handle: test_case = pickle.load(handle) - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - config.read(test_case.config_filename) + config = CompassConfigParser() + config.add_from_file(test_case.config_filename) check_parallel_system(config) @@ -278,9 +275,8 @@ def run_step(): test_case.steps_to_run = [step.name] test_case.new_step_log_file = False - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - config.read(step.config_filename) + config = CompassConfigParser() + config.add_from_file(step.config_filename) check_parallel_system(config) diff --git a/compass/setup.py b/compass/setup.py index 3d7f61645e..9f125c3f12 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -1,15 +1,13 @@ import argparse import sys -import configparser import os import pickle import warnings -from mache import MachineInfo, discover_machine +from mache import discover_machine from compass.mpas_cores import get_mpas_cores -from compass.config import add_config, merge_other_config, \ - ensure_absolute_paths +from compass.config import CompassConfigParser from compass.io import symlink from compass import provenance @@ -69,11 +67,6 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, if machine is None: machine = discover_machine() - if machine is None: - machine_info = MachineInfo(machine='unknown') - else: - machine_info = MachineInfo(machine=machine) - if config_file is None and machine is None: raise ValueError('At least one of config_file and machine is needed.') @@ -147,8 +140,8 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, print('Setting up test cases:') for path, test_case in test_cases.items(): - setup_case(path, test_case, config_file, machine, machine_info, - work_dir, baseline_dir, mpas_model_path, + setup_case(path, test_case, config_file, machine, work_dir, + baseline_dir, mpas_model_path, cached_steps=cached_steps[path]) test_suite = {'name': suite_name, @@ -174,8 +167,8 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, return test_cases -def setup_case(path, test_case, config_file, machine, machine_info, work_dir, - baseline_dir, mpas_model_path, cached_steps): +def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, + mpas_model_path, cached_steps): """ Set up one or more test cases @@ -195,10 +188,6 @@ def setup_case(path, test_case, config_file, machine, machine_info, work_dir, The name of one of the machines with defined config options, which can be listed with ``compass list --machines`` - machine_info : mache.MachineInfo - Information about the machine, to be included in the config options - and passed along to each step - work_dir : str A directory that will serve as the base for creating case directories @@ -216,32 +205,35 @@ def setup_case(path, test_case, config_file, machine, machine_info, work_dir, print(' {}'.format(path)) - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) + config = CompassConfigParser() + + if config_file is not None: + config.add_user_config(config_file) # start with default compass config options - add_config(config, 'compass', 'default.cfg') + config.add_from_package('compass', 'default.cfg') # add the E3SM config options from mache - merge_other_config(config, machine_info.config) + if machine is not None: + config.add_from_package('mache', f'{machine}.cfg') + # add the compass machine config file if machine is None: machine = 'default' - add_config(config, 'compass.machines', '{}.cfg'.format(machine)) + config.add_from_package('compass.machines', f'{machine}.cfg') # add the config options for the MPAS core mpas_core = test_case.mpas_core.name - add_config(config, 'compass.{}'.format(mpas_core), - '{}.cfg'.format(mpas_core)) + config.add_from_package(f'compass.{mpas_core}', f'{mpas_core}.cfg') # add the config options for the test group (if defined) test_group = test_case.test_group.name - add_config(config, 'compass.{}.tests.{}'.format(mpas_core, test_group), - '{}.cfg'.format(test_group), exception=False) + config.add_from_package(f'compass.{mpas_core}.tests.{test_group}', + f'{test_group}.cfg', exception=False) # add the config options for the test case (if defined) - add_config(config, test_case.__module__, - '{}.cfg'.format(test_case.name), exception=False) + config.add_from_package(test_case.__module__, + f'{test_case.name}.cfg', exception=False) test_case_dir = os.path.join(work_dir, path) try: @@ -251,20 +243,10 @@ def setup_case(path, test_case, config_file, machine, machine_info, work_dir, test_case.work_dir = test_case_dir test_case.base_work_dir = work_dir - # add the custom config file once before calling configure() in case we - # need to use the config options from there - if config_file is not None: - config.read(config_file) - # add config options specific to the test case test_case.config = config test_case.configure() - # add the custom config file (again) last, so these options are the - # defaults - if config_file is not None: - config.read(config_file) - # add the baseline directory for this test case if baseline_dir is not None: test_case.baseline_dir = os.path.join(baseline_dir, path) @@ -275,10 +257,6 @@ def setup_case(path, test_case, config_file, machine, machine_info, work_dir, config.set('test_case', 'steps_to_run', ' '.join(test_case.steps_to_run)) - # make sure all paths in the paths, namelists and streams sections are - # absolute paths - ensure_absolute_paths(config) - # write out the config file test_case_config = '{}.cfg'.format(test_case.name) test_case.config_filename = test_case_config diff --git a/compass/step.py b/compass/step.py index e307e7bc4e..a62ffb5de4 100644 --- a/compass/step.py +++ b/compass/step.py @@ -1,6 +1,5 @@ import os from lxml import etree -import configparser from importlib.resources import path import shutil import numpy @@ -94,7 +93,7 @@ class Step(ABC): a dictionary used internally to keep track of updates to the default streams from calls to :py:meth:`compass.Step.add_streams_file` - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration diff --git a/compass/testcase.py b/compass/testcase.py index c6a8f6db00..f73d8f95e4 100644 --- a/compass/testcase.py +++ b/compass/testcase.py @@ -1,5 +1,4 @@ import os -import configparser from mpas_tools.logging import LoggingContext from compass.parallel import get_available_cores_and_nodes @@ -38,7 +37,7 @@ class TestCase: the path within the base work directory of the test case, made up of ``mpas_core``, ``test_group``, and the test case's ``subdir`` - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration From af6f4bae5b70f8dad88337819beb33254ccd0fa3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 28 Mar 2022 21:49:04 +0200 Subject: [PATCH 03/15] Update config parsing in ocean framework --- compass/ocean/plot.py | 2 +- compass/ocean/vertical/__init__.py | 2 +- compass/ocean/vertical/grid_1d.py | 4 ++-- compass/ocean/vertical/partial_cells.py | 4 ++-- compass/ocean/vertical/zlevel.py | 2 +- compass/ocean/vertical/zstar.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compass/ocean/plot.py b/compass/ocean/plot.py index 14b7fd3d78..b356d53873 100644 --- a/compass/ocean/plot.py +++ b/compass/ocean/plot.py @@ -132,7 +132,7 @@ def plot_vertical_grid(grid_filename, config, grid_filename : str The name of the NetCDF file containing the vertical grid - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for the vertical grid out_filename : str, optional diff --git a/compass/ocean/vertical/__init__.py b/compass/ocean/vertical/__init__.py index 41f9927e23..5264da644d 100644 --- a/compass/ocean/vertical/__init__.py +++ b/compass/ocean/vertical/__init__.py @@ -48,7 +48,7 @@ def init_vertical_coord(config, ds): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid diff --git a/compass/ocean/vertical/grid_1d.py b/compass/ocean/vertical/grid_1d.py index f0c99fd820..69e02e2019 100644 --- a/compass/ocean/vertical/grid_1d.py +++ b/compass/ocean/vertical/grid_1d.py @@ -13,7 +13,7 @@ def generate_1d_grid(config): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid @@ -103,7 +103,7 @@ def add_1d_grid(config, ds): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid diff --git a/compass/ocean/vertical/partial_cells.py b/compass/ocean/vertical/partial_cells.py index d2c9e7a488..f9ec9c4b1b 100644 --- a/compass/ocean/vertical/partial_cells.py +++ b/compass/ocean/vertical/partial_cells.py @@ -9,7 +9,7 @@ def alter_bottom_depth(config, bottomDepth, refBottomDepth, maxLevelCell): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid @@ -58,7 +58,7 @@ def alter_ssh(config, ssh, refBottomDepth, minLevelCell): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid diff --git a/compass/ocean/vertical/zlevel.py b/compass/ocean/vertical/zlevel.py index b7c5d3e4f0..b12b7deb19 100644 --- a/compass/ocean/vertical/zlevel.py +++ b/compass/ocean/vertical/zlevel.py @@ -48,7 +48,7 @@ def init_z_level_vertical_coord(config, ds): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid diff --git a/compass/ocean/vertical/zstar.py b/compass/ocean/vertical/zstar.py index 67aa1024d3..93ae4da64f 100644 --- a/compass/ocean/vertical/zstar.py +++ b/compass/ocean/vertical/zstar.py @@ -50,7 +50,7 @@ def init_z_star_vertical_coord(config, ds): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options with parameters used to construct the vertical grid From bcaaae24c7f00f3735034a4ac503975b566c3a10 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 28 Mar 2022 21:49:31 +0200 Subject: [PATCH 04/15] Update config parsing in ocean test groups --- compass/ocean/tests/baroclinic_channel/__init__.py | 2 +- .../tests/global_convergence/cosine_bell/__init__.py | 9 +++------ compass/ocean/tests/global_ocean/configure.py | 5 ++--- .../global_ocean/files_for_e3sm/diagnostics_files.py | 2 +- .../global_ocean/make_diagnostics_files/__init__.py | 7 +++---- compass/ocean/tests/global_ocean/metadata.py | 8 ++++---- compass/ocean/tests/ice_shelf_2d/__init__.py | 2 +- .../ocean/tests/planar_convergence/conv_test_case.py | 11 ++++------- .../correlated_tracers_2d/__init__.py | 2 -- .../tests/sphere_transport/divergent_2d/__init__.py | 2 -- .../sphere_transport/nondivergent_2d/__init__.py | 2 -- .../tests/sphere_transport/rotation_2d/__init__.py | 2 -- compass/ocean/tests/ziso/__init__.py | 3 +-- 13 files changed, 20 insertions(+), 37 deletions(-) diff --git a/compass/ocean/tests/baroclinic_channel/__init__.py b/compass/ocean/tests/baroclinic_channel/__init__.py index 8bde84f10f..4fdc1cc76f 100644 --- a/compass/ocean/tests/baroclinic_channel/__init__.py +++ b/compass/ocean/tests/baroclinic_channel/__init__.py @@ -40,7 +40,7 @@ def configure(resolution, config): resolution : str The resolution of the test case - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case """ res_params = {'10km': {'nx': 16, diff --git a/compass/ocean/tests/global_convergence/cosine_bell/__init__.py b/compass/ocean/tests/global_convergence/cosine_bell/__init__.py index 9a560ce908..e213fe7db6 100644 --- a/compass/ocean/tests/global_convergence/cosine_bell/__init__.py +++ b/compass/ocean/tests/global_convergence/cosine_bell/__init__.py @@ -1,6 +1,4 @@ -import configparser - -from compass.config import add_config +from compass.config import CompassConfigParser from compass.testcase import TestCase from compass.ocean.tests.global_convergence.cosine_bell.mesh import Mesh @@ -31,9 +29,8 @@ def __init__(self, test_group): self.resolutions = None # add the steps with default resolutions so they can be listed - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) - add_config(config, self.__module__, '{}.cfg'.format(self.name)) + config = CompassConfigParser() + config.add_from_package(self.__module__, '{}.cfg'.format(self.name)) self._setup_steps(config) def configure(self): diff --git a/compass/ocean/tests/global_ocean/configure.py b/compass/ocean/tests/global_ocean/configure.py index c65cd5446e..043a358c52 100644 --- a/compass/ocean/tests/global_ocean/configure.py +++ b/compass/ocean/tests/global_ocean/configure.py @@ -1,4 +1,3 @@ -from compass.config import add_config from compass.ocean.tests.global_ocean.metadata import \ get_author_and_email_from_git @@ -20,8 +19,8 @@ def configure_global_ocean(test_case, mesh, init=None): """ config = test_case.config mesh_step = mesh.mesh_step - add_config(config, mesh_step.package, mesh_step.mesh_config_filename, - exception=True) + config.add_from_package(mesh_step.package, mesh_step.mesh_config_filename, + exception=True) if mesh.with_ice_shelf_cavities: config.set('global_ocean', 'prefix', '{}wISC'.format( diff --git a/compass/ocean/tests/global_ocean/files_for_e3sm/diagnostics_files.py b/compass/ocean/tests/global_ocean/files_for_e3sm/diagnostics_files.py index c9981c0dbc..ebe690e7f3 100644 --- a/compass/ocean/tests/global_ocean/files_for_e3sm/diagnostics_files.py +++ b/compass/ocean/tests/global_ocean/files_for_e3sm/diagnostics_files.py @@ -74,7 +74,7 @@ def make_diagnostics_files(config, logger, mesh_short_name, Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case logger : logging.Logger diff --git a/compass/ocean/tests/global_ocean/make_diagnostics_files/__init__.py b/compass/ocean/tests/global_ocean/make_diagnostics_files/__init__.py index 7668f2ce5c..e3aaea68be 100644 --- a/compass/ocean/tests/global_ocean/make_diagnostics_files/__init__.py +++ b/compass/ocean/tests/global_ocean/make_diagnostics_files/__init__.py @@ -1,6 +1,5 @@ import os -from compass.config import add_config from compass.io import symlink from compass.testcase import TestCase from compass.step import Step @@ -30,9 +29,9 @@ def configure(self): """ Modify the configuration options for this test case """ - add_config(self.config, - 'compass.ocean.tests.global_ocean.make_diagnostics_files', - 'make_diagnostics_files.cfg', exception=True) + self.config.add_from_package( + 'compass.ocean.tests.global_ocean.make_diagnostics_files', + 'make_diagnostics_files.cfg', exception=True) def run(self): """ diff --git a/compass/ocean/tests/global_ocean/metadata.py b/compass/ocean/tests/global_ocean/metadata.py index 8bb59da78f..602c99211b 100644 --- a/compass/ocean/tests/global_ocean/metadata.py +++ b/compass/ocean/tests/global_ocean/metadata.py @@ -13,7 +13,7 @@ def get_author_and_email_from_git(config): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case """ author = config.get('global_ocean', 'author') @@ -49,7 +49,7 @@ def get_e3sm_mesh_names(config, levels): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case levels : int @@ -64,10 +64,10 @@ def get_e3sm_mesh_names(config, levels): The long E3SM name of the ocean and sea-ice mesh """ + config.set('global_ocean', 'levels', '{}'.format(levels)) mesh_prefix = config.get('global_ocean', 'prefix') min_res = config.get('global_ocean', 'min_res') max_res = config.get('global_ocean', 'max_res') - config.set('global_ocean', 'levels', '{}'.format(levels)) e3sm_version = config.get('global_ocean', 'e3sm_version') mesh_revision = config.get('global_ocean', 'mesh_revision') @@ -94,7 +94,7 @@ def add_mesh_and_init_metadata(output_filenames, config, init_filename): output_filenames : list A list of output files. - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case init_filename : str diff --git a/compass/ocean/tests/ice_shelf_2d/__init__.py b/compass/ocean/tests/ice_shelf_2d/__init__.py index c492e0155e..dac62ce20a 100644 --- a/compass/ocean/tests/ice_shelf_2d/__init__.py +++ b/compass/ocean/tests/ice_shelf_2d/__init__.py @@ -36,7 +36,7 @@ def configure(resolution, coord_type, config): coord_type : str The type of vertical coordinate (``z-star``, ``z-level``, etc.) - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case """ res_params = {'5km': {'nx': 10, 'ny': 44, 'dc': 5e3}} diff --git a/compass/ocean/tests/planar_convergence/conv_test_case.py b/compass/ocean/tests/planar_convergence/conv_test_case.py index 22f47f4ba4..8bea1ded9d 100644 --- a/compass/ocean/tests/planar_convergence/conv_test_case.py +++ b/compass/ocean/tests/planar_convergence/conv_test_case.py @@ -1,6 +1,4 @@ -import configparser - -from compass.config import add_config +from compass.config import CompassConfigParser from compass.testcase import TestCase from compass.ocean.tests.planar_convergence.forward import \ Forward @@ -31,10 +29,9 @@ def __init__(self, test_group, name): self.resolutions = None # add the steps with default resolutions so they can be listed - config = configparser.ConfigParser( - interpolation=configparser.ExtendedInterpolation()) + config = CompassConfigParser() module = 'compass.ocean.tests.planar_convergence' - add_config(config, module, 'planar_convergence.cfg') + config.add_from_package(module, 'planar_convergence.cfg') self._setup_steps(config) def configure(self): @@ -104,7 +101,7 @@ def _setup_steps(self, config): Parameters ---------- - config : configparser.ConfigParse + config : compass.config.CompassConfigParser The config options containing the resolutions """ diff --git a/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py b/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py index bf4ac324ed..15926e7351 100644 --- a/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py @@ -1,5 +1,3 @@ -import configparser - from compass.testcase import TestCase from compass.ocean.tests.sphere_transport.correlated_tracers_2d.mesh \ diff --git a/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py b/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py index a8c63aeae8..0995510530 100644 --- a/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py @@ -1,5 +1,3 @@ -import configparser - from compass.testcase import TestCase from compass.ocean.tests.sphere_transport.divergent_2d.mesh import Mesh diff --git a/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py b/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py index 6a700d4715..1eef8b98ae 100644 --- a/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py @@ -1,5 +1,3 @@ -import configparser - from compass.testcase import TestCase from compass.ocean.tests.sphere_transport.nondivergent_2d.mesh import Mesh diff --git a/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py b/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py index 49468173d8..bbeb7849c6 100644 --- a/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py @@ -1,5 +1,3 @@ -import configparser - from compass.testcase import TestCase from compass.ocean.tests.sphere_transport.rotation_2d.mesh import Mesh diff --git a/compass/ocean/tests/ziso/__init__.py b/compass/ocean/tests/ziso/__init__.py index c0319577bd..130a2cde42 100644 --- a/compass/ocean/tests/ziso/__init__.py +++ b/compass/ocean/tests/ziso/__init__.py @@ -1,6 +1,5 @@ from compass.testgroup import TestGroup from compass.ocean.tests.ziso.with_frazil import WithFrazil -from compass.config import add_config from compass.ocean.tests.ziso.ziso_test_case import ZisoTestCase @@ -42,7 +41,7 @@ def configure(name, resolution, config): resolution : str The resolution of the test case - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case """ res_params = {'20km': {'nx': 50, From 28b4cc29439b0fd1bc2a6d44b24e7beb0d4db739 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 28 Mar 2022 21:49:44 +0200 Subject: [PATCH 05/15] Update config parsing in landice test groups --- compass/landice/tests/circular_shelf/setup_mesh.py | 2 +- compass/landice/tests/circular_shelf/visualize.py | 2 +- compass/landice/tests/dome/setup_mesh.py | 2 +- compass/landice/tests/dome/visualize.py | 2 +- .../landice/tests/eismint2/standard_experiments/visualize.py | 2 +- compass/landice/tests/hydro_radial/visualize.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compass/landice/tests/circular_shelf/setup_mesh.py b/compass/landice/tests/circular_shelf/setup_mesh.py index fe7382f766..a678e0f784 100644 --- a/compass/landice/tests/circular_shelf/setup_mesh.py +++ b/compass/landice/tests/circular_shelf/setup_mesh.py @@ -81,7 +81,7 @@ def _setup_circular_shelf_initial_conditions(config, logger, filename): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration diff --git a/compass/landice/tests/circular_shelf/visualize.py b/compass/landice/tests/circular_shelf/visualize.py index b5bff38607..ed76d26f96 100644 --- a/compass/landice/tests/circular_shelf/visualize.py +++ b/compass/landice/tests/circular_shelf/visualize.py @@ -58,7 +58,7 @@ def visualize_circular_shelf(config, logger, filename): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration diff --git a/compass/landice/tests/dome/setup_mesh.py b/compass/landice/tests/dome/setup_mesh.py index b2ac0a6922..912cf3dac8 100644 --- a/compass/landice/tests/dome/setup_mesh.py +++ b/compass/landice/tests/dome/setup_mesh.py @@ -91,7 +91,7 @@ def _setup_dome_initial_conditions(config, logger, filename): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration diff --git a/compass/landice/tests/dome/visualize.py b/compass/landice/tests/dome/visualize.py index 5d4d0231c5..98ea0e1c56 100644 --- a/compass/landice/tests/dome/visualize.py +++ b/compass/landice/tests/dome/visualize.py @@ -61,7 +61,7 @@ def visualize_dome(config, logger, filename): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration diff --git a/compass/landice/tests/eismint2/standard_experiments/visualize.py b/compass/landice/tests/eismint2/standard_experiments/visualize.py index 20e3a7b991..8e813220f4 100644 --- a/compass/landice/tests/eismint2/standard_experiments/visualize.py +++ b/compass/landice/tests/eismint2/standard_experiments/visualize.py @@ -51,7 +51,7 @@ def visualize_eismint2(config, logger, experiment): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration diff --git a/compass/landice/tests/hydro_radial/visualize.py b/compass/landice/tests/hydro_radial/visualize.py index 5b45fa7140..d2acf027cc 100644 --- a/compass/landice/tests/hydro_radial/visualize.py +++ b/compass/landice/tests/hydro_radial/visualize.py @@ -57,7 +57,7 @@ def visualize_hydro_radial(config, logger): Parameters ---------- - config : configparser.ConfigParser + config : compass.config.CompassConfigParser Configuration options for this test case, a combination of the defaults for the machine, core and configuration From 7ae8d2406c935335f4e48841c923ec5d209ac836 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 28 Mar 2022 22:18:42 +0200 Subject: [PATCH 06/15] Parse config lists with getlist --- .../tests/global_convergence/cosine_bell/__init__.py | 4 +--- compass/ocean/tests/planar_convergence/conv_test_case.py | 5 ++--- .../sphere_transport/correlated_tracers_2d/__init__.py | 9 ++++----- .../tests/sphere_transport/divergent_2d/__init__.py | 7 ++----- .../tests/sphere_transport/nondivergent_2d/__init__.py | 9 ++++----- .../ocean/tests/sphere_transport/rotation_2d/__init__.py | 7 ++----- 6 files changed, 15 insertions(+), 26 deletions(-) diff --git a/compass/ocean/tests/global_convergence/cosine_bell/__init__.py b/compass/ocean/tests/global_convergence/cosine_bell/__init__.py index e213fe7db6..781a18dad7 100644 --- a/compass/ocean/tests/global_convergence/cosine_bell/__init__.py +++ b/compass/ocean/tests/global_convergence/cosine_bell/__init__.py @@ -101,9 +101,7 @@ def update_cores(self): def _setup_steps(self, config): """ setup steps given resolutions """ - resolutions = config.get('cosine_bell', 'resolutions') - resolutions = [int(resolution) for resolution in - resolutions.replace(',', ' ').split()] + resolutions = config.getlist('cosine_bell', 'resolutions', dtype=int) if self.resolutions is not None and self.resolutions == resolutions: return diff --git a/compass/ocean/tests/planar_convergence/conv_test_case.py b/compass/ocean/tests/planar_convergence/conv_test_case.py index 8bea1ded9d..2e8ad79e36 100644 --- a/compass/ocean/tests/planar_convergence/conv_test_case.py +++ b/compass/ocean/tests/planar_convergence/conv_test_case.py @@ -105,9 +105,8 @@ def _setup_steps(self, config): The config options containing the resolutions """ - resolutions = config.get('planar_convergence', 'resolutions') - resolutions = [int(resolution) for resolution in - resolutions.replace(',', ' ').split()] + resolutions = config.getlist('planar_convergence', 'resolutions', + dtype=int) if self.resolutions is not None and self.resolutions == resolutions: return diff --git a/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py b/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py index 15926e7351..6188462396 100644 --- a/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/correlated_tracers_2d/__init__.py @@ -36,11 +36,10 @@ def configure(self): Set config options for the test case """ config = self.config - resolutions = config.get('correlated_tracers_2d', 'resolutions') - resolutions = [int(resolution) for resolution in - resolutions.replace(',', ' ').split()] - dtmin = config.get('correlated_tracers_2d', 'timestep_minutes') - dtmin = [int(dt) for dt in dtmin.replace(',', ' ').split()] + resolutions = config.getlist('correlated_tracers_2d', 'resolutions', + dtype=int) + dtmin = config.getlist('correlated_tracers_2d', 'timestep_minutes', + dtype=int) self.resolutions = resolutions self.timesteps = dtmin diff --git a/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py b/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py index 0995510530..c94c07e799 100644 --- a/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/divergent_2d/__init__.py @@ -34,11 +34,8 @@ def configure(self): Set config options for the test case """ config = self.config - resolutions = config.get('divergent_2d', 'resolutions') - resolutions = [int(resolution) for resolution in - resolutions.replace(',', ' ').split()] - dtmin = config.get('divergent_2d', 'timestep_minutes') - dtmin = [int(dt) for dt in dtmin.replace(',', ' ').split()] + resolutions = config.getlist('divergent_2d', 'resolutions', dtype=int) + dtmin = config.getlist('divergent_2d', 'timestep_minutes', dtype=int) self.resolutions = resolutions self.timesteps = dtmin diff --git a/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py b/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py index 1eef8b98ae..1c7df0098b 100644 --- a/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/nondivergent_2d/__init__.py @@ -34,11 +34,10 @@ def configure(self): Set config options for the test case """ config = self.config - resolutions = config.get('nondivergent_2d', 'resolutions') - resolutions = [int(resolution) for resolution in - resolutions.replace(',', ' ').split()] - dtmin = config.get('nondivergent_2d', 'timestep_minutes') - dtmin = [int(dt) for dt in dtmin.replace(',', ' ').split()] + resolutions = config.getlist('nondivergent_2d', 'resolutions', + dtype=int) + dtmin = config.getlist('nondivergent_2d', 'timestep_minutes', + dtype=int) self.resolutions = resolutions self.timesteps = dtmin diff --git a/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py b/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py index bbeb7849c6..228ceab963 100644 --- a/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py +++ b/compass/ocean/tests/sphere_transport/rotation_2d/__init__.py @@ -33,11 +33,8 @@ def configure(self): Set config options for the test case """ config = self.config - resolutions = config.get('rotation_2d', 'resolutions') - resolutions = [int(resolution) for resolution in - resolutions.replace(',', ' ').split()] - dtmin = config.get('rotation_2d', 'timestep_minutes') - dtmin = [int(dt) for dt in dtmin.replace(',', ' ').split()] + resolutions = config.getlist('rotation_2d', 'resolutions', dtype=int) + dtmin = config.getlist('rotation_2d', 'timestep_minutes', dtype=int) self.resolutions = resolutions self.timesteps = dtmin From 211fb5928200fae972c70562e80b09d623ae9f0a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 29 Mar 2022 14:38:15 +0200 Subject: [PATCH 07/15] Update the docs on config options --- docs/developers_guide/api.rst | 17 +- docs/developers_guide/framework.rst | 103 +++++----- docs/users_guide/config_files.rst | 300 ++++++++++++++++++++++++++-- 3 files changed, 350 insertions(+), 70 deletions(-) diff --git a/docs/developers_guide/api.rst b/docs/developers_guide/api.rst index aefeef1103..135007aa2f 100644 --- a/docs/developers_guide/api.rst +++ b/docs/developers_guide/api.rst @@ -169,10 +169,19 @@ config .. autosummary:: :toctree: generated/ - duplicate_config - add_config - ensure_absolute_paths - get_source_file + CompassConfigParser + CompassConfigParser.add_user_config + CompassConfigParser.add_from_file + CompassConfigParser.add_from_package + CompassConfigParser.get + CompassConfigParser.getint + CompassConfigParser.getfloat + CompassConfigParser.getboolean + CompassConfigParser.getlist + CompassConfigParser.has_option + CompassConfigParser.set + CompassConfigParser.write + CompassConfigParser.__getitem__ io ^^ diff --git a/docs/developers_guide/framework.rst b/docs/developers_guide/framework.rst index be4a29e70d..5d15a6279a 100644 --- a/docs/developers_guide/framework.rst +++ b/docs/developers_guide/framework.rst @@ -117,15 +117,16 @@ and debugging. Config files ------------ -The ``compass.config`` module includes functions for creating and manipulating -config options and :ref:`config_files`. +The ``compass.config`` module includes the +:py:class:`compass.config.CompassConfigParser` class reading, getting, setting, +and writing config options and :ref:`config_files`. -The :py:func:`compass.config.add_config()` function can be used to add the -contents of a config file within a package to the current config parser. -Examples of this can be found in most test cases as well as +The :py:meth:`compass.config.CompassConfigParser.add_from_package()` method can +be used to add the contents of a config file within a package to the config +options. Examples of this can be found in many test cases as well as :py:func:`compass.setup.setup_case()`. Here is a typical example from -:py:func:`compass.landice.tests.enthalpy_benchmark.A.A.configure()`: +:py:func:`compass.ocean.tests.global_ocean.make_diagnostics_files.MakeDiagnosticsFiles.configure()`: .. code-block:: python @@ -133,13 +134,13 @@ Examples of this can be found in most test cases as well as """ Modify the configuration options for this test case """ - add_config(self.config, 'compass.landice.tests.enthalpy_benchmark.A', - 'A.cfg', exception=True) - ... + self.config.add_from_package( + 'compass.ocean.tests.global_ocean.make_diagnostics_files', + 'make_diagnostics_files.cfg', exception=True) -The second and third arguments are the name of a package containing the config +The first and second arguments are the name of a package containing the config file and the name of the config file itself, respectively. You can see that -the file is in the path ``compass/landice/tests/enthalpy_benchmark/A`` +the file is in the path ``compass/ocean/tests/global_ocean/make_diagnostics_files`` (replacing the ``.`` in the module name with ``/``). In this case, we know that the config file should always exist, so we would like the code to raise an exception (``exception=True``) if the file is not found. This is the @@ -147,50 +148,56 @@ default behavior. In some cases, you would like the code to add the config options if the config file exists and do nothing if it does not. This can be useful if a common configure function is being used for all test cases in a configuration, as in this example from -:py:func:`compass.ocean.tests.global_ocean.configure.configure_global_ocean()`: +:py:func:`setup.setup_case()`: .. code-block:: python - add_config(config, test_case.__module__, '{}.cfg'.format(test_case.name), - exception=False) + # add the config options for the test group (if defined) + test_group = test_case.test_group.name + config.add_from_package(f'compass.{mpas_core}.tests.{test_group}', + f'{test_group}.cfg', exception=False) + +If a test group doesn't have any config options, nothing will happen. + +The ``CompassConfigParser`` class also includes methods for adding a user +config file and other config files by file name, but these are largely intended +for use by the framework rather than individual test cases. -When this is called within the ``mesh`` test case, nothing will happen because -``compass/ocean/tests/global_ocean/mesh`` does not contain a ``mesh.cfg`` file. -The config files for meshes are handled differently, since they aren't -associated with a particular test case: +Other methods for the ``CompassConfigParser`` are similar to those for +:py:class:`configparser.ConfigParser`. In addition to ``get()``, +``getinteger()``, ``getfloat()`` and ``getboolean()`` methods, this class +implements :py:meth:`compass.config.CompassConfigParser.getlist()`, which +can be used to parse a config value separated by spaces and/or commas into +a list of strings, floats, integers, booleans, etc. + +Currently, ``CompassConfigParser`` supports accessing a config section using +section names as keys, e.g.: .. code-block:: python - mesh_step = mesh.mesh_step - add_config(config, mesh_step.package, mesh_step.mesh_config_filename, - exception=True) - -In this case, the mesh step keeps track of the package and config file in its -attributes (e.g. ``compass.ocean.tests.global_ocean.mesh.qu240`` and -``qu240.cfg`` for the ``QU240`` and ``QUwISC240`` meshes). Since we require -each mesh to have config options (to define the vertical grid and the metadata -to be added to the mesh, at the very least), we use ``exception=True`` so an -exception will be raised if no config file is found. - -The ``config`` module also contains 3 functions that are intended for internal -use by the framework itself. Test-case developers will typically not need to -call these functions directly. - -The :py:func:`compass.config.duplicate_config()` function can be used to make a -deep copy of a ``config`` object so changes can be made without affecting the -original. - -The :py:func:`compass.config.ensure_absolute_paths()` function is used -internally by the framework to check and update config options in the -``paths``, ``namelists``, ``streams``, and ``executables`` sections of the -config file to make sure they have absolute paths. The absolute paths are -determined from the location where one of the tools from the compass -:ref:`dev_command_line` was called. - -The :py:func:`compass.config.get_source_file()` function is used to get an -absolute path for a file using one of the config options defined in the -``paths`` section. This function is used by the framework as part of -downloading files (e.g. to a defined database), see :ref:`dev_io`. + section = self.config['enthalpy_benchmark_viz'] + display_image = section.getboolean('display_image') + ... + +But it does not allow assignment of a section or many of the other +dictionary-like features supported by :py:class:`configparser.ConfigParser`. + +Comments in config files +~~~~~~~~~~~~~~~~~~~~~~~~ + +One of the main advantages of :py:class:`compass.config.CompassConfigParser` +over :py:class:`configparser.ConfigParser` is that it keeps track of comments +that are associated with config sections and options. There are a few "rules" +that make this possible. + +Comments must be with the ``#`` character. They must be placed *before* the +config section or option in question (preferably without blank lines between). +The comments can be any number of lines. + +.. note:: + + Inline comments (after a config option on the same line) are not allowed + and will be parsed as part of the config option itself. .. _dev_logging: diff --git a/docs/users_guide/config_files.rst b/docs/users_guide/config_files.rst index 7fe330d34c..936d6f9423 100644 --- a/docs/users_guide/config_files.rst +++ b/docs/users_guide/config_files.rst @@ -143,96 +143,360 @@ looks like: .. code-block:: cfg + # Options related to the current test case + [test_case] + + # source: /home/xylar/code/compass/customize_config_parser/compass/setup.py + steps_to_run = mesh + + + # Options related to downloading files [download] + + # the base url for the server from which meshes, initial conditions, and other + # data sets can be downloaded + # source: /home/xylar/code/compass/customize_config_parser/compass/default.cfg server_base_url = https://web.lcrc.anl.gov/public/e3sm/mpas_standalonedata + + # whether to download files during setup that have not been cached locally + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg download = True + + # whether to check the size of files that have been downloaded to make sure + # they are the right size + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg check_size = False + + # whether to verify SSL certificates for HTTPS requests + # source: /home/xylar/code/compass/customize_config_parser/compass/default.cfg verify = True + + # the path on the server for MPAS-Ocean + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg core_path = mpas-ocean + + # The parallel section describes options related to running tests in parallel [parallel] + + # the program to use for graph partitioning + # source: /home/xylar/code/compass/customize_config_parser/compass/default.cfg partition_executable = gpmetis + + # parallel system of execution: slurm or single_node + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg system = single_node + + # whether to use mpirun or srun to run the model + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg parallel_executable = mpirun + + # cores per node on the machine + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg cores_per_node = 8 + + # the number of multiprocessing or dask threads to use + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg threads = 8 + + # The io section describes options related to file i/o + [io] + + # the NetCDF file format: NETCDF4, NETCDF4_CLASSIC, NETCDF3_64BIT, or + # NETCDF3_CLASSIC + # source: /home/xylar/code/compass/customize_config_parser/compass/default.cfg + format = NETCDF3_64BIT + + # the NetCDF output engine: netcdf4 or scipy + # the netcdf4 engine is not performing well on Chrysalis and Anvil, so we will + # try scipy for now. If we can switch to NETCDF4 format, netcdf4 will be + # required + # source: /home/xylar/code/compass/customize_config_parser/compass/default.cfg + engine = scipy + + + # This file contains some common config options you might want to set + # if you're working with the compass ocean core and MPAS-Ocean. + # The paths section describes paths that are used within the ocean core test + # cases. [paths] - mpas_model = /home/xylar/code/mpas-work/compass/compass_1.0/E3SM-Project/components/mpas-ocean + + # source: /home/xylar/code/compass/customize_config_parser/compass/setup.py + mpas_model = /home/xylar/code/compass/customize_config_parser/E3SM-Project/components/mpas-ocean + + # The root to a location where the mesh_database, initial_condition_database, + # and bathymetry_database for MPAS-Ocean will be cached + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg ocean_database_root = /home/xylar/data/mpas/mpas_standalonedata/mpas-ocean + + # The root to a location where data files for MALI will be cached + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg landice_database_root = /home/xylar/data/mpas/mpas_standalonedata/mpas-albany-landice - baseline_dir = /home/xylar/data/mpas/test_20210413/compass_classes/ocean/global_ocean/QU240/PHC/init + + # The namelists section defines paths to example_compact namelists that will be used + # to generate specific namelists. By default, these point to the forward and + # init namelists in the default_inputs directory after a successful build of + # the ocean model. Change these in a custom config file if you need a different + # example_compact. [namelists] - forward = /home/xylar/code/mpas-work/compass/compass_1.0/E3SM-Project/components/mpas-ocean/default_inputs/namelist.ocean.forward - init = /home/xylar/code/mpas-work/compass/compass_1.0/E3SM-Project/components/mpas-ocean/default_inputs/namelist.ocean.init + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg + forward = /home/xylar/code/compass/customize_config_parser/E3SM-Project/components/mpas-ocean/default_inputs/namelist.ocean.forward + + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg + init = /home/xylar/code/compass/customize_config_parser/E3SM-Project/components/mpas-ocean/default_inputs/namelist.ocean.init + + + # The streams section defines paths to example_compact streams files that will be used + # to generate specific streams files. By default, these point to the forward and + # init streams files in the default_inputs directory after a successful build of + # the ocean model. Change these in a custom config file if you need a different + # example_compact. [streams] - forward = /home/xylar/code/mpas-work/compass/compass_1.0/E3SM-Project/components/mpas-ocean/default_inputs/streams.ocean.forward - init = /home/xylar/code/mpas-work/compass/compass_1.0/E3SM-Project/components/mpas-ocean/default_inputs/streams.ocean.init + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg + forward = /home/xylar/code/compass/customize_config_parser/E3SM-Project/components/mpas-ocean/default_inputs/streams.ocean.forward + + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg + init = /home/xylar/code/compass/customize_config_parser/E3SM-Project/components/mpas-ocean/default_inputs/streams.ocean.init + + + # The executables section defines paths to required executables. These + # executables are provided for use by specific test cases. Most tools that + # compass needs should be in the conda environment, so this is only the path + # to the MPAS-Ocean executable by default. [executables] - model = /home/xylar/code/mpas-work/compass/compass_1.0/E3SM-Project/components/mpas-ocean/ocean_model + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg + model = /home/xylar/code/compass/customize_config_parser/E3SM-Project/components/mpas-ocean/ocean_model + + + # Options relate to adjusting the sea-surface height or land-ice pressure + # below ice shelves to they are dynamically consistent with one another [ssh_adjustment] + + # the number of iterations of ssh adjustment to perform + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/ocean.cfg iterations = 10 + + # options for global ocean testcases [global_ocean] - mesh_cores = 1 + + ## each mesh should replace these with appropriate values in its config file + ## config options related to the mesh step + # number of cores to use + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg + mesh_cores = 18 + + # minimum of cores, below which the step fails + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg mesh_min_cores = 1 + + # maximum memory usage allowed (in MB) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg mesh_max_memory = 1000 + + # maximum disk usage allowed (in MB) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg mesh_max_disk = 1000 + + ## config options related to the initial_state step + # number of cores to use + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg init_cores = 4 + + # minimum of cores, below which the step fails + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg init_min_cores = 1 + + # maximum memory usage allowed (in MB) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg init_max_memory = 1000 + + # maximum disk usage allowed (in MB) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg init_max_disk = 1000 + + # number of threads + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg init_threads = 1 + + ## config options related to the forward steps + # number of cores to use + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg forward_cores = 4 + + # minimum of cores, below which the step fails + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg forward_min_cores = 1 + + # number of threads + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg forward_threads = 1 + + # maximum memory usage allowed (in MB) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg forward_max_memory = 1000 + + # maximum disk usage allowed (in MB) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg forward_max_disk = 1000 + + ## metadata related to the mesh + # whether to add metadata to output files + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg add_metadata = True + + ## metadata related to the mesh + # the prefix (e.g. QU, EC, WC, SO) + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg prefix = QU - mesh_description = MPAS quasi-uniform mesh for E3SM version ${e3sm_version} at - ${min_res}-km global resolution with ${levels} vertical + + # a description of the mesh + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg + mesh_description = MPAS quasi-uniform mesh for E3SM version 2 at + 240-km global resolution with autodetect vertical level + + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/configure.py bathy_description = Bathymetry is from GEBCO 2019, combined with BedMachine Antarctica around Antarctica. - init_description = Polar science center Hydrographic Climatology (PHC) + + # a description of the mesh with ice-shelf cavities + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg + init_description = <<>> + + # E3SM version that the mesh is intended for + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg e3sm_version = 2 + + # The revision number of the mesh, which should be incremented each time the + # mesh is revised + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg mesh_revision = 1 + + # the minimum (finest) resolution in the mesh + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg min_res = 240 + + # the maximum (coarsest) resolution in the mesh, can be the same as min_res + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg max_res = 240 + + # the maximum depth of the ocean, always detected automatically + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg max_depth = autodetect + + # the number of vertical levels, always detected automatically + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg levels = autodetect + + # the date the mesh was created as YYMMDD, typically detected automatically + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg creation_date = autodetect + + # These options are used in the metadata for global ocean initial conditions. + # You can indicated that you are the "author" of a mesh and give your preferred + # email address for people to contact your if they have questions about the + # mesh. Or you can let compass figure out who you are from your git + # configuration + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg author = Xylar Asay-Davis + + # source: /home/xylar/code/compass/customize_config_parser/inej.cfg email = xylar@lanl.gov - pull_request = https://github.com/MPAS-Dev/compass/pull/28 + # The URL of the pull request documenting the creation of the mesh + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg + pull_request = <<>> + + + # config options related to dynamic adjustment + [dynamic_adjustment] + + # the maximum allowed value of temperatureMax in global statistics + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg + temperature_max = 33.0 + + + # config options related to initial condition and diagnostics support files + # for E3SM [files_for_e3sm] + + # whether to generate an ocean initial condition in E3SM + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg enable_ocean_initial_condition = true + + # whether to generate graph partitions for different numbers of ocean cores in + # E3SM + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg enable_ocean_graph_partition = true + + # whether to generate a sea-ice initial condition in E3SM + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg enable_seaice_initial_condition = true + + # whether to generate SCRIP files for later use in creating E3SM mapping files + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg enable_scrip = true + + # whether to generate region masks, transects and mapping files for use in both + # online analysis members and offline with MPAS-Analysis + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg enable_diagnostics_files = true + + ## the following relate to the comparison grids in MPAS-Analysis to generate + ## mapping files for. The default values are also the defaults in + ## MPAS-Analysis. Coarser or finer resolution may be desirable for some MPAS + ## meshes. + # The comparison lat/lon grid resolution in degrees + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg comparisonlatresolution = 0.5 + + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg comparisonlonresolution = 0.5 + + # The comparison Antarctic polar stereographic grid size and resolution in km + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg comparisonantarcticstereowidth = 6000. + + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg comparisonantarcticstereoresolution = 10. + + # The comparison Arctic polar stereographic grid size and resolution in km + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg comparisonarcticstereowidth = 6000. + + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/global_ocean.cfg comparisonarcticstereoresolution = 10. + + # Options related to the vertical grid [vertical_grid] + + # the type of vertical grid + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg grid_type = tanh_dz + + # Number of vertical levels + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg vert_levels = 16 + + # Depth of the bottom of the ocean + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg bottom_depth = 3000.0 + + # The minimum layer thickness + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg min_layer_thickness = 3.0 + + # The maximum layer thickness + # source: /home/xylar/code/compass/customize_config_parser/compass/ocean/tests/global_ocean/mesh/qu240/qu240.cfg max_layer_thickness = 500.0 -Unfortunately, all comments are lost in the process of combining config -options. Comments are not parsed by ``ConfigParser``, and there is not a -standard for which comments are associated with which options. So users -will need to search through this documentation (or the code on the -`compass repo `_) to know what the config -options are used for. +The comments are retained (unlike in the previous version of compass) and the +config file or python module where they were defined is also included as a +a comment for provenance and to make it easier for users and developers to +understand how the config file is built up. From 8da92b0e7a5e46a53db67866d334726200f4e5ef Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 29 Mar 2022 15:12:50 +0200 Subject: [PATCH 08/15] Add source provenance when writing config files --- compass/config.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/compass/config.py b/compass/config.py index e65438d601..938ec8dc6e 100644 --- a/compass/config.py +++ b/compass/config.py @@ -20,6 +20,7 @@ def __init__(self): self._configs = dict() self._user_config = dict() self._combined = None + self._sources = None def add_user_config(self, filename): """ @@ -221,7 +222,7 @@ def set(self, section, option, value=None): config.set(section, option, value) self._combined = None - def write(self, fp): + def write(self, fp, include_sources=True): """ Write the config options to the given file pointer. @@ -229,10 +230,23 @@ def write(self, fp): ---------- fp : typing.TestIO The file pointer to write to. + + include_sources : bool, optional + Whether to include a comment above each option indicating the + source file where it was defined """ if self._combined is None: self._combine() - self._combined.write(fp) + for section in self._combined.sections(): + section_items = self._combined.items(section=section) + fp.write(f'[{section}]\n') + for key, value in section_items: + if include_sources: + source = self._sources[(section, key)] + fp.write(f'# source: {source}\n') + value = str(value).replace('\n', '\n\t') + fp.write(f'{key} = {value}\n') + fp.write('\n') def __getitem__(self, section): """ @@ -268,11 +282,13 @@ def _combine(self): self._combined = ConfigParser(interpolation=ExtendedInterpolation()) configs = dict(self._configs) configs.update(self._user_config) + self._sources = dict() for source, config in configs.items(): for section in config.sections(): if not self._combined.has_section(section): self._combined.add_section(section) for key, value in config.items(section): + self._sources[(section, key)] = source self._combined.set(section, key, value) self._ensure_absolute_paths() From 3dcce4b16a2c2ff0ac13bfae82bc86388544250e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 30 Mar 2022 00:42:26 +0200 Subject: [PATCH 09/15] Add parse and write out config file comments --- compass/config.py | 111 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 9 deletions(-) diff --git a/compass/config.py b/compass/config.py index 938ec8dc6e..dd7d69a76d 100644 --- a/compass/config.py +++ b/compass/config.py @@ -2,6 +2,7 @@ import os from importlib import resources import inspect +import sys class CompassConfigParser: @@ -19,7 +20,9 @@ def __init__(self): self._configs = dict() self._user_config = dict() + self._comments = dict() self._combined = None + self._combined_comments = None self._sources = None def add_user_config(self, filename): @@ -196,7 +199,7 @@ def has_option(self, section, option): self._combine() return self._combined.has_option(section, option) - def set(self, section, option, value=None): + def set(self, section, option, value=None, comment=''): """ Set the value of the given option in the given section. The file from which this function was called is also retained for provenance. @@ -211,6 +214,10 @@ def set(self, section, option, value=None): value : str, optional The value to set the option to + + comment : str, optional + A comment to include with the config option when it is written + to a file """ calling_frame = inspect.stack(context=2)[1] filename = os.path.abspath(calling_frame.filename) @@ -221,8 +228,13 @@ def set(self, section, option, value=None): config.add_section(section) config.set(section, option, value) self._combined = None + self._combined_comments = None + self._sources = None + if filename not in self._comments: + self._comments[filename] = dict() + self._comments[filename][(section, option)] = comment - def write(self, fp, include_sources=True): + def write(self, fp, include_sources=True, include_comments=True): """ Write the config options to the given file pointer. @@ -234,18 +246,26 @@ def write(self, fp, include_sources=True): include_sources : bool, optional Whether to include a comment above each option indicating the source file where it was defined + + include_comments : bool, optional + Whether to include the original comments associated with each + section or option """ if self._combined is None: self._combine() for section in self._combined.sections(): section_items = self._combined.items(section=section) - fp.write(f'[{section}]\n') - for key, value in section_items: + if include_comments and section in self._combined_comments: + fp.write(self._combined_comments[section]) + fp.write(f'[{section}]\n\n') + for option, value in section_items: + if include_comments: + fp.write(self._combined_comments[(section, option)]) if include_sources: - source = self._sources[(section, key)] + source = self._sources[(section, option)] fp.write(f'# source: {source}\n') value = str(value).replace('\n', '\n\t') - fp.write(f'{key} = {value}\n') + fp.write(f'{option} = {value}\n\n') fp.write('\n') def __getitem__(self, section): @@ -272,24 +292,36 @@ def _add(self, filename, user): if not os.path.exists(filename): raise FileNotFoundError(f'Config file does not exist: {filename}') config.read(filenames=filename) + with open(filename) as fp: + comments = self._parse_comments(fp, filename, comments_before=True) + if user: self._user_config = {filename: config} else: self._configs[filename] = config + self._comments[filename] = comments self._combined = None + self._combined_comments = None + self._sources = None def _combine(self): self._combined = ConfigParser(interpolation=ExtendedInterpolation()) configs = dict(self._configs) configs.update(self._user_config) self._sources = dict() + self._combined_comments = dict() for source, config in configs.items(): for section in config.sections(): + if section in self._comments[source]: + self._combined_comments[section] = \ + self._comments[source][section] if not self._combined.has_section(section): self._combined.add_section(section) - for key, value in config.items(section): - self._sources[(section, key)] = source - self._combined.set(section, key, value) + for option, value in config.items(section): + self._sources[(section, option)] = source + self._combined.set(section, option, value) + self._combined_comments[(section, option)] = \ + self._comments[source][(section, option)] self._ensure_absolute_paths() def _ensure_absolute_paths(self): @@ -304,3 +336,64 @@ def _ensure_absolute_paths(self): for option, value in config.items(section): value = os.path.abspath(value) config.set(section, option, value) + + @staticmethod + def _parse_comments(fp, filename, comments_before=True): + """ Parse the comments in a config file into a dictionary """ + comments = dict() + current_comment = '' + section_name = None + option_name = None + indent_level = 0 + for line_number, line in enumerate(fp, start=1): + value = line.strip() + is_comment = value.startswith('#') + if is_comment: + current_comment = current_comment + line + if len(value) == 0 or is_comment: + # end of value + indent_level = sys.maxsize + continue + + cur_indent_level = len(line) - len(line.lstrip()) + is_continuation = cur_indent_level > indent_level + # a section header or option header? + if section_name is None or option_name is None or \ + not is_continuation: + indent_level = cur_indent_level + # is it a section header? + is_section = value.startswith('[') and value.endswith(']') + if is_section: + if not comments_before: + if option_name is None: + comments[section_name] = current_comment + else: + comments[(section_name, option_name)] = \ + current_comment + section_name = value[1:-1].strip().lower() + option_name = None + + if comments_before: + comments[section_name] = current_comment + current_comment = '' + # an option line? + else: + delimiter_index = value.find('=') + if delimiter_index == -1: + raise ValueError(f'Expected to find "=" on line ' + f'{line_number} of {filename}') + + if not comments_before: + if option_name is None: + comments[section_name] = current_comment + else: + comments[(section_name, option_name)] = \ + current_comment + + option_name = value[:delimiter_index].strip().lower() + + if comments_before: + comments[(section_name, option_name)] = current_comment + current_comment = '' + + return comments From 100da75c05e3a88eddb2d78f28e2b365c310f87c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 31 Mar 2022 21:10:51 +0200 Subject: [PATCH 10/15] Fix comments in set() method --- compass/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compass/config.py b/compass/config.py index dd7d69a76d..9fbb884b44 100644 --- a/compass/config.py +++ b/compass/config.py @@ -219,6 +219,8 @@ def set(self, section, option, value=None, comment=''): A comment to include with the config option when it is written to a file """ + section = section.lower() + option = option.lower() calling_frame = inspect.stack(context=2)[1] filename = os.path.abspath(calling_frame.filename) if filename not in self._configs: @@ -232,6 +234,7 @@ def set(self, section, option, value=None, comment=''): self._sources = None if filename not in self._comments: self._comments[filename] = dict() + comment = ''.join([f'# {line}\n' for line in comment.split('\n')]) self._comments[filename][(section, option)] = comment def write(self, fp, include_sources=True, include_comments=True): From 9919abf8deba890372662e97a82b79c6ff7757ed Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 31 Mar 2022 21:11:10 +0200 Subject: [PATCH 11/15] Add a few example comments in calls to config.set() --- .../tests/baroclinic_channel/__init__.py | 11 +++++++--- .../cosine_bell/__init__.py | 22 +++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/compass/ocean/tests/baroclinic_channel/__init__.py b/compass/ocean/tests/baroclinic_channel/__init__.py index 4fdc1cc76f..4b3d94ae73 100644 --- a/compass/ocean/tests/baroclinic_channel/__init__.py +++ b/compass/ocean/tests/baroclinic_channel/__init__.py @@ -53,9 +53,14 @@ def configure(resolution, config): 'ny': 500, 'dc': 1e3}} + comment = {'nx': 'the number of mesh cells in the x direction', + 'ny': 'the number of mesh cells in the y direction', + 'dc': 'the distance between adjacent cell centers'} + if resolution not in res_params: - raise ValueError('Unsupported resolution {}. Supported values are: ' - '{}'.format(resolution, list(res_params))) + raise ValueError(f'Unsupported resolution {resolution}. Supported ' + f'values are: {list(res_params)}') res_params = res_params[resolution] for param in res_params: - config.set('baroclinic_channel', param, '{}'.format(res_params[param])) + config.set('baroclinic_channel', param, str(res_params[param]), + comment=comment[param]) diff --git a/compass/ocean/tests/global_convergence/cosine_bell/__init__.py b/compass/ocean/tests/global_convergence/cosine_bell/__init__.py index 781a18dad7..70d5e6ccd2 100644 --- a/compass/ocean/tests/global_convergence/cosine_bell/__init__.py +++ b/compass/ocean/tests/global_convergence/cosine_bell/__init__.py @@ -30,7 +30,7 @@ def __init__(self, test_group): # add the steps with default resolutions so they can be listed config = CompassConfigParser() - config.add_from_package(self.__module__, '{}.cfg'.format(self.name)) + config.add_from_package(self.__module__, f'{self.name}.cfg') self._setup_steps(config) def configure(self): @@ -44,7 +44,7 @@ def configure(self): init_options = dict() for option in ['temperature', 'salinity', 'lat_center', 'lon_center', 'radius', 'psi0', 'vel_pd']: - init_options['config_cosine_bell_{}'.format(option)] = \ + init_options[f'config_cosine_bell_{option}'] = \ config.get('cosine_bell', option) for step in self.steps.values(): @@ -59,11 +59,10 @@ def run(self): """ config = self.config for resolution in self.resolutions: - cores = config.getint('cosine_bell', - 'QU{}_cores'.format(resolution)) + cores = config.getint('cosine_bell', f'QU{resolution}_cores') min_cores = config.getint('cosine_bell', - 'QU{}_min_cores'.format(resolution)) - step = self.steps['QU{}_forward'.format(resolution)] + f'QU{resolution}_min_cores') + step = self.steps[f'QU{resolution}_forward'] step.cores = cores step.min_cores = min_cores @@ -90,14 +89,15 @@ def update_cores(self): # In a pinch, about 3000 cells per core min_cores = max(1, round(approx_cells / max_cells_per_core)) - step = self.steps['QU{}_forward'.format(resolution)] + step = self.steps[f'QU{resolution}_forward'] step.cores = cores step.min_cores = min_cores - config.set('cosine_bell', 'QU{}_cores'.format(resolution), - str(cores)) - config.set('cosine_bell', 'QU{}_min_cores'.format(resolution), - str(min_cores)) + config.set('cosine_bell', f'QU{resolution}_cores', str(cores), + comment=f'Target core count for {resolution} km mesh') + config.set('cosine_bell', f'QU{resolution}_min_cores', + str(min_cores), + comment=f'Minimum core count for {resolution} km mesh') def _setup_steps(self, config): """ setup steps given resolutions """ From fbbc2f878456e4972679ff8116f454bc9b6abba9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 31 Mar 2022 22:31:45 +0200 Subject: [PATCH 12/15] Fix mache.machines package name --- compass/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/setup.py b/compass/setup.py index 9f125c3f12..5e60ed1eaf 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -215,7 +215,7 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, # add the E3SM config options from mache if machine is not None: - config.add_from_package('mache', f'{machine}.cfg') + config.add_from_package('mache.machines', f'{machine}.cfg') # add the compass machine config file if machine is None: From 49aa9385798f637d8d21a5003a934d218c40656f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 31 Mar 2022 22:40:21 +0200 Subject: [PATCH 13/15] Fix set() without comments --- compass/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compass/config.py b/compass/config.py index 9fbb884b44..0507f76555 100644 --- a/compass/config.py +++ b/compass/config.py @@ -199,7 +199,7 @@ def has_option(self, section, option): self._combine() return self._combined.has_option(section, option) - def set(self, section, option, value=None, comment=''): + def set(self, section, option, value=None, comment=None): """ Set the value of the given option in the given section. The file from which this function was called is also retained for provenance. @@ -234,7 +234,10 @@ def set(self, section, option, value=None, comment=''): self._sources = None if filename not in self._comments: self._comments[filename] = dict() - comment = ''.join([f'# {line}\n' for line in comment.split('\n')]) + if comment is None: + comment = '' + else: + comment = ''.join([f'# {line}\n' for line in comment.split('\n')]) self._comments[filename][(section, option)] = comment def write(self, fp, include_sources=True, include_comments=True): From 631af5946f82d275755d22a2742e56f625018da9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 13 Apr 2022 10:03:56 +0200 Subject: [PATCH 14/15] Add a way to set user config options This is needed to set user config options that come from the command-line and not from config files (e.g. the MPAS path) --- compass/config.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/compass/config.py b/compass/config.py index 0507f76555..ae9e56d6e4 100644 --- a/compass/config.py +++ b/compass/config.py @@ -199,7 +199,7 @@ def has_option(self, section, option): self._combine() return self._combined.has_option(section, option) - def set(self, section, option, value=None, comment=None): + def set(self, section, option, value=None, comment=None, user=False): """ Set the value of the given option in the given section. The file from which this function was called is also retained for provenance. @@ -218,14 +218,23 @@ def set(self, section, option, value=None, comment=None): comment : str, optional A comment to include with the config option when it is written to a file + + user : bool, optional + Whether this config option was supplied by the user (e.g. through + a command-line flag) and should take priority over other sources """ section = section.lower() option = option.lower() calling_frame = inspect.stack(context=2)[1] filename = os.path.abspath(calling_frame.filename) - if filename not in self._configs: - self._configs[filename] = RawConfigParser() - config = self._configs[filename] + + if user: + config_dict = self._user_config + else: + config_dict = self._configs + if filename not in config_dict: + config_dict[filename] = RawConfigParser() + config = config_dict[filename] if not config.has_section(section): config.add_section(section) config.set(section, option, value) @@ -312,22 +321,21 @@ def _add(self, filename, user): def _combine(self): self._combined = ConfigParser(interpolation=ExtendedInterpolation()) - configs = dict(self._configs) - configs.update(self._user_config) self._sources = dict() self._combined_comments = dict() - for source, config in configs.items(): - for section in config.sections(): - if section in self._comments[source]: - self._combined_comments[section] = \ - self._comments[source][section] - if not self._combined.has_section(section): - self._combined.add_section(section) - for option, value in config.items(section): - self._sources[(section, option)] = source - self._combined.set(section, option, value) - self._combined_comments[(section, option)] = \ - self._comments[source][(section, option)] + for configs in [self._configs, self._user_config]: + for source, config in configs.items(): + for section in config.sections(): + if section in self._comments[source]: + self._combined_comments[section] = \ + self._comments[source][section] + if not self._combined.has_section(section): + self._combined.add_section(section) + for option, value in config.items(section): + self._sources[(section, option)] = source + self._combined.set(section, option, value) + self._combined_comments[(section, option)] = \ + self._comments[source][(section, option)] self._ensure_absolute_paths() def _ensure_absolute_paths(self): From a72aed65eb87865fc6c9e61ad7e7e07b65e83728 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 13 Apr 2022 10:05:08 +0200 Subject: [PATCH 15/15] Set the MPAS path as a "user" config option --- compass/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/setup.py b/compass/setup.py index 5e60ed1eaf..b5a72ec51b 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -253,7 +253,7 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, # set the mpas_model path from the command line if provided if mpas_model_path is not None: - config.set('paths', 'mpas_model', mpas_model_path) + config.set('paths', 'mpas_model', mpas_model_path, user=True) config.set('test_case', 'steps_to_run', ' '.join(test_case.steps_to_run))