From 7971a5f6dcca8740fe20d36680a2fe0a479c9b1b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 6 Dec 2022 11:05:38 -0500 Subject: [PATCH] Allow customizations based on user/configuration (#1146) * allow for modifying any config file from a higher yaml --- parm/config/config.aeroanl | 4 +- parm/config/yaml/defaults.yaml | 3 + workflow/hosts.py | 17 ++---- workflow/hosts/hera.yaml | 46 +++++++------- workflow/hosts/orion.yaml | 46 +++++++------- workflow/hosts/s4.yaml | 46 +++++++------- workflow/hosts/wcoss2.yaml | 46 +++++++------- workflow/setup_expt.py | 106 ++++++++++++++++++++------------- 8 files changed, 165 insertions(+), 149 deletions(-) create mode 100644 parm/config/yaml/defaults.yaml diff --git a/parm/config/config.aeroanl b/parm/config/config.aeroanl index 796c0bea1a..edba514721 100755 --- a/parm/config/config.aeroanl +++ b/parm/config/config.aeroanl @@ -11,7 +11,7 @@ export AEROVARYAML=$HOMEgfs/sorc/gdas.cd/parm/aero/variational/3dvar_dripcg.yaml export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/aero/berror/static_bump.yaml export FV3JEDI_FIX=$HOMEgfs/fix/gdas -export io_layout_x=1 -export io_layout_y=1 +export io_layout_x=@IO_LAYOUT_X@ +export io_layout_y=@IO_LAYOUT_Y@ echo "END: config.aeroanl" diff --git a/parm/config/yaml/defaults.yaml b/parm/config/yaml/defaults.yaml new file mode 100644 index 0000000000..7537454957 --- /dev/null +++ b/parm/config/yaml/defaults.yaml @@ -0,0 +1,3 @@ +aeroanl: + IO_LAYOUT_X: 1 + IO_LAYOUT_Y: 1 diff --git a/workflow/hosts.py b/workflow/hosts.py index 968b3bc1b4..91925da71a 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -2,20 +2,11 @@ import os from pathlib import Path -from yaml import load -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader - -__all__ = ['Host'] +from pygw.yaml_file import YAMLFile -def load_yaml(_path: Path): - with open(_path, "r") as _file: - yaml_dict = load(_file, Loader=Loader) - return yaml_dict +__all__ = ['Host'] class Host: @@ -35,7 +26,7 @@ def __init__(self, host=None): self.machine = detected_host self.info = self._get_info - self.scheduler = self.info['scheduler'] + self.scheduler = self.info['SCHEDULER'] @classmethod def detect(cls): @@ -65,7 +56,7 @@ def _get_info(self) -> dict: hostfile = Path(os.path.join(os.path.dirname(__file__), f'hosts/{self.machine.lower()}.yaml')) try: - info = load_yaml(hostfile) + info = YAMLFile(path=hostfile) except FileNotFoundError: raise FileNotFoundError(f'{hostfile} does not exist!') except IOError: diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 19cd1157e7..d6e3b83b3d 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -1,23 +1,23 @@ -base_git: '/scratch1/NCEPDEV/global/glopara/git' -dmpdir: '/scratch1/NCEPDEV/global/glopara/dump' -packageroot: '/scratch1/NCEPDEV/global/glopara/nwpara' -comroot: '/scratch1/NCEPDEV/global/glopara/com' -cominsyn: '${COMROOT}/gfs/prod/syndat' -homedir: '/scratch1/NCEPDEV/global/$USER' -stmp: '/scratch1/NCEPDEV/stmp2/$USER' -ptmp: '/scratch1/NCEPDEV/stmp4/$USER' -noscrub: $HOMEDIR -account: fv3-cpu -scheduler: slurm -queue: batch -queue_service: batch -partition_batch: hera -partition_service: service -chgrp_rstprod: 'YES' -chgrp_cmd: 'chgrp rstprod' -hpssarch: 'YES' -localarch: 'NO' -atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' -make_nsstbufr: 'NO' -make_acftbufr: 'NO' -supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48'] +BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' +DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump' +PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' +COMROOT: '/scratch1/NCEPDEV/global/glopara/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' +STMP: '/scratch1/NCEPDEV/stmp2/${USER}' +PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' +NOSCRUB: $HOMEDIR +ACCOUNT: fv3-cpu +SCHEDULER: slurm +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: hera +PARTITION_SERVICE: service +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'YES' +LOCALARCH: 'NO' +ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 49c905538b..b1f0a8ad97 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -1,23 +1,23 @@ -base_git: '/work/noaa/global/glopara/git' -dmpdir: '/work/noaa/rstprod/dump' -packageroot: '/work/noaa/global/glopara/nwpara' -comroot: '/work/noaa/global/glopara/com' -cominsyn: '${COMROOT}/gfs/prod/syndat' -homedir: '/work/noaa/global/$USER' -stmp: '/work/noaa/stmp/$USER' -ptmp: '/work/noaa/stmp/$USER' -noscrub: $HOMEDIR -scheduler: slurm -account: fv3-cpu -queue: batch -queue_service: batch -partition_batch: orion -partition_service: service -chgrp_rstprod: 'YES' -chgrp_cmd: 'chgrp rstprod' -hpssarch: 'NO' -localarch: 'NO' -atardir: '$NOSCRUB/archive_rotdir/$PSLOT' -make_nsstbufr: 'NO' -make_acftbufr: 'NO' -supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48'] +BASE_GIT: '/work/noaa/global/glopara/git' +DMPDIR: '/work/noaa/rstprod/dump' +PACKAGEROOT: '/work/noaa/global/glopara/nwpara' +COMROOT: '/work/noaa/global/glopara/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/work/noaa/global/${USER}' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' +NOSCRUB: $HOMEDIR +SCHEDULER: slurm +ACCOUNT: fv3-cpu +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: orion +PARTITION_SERVICE: service +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'NO' +LOCALARCH: 'NO' +ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index 38a434b353..d200eaa519 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -1,23 +1,23 @@ -base_git: '/data/prod/glopara/git' -dmpdir: '/data/prod/glopara/dump' -packageroot: '/data/prod/glopara/nwpara' -comroot: '/data/prod/glopara/com' -cominsyn: '${COMROOT}/gfs/prod/syndat' -homedir: '/data/users/$USER' -stmp: '/scratch/users/$USER' -ptmp: '/scratch/users/$USER' -noscrub: $HOMEDIR -account: star -scheduler: slurm -queue: s4 -queue_service: serial -partition_batch: s4 -partition_service: serial -chgrp_rstprod: 'NO' -chgrp_cmd: 'ls' -hpssarch: 'NO' -localarch: 'NO' -atardir: '$NOSCRUB/archive_rotdir/$PSLOT' -make_nsstbufr: 'YES' -make_acftbufr: 'YES' -supported_resolutions: ['C384', 'C192', 'C96', 'C48'] +BASE_GIT: '/data/prod/glopara/git' +DMPDIR: '/data/prod/glopara/dump' +PACKAGEROOT: '/data/prod/glopara/nwpara' +COMROOT: '/data/prod/glopara/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/data/users/${USER}' +STMP: '/scratch/users/${USER}' +PTMP: '/scratch/users/${USER}' +NOSCRUB: ${HOMEDIR} +ACCOUNT: star +SCHEDULER: slurm +QUEUE: s4 +QUEUE_SERVICE: serial +PARTITION_BATCH: s4 +PARTITION_SERVICE: serial +CHGRP_RSTPROD: 'NO' +CHGRP_CMD: 'ls' +HPSSARCH: 'NO' +LOCALARCH: 'NO' +ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' +MAKE_NSSTBUFR: 'YES' +MAKE_ACFTBUFR: 'YES' +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 592fa3db79..a049f6ae1f 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -1,23 +1,23 @@ -base_git: '/lfs/h2/emc/global/save/emc.global/git' -dmpdir: '/lfs/h2/emc/global/noscrub/emc.global/dump' -packageroot: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}' -comroot: '${COMROOT:-"/lfs/h1/ops/prod/com"}' -cominsyn: '${COMROOT}/gfs/${gfs_ver:-"v16.2"}/syndat' -homedir: '/lfs/h2/emc/global/noscrub/$USER' -stmp: '/lfs/h2/emc/stmp/$USER' -ptmp: '/lfs/h2/emc/ptmp/$USER' -noscrub: $HOMEDIR -account: 'GFS-DEV' -scheduler: pbspro -queue: 'dev' -queue_service: 'dev_transfer' -partition_batch: '' -partition_service: '' -chgrp_rstprod: 'YES' -chgrp_cmd: 'chgrp rstprod' -hpssarch: 'NO' -localarch: 'NO' -atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' -make_nsstbufr: 'NO' -make_acftbufr: 'NO' -supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48'] +BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git' +DMPDIR: '/lfs/h2/emc/global/noscrub/emc.global/dump' +PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}' +COMROOT: '${COMROOT:-"/lfs/h1/ops/prod/com"}' +COMINsyn: '${COMROOT}/gfs/${gfs_ver:-"v16.2"}/syndat' +HOMEDIR: '/lfs/h2/emc/global/noscrub/${USER}' +STMP: '/lfs/h2/emc/stmp/${USER}' +PTMP: '/lfs/h2/emc/ptmp/${USER}' +NOSCRUB: $HOMEDIR +ACCOUNT: 'GFS-DEV' +SCHEDULER: pbspro +QUEUE: 'dev' +QUEUE_SERVICE: 'dev_transfer' +PARTITION_BATCH: '' +PARTITION_SERVICE: '' +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'NO' +LOCALARCH: 'NO' +ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index ac9f1a8dc7..02a8a6f959 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -9,8 +9,11 @@ import shutil from datetime import datetime from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + from hosts import Host +from pygw.yaml_file import YAMLFile + _here = os.path.dirname(__file__) _top = os.path.abspath(os.path.join(os.path.abspath(_here), '..')) @@ -108,47 +111,51 @@ def fill_EXPDIR(inputs): return +def update_configs(host, inputs): + + # First update config.base + edit_baseconfig(host, inputs) + + yaml_path = inputs.yaml + yaml_dict = YAMLFile(path=yaml_path) + + # loop over other configs and update them + for cfg in yaml_dict.keys(): + cfg_file = f'{inputs.expdir}/{inputs.pslot}/config.{cfg}' + cfg_dict = get_template_dict(yaml_dict[cfg]) + edit_config(cfg_file, cfg_file, cfg_dict) + + return + + def edit_baseconfig(host, inputs): """ Parses and populates the templated `config.base.emc.dyn` to `config.base` """ tmpl_dict = { - "@MACHINE@": host.machine.upper(), + "@HOMEgfs@": _top, + "@MACHINE@": host.machine.upper()} + + # Replace host related items + extend_dict = get_template_dict(host.info) + tmpl_dict = dict(tmpl_dict, **extend_dict) + + extend_dict = dict() + extend_dict = { "@PSLOT@": inputs.pslot, "@SDATE@": inputs.idate.strftime('%Y%m%d%H'), "@EDATE@": inputs.edate.strftime('%Y%m%d%H'), "@CASECTL@": f'C{inputs.resdet}', - "@HOMEgfs@": _top, - "@BASE_GIT@": host.info["base_git"], - "@DMPDIR@": host.info["dmpdir"], - "@PACKAGEROOT@": host.info["packageroot"], - "@COMROOT@": host.info["comroot"], - "@COMINsyn@": host.info["cominsyn"], - "@HOMEDIR@": host.info["homedir"], "@EXPDIR@": inputs.expdir, "@ROTDIR@": inputs.comrot, "@ICSDIR@": inputs.icsdir, - "@STMP@": host.info["stmp"], - "@PTMP@": host.info["ptmp"], - "@NOSCRUB@": host.info["noscrub"], - "@ACCOUNT@": host.info["account"], - "@QUEUE@": host.info["queue"], - "@QUEUE_SERVICE@": host.info["queue_service"], - "@PARTITION_BATCH@": host.info["partition_batch"], - "@PARTITION_SERVICE@": host.info["partition_service"], "@EXP_WARM_START@": inputs.warm_start, "@MODE@": inputs.mode, - "@CHGRP_RSTPROD@": host.info["chgrp_rstprod"], - "@CHGRP_CMD@": host.info["chgrp_cmd"], - "@HPSSARCH@": host.info["hpssarch"], - "@LOCALARCH@": host.info["localarch"], - "@ATARDIR@": host.info["atardir"], - "@MAKE_NSSTBUFR@": host.info["make_nsstbufr"], - "@MAKE_ACFTBUFR@": host.info["make_acftbufr"], "@gfs_cyc@": inputs.gfs_cyc, - "@APP@": inputs.app, + "@APP@": inputs.app } + tmpl_dict = dict(tmpl_dict, **extend_dict) extend_dict = dict() if inputs.mode in ['cycled']: @@ -171,29 +178,41 @@ def edit_baseconfig(host, inputs): tmpl_dict = dict(tmpl_dict, **extend_dict) - # Open and read the templated config.base.emc.dyn - base_tmpl = f'{inputs.configdir}/config.base.emc.dyn' - with open(base_tmpl, 'rt') as fi: - basestr = fi.read() + base_input = f'{inputs.configdir}/config.base.emc.dyn' + base_output = f'{inputs.expdir}/{inputs.pslot}/config.base' + edit_config(base_input, base_output, tmpl_dict) + + return - for key, val in tmpl_dict.items(): - basestr = basestr.replace(key, str(val)) - # Write and clobber the experiment config.base - base_config = f'{inputs.expdir}/{inputs.pslot}/config.base' - if os.path.exists(base_config): - os.unlink(base_config) +def edit_config(input_config, output_config, config_dict): - with open(base_config, 'wt') as fo: - fo.write(basestr) + # Read input config + with open(input_config, 'rt') as fi: + config_str = fi.read() - print('') - print(f'EDITED: {base_config} as per user input.') - print(f'DEFAULT: {base_tmpl} is for reference only.') - print('') + # Substitute from config_dict + for key, val in config_dict.items(): + config_str = config_str.replace(key, str(val)) + + # Ensure no output_config file exists + if os.path.exists(output_config): + os.unlink(output_config) + + # Write output config + with open(output_config, 'wt') as fo: + fo.write(config_str) + + print(f'EDITED: {output_config} as per user input.') return +def get_template_dict(input_dict): + output_dict = dict() + for key, value in input_dict.items(): + output_dict[f'@{key}@'] = value + + return output_dict def input_args(): """ @@ -239,6 +258,9 @@ def input_args(): subp.add_argument('--start', help='restart mode: warm or cold', type=str, choices=['warm', 'cold'], required=False, default='cold') + subp.add_argument('--yaml', help='Defaults to substitute from', type=str, + required=False, default=os.path.join(_top, 'parm/config/yaml/defaults.yaml')) + # cycled mode additional arguments cycled.add_argument('--resens', help='resolution of the ensemble model forecast', type=int, required=False, default=192) @@ -284,7 +306,7 @@ def query_and_clean(dirname): def validate_user_request(host, inputs): expt_res = f'C{inputs.resdet}' - supp_res = host.info['supported_resolutions'] + supp_res = host.info['SUPPORTED_RESOLUTIONS'] machine = host.machine if expt_res not in supp_res: raise NotImplementedError(f"Supported resolutions on {machine} are:\n{', '.join(supp_res)}") @@ -310,4 +332,4 @@ def validate_user_request(host, inputs): if create_expdir: makedirs_if_missing(expdir) fill_EXPDIR(user_inputs) - edit_baseconfig(host, user_inputs) + update_configs(host, user_inputs)