Allow customizations based on user/configuration (#1146)

* allow for modifying any config file from a higher yaml

parm/config/config.aeroanl

@@ -11,7 +11,7 @@ export AEROVARYAML=$HOMEgfs/sorc/gdas.cd/parm/aero/variational/3dvar_dripcg.yaml
 export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/aero/berror/static_bump.yaml
 export FV3JEDI_FIX=$HOMEgfs/fix/gdas
 
-export io_layout_x=1
-export io_layout_y=1
+export io_layout_x=@IO_LAYOUT_X@
+export io_layout_y=@IO_LAYOUT_Y@
 
 echo "END: config.aeroanl"

parm/config/yaml/defaults.yaml

@@ -0,0 +1,3 @@
+aeroanl:
+  IO_LAYOUT_X: 1
+  IO_LAYOUT_Y: 1

workflow/hosts.py

@@ -2,20 +2,11 @@
 
 import os
 from pathlib import Path
-from yaml import load
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader
 
-
-__all__ = ['Host']
+from pygw.yaml_file import YAMLFile
 
 
-def load_yaml(_path: Path):
-    with open(_path, "r") as _file:
-        yaml_dict = load(_file, Loader=Loader)
-        return yaml_dict
+__all__ = ['Host']
 
 
 class Host:
@@ -35,7 +26,7 @@ def __init__(self, host=None):
 
         self.machine = detected_host
         self.info = self._get_info
-        self.scheduler = self.info['scheduler']
+        self.scheduler = self.info['SCHEDULER']
 
     @classmethod
     def detect(cls):
@@ -65,7 +56,7 @@ def _get_info(self) -> dict:
 
         hostfile = Path(os.path.join(os.path.dirname(__file__), f'hosts/{self.machine.lower()}.yaml'))
         try:
-            info = load_yaml(hostfile)
+            info = YAMLFile(path=hostfile)
         except FileNotFoundError:
             raise FileNotFoundError(f'{hostfile} does not exist!')
         except IOError:

workflow/hosts/hera.yaml

@@ -1,23 +1,23 @@
-base_git: '/scratch1/NCEPDEV/global/glopara/git'
-dmpdir: '/scratch1/NCEPDEV/global/glopara/dump'
-packageroot: '/scratch1/NCEPDEV/global/glopara/nwpara'
-comroot: '/scratch1/NCEPDEV/global/glopara/com'
-cominsyn: '${COMROOT}/gfs/prod/syndat'
-homedir: '/scratch1/NCEPDEV/global/$USER'
-stmp: '/scratch1/NCEPDEV/stmp2/$USER'
-ptmp: '/scratch1/NCEPDEV/stmp4/$USER'
-noscrub: $HOMEDIR
-account: fv3-cpu
-scheduler: slurm
-queue: batch
-queue_service: batch
-partition_batch: hera
-partition_service: service
-chgrp_rstprod: 'YES'
-chgrp_cmd: 'chgrp rstprod'
-hpssarch: 'YES'
-localarch: 'NO'
-atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT'
-make_nsstbufr: 'NO'
-make_acftbufr: 'NO'
-supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48']
+BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git'
+DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump'
+PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara'
+COMROOT: '/scratch1/NCEPDEV/global/glopara/com'
+COMINsyn: '${COMROOT}/gfs/prod/syndat'
+HOMEDIR: '/scratch1/NCEPDEV/global/${USER}'
+STMP: '/scratch1/NCEPDEV/stmp2/${USER}'
+PTMP: '/scratch1/NCEPDEV/stmp4/${USER}'
+NOSCRUB: $HOMEDIR
+ACCOUNT: fv3-cpu
+SCHEDULER: slurm
+QUEUE: batch
+QUEUE_SERVICE: batch
+PARTITION_BATCH: hera
+PARTITION_SERVICE: service
+CHGRP_RSTPROD: 'YES'
+CHGRP_CMD: 'chgrp rstprod'
+HPSSARCH: 'YES'
+LOCALARCH: 'NO'
+ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
+MAKE_NSSTBUFR: 'NO'
+MAKE_ACFTBUFR: 'NO'
+SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']

workflow/hosts/orion.yaml

@@ -1,23 +1,23 @@
-base_git: '/work/noaa/global/glopara/git'
-dmpdir: '/work/noaa/rstprod/dump'
-packageroot: '/work/noaa/global/glopara/nwpara'
-comroot: '/work/noaa/global/glopara/com'
-cominsyn: '${COMROOT}/gfs/prod/syndat'
-homedir: '/work/noaa/global/$USER'
-stmp: '/work/noaa/stmp/$USER'
-ptmp: '/work/noaa/stmp/$USER'
-noscrub: $HOMEDIR
-scheduler: slurm
-account: fv3-cpu
-queue: batch
-queue_service: batch
-partition_batch: orion
-partition_service: service
-chgrp_rstprod: 'YES'
-chgrp_cmd: 'chgrp rstprod'
-hpssarch: 'NO'
-localarch: 'NO'
-atardir: '$NOSCRUB/archive_rotdir/$PSLOT'
-make_nsstbufr: 'NO'
-make_acftbufr: 'NO'
-supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48']
+BASE_GIT: '/work/noaa/global/glopara/git'
+DMPDIR: '/work/noaa/rstprod/dump'
+PACKAGEROOT: '/work/noaa/global/glopara/nwpara'
+COMROOT: '/work/noaa/global/glopara/com'
+COMINsyn: '${COMROOT}/gfs/prod/syndat'
+HOMEDIR: '/work/noaa/global/${USER}'
+STMP: '/work/noaa/stmp/${USER}'
+PTMP: '/work/noaa/stmp/${USER}'
+NOSCRUB: $HOMEDIR
+SCHEDULER: slurm
+ACCOUNT: fv3-cpu
+QUEUE: batch
+QUEUE_SERVICE: batch
+PARTITION_BATCH: orion
+PARTITION_SERVICE: service
+CHGRP_RSTPROD: 'YES'
+CHGRP_CMD: 'chgrp rstprod'
+HPSSARCH: 'NO'
+LOCALARCH: 'NO'
+ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}'
+MAKE_NSSTBUFR: 'NO'
+MAKE_ACFTBUFR: 'NO'
+SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']

workflow/hosts/s4.yaml

@@ -1,23 +1,23 @@
-base_git: '/data/prod/glopara/git'
-dmpdir: '/data/prod/glopara/dump'
-packageroot: '/data/prod/glopara/nwpara'
-comroot: '/data/prod/glopara/com'
-cominsyn: '${COMROOT}/gfs/prod/syndat'
-homedir: '/data/users/$USER'
-stmp: '/scratch/users/$USER'
-ptmp: '/scratch/users/$USER'
-noscrub: $HOMEDIR
-account: star
-scheduler: slurm
-queue: s4
-queue_service: serial
-partition_batch: s4
-partition_service: serial
-chgrp_rstprod: 'NO'
-chgrp_cmd: 'ls'
-hpssarch: 'NO'
-localarch: 'NO'
-atardir: '$NOSCRUB/archive_rotdir/$PSLOT'
-make_nsstbufr: 'YES'
-make_acftbufr: 'YES'
-supported_resolutions: ['C384', 'C192', 'C96', 'C48']
+BASE_GIT: '/data/prod/glopara/git'
+DMPDIR: '/data/prod/glopara/dump'
+PACKAGEROOT: '/data/prod/glopara/nwpara'
+COMROOT: '/data/prod/glopara/com'
+COMINsyn: '${COMROOT}/gfs/prod/syndat'
+HOMEDIR: '/data/users/${USER}'
+STMP: '/scratch/users/${USER}'
+PTMP: '/scratch/users/${USER}'
+NOSCRUB: ${HOMEDIR}
+ACCOUNT: star
+SCHEDULER: slurm
+QUEUE: s4
+QUEUE_SERVICE: serial
+PARTITION_BATCH: s4
+PARTITION_SERVICE: serial
+CHGRP_RSTPROD: 'NO'
+CHGRP_CMD: 'ls'
+HPSSARCH: 'NO'
+LOCALARCH: 'NO'
+ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}'
+MAKE_NSSTBUFR: 'YES'
+MAKE_ACFTBUFR: 'YES'
+SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48']

workflow/hosts/wcoss2.yaml

@@ -1,23 +1,23 @@
-base_git: '/lfs/h2/emc/global/save/emc.global/git'
-dmpdir: '/lfs/h2/emc/global/noscrub/emc.global/dump'
-packageroot: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}'
-comroot: '${COMROOT:-"/lfs/h1/ops/prod/com"}'
-cominsyn: '${COMROOT}/gfs/${gfs_ver:-"v16.2"}/syndat'
-homedir: '/lfs/h2/emc/global/noscrub/$USER'
-stmp: '/lfs/h2/emc/stmp/$USER'
-ptmp: '/lfs/h2/emc/ptmp/$USER'
-noscrub: $HOMEDIR
-account: 'GFS-DEV'
-scheduler: pbspro
-queue: 'dev'
-queue_service: 'dev_transfer'
-partition_batch: ''
-partition_service: ''
-chgrp_rstprod: 'YES'
-chgrp_cmd: 'chgrp rstprod'
-hpssarch: 'NO'
-localarch: 'NO'
-atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT'
-make_nsstbufr: 'NO'
-make_acftbufr: 'NO'
-supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48']
+BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git'
+DMPDIR: '/lfs/h2/emc/global/noscrub/emc.global/dump'
+PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}'
+COMROOT: '${COMROOT:-"/lfs/h1/ops/prod/com"}'
+COMINsyn: '${COMROOT}/gfs/${gfs_ver:-"v16.2"}/syndat'
+HOMEDIR: '/lfs/h2/emc/global/noscrub/${USER}'
+STMP: '/lfs/h2/emc/stmp/${USER}'
+PTMP: '/lfs/h2/emc/ptmp/${USER}'
+NOSCRUB: $HOMEDIR
+ACCOUNT: 'GFS-DEV'
+SCHEDULER: pbspro
+QUEUE: 'dev'
+QUEUE_SERVICE: 'dev_transfer'
+PARTITION_BATCH: ''
+PARTITION_SERVICE: ''
+CHGRP_RSTPROD: 'YES'
+CHGRP_CMD: 'chgrp rstprod'
+HPSSARCH: 'NO'
+LOCALARCH: 'NO'
+ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
+MAKE_NSSTBUFR: 'NO'
+MAKE_ACFTBUFR: 'NO'
+SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']