Testing of the Marine DA jjobs (#234)

* blank ctests for jjobs

* works but needs tidy

* fname too long for fms ...

* some code tidy

* removed debug prints

* replacing machine in config

* copy bkg

* manual merge of prep script

* adjusted a few things related to jjob refactor ...

* tidy ... some and local2hpc

* code tidy and some shellcheck

* bomb obs, fix yaml

* clean exp

parm/soca/obs/config/sst_noaa19_l3u.yaml

@@ -3,7 +3,7 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs${OPREFIX}sst_noaa19_l3u.${CDATE}.nc4
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_noaa19_l3u.${CDATE}.nc4
   obsdataout:
     engine:
       type: H5File

scripts/exgdas_global_marine_analysis_prep.py

@@ -44,8 +44,7 @@
 my_home = os.path.dirname(os.path.dirname(my_dir))
 gdas_home = os.path.join(os.getenv('HOMEgfs'), 'sorc', 'gdas.cd')
 sys.path.append(os.path.join(os.getenv('HOMEgfs', my_home), 'ush'))
-print(f"sys.path={sys.path}")
-
+sys.path.append(os.path.join(gdas_home, 'ush'))
 
 # import UFSDA utilities
 import ufsda
@@ -97,7 +96,7 @@ def test_hist_date(histfile, ref_date):
     assert hist_date == ref_date, 'Inconsistent bkg date'
 
 
-def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]', yaml_name='bkg.yaml'):
+def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00[3-9]', yaml_name='bkg.yaml'):
     """
     Generate a YAML of the list of backgrounds for the pseudo model
     TODO: [3-9] shouldn't be hard-coded. Instead construct the list of background dates for the cycle
@@ -113,18 +112,27 @@ def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]'
     test_hist_date(os.path.join(bkg_path, ocn_filename_ic), bkg_date)  # assert date of the history file is correct
 
     for bkg in files:
-        test_hist_date(bkg, bkg_date)  # assert date of the history file is correct
+
+        # assert validity of ocean bkg, remove basename
+        test_hist_date(bkg, bkg_date)
         ocn_filename = os.path.splitext(os.path.basename(bkg))[0]+'.nc'
+
+        # aggregate seaice variables and dump aggregated ice bkg in out_path
         ice_filename = ocn_filename.replace("ocn", "ice")
         agg_ice_filename = ocn_filename.replace("ocn", "agg_ice")
         agg_seaice(os.path.join(bkg_path, ice_filename),
-                   os.path.join(bkg_path, agg_ice_filename))  # aggregate seaice variables
+                   os.path.join(out_path, agg_ice_filename))
+
+        # copy ocean bkg to out_path
+        ufsda.disk_utils.copyfile(os.path.join(bkg_path, ocn_filename),
+                                  os.path.join(out_path, ocn_filename))
+
         bkg_dict = {'date': bkg_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
-                    'basename': bkg_path+'/',
+                    'basename': './bkg/',
                     'ocn_filename': ocn_filename,
                     'ice_filename': agg_ice_filename,
                     'read_from_file': 1,
-                    'remap_filename': os.path.join(bkg_path, ocn_filename_ic)}
+                    'remap_filename': './bkg/'+ocn_filename_ic}
         bkg_date = bkg_date + timedelta(hours=1)  # TODO: make the bkg interval a configurable
         bkg_list.append(bkg_dict)
     f = open(yaml_name, 'w')
@@ -148,11 +156,14 @@ def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]'
 diags = os.path.join(anl_dir, 'diags')
 ufsda.mkdir(diags)
 
+# create output directory for obs
+bkg_dir = os.path.join(anl_dir, 'bkg')
+ufsda.mkdir(bkg_dir)
+
 # create output directory for soca DA
 anl_out = os.path.join(comout, 'ocnanal_'+os.getenv('CDATE'), 'Data')
 ufsda.mkdir(anl_out)
-ufsda.symlink(os.path.join(anl_dir, 'Data'), anl_out, remove=False)
-
+ufsda.symlink(anl_out, os.path.join(anl_dir, 'Data'), remove=False)
 
 ################################################################################
 # fetch observations
@@ -164,7 +175,6 @@ def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]'
 
 # create config dict from runtime env
 envconfig = ufsda.misc_utils.get_env_config(component='notatm')
-
 stage_cfg = YAMLFile(path=os.path.join(gdas_home,
                                        'parm',
                                        'templates',
@@ -175,13 +185,6 @@ def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]'
 # stage observations from R2D2 to COMIN_OBS and then link to analysis subdir
 ufsda.stage.obs(stage_cfg)
 
-################################################################################
-# stage backgrounds from COMIN_GES to analysis subdir
-logging.info(f"---------------- Stage backgrounds")
-
-stage_cfg['background_dir'] = os.getenv('COMIN_GES')
-ufsda.stage.background(stage_cfg)
-
 ################################################################################
 # stage static files
 logging.info(f"---------------- Stage static files")
@@ -275,9 +278,13 @@ def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]'
                                  'soca',
                                  'variational',
                                  '3dvarfgat.yaml')
+
 half_assim_freq = timedelta(hours=int(os.getenv('assim_freq'))/2)
 window_begin = datetime.strptime(os.getenv('CDATE'), '%Y%m%d%H') - half_assim_freq
-gen_bkg_list(window_begin=window_begin, bkg_path=os.getenv('COMIN_GES'), yaml_name='bkg_list.yaml')
+gen_bkg_list(bkg_path=os.getenv('COMIN_GES'),
+             out_path=bkg_dir,
+             window_begin=window_begin,
+             yaml_name='bkg_list.yaml')
 os.environ['BKG_LIST'] = 'bkg_list.yaml'
 os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml')
 logging.info(f"{config}")
@@ -289,10 +296,10 @@ def gen_bkg_list(window_begin=' ', bkg_path='.', file_type='gdas.t*.ocnf00[3-9]'
 varconfig.save(var_yaml)
 
 # link of convenience
-mom_ic = glob.glob(os.path.join(os.getenv('COMIN_GES'), 'gdas.*.ocnf003.nc'))[0]
+mom_ic = glob.glob(os.path.join(bkg_dir, 'gdas.*.ocnf003.nc'))[0]
 ufsda.disk_utils.symlink(mom_ic, os.path.join(anl_dir, 'INPUT', 'MOM.res.nc'))
 
-cice_ic = glob.glob(os.path.join(os.getenv('COMIN_GES'), 'gdas.*.agg_icef003.nc'))[0]
+cice_ic = glob.glob(os.path.join(bkg_dir, 'gdas.*.agg_icef003.nc'))[0]
 ufsda.disk_utils.symlink(cice_ic, os.path.join(anl_dir, 'INPUT', 'cice.res.nc'))
 
 # prepare input.nml

test/setup_workflow_exp.sh

@@ -17,6 +17,8 @@ configdir=$srcdir/../../parm/config
 comrot=$bindir/test/testrun/ROTDIRS
 expdir=$bindir/test/testrun/experiments
 
+# clean previous experiment
+rm -rf "${comrot}" "${expdir}"
 
 # run the script
 cd $srcdir/../../workflow

test/soca/CMakeLists.txt

@@ -97,3 +97,8 @@ set_tests_properties(test_gdasapp_soca_ana_run
       PROPERTIES
           DEPENDS "test_gdasapp_soca_ana_prep"
         )
+
+# Test exgdas scripts from the global-worflow
+if (WORKFLOW_TESTS)
+  add_subdirectory(gw)
+endif()

test/soca/gw/CMakeLists.txt

@@ -0,0 +1,31 @@
+# test for creating an experiment directory within the global-workflow
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
+
+add_test(NAME test_gdasapp_setup_soca_cycled_exp
+  COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_workflow_exp.sh
+          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
+
+add_test(NAME test_gdasapp_soca_bkg2comrot
+  COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/bkg2comrot.sh
+          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
+set_tests_properties(test_gdasapp_soca_bkg2comrot
+  PROPERTIES
+DEPENDS "test_gdasapp_soca_cycled_exp")
+
+add_test(NAME test_gdasapp_soca_jjob_soca_prep
+  COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/jjob_prep.sh
+          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
+set_tests_properties(test_gdasapp_soca_jjob_soca_prep
+  PROPERTIES
+    DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot")
+
+add_test(NAME test_gdasapp_soca_jjob_soca_run
+  COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/jjob_run.sh
+          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
+  set_tests_properties(test_gdasapp_soca_jjob_soca_run
+  PROPERTIES
+    DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot;test_gdasapp_soca_jjob_soca_prep")

test/soca/gw/bkg2comrot.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+bindir=$1
+srcdir=$2
+
+# export env. var.
+source "${srcdir}/test/soca/gw/runtime_vars.sh" "${bindir}" "${srcdir}"
+
+# prepare background from previous cycle
+mkdir -p "${ROTDIR}/${PSLOT}/gdas.${PDY}/${gcyc}/ocean/"
+cp -r "${bindir}/test/soca/bkg/"* "${ROTDIR}/${PSLOT}/gdas.${PDY}/${gcyc}/ocean/"