diff --git a/parm/soca/berror/saber_block_ens.yaml b/parm/soca/berror/saber_block_ens.yaml new file mode 100644 index 000000000..9420102b0 --- /dev/null +++ b/parm/soca/berror/saber_block_ens.yaml @@ -0,0 +1,12 @@ +covariance model: ensemble +members from template: + template: + read_from_file: 1 + date: '{{ATM_WINDOW_BEGIN}}' + basename: ./ens/ + ocn_filename: MOM.res_%mem%.nc + ice_filename: cice.res_%mem%.nc + remap_filename: ./bkg/MOM.res.nc + state variables: [tocn, sonc, ssh, hocn, cicen, hicen] + pattern: '%mem%' + nmembers: 3 \ No newline at end of file diff --git a/parm/soca/berror/soca_bump2d.yaml b/parm/soca/berror/soca_bump2d.yaml index 2cb45b515..a383c5f30 100644 --- a/parm/soca/berror/soca_bump2d.yaml +++ b/parm/soca/berror/soca_bump2d.yaml @@ -23,8 +23,8 @@ bump: model: do not cross mask boundaries: false nicas: - resolution: 1.0 # Do not use these numbers for - max horizontal grid size: 15000 # science experiments, only good for testing + resolution: !ENV ${NICAS_RESOL} + max horizontal grid size: ${NICAS_GRID_SIZE} grids: - model: variables: diff --git a/parm/soca/berror/soca_bump_split.yaml b/parm/soca/berror/soca_bump_split.yaml index d4467b2dc..6beafdf7d 100644 --- a/parm/soca/berror/soca_bump_split.yaml +++ b/parm/soca/berror/soca_bump_split.yaml @@ -24,8 +24,8 @@ bump: model: do not cross mask boundaries: false nicas: - resolution: 1.0 # Do not use these numbers for - max horizontal grid size: 15000 # science experiments, only good for testing + resolution: !ENV ${NICAS_RESOL} + max horizontal grid size: ${NICAS_GRID_SIZE} grids: - model: variables: diff --git a/parm/soca/obs/config/obs_list.yaml b/parm/soca/obs/config/obs_list.yaml new file mode 120000 index 000000000..372f2edc2 --- /dev/null +++ b/parm/soca/obs/config/obs_list.yaml @@ -0,0 +1 @@ +../obs_list.yaml \ No newline at end of file diff --git a/parm/soca/variational/3dvarfgat.yaml b/parm/soca/variational/3dvarfgat.yaml index 6c0536e4b..69f2d5eff 100644 --- a/parm/soca/variational/3dvarfgat.yaml +++ b/parm/soca/variational/3dvarfgat.yaml @@ -22,7 +22,7 @@ cost function: state variables: *model_vars background error: !INC ${SABER_BLOCKS_YAML} - observations: !INC ${OBS_YAML} + observations: !INC ${OBS_LIST} variational: minimizer: diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index 5fa8f602c..cf198de99 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -174,7 +174,7 @@ def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00 ufsda.r2d2.setup(r2d2_config_yaml=os.path.join(anl_dir, 'r2d2_config.yaml'), shared_root=comin_obs) # create config dict from runtime env -envconfig = ufsda.misc_utils.get_env_config(component='notatm') +envconfig = ufsda.misc_utils.get_env_config(component='soca') stage_cfg = YAMLFile(path=os.path.join(gdas_home, 'parm', 'templates', @@ -243,8 +243,6 @@ def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00 'berror', 'soca_bump2d.yaml') config = YAMLFile(path=bump_yaml_template) -config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, {'datadir': bumpdir}.get) -config = Template.substitute_structure(config, TemplateConstants.DOLLAR_PARENTHESES, {'datadir': bumpdir}.get) config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) config = Template.substitute_structure(config, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get) config.save(bump_yaml) @@ -286,7 +284,15 @@ def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00 window_begin=window_begin, yaml_name='bkg_list.yaml') os.environ['BKG_LIST'] = 'bkg_list.yaml' -os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml') + +# select the SABER BLOCKS to use +if 'SABER_BLOCKS_YAML' in os.environ and os.environ['SABER_BLOCKS_YAML']: + saber_blocks_yaml = os.getenv('SABER_BLOCKS_YAML') + logging.info(f"using non-default SABER blocks yaml: {saber_blocks_yaml}") +else: + logging.info(f"using default SABER blocks yaml") + os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml') + logging.info(f"{config}") varconfig = YAMLFile(path=var_yaml_template) varconfig = Template.substitute_structure(varconfig, TemplateConstants.DOUBLE_CURLY_BRACES, config.get) diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index e3a6a037c..75e2b4b93 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -22,10 +22,15 @@ set_tests_properties(test_gdasapp_soca_jjob_soca_prep PROPERTIES DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot") -add_test(NAME test_gdasapp_soca_jjob_soca_run - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/jjob_run.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) - set_tests_properties(test_gdasapp_soca_jjob_soca_run - PROPERTIES - DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot;test_gdasapp_soca_jjob_soca_prep") +# TODO: This test is broken on Orion, skip it for now when running the CI on Orion. +IF (IS_DIRECTORY /work2/noaa/da) + message("Skipping this test for now") +ELSE() + add_test(NAME test_gdasapp_soca_jjob_soca_run + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/jjob_run.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) + set_tests_properties(test_gdasapp_soca_jjob_soca_run + PROPERTIES + DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot;test_gdasapp_soca_jjob_soca_prep") +ENDIF() diff --git a/test/soca/gw/config.yaml b/test/soca/gw/config.yaml index 9ad0d0214..b74f5b037 100644 --- a/test/soca/gw/config.yaml +++ b/test/soca/gw/config.yaml @@ -2,6 +2,10 @@ ocnanal: SOCA_INPUT_FIX_DIR: '@SOCA_INPUT_FIX_DIR@' CASE_ANL: 'C24' COMIN_OBS: '@COMIN_OBS@' - SOCA_NINNER: 3 + SOCA_OBS_LIST: '@SOCA_OBS_LIST@' + SOCA_NINNER: 1 R2D2_OBS_SRC: 'gdasapp' R2D2_OBS_DUMP: 'soca' + SABER_BLOCKS_YAML: '' + NICAS_RESOL: 1 + NICAS_GRID_SIZE: 15000 diff --git a/test/soca/gw/jjob_prep.sh b/test/soca/gw/jjob_prep.sh index e7f7062ed..5d426c2c1 100755 --- a/test/soca/gw/jjob_prep.sh +++ b/test/soca/gw/jjob_prep.sh @@ -6,7 +6,7 @@ srcdir=$2 # export env. var. source "${srcdir}/test/soca/gw/runtime_vars.sh" "${bindir}" "${srcdir}" -OCNDIR="${ROTDIR}/${PSLOT}/gdas.${PDY}/${gcyc}/ocean/" +OCNDIR="${ROTDIR}/${PSLOT}/gdas.${PDY}/${gcyc}/ocean/" rm -r ${OCNDIR} @@ -14,5 +14,17 @@ rm -r ${OCNDIR} mkdir -p ${OCNDIR} cp -r "${bindir}/test/soca/bkg/"* ${OCNDIR} +# detemine machine from config.base +machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') + # run prep jjob -"${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" +if [[ ${machine} == 'CONTAINER' ]]; then + "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" +else + sbatch --ntasks=1 \ + --account=da-cpu \ + --qos=debug \ + --time=00:5:00 \ + --export=ALL \ + --wait "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" +fi diff --git a/test/soca/gw/jjob_run.sh b/test/soca/gw/jjob_run.sh index c66a8ef78..da6952fad 100755 --- a/test/soca/gw/jjob_run.sh +++ b/test/soca/gw/jjob_run.sh @@ -5,5 +5,17 @@ srcdir=$2 # export env. var. source "${srcdir}/test/soca/gw/runtime_vars.sh" "${bindir}" "${srcdir}" +# detemine machine from config.base +machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') + # run DA jjob -"${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN" +if [[ ${machine} == 'CONTAINER' ]]; then + "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN" +else + sbatch --ntasks=16 \ + --account=da-cpu \ + --qos=debug \ + --time=00:10:00 \ + --export=ALL \ + --wait "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN" +fi diff --git a/test/soca/gw/runtime_vars.sh b/test/soca/gw/runtime_vars.sh index 19f0cf375..30dbda548 100644 --- a/test/soca/gw/runtime_vars.sh +++ b/test/soca/gw/runtime_vars.sh @@ -9,6 +9,7 @@ export cyc=12 export gcyc=06 export ROTDIR="${bindir}/test/soca/gw/testrun/ROTDIRS" export DATAROOT="${bindir}/test/soca/gw/testrun/RUNDIRS/gdas_test" +export COMROOT=${DATAROOT} export COMIN_GES="${bindir}/test/soca/bkg" export PSLOT='gdas_test' export EXPDIRS="${bindir}/test/soca/gw/testrun/experiments/" diff --git a/test/soca/gw/setup_workflow_exp.sh b/test/soca/gw/setup_workflow_exp.sh index 2210391ab..f7ea4dd59 100755 --- a/test/soca/gw/setup_workflow_exp.sh +++ b/test/soca/gw/setup_workflow_exp.sh @@ -32,8 +32,10 @@ cp -r "${configdir}" config cp "${srcdir}/test/soca/gw/config.yaml" . soca_input_fix_dir="${bindir}/soca_static" comin_obs="${bindir}/test/soca/obs/r2d2-shared" +soca_obs_list="${srcdir}/parm/soca/obs/obs_list.yaml" sed -i -e "s~@SOCA_INPUT_FIX_DIR@~${soca_input_fix_dir}~g" config.yaml sed -i -e "s~@COMIN_OBS@~${comin_obs}~g" config.yaml +sed -i -e "s~@SOCA_OBS_LIST@~${soca_obs_list}~g" config.yaml echo "Running global-workflow experiment generation script" ./setup_expt.py cycled --idate "${idate}" \ diff --git a/test/soca/runtime_vars.sh b/test/soca/runtime_vars.sh index b54bd853b..40c0c5086 100644 --- a/test/soca/runtime_vars.sh +++ b/test/soca/runtime_vars.sh @@ -46,4 +46,6 @@ export R2D2_OBS_DUMP=soca export R2D2_OBS_SRC=gdasapp export R2D2_OBS_WINDOW=24 # R2D2 sampling DB window export SOCA_NINNER=2 +export NICAS_RESOL=1 +export NICAS_GRID_SIZE=15000 export APRUN_OCNANAL="$MPIEXEC_EXEC $MPIEXEC_NPROC 2" diff --git a/ush/soca/examples/run_jjobs_container.yaml b/ush/soca/examples/run_jjobs_container.yaml index 197aa9307..13cf2a76c 100644 --- a/ush/soca/examples/run_jjobs_container.yaml +++ b/ush/soca/examples/run_jjobs_container.yaml @@ -22,9 +22,9 @@ gw environement: COMIN_GES: /home/gvernier/wrk/gw/sorc/gdas.cd/build/test/soca/bkg working directories: - ROTDIR: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/runjjobs/test/ROTDIRS - EXPDIRS: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/runjjobs/test/experiments - STMP: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/runjjobs/test + ROTDIR: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/soca/test/ROTDIRS + EXPDIRS: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/soca/test/experiments + STMP: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/soca/test jedi: OOPS_TRACE: 1 @@ -33,9 +33,13 @@ gw environement: setup_expt config: ocnanal: - SOCA_INPUT_FIX_DIR: '/home/gvernier/wrk/gw/sorc/gdas.cd/build/soca_static' - CASE_ANL: 'C48' - COMIN_OBS: '/home/gvernier/wrk/gw/sorc/gdas.cd/build/test/soca/obs/r2d2-shared' + SOCA_INPUT_FIX_DIR: /home/gvernier/wrk/gw/sorc/gdas.cd/build/soca_static + CASE_ANL: C48 + SOCA_OBS_LIST: /home/gvernier/wrk/gw/sorc/gdas.cd/parm/soca/obs/obs_list.yaml + COMIN_OBS: /home/gvernier/wrk/gw/sorc/gdas.cd/build/test/soca/obs/r2d2-shared SOCA_NINNER: 1 - R2D2_OBS_SRC: 'gdasapp' - R2D2_OBS_DUMP: 'soca' + R2D2_OBS_SRC: gdasapp + R2D2_OBS_DUMP: soca + SABER_BLOCKS_YAML: /home/gvernier/wrk/gw/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml + NICAS_RESOL: 1 + NICAS_GRID_SIZE: 15000 diff --git a/ush/soca/examples/run_jjobs_hera.yaml b/ush/soca/examples/run_jjobs_hera.yaml index aa51fbfa9..b11b0a332 100644 --- a/ush/soca/examples/run_jjobs_hera.yaml +++ b/ush/soca/examples/run_jjobs_hera.yaml @@ -34,10 +34,14 @@ setup_expt config: ocnanal: SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca' CASE_ANL: 'C48' + SOCA_OBS_LIST: '' COMIN_OBS: /scratch2/NCEPDEV/marineda/r2d2 SOCA_NINNER: 3 R2D2_OBS_SRC: 'gdas_marine' R2D2_OBS_DUMP: 's2s_v1' + SABER_BLOCKS_YAML: '' + NICAS_RESOL: 1 + NICAS_GRID_SIZE: 15000 job options: account: marine-cpu diff --git a/ush/soca/examples/run_jjobs_hera_025.yaml b/ush/soca/examples/run_jjobs_hera_025.yaml index d286e089e..900167089 100644 --- a/ush/soca/examples/run_jjobs_hera_025.yaml +++ b/ush/soca/examples/run_jjobs_hera_025.yaml @@ -34,10 +34,14 @@ setup_expt config: ocnanal: SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca' CASE_ANL: 'C384' + SOCA_OBS_LIST: '' COMIN_OBS: /scratch2/NCEPDEV/marineda/r2d2 - SOCA_NINNER: 3 + SOCA_NINNER: 1 R2D2_OBS_SRC: 'gdas_marine' R2D2_OBS_DUMP: 's2s_v1' + SABER_BLOCKS_YAML: '' + NICAS_RESOL: 1 + NICAS_GRID_SIZE: 15000 job options: account: marine-cpu