Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix time offset issue on ICS with GFS nemsio and netcdf files and add new archive file name on HPSS #457

Merged
merged 12 commits into from
Nov 9, 2022
Merged

Fix time offset issue on ICS with GFS nemsio and netcdf files and add new archive file name on HPSS #457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a7fef59
fix time offset with gfs
chan-hoo Nov 4, 2022
9b9658a
add new archive file name and we2e test
chan-hoo Nov 5, 2022
7a80c20
Merge branch 'develop' of https://github.com/ufs-community/ufs-srweat…
chan-hoo Nov 5, 2022
7d6d252
add ics_or_lbcs to test_retrieve.py
chan-hoo Nov 5, 2022
313a3bb
Add check availability of nemsio and netcdf
chan-hoo Nov 5, 2022
37ed64f
Copy file_templates dict before modifying it.
danielabdi-noaa Nov 6, 2022
f5e0918
remove native from xml for dev_transfer on wcoss2
chan-hoo Nov 7, 2022
5c8844e
Merge branch 'bugfix/timeoffset_gfs' of https://github.com/chan-hoo/u…
chan-hoo Nov 7, 2022
1d38fca
correct data locations on wcoss2
chan-hoo Nov 7, 2022
a021690
remove met_verification from fundamental.cheyenne.gnu
chan-hoo Nov 7, 2022
593cde9
replace cyc to hh in data locations
chan-hoo Nov 8, 2022
4a3e92d
make lbcs condition general
chan-hoo Nov 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions jobs/JREGIONAL_GET_EXTRN_MDL_FILES
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,11 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS"
if [ "${ICS_OR_LBCS}" = "ICS" ]; then
export TIME_OFFSET_HRS=${EXTRN_MDL_ICS_OFFSET_HRS:-0}
export EXTRN_MDL_NAME=${EXTRN_MDL_NAME_ICS}

GFS_FILE_FMT=${FV3GFS_FILE_FMT_ICS}
elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then
export TIME_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-0}
export EXTRN_MDL_NAME=${EXTRN_MDL_NAME_LBCS}
GFS_FILE_FMT=${FV3GFS_FILE_FMT_LBCS}
fi

#
Expand Down Expand Up @@ -159,10 +160,10 @@ case ${EXTRN_MDL_NAME} in
# The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e.
# this was the first official forecast with the FV3GFS. So we set the
# last CDATE for the GSMGFS to the one 6 hours before this.
cdate_max="2019061206"
if [ "$EXTRN_MDL_CDATE" -gt "$cdate_max" ]; then
CDATE_max="2019061206"
if [ "$EXTRN_MDL_CDATE" -gt "$CDATE_max" ]; then
print_err_msg_exit "\
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_max max)"
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $CDATE_max max)"
fi
;;

Expand All @@ -171,11 +172,20 @@ case ${EXTRN_MDL_NAME} in
# this was the first official forecast with the FV3GFS. However, paral-
# lel runs with the FV3GFS go back to 2018121500. So we set the first
# EXTRN_MDL_CDATE for the FV3GFS to this date and time.
# CDATE_min="2019061212"
CDATE_min="2018121500"
CDATE_min_netcdf="2021032100"
CDATE_max_nemsio="2021032018"
if [ "$EXTRN_MDL_CDATE" -lt "$CDATE_min" ]; then
print_err_msg_exit "\
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_min min)"
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $CDATE_min min)"
elif [ "${GFS_FILE_FMT}" = "netcdf" ] && [ "${EXTRN_MDL_CDATE}" -lt "${CDATE_min_netcdf}" ]; then
print_err_msg_exit "\
NETCDF is not available for this date:: However, NEMSIO is available:: \
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $CDATE_min_netcdf min)"
elif [ "${GFS_FILE_FMT}" = "nemsio" ] && [ "${EXTRN_MDL_CDATE}" -gt "${CDATE_max_nemsio}" ]; then
print_err_msg_exit "\
NEMSIO is not available for this date:: However, NETCDF is available:: \
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $CDATE_max_nemsio max)"
fi
;;

Expand All @@ -185,7 +195,7 @@ case ${EXTRN_MDL_NAME} in
CDATE_min="2015070100"
if [ "$EXTRN_MDL_CDATE" -lt "$CDATE_min" ]; then
print_err_msg_exit "\
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_min min)"
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $CDATE_min min)"
fi
;;

Expand All @@ -196,7 +206,7 @@ case ${EXTRN_MDL_NAME} in
CDATE_min="2014103000"
if [ "$EXTRN_MDL_CDATE" -lt "$CDATE_min" ]; then
print_err_msg_exit "\
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_min min)"
$(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $CDATE_min min)"
fi
;;

Expand Down
4 changes: 4 additions & 0 deletions parm/FV3LAM_wflow.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,9 @@ MODULES_RUN_TASK_FP script.
{%- endif %}
<walltime>{{ wtime_get_extrn_ics }}</walltime>
<nodesize>&NCORES_PER_NODE;</nodesize>
{%- if machine not in ["WCOSS2"] %}
<native>&SCHED_NATIVE_CMD;</native>
{%- endif %}
<jobname>&GET_EXTRN_ICS_TN;</jobname>
<join>&LOGDIR;/&GET_EXTRN_ICS_TN;<cyclestr>_@Y@m@d@H</cyclestr>&LOGEXT;</join>

Expand Down Expand Up @@ -335,7 +337,9 @@ MODULES_RUN_TASK_FP script.
{%- endif %}
<walltime>{{ wtime_get_extrn_lbcs }}</walltime>
<nodesize>&NCORES_PER_NODE;</nodesize>
{%- if machine not in ["WCOSS2"] %}
<native>&SCHED_NATIVE_CMD;</native>
{%- endif %}
<jobname>&GET_EXTRN_LBCS_TN;</jobname>
<join>&LOGDIR;/&GET_EXTRN_LBCS_TN;<cyclestr>_@Y@m@d@H</cyclestr>&LOGEXT;</join>

Expand Down
7 changes: 7 additions & 0 deletions parm/data_locations.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,17 +64,20 @@ FV3GFS:
- gfs.t{hh}z.sfcanl.nemsio
fcst:
- gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio
- gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio
netcdf:
anl:
- gfs.t{hh}z.atmanl.nc
- gfs.t{hh}z.sfcanl.nc
fcst:
- gfs.t{hh}z.atmf{fcst_hr:03d}.nc
- gfs.t{hh}z.sfcf{fcst_hr:03d}.nc
hpss:
protocol: htar
archive_path:
- /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
- /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
- /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
archive_internal_dir:
- ./gfs.{yyyymmdd}/{hh}
- ./gfs.{yyyymmdd}/{hh}/atmos
Expand All @@ -83,9 +86,11 @@ FV3GFS:
anl:
- gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
- com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
- com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
fcst:
- gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
- com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
- com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
nemsio:
anl:
- gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsioa.tar
Expand All @@ -97,9 +102,11 @@ FV3GFS:
anl:
- gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar
- com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar
- com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_nca.tar
fcst:
- ['gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
- ['com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
- ['com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
file_names:
<<: *gfs_file_names
aws:
Expand Down
1 change: 1 addition & 0 deletions scripts/exregional_get_extrn_mdl_files.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ python3 -u ${USHdir}/retrieve_data.py \
--data_stores ${data_stores} \
--external_model ${EXTRN_MDL_NAME} \
--fcst_hrs ${fcst_hrs[@]} \
--ics_or_lbcs ${ICS_OR_LBCS} \
--output_path ${EXTRN_MDL_STAGING_DIR} \
--summary_file ${EXTRN_DEFNS} \
$additional_flags"
Expand Down
1 change: 0 additions & 1 deletion tests/WE2E/machine_suites/fundamental.cheyenne.gnu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
MET_verification
grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
Expand Down
28 changes: 28 additions & 0 deletions ..._nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
metadata:
description: |-
This test is to ensure that the workflow running in nco mode completes
successfully on the RRFS_CONUS_25km grid using the FV3_GFS_v16 physics
suite with time-offset ICs/LBCs derived from the FV3GFS.
user:
RUN_ENVIR: nco
workflow:
CCPP_PHYS_SUITE: FV3_GFS_v16
DATE_FIRST_CYCL: '2022081012'
DATE_LAST_CYCL: '2022081012'
FCST_LEN_HRS: 6
PREEXISTING_DIR_METHOD: rename
workflow_switches:
RUN_TASK_MAKE_GRID: false
RUN_TASK_MAKE_OROG: false
RUN_TASK_MAKE_SFC_CLIMO: false
task_get_extrn_ics:
EXTRN_MDL_NAME_ICS: FV3GFS
FV3GFS_FILE_FMT_ICS: netcdf
EXTRN_MDL_ICS_OFFSET_HRS: 6
task_get_extrn_lbcs:
EXTRN_MDL_NAME_LBCS: FV3GFS
FV3GFS_FILE_FMT_LBCS: netcdf
LBC_SPEC_INTVL_HRS: 3
EXTRN_MDL_LBCS_OFFSET_HRS: 6
task_run_fcst:
PREDEF_GRID_NAME: RRFS_CONUS_25km
4 changes: 2 additions & 2 deletions ush/machine/wcoss2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ task_run_fcst:
FIXlut: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/fix/fix_lut
data:
GSMGFS: compath.py ${envir}/gsmgfs/${gsmgfs_ver}/gsmgfs.${PDY}
FV3GFS: compath.py ${envir}/gfs/${gfs_ver}/gfs.${PDY}
FV3GFS: compath.py ${envir}/gfs/${gfs_ver}/gfs.${PDY}/${hh}/atmos
RAP: compath.py ${envir}/rap/${rap_ver}/rap.${PDY}
NAM: compath.py ${envir}/nam/${nam_ver}/nam.${PDY}
HRRR: compath.py ${envir}/hrrr/${hrrr_ver}/hrrr.${PDY}
HRRR: compath.py ${envir}/hrrr/${hrrr_ver}/hrrr.${PDY}/conus

17 changes: 17 additions & 0 deletions ush/retrieve_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import sys
from textwrap import dedent
import time
from copy import deepcopy

import yaml

Expand Down Expand Up @@ -289,6 +290,16 @@ def get_file_templates(cla, known_data_info, data_store, use_cla_tmpl=False):
"""

file_templates = known_data_info.get(data_store, {}).get("file_names")
file_templates = deepcopy(file_templates)

# Remove sfc files from fcst in file_names of external models for LBCs
# sfc files needed in fcst when time_offset is not zero.
if cla.ics_or_lbcs == "LBCS":
for format in ['netcdf', 'nemsio']:
for i, tmpl in enumerate(file_templates.get(format, {}).get('fcst', [])):
if "sfc" in tmpl:
del file_templates[format]['fcst'][i]

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you may need to add a similar entry to the "nemsio" format for this to work. Changing the format to nemsio for the new test case you added I get this error:

Traceback (most recent call last):
  File "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/ufs-srweather-app/ush/retrieve_data.py", line 1012, in <module>
    main(sys.argv[1:])
  File "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/ufs-srweather-app/ush/retrieve_data.py", line 814, in main 
    file_templates = get_file_templates(
  File "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/ufs-srweather-app/ush/retrieve_data.py", line 296, in get_file_templates
    del file_templates['nemsio']['fcst'][1]
IndexError: list assignment index out of range

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is happening because the gfs_file_names anchor is shared for all sources, so when it tries hpss and doesn't succeed it deletes the sfc entry. Then when it tries aws, the entry is already gone. I think you can solve this by not using anchors in the yaml file (bad) or by making a deep copy of the dictionary before deleting the entries.

Copy link
Collaborator Author

@chan-hoo chan-hoo Nov 5, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@danielabdi-noaa, nemsio is not available for the date (08/2022). netcdf and grib2 are only available. Do we need to add any other conditions for this?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@danielabdi-noaa, I've added if-statements to check the availability of netcdf and nemsio for the cycle date in jobs/JREGIONAL_GET_EXTRN_MDL_FILES. If nemsio (or netcdf) is not available, the get_extrn_ics/lbcs will fail with an error message.

Copy link
Collaborator

@danielabdi-noaa danielabdi-noaa Nov 6, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@chan-hoo I still think we need a safeguard against multiple deletions of the sfc file -- best would be to find another way that does not delete entries in the data_locations dictionary. If hpss does not have the file for some reason (maybe it is down), and we want to try aws next, it will fail because the sfc entry has been deleted by HPSS. The invalid nemsio date run i did (although wrong) shows what could happen if we can't find the file on hpss but could be available on aws. I added this before the del if statement to make it try aws successfully when it could not find it in hpss

file_templates = deepcopy(file_templates)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@danielabdi-noaa, yes, I agree with you. Can you help me? I've sent an invitation to you.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@chan-hoo No big deal, i've pushed the change i think will avoid multiple deletions of the sfc file. It is not an ideal solution so lets wait for @christinaholtNOAA .

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you!

if use_cla_tmpl:
file_templates = cla.file_templates if cla.file_templates else file_templates

Expand Down Expand Up @@ -946,6 +957,12 @@ def parse_args(argv):
required=True,
type=os.path.abspath,
)
parser.add_argument(
"--ics_or_lbcs",
choices=("ICS", "LBCS"),
help="Flag for whether ICS or LBCS.",
required=True,
)

# Optional
parser.add_argument(
Expand Down
10 changes: 10 additions & 0 deletions ush/test_retrieve_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def test_fv3gfs_grib2_lbcs_from_hpss(self):
'--external_model', 'FV3GFS',
'--fcst_hrs', '6', '12', '3',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'LBCS',
'--debug',
'--file_type', 'grib2',
]
Expand Down Expand Up @@ -80,6 +81,7 @@ def test_fv3gfs_netcdf_lbcs_from_hpss(self):
'--external_model', 'FV3GFS',
'--fcst_hrs', '24', '48', '24',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'LBCS',
'--debug',
'--file_type', 'netcdf',
]
Expand Down Expand Up @@ -112,6 +114,7 @@ def test_gdas_ics_from_aws(self):
'--external_model', 'GDAS',
'--fcst_hrs', '6', '9', '3',
'--output_path', out_path_tmpl,
'--ics_or_lbcs', 'LBCS',
'--debug',
'--file_type', 'netcdf',
'--members', '9', '10',
Expand Down Expand Up @@ -147,6 +150,7 @@ def test_gefs_grib2_ics_from_aws(self):
'--external_model', 'GEFS',
'--fcst_hrs', '6',
'--output_path', out_path_tmpl,
'--ics_or_lbcs', 'ICS',
'--debug',
'--file_type', 'netcdf',
'--members', '1', '2',
Expand Down Expand Up @@ -180,6 +184,7 @@ def test_hrrr_ics_from_hpss(self):
'--external_model', 'HRRR',
'--fcst_hrs', '0',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'ICS',
'--debug',
]
# fmt: on
Expand Down Expand Up @@ -209,6 +214,7 @@ def test_hrrr_lbcs_from_hpss(self):
'--external_model', 'HRRR',
'--fcst_hrs', '3', '24', '3',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'LBCS',
'--debug',
]
# fmt: on
Expand Down Expand Up @@ -237,6 +243,7 @@ def test_hrrr_ics_from_aws(self):
'--external_model', 'HRRR',
'--fcst_hrs', '0',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'ICS',
'--debug',
]
# fmt: on
Expand Down Expand Up @@ -265,6 +272,7 @@ def test_hrrr_lbcs_from_aws(self):
'--external_model', 'HRRR',
'--fcst_hrs', '3', '24', '3',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'LBCS',
'--debug',
]
# fmt: on
Expand Down Expand Up @@ -294,6 +302,7 @@ def test_rap_ics_from_aws(self):
'--external_model', 'RAP',
'--fcst_hrs', '3',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'ICS',
'--debug',
]
# fmt: on
Expand Down Expand Up @@ -323,6 +332,7 @@ def test_rap_lbcs_from_aws(self):
'--external_model', 'RAP',
'--fcst_hrs', '3', '30', '6',
'--output_path', tmp_dir,
'--ics_or_lbcs', 'LBCS',
'--debug',
]
# fmt: on
Expand Down